Compare commits
No commits in common. "main" and "release-ea887c8f56cd" have entirely different histories.
main
...
release-ea
132 changed files with 19425 additions and 7991 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -1,5 +1,8 @@
|
||||||
/target/
|
/target/
|
||||||
/web/target/
|
|
||||||
/.clusterflux/
|
/.clusterflux/
|
||||||
**/.clusterflux/
|
**/.clusterflux/
|
||||||
/vscode-extension/node_modules/
|
/vscode-extension/node_modules/
|
||||||
|
/private/*/Cargo.lock
|
||||||
|
!/private/hosted-policy/Cargo.lock
|
||||||
|
/private/*/target/
|
||||||
|
/scripts/containers-home/
|
||||||
|
|
|
||||||
22
CLUSTERFLUX_PUBLIC_TREE.json
Normal file
22
CLUSTERFLUX_PUBLIC_TREE.json
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
{
|
||||||
|
"kind": "clusterflux-filtered-public-tree",
|
||||||
|
"source_commit": "ea887c8f56cd53985a1179b13e5f1b85c485f584",
|
||||||
|
"release_name": "release-ea887c8f56cd",
|
||||||
|
"filtered_out": [
|
||||||
|
"private/**",
|
||||||
|
"internal/**",
|
||||||
|
"experiments/**",
|
||||||
|
".git",
|
||||||
|
"target",
|
||||||
|
"git-ignored source paths",
|
||||||
|
"root/*.md except README.md",
|
||||||
|
"**/.clusterflux/**",
|
||||||
|
".forgejo/**"
|
||||||
|
],
|
||||||
|
"public_export": {
|
||||||
|
"host_neutral": true,
|
||||||
|
"include_forgejo_workflows": false
|
||||||
|
},
|
||||||
|
"forgejo_host": "git.michelpaulissen.com",
|
||||||
|
"default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
|
||||||
|
}
|
||||||
24
Cargo.lock
generated
24
Cargo.lock
generated
|
|
@ -290,20 +290,6 @@ dependencies = [
|
||||||
"wasmparser 0.245.1",
|
"wasmparser 0.245.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "clusterflux-client"
|
|
||||||
version = "0.1.0"
|
|
||||||
dependencies = [
|
|
||||||
"base64",
|
|
||||||
"clusterflux-control",
|
|
||||||
"clusterflux-coordinator",
|
|
||||||
"clusterflux-core",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"thiserror 1.0.69",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "clusterflux-control"
|
name = "clusterflux-control"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
|
|
@ -1726,6 +1712,16 @@ dependencies = [
|
||||||
"windows-sys 0.52.0",
|
"windows-sys 0.52.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "runtime-conformance"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"clusterflux-sdk",
|
||||||
|
"futures-executor",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.3"
|
version = "2.1.3"
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,7 @@
|
||||||
[workspace]
|
[workspace]
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
exclude = ["web"]
|
|
||||||
members = [
|
members = [
|
||||||
"crates/clusterflux-cli",
|
"crates/clusterflux-cli",
|
||||||
"crates/clusterflux-client",
|
|
||||||
"crates/clusterflux-control",
|
"crates/clusterflux-control",
|
||||||
"crates/clusterflux-coordinator",
|
"crates/clusterflux-coordinator",
|
||||||
"crates/clusterflux-core",
|
"crates/clusterflux-core",
|
||||||
|
|
@ -14,12 +12,13 @@ members = [
|
||||||
"crates/clusterflux-wasm-runtime",
|
"crates/clusterflux-wasm-runtime",
|
||||||
"examples/hello-build",
|
"examples/hello-build",
|
||||||
"examples/recovery-build",
|
"examples/recovery-build",
|
||||||
|
"tests/fixtures/runtime-conformance",
|
||||||
]
|
]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
license = "Apache-2.0 OR MIT"
|
license = "Apache-2.0 OR MIT"
|
||||||
repository = "https://clusterflux.lesstuff.com"
|
repository = "https://git.michelpaulissen.com/michel/clusterflux-public"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
anyhow = "1.0"
|
anyhow = "1.0"
|
||||||
|
|
|
||||||
59
README.md
59
README.md
|
|
@ -1,57 +1,9 @@
|
||||||
# Clusterflux
|
# Clusterflux
|
||||||
|
|
||||||
Clusterflux runs a Rust-defined workflow as one distributed virtual process. The
|
Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
|
||||||
async main runs serverless, provisioning nodes to run tasks through rootless
|
coordinator hosts the async main, while attached nodes execute Wasm tasks,
|
||||||
containers. Tasks on nodes exchange data simply and efficiently.
|
rootless containers, and native commands. Tasks exchange canonical values and
|
||||||
|
portable typed handles instead of sharing host memory.
|
||||||
The user experience is built to be as much as possible like building a regular
|
|
||||||
program. The processes are debuggable as normal through a debugger adapter.
|
|
||||||
|
|
||||||
The primary use case is to consolidate build processes into a single streamlined
|
|
||||||
developer experience. An example program can be seen below:
|
|
||||||
|
|
||||||
~~~rust
|
|
||||||
use clusterflux::prelude::*;
|
|
||||||
|
|
||||||
#[clusterflux::task(capabilities = "command")]
|
|
||||||
pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
|
|
||||||
let executable = fs::output("hello-clusterflux")?;
|
|
||||||
Command::new("cc")
|
|
||||||
.args([
|
|
||||||
"-Os",
|
|
||||||
"-static",
|
|
||||||
"-s",
|
|
||||||
"fixture/hello-clusterflux.c",
|
|
||||||
"-o",
|
|
||||||
executable.as_str(),
|
|
||||||
])
|
|
||||||
.cwd(source.mount()?)
|
|
||||||
.env("SOURCE_DATE_EPOCH", "0")
|
|
||||||
.network_disabled()
|
|
||||||
.run()
|
|
||||||
.await?;
|
|
||||||
fs::publish(&executable).await
|
|
||||||
}
|
|
||||||
|
|
||||||
#[clusterflux::main]
|
|
||||||
pub async fn build() -> Result<Artifact> {
|
|
||||||
let source = source::current_project().snapshot().await?;
|
|
||||||
let compile = clusterflux::spawn!(compile(source))
|
|
||||||
.on(clusterflux::env!("linux"))
|
|
||||||
.await?;
|
|
||||||
compile.join().await
|
|
||||||
}
|
|
||||||
~~~
|
|
||||||
|
|
||||||
After setup, this build pipeline could be deployed as easily as launching it
|
|
||||||
through your IDE. This repository includes a VS Code extension to make
|
|
||||||
development as straightforward as possible. A full collection of CLI tools is
|
|
||||||
included for advanced usage.
|
|
||||||
|
|
||||||
Clusterflux is explicitly local-first. It is trivial to provision existing
|
|
||||||
hardware as resources. Bulk data will typically not leave the local network,
|
|
||||||
allowing maximum throughput. The same capability, however, also makes it
|
|
||||||
possible to leverage cloud resources easily.
|
|
||||||
|
|
||||||
Start with [Getting started](docs/getting-started.md). It takes you through
|
Start with [Getting started](docs/getting-started.md). It takes you through
|
||||||
authentication, project setup, node enrollment, a run, debugging, task restart,
|
authentication, project setup, node enrollment, a run, debugging, task restart,
|
||||||
|
|
@ -97,9 +49,6 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato
|
||||||
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
|
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
On NixOS or another system with Nix, the equivalent package is available with
|
|
||||||
`nix profile install .#clusterflux-tools`.
|
|
||||||
|
|
||||||
Rootless Podman is required on Linux nodes that build or run a declared
|
Rootless Podman is required on Linux nodes that build or run a declared
|
||||||
Containerfile environment. Install VS Code when you want the graphical debug
|
Containerfile environment. Install VS Code when you want the graphical debug
|
||||||
workflow.
|
workflow.
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained.
|
||||||
|
|
||||||
## Report a vulnerability
|
## Report a vulnerability
|
||||||
|
|
||||||
Email security@clusterflux.lesstuff.com with a concise description, affected version
|
Email security@michelpaulissen.com with a concise description, affected version
|
||||||
or source revision, reproduction steps, and impact. Do not open a public issue
|
or source revision, reproduction steps, and impact. Do not open a public issue
|
||||||
for an unpatched vulnerability or include credentials, session tokens, private
|
for an unpatched vulnerability or include credentials, session tokens, private
|
||||||
keys, provider tokens, customer data, or operator secrets in a public report.
|
keys, provider tokens, customer data, or operator secrets in a public report.
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
|
||||||
.unwrap_or(json!(false)),
|
.unwrap_or(json!(false)),
|
||||||
"response": response,
|
"response": response,
|
||||||
"safe_default": "read_only",
|
"safe_default": "read_only",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
|
||||||
"command": "admin status",
|
"command": "admin status",
|
||||||
"mode": "self_hosted_local",
|
"mode": "self_hosted_local",
|
||||||
"safe_default": "read_only",
|
"safe_default": "read_only",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
"project": project.clone(),
|
"project": project.clone(),
|
||||||
"user": user,
|
"user": user,
|
||||||
"coordinator": scope.coordinator,
|
"coordinator": scope.coordinator,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"self_hosted_cli_only": true,
|
"self_hosted_cli_only": true,
|
||||||
"project_config_written": project_init
|
"project_config_written": project_init
|
||||||
.get("project_config_written")
|
.get("project_config_written")
|
||||||
|
|
@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
{
|
{
|
||||||
"step": "start_self_hosted_coordinator",
|
"step": "start_self_hosted_coordinator",
|
||||||
"command": "clusterflux-coordinator --listen 127.0.0.1:0",
|
"command": "clusterflux-coordinator --listen 127.0.0.1:0",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "create_or_link_project",
|
"step": "create_or_link_project",
|
||||||
"command": "clusterflux project init --yes",
|
"command": "clusterflux project init --yes",
|
||||||
"completed": true,
|
"completed": true,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "create_node_enrollment_grant",
|
"step": "create_node_enrollment_grant",
|
||||||
|
|
@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
"clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
|
"clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
|
||||||
tenant, project
|
tenant, project
|
||||||
),
|
),
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "attach_worker_node",
|
"step": "attach_worker_node",
|
||||||
|
|
@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
"clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
|
"clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
|
||||||
tenant, project
|
tenant, project
|
||||||
),
|
),
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "run_process",
|
"step": "run_process",
|
||||||
|
|
@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
"clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
|
"clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
|
||||||
tenant, project
|
tenant, project
|
||||||
),
|
),
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "inspect_status_logs_artifacts",
|
"step": "inspect_status_logs_artifacts",
|
||||||
|
|
@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
|
||||||
"clusterflux artifact list",
|
"clusterflux artifact list",
|
||||||
"clusterflux quota status",
|
"clusterflux quota status",
|
||||||
],
|
],
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"step": "revoke_access",
|
"step": "revoke_access",
|
||||||
"command": "clusterflux admin revoke-node --node <node-id> --yes",
|
"command": "clusterflux admin revoke-node --node <node-id> --yes",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
}))
|
}))
|
||||||
|
|
@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
|
||||||
"actor_tenant": actor_tenant,
|
"actor_tenant": actor_tenant,
|
||||||
"actor_user": actor_user,
|
"actor_user": actor_user,
|
||||||
"suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
|
"suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"response": response,
|
"response": response,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
}));
|
}));
|
||||||
|
|
@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
|
||||||
"status": "requires_coordinator",
|
"status": "requires_coordinator",
|
||||||
"requires_confirmation": !args.yes,
|
"requires_confirmation": !args.yes,
|
||||||
"tenant": tenant,
|
"tenant": tenant,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,6 @@ use crate::client::{
|
||||||
};
|
};
|
||||||
use crate::config::StoredCliSession;
|
use crate::config::StoredCliSession;
|
||||||
use crate::errors::cli_error_summary_for_category;
|
use crate::errors::cli_error_summary_for_category;
|
||||||
use crate::process::hydrate_process_scope;
|
|
||||||
use crate::process_events::{
|
use crate::process_events::{
|
||||||
artifact_download_grant_disclosures, artifact_download_session_summary,
|
artifact_download_grant_disclosures, artifact_download_session_summary,
|
||||||
artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
|
artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
|
||||||
|
|
@ -28,10 +27,9 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn artifact_list_report_with_session(
|
pub(crate) fn artifact_list_report_with_session(
|
||||||
mut args: ArtifactListArgs,
|
args: ArtifactListArgs,
|
||||||
stored_session: Option<&StoredCliSession>,
|
stored_session: Option<&StoredCliSession>,
|
||||||
) -> Result<Value> {
|
) -> Result<Value> {
|
||||||
hydrate_process_scope(&mut args.scope, stored_session);
|
|
||||||
let events = list_task_events_if_available_with_session(
|
let events = list_task_events_if_available_with_session(
|
||||||
args.scope.coordinator.as_deref(),
|
args.scope.coordinator.as_deref(),
|
||||||
&args.scope,
|
&args.scope,
|
||||||
|
|
@ -55,10 +53,9 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Val
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn artifact_download_report_with_session(
|
pub(crate) fn artifact_download_report_with_session(
|
||||||
mut args: ArtifactDownloadArgs,
|
args: ArtifactDownloadArgs,
|
||||||
stored_session: Option<&StoredCliSession>,
|
stored_session: Option<&StoredCliSession>,
|
||||||
) -> Result<Value> {
|
) -> Result<Value> {
|
||||||
hydrate_process_scope(&mut args.scope, stored_session);
|
|
||||||
if let Some(coordinator) = &args.scope.coordinator {
|
if let Some(coordinator) = &args.scope.coordinator {
|
||||||
let mut session = JsonLineSession::connect(coordinator)?;
|
let mut session = JsonLineSession::connect(coordinator)?;
|
||||||
let response = session.request(authenticated_or_local_trusted_request(
|
let response = session.request(authenticated_or_local_trusted_request(
|
||||||
|
|
@ -146,10 +143,9 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value>
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn artifact_export_report_with_session(
|
pub(crate) fn artifact_export_report_with_session(
|
||||||
mut args: ArtifactExportArgs,
|
args: ArtifactExportArgs,
|
||||||
stored_session: Option<&StoredCliSession>,
|
stored_session: Option<&StoredCliSession>,
|
||||||
) -> Result<Value> {
|
) -> Result<Value> {
|
||||||
hydrate_process_scope(&mut args.scope, stored_session);
|
|
||||||
if let Some(coordinator) = &args.scope.coordinator {
|
if let Some(coordinator) = &args.scope.coordinator {
|
||||||
let mut session = JsonLineSession::connect(coordinator)?;
|
let mut session = JsonLineSession::connect(coordinator)?;
|
||||||
let response = session.request(authenticated_or_local_trusted_request(
|
let response = session.request(authenticated_or_local_trusted_request(
|
||||||
|
|
|
||||||
|
|
@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<V
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
|
|
@ -263,7 +263,7 @@ fn coordinator_auth_status_summary(
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"machine_error": cli_error_summary(&message),
|
"machine_error": cli_error_summary(&message),
|
||||||
"error": message,
|
"error": message,
|
||||||
|
|
@ -296,7 +296,7 @@ fn coordinator_auth_status_summary(
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"machine_error": cli_error_summary(&message),
|
"machine_error": cli_error_summary(&message),
|
||||||
"error": message,
|
"error": message,
|
||||||
|
|
@ -316,7 +316,7 @@ fn coordinator_auth_status_summary(
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"machine_error": cli_error_summary(&message),
|
"machine_error": cli_error_summary(&message),
|
||||||
"error": message,
|
"error": message,
|
||||||
|
|
@ -338,7 +338,7 @@ fn coordinator_auth_status_summary(
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"machine_error": cli_error_summary(message),
|
"machine_error": cli_error_summary(message),
|
||||||
"coordinator_response_type": "error",
|
"coordinator_response_type": "error",
|
||||||
|
|
@ -403,7 +403,7 @@ fn coordinator_auth_status_summary(
|
||||||
"manual_review": manual_review,
|
"manual_review": manual_review,
|
||||||
"sanitized_reason": sanitized_reason,
|
"sanitized_reason": sanitized_reason,
|
||||||
"next_actions": next_actions,
|
"next_actions": next_actions,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
|
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
|
||||||
"coordinator_session_requests": coordinator_session_requests,
|
"coordinator_session_requests": coordinator_session_requests,
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
|
||||||
"account_status": "unknown",
|
"account_status": "unknown",
|
||||||
"suspension_known": false,
|
"suspension_known": false,
|
||||||
"account_state_known": false,
|
"account_state_known": false,
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"next_actions": ["clusterflux login --browser"],
|
"next_actions": ["clusterflux login --browser"],
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::CliScopeArgs;
|
use crate::CliScopeArgs;
|
||||||
|
|
||||||
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com";
|
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
|
||||||
|
"https://clusterflux.michelpaulissen.com";
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub(crate) struct ProjectConfig {
|
pub(crate) struct ProjectConfig {
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
|
||||||
"command": "dap",
|
"command": "dap",
|
||||||
"adapter": dap_binary_path()?.display().to_string(),
|
"adapter": dap_binary_path()?.display().to_string(),
|
||||||
"args": args.args,
|
"args": args.args,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_else(|| json!(0)),
|
.unwrap_or_else(|| json!(0)),
|
||||||
"debug_reads_quota_limited": true,
|
"debug_reads_quota_limited": true,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
|
||||||
"dap": dap,
|
"dap": dap,
|
||||||
"authorized": "unknown_without_coordinator",
|
"authorized": "unknown_without_coordinator",
|
||||||
"debug_reads_quota_limited": "unknown_without_coordinator",
|
"debug_reads_quota_limited": "unknown_without_coordinator",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -38,10 +38,7 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s
|
||||||
);
|
);
|
||||||
object.insert("community_tier_language".to_owned(), json!(true));
|
object.insert("community_tier_language".to_owned(), json!(true));
|
||||||
object.insert("community_tier_label".to_owned(), json!("community tier"));
|
object.insert("community_tier_label".to_owned(), json!("community tier"));
|
||||||
object.insert(
|
object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
|
||||||
"sensitive_abuse_heuristics_exposed".to_owned(),
|
|
||||||
json!(false),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
summary
|
summary
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ use serde_json::{json, Value};
|
||||||
|
|
||||||
use crate::client::list_task_events_if_available_with_session;
|
use crate::client::list_task_events_if_available_with_session;
|
||||||
use crate::config::StoredCliSession;
|
use crate::config::StoredCliSession;
|
||||||
use crate::process::hydrate_process_scope;
|
|
||||||
use crate::process_events::log_entries;
|
use crate::process_events::log_entries;
|
||||||
use crate::LogsArgs;
|
use crate::LogsArgs;
|
||||||
|
|
||||||
|
|
@ -13,10 +12,9 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn logs_report_with_session(
|
pub(crate) fn logs_report_with_session(
|
||||||
mut args: LogsArgs,
|
args: LogsArgs,
|
||||||
stored_session: Option<&StoredCliSession>,
|
stored_session: Option<&StoredCliSession>,
|
||||||
) -> Result<Value> {
|
) -> Result<Value> {
|
||||||
hydrate_process_scope(&mut args.scope, stored_session);
|
|
||||||
let events = list_task_events_if_available_with_session(
|
let events = list_task_events_if_available_with_session(
|
||||||
args.scope.coordinator.as_deref(),
|
args.scope.coordinator.as_deref(),
|
||||||
&args.scope,
|
&args.scope,
|
||||||
|
|
|
||||||
|
|
@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
|
||||||
"tenant": tenant,
|
"tenant": tenant,
|
||||||
"project": project,
|
"project": project,
|
||||||
"user": user,
|
"user": user,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"enrollment_grant": enrollment_grant,
|
"enrollment_grant": enrollment_grant,
|
||||||
"response": response,
|
"response": response,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
|
|
@ -161,7 +161,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
|
||||||
Ok(json!({
|
Ok(json!({
|
||||||
"command": "node enroll",
|
"command": "node enroll",
|
||||||
"status": "requires_coordinator",
|
"status": "requires_coordinator",
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"requested_ttl_seconds": args.ttl_seconds,
|
"requested_ttl_seconds": args.ttl_seconds,
|
||||||
"enrollment_grant": null,
|
"enrollment_grant": null,
|
||||||
"reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
|
"reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
|
||||||
|
|
|
||||||
|
|
@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String {
|
||||||
}
|
}
|
||||||
push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
|
push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
|
||||||
if let Some(exposed) = account
|
if let Some(exposed) = account
|
||||||
.get("sensitive_moderation_details_exposed")
|
.get("private_moderation_details_exposed")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
{
|
{
|
||||||
lines.push(format!("sensitive moderation details exposed: {exposed}"));
|
lines.push(format!("private moderation details exposed: {exposed}"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(coordinator_selection) = value.get("coordinator") {
|
if let Some(coordinator_selection) = value.get("coordinator") {
|
||||||
|
|
@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if let Some(flag) = value
|
if let Some(flag) = value
|
||||||
.get("external_website_required")
|
.get("private_website_required")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
{
|
{
|
||||||
lines.push(format!("external website required: {flag}"));
|
lines.push(format!("private website required: {flag}"));
|
||||||
}
|
}
|
||||||
if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
|
if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
|
||||||
let actions = next_actions
|
let actions = next_actions
|
||||||
|
|
|
||||||
|
|
@ -15,10 +15,7 @@ use crate::{
|
||||||
ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
|
ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(crate) fn hydrate_process_scope(
|
fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
|
||||||
scope: &mut CliScopeArgs,
|
|
||||||
stored_session: Option<&StoredCliSession>,
|
|
||||||
) {
|
|
||||||
if scope.coordinator.is_none() {
|
if scope.coordinator.is_none() {
|
||||||
scope.coordinator = stored_session
|
scope.coordinator = stored_session
|
||||||
.filter(|session| session.session_secret.is_some())
|
.filter(|session| session.session_secret.is_some())
|
||||||
|
|
|
||||||
|
|
@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
|
||||||
"cross_tenant_reuse_allowed": false,
|
"cross_tenant_reuse_allowed": false,
|
||||||
"unauthorized_project_reuse_allowed": false,
|
"unauthorized_project_reuse_allowed": false,
|
||||||
"default_durable_store_assumed": false,
|
"default_durable_store_assumed": false,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}])
|
}])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<
|
||||||
"current_usage": current_usage,
|
"current_usage": current_usage,
|
||||||
"limits": quota_limits_value(),
|
"limits": quota_limits_value(),
|
||||||
"next_blocked_action": next_blocked_action,
|
"next_blocked_action": next_blocked_action,
|
||||||
"sensitive_abuse_heuristics_exposed": false,
|
"private_abuse_heuristics_exposed": false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
|
||||||
Ok(json!({
|
Ok(json!({
|
||||||
"command": "project init",
|
"command": "project init",
|
||||||
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
|
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"project_config_written": true,
|
"project_config_written": true,
|
||||||
"project_config_write_after_coordinator_acceptance": coordinator.is_some(),
|
"project_config_write_after_coordinator_acceptance": coordinator.is_some(),
|
||||||
"coordinator_create_before_local_write": coordinator.is_some(),
|
"coordinator_create_before_local_write": coordinator.is_some(),
|
||||||
|
|
@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
|
||||||
"config_format": "clusterflux_project_config_v1",
|
"config_format": "clusterflux_project_config_v1",
|
||||||
"links_current_directory": true,
|
"links_current_directory": true,
|
||||||
"writes_current_directory_only": true,
|
"writes_current_directory_only": true,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
"safe_defaults": {
|
"safe_defaults": {
|
||||||
"tenant": config.tenant.clone(),
|
"tenant": config.tenant.clone(),
|
||||||
|
|
@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
|
||||||
"default_project_id_used": args.new_project == "project",
|
"default_project_id_used": args.new_project == "project",
|
||||||
"default_project_name_used": args.name == "Clusterflux Project",
|
"default_project_name_used": args.name == "Clusterflux Project",
|
||||||
"browser_interaction_required": false,
|
"browser_interaction_required": false,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
},
|
},
|
||||||
"project_config": config,
|
"project_config": config,
|
||||||
"config_file": project_config_file(&cwd),
|
"config_file": project_config_file(&cwd),
|
||||||
|
|
@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
|
||||||
"user": user,
|
"user": user,
|
||||||
"projects": projects,
|
"projects": projects,
|
||||||
"project_count": project_count,
|
"project_count": project_count,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"response": response,
|
"response": response,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
}));
|
}));
|
||||||
|
|
@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
|
||||||
"source": "local_project_config",
|
"source": "local_project_config",
|
||||||
"projects": projects,
|
"projects": projects,
|
||||||
"project_count": project_count,
|
"project_count": project_count,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re
|
||||||
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
|
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
|
||||||
"selected_project": selected_project,
|
"selected_project": selected_project,
|
||||||
"project_config_written": true,
|
"project_config_written": true,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"project_config": config,
|
"project_config": config,
|
||||||
"coordinator_response": coordinator_response,
|
"coordinator_response": coordinator_response,
|
||||||
}))
|
}))
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
|
||||||
"user": effective_scope.user,
|
"user": effective_scope.user,
|
||||||
"coordinator": coordinator,
|
"coordinator": coordinator,
|
||||||
"project_config": config,
|
"project_config": config,
|
||||||
"policy_surface": "generic public quota categories; hosted tuning is coordinator-defined",
|
"policy_surface": "generic public quota categories; hosted tuning remains private policy",
|
||||||
"limits": limits,
|
"limits": limits,
|
||||||
"window_seconds": window_seconds,
|
"window_seconds": window_seconds,
|
||||||
"current_usage": current_usage,
|
"current_usage": current_usage,
|
||||||
|
|
@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
|
||||||
"next_blocked_action": quota_next_blocked_action(¤t_usage),
|
"next_blocked_action": quota_next_blocked_action(¤t_usage),
|
||||||
"quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
|
"quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
|
||||||
"quota_tier": quota_tier,
|
"quota_tier": quota_tier,
|
||||||
"sensitive_abuse_heuristics_exposed": false,
|
"private_abuse_heuristics_exposed": false,
|
||||||
"quota_response": quota_status,
|
"quota_response": quota_status,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
|
||||||
"safe_failure": true,
|
"safe_failure": true,
|
||||||
"message": message,
|
"message": message,
|
||||||
"next_actions": next_actions,
|
"next_actions": next_actions,
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
"machine_error": machine_error,
|
"machine_error": machine_error,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -329,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
|
||||||
"task_launch": launch_task_response,
|
"task_launch": launch_task_response,
|
||||||
"coordinator_response": response,
|
"coordinator_response": response,
|
||||||
"coordinator_session_requests": session.requests(),
|
"coordinator_session_requests": session.requests(),
|
||||||
"external_website_required": false,
|
"private_website_required": false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -124,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
|
||||||
assert_eq!(summary["resource_category"], "api_calls");
|
assert_eq!(summary["resource_category"], "api_calls");
|
||||||
assert_eq!(summary["community_tier_language"], true);
|
assert_eq!(summary["community_tier_language"], true);
|
||||||
assert_eq!(summary["community_tier_label"], "community tier");
|
assert_eq!(summary["community_tier_label"], "community tier");
|
||||||
assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false);
|
assert_eq!(summary["private_abuse_heuristics_exposed"], false);
|
||||||
let rendered = human_report(&json!({
|
let rendered = human_report(&json!({
|
||||||
"command": "run",
|
"command": "run",
|
||||||
"machine_error": summary,
|
"machine_error": summary,
|
||||||
|
|
@ -207,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
|
||||||
assert_eq!(report["status"], "authentication_required");
|
assert_eq!(report["status"], "authentication_required");
|
||||||
assert_eq!(report["non_interactive"], true);
|
assert_eq!(report["non_interactive"], true);
|
||||||
assert_eq!(report["browser_opened"], false);
|
assert_eq!(report["browser_opened"], false);
|
||||||
assert_eq!(report["external_website_required"], false);
|
assert_eq!(report["private_website_required"], false);
|
||||||
assert_eq!(report["machine_error"]["category"], "authentication");
|
assert_eq!(report["machine_error"]["category"], "authentication");
|
||||||
assert_eq!(report["machine_error"]["stable_exit_code"], 20);
|
assert_eq!(report["machine_error"]["stable_exit_code"], 20);
|
||||||
assert_eq!(report["machine_error"]["browser_opened"], false);
|
assert_eq!(report["machine_error"]["browser_opened"], false);
|
||||||
|
|
@ -827,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() {
|
||||||
assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
|
assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
|
||||||
assert_eq!(rejected["machine_error"]["community_tier_language"], true);
|
assert_eq!(rejected["machine_error"]["community_tier_language"], true);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
rejected["machine_error"]["sensitive_abuse_heuristics_exposed"],
|
rejected["machine_error"]["private_abuse_heuristics_exposed"],
|
||||||
false
|
false
|
||||||
);
|
);
|
||||||
assert!(rejected["machine_error"]["next_actions"]
|
assert!(rejected["machine_error"]["next_actions"]
|
||||||
|
|
@ -1665,11 +1665,12 @@ fn node_attach_refuses_a_symlink_credential_target() {
|
||||||
fn hosted_coordinator_remains_a_real_https_control_endpoint() {
|
fn hosted_coordinator_remains_a_real_https_control_endpoint() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
|
control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
|
||||||
"https://clusterflux.lesstuff.com/api/v1/control"
|
"https://clusterflux.michelpaulissen.com/api/v1/control"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
|
control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
|
||||||
"https://clusterflux.lesstuff.com/api/v1/control"
|
.unwrap(),
|
||||||
|
"https://clusterflux.michelpaulissen.com/api/v1/control"
|
||||||
);
|
);
|
||||||
assert!(control_endpoint_identity("http://operator.example.test").is_err());
|
assert!(control_endpoint_identity("http://operator.example.test").is_err());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
@ -1787,7 +1788,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
|
||||||
assert!(!line.contains(r#""actor_user":"user-session""#));
|
assert!(!line.contains(r#""actor_user":"user-session""#));
|
||||||
stream
|
stream
|
||||||
.write_all(
|
.write_all(
|
||||||
br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
|
br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
stream.write_all(b"\n").unwrap();
|
stream.write_all(b"\n").unwrap();
|
||||||
|
|
@ -1850,7 +1851,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
|
||||||
"active"
|
"active"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
|
report["coordinator_account_status"]["private_moderation_details_exposed"],
|
||||||
false
|
false
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -1932,7 +1933,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() {
|
fn auth_status_queries_coordinator_account_state_without_private_moderation_details() {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
let addr = listener.local_addr().unwrap().to_string();
|
let addr = listener.local_addr().unwrap().to_string();
|
||||||
let server = std::thread::spawn(move || {
|
let server = std::thread::spawn(move || {
|
||||||
|
|
@ -1946,7 +1947,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de
|
||||||
assert!(line.contains(r#""actor_user":"user-live""#));
|
assert!(line.contains(r#""actor_user":"user-live""#));
|
||||||
stream
|
stream
|
||||||
.write_all(
|
.write_all(
|
||||||
br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#,
|
br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#,
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
stream.write_all(b"\n").unwrap();
|
stream.write_all(b"\n").unwrap();
|
||||||
|
|
@ -1986,7 +1987,7 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de
|
||||||
"account or tenant is suspended by hosted policy"
|
"account or tenant is suspended by hosted policy"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
|
report["coordinator_account_status"]["private_moderation_details_exposed"],
|
||||||
false
|
false
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
@ -2004,13 +2005,13 @@ fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_de
|
||||||
let serialized = serde_json::to_string(&report).unwrap();
|
let serialized = serde_json::to_string(&report).unwrap();
|
||||||
assert!(!serialized.contains("abuse_score"));
|
assert!(!serialized.contains("abuse_score"));
|
||||||
assert!(!serialized.contains("moderation_notes"));
|
assert!(!serialized.contains("moderation_notes"));
|
||||||
assert!(!serialized.contains("sensitive moderation note"));
|
assert!(!serialized.contains("private moderation note"));
|
||||||
|
|
||||||
let rendered = human_report(&report);
|
let rendered = human_report(&report);
|
||||||
assert!(rendered.contains("account status: suspended"));
|
assert!(rendered.contains("account status: suspended"));
|
||||||
assert!(rendered.contains("account suspended: true"));
|
assert!(rendered.contains("account suspended: true"));
|
||||||
assert!(rendered.contains("sensitive moderation details exposed: false"));
|
assert!(rendered.contains("private moderation details exposed: false"));
|
||||||
assert!(!rendered.contains("sensitive moderation note"));
|
assert!(!rendered.contains("private moderation note"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -2061,11 +2062,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
|
||||||
"manual_review": manual_review,
|
"manual_review": manual_review,
|
||||||
"sanitized_reason": reason,
|
"sanitized_reason": reason,
|
||||||
"next_actions": ["contact the hosted operator"],
|
"next_actions": ["contact the hosted operator"],
|
||||||
"sensitive_moderation_details_exposed": false,
|
"private_moderation_details_exposed": false,
|
||||||
"signup_failure_details_exposed": false,
|
"signup_failure_details_exposed": false,
|
||||||
"abuse_score": 99,
|
"abuse_score": 99,
|
||||||
"moderation_notes": "sensitive moderation note",
|
"moderation_notes": "private moderation note",
|
||||||
"signup_policy_trace": "sensitive signup trace",
|
"signup_policy_trace": "private signup trace",
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
@ -2104,7 +2105,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
|
report["coordinator_account_status"]["private_moderation_details_exposed"],
|
||||||
false
|
false
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|
@ -2115,11 +2116,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
|
||||||
assert!(!serialized.contains("abuse_score"));
|
assert!(!serialized.contains("abuse_score"));
|
||||||
assert!(!serialized.contains("moderation_notes"));
|
assert!(!serialized.contains("moderation_notes"));
|
||||||
assert!(!serialized.contains("signup_policy_trace"));
|
assert!(!serialized.contains("signup_policy_trace"));
|
||||||
assert!(!serialized.contains("sensitive moderation note"));
|
assert!(!serialized.contains("private moderation note"));
|
||||||
let rendered = human_report(&report);
|
let rendered = human_report(&report);
|
||||||
assert!(rendered.contains(&format!("account status: {status}")));
|
assert!(rendered.contains(&format!("account status: {status}")));
|
||||||
assert!(rendered.contains(rendered_marker));
|
assert!(rendered.contains(rendered_marker));
|
||||||
assert!(!rendered.contains("sensitive moderation note"));
|
assert!(!rendered.contains("private moderation note"));
|
||||||
}
|
}
|
||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
@ -2234,11 +2235,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
|
||||||
|
|
||||||
assert_eq!(report["command"], "admin bootstrap");
|
assert_eq!(report["command"], "admin bootstrap");
|
||||||
assert_eq!(report["mode"], "self_hosted_local");
|
assert_eq!(report["mode"], "self_hosted_local");
|
||||||
assert_eq!(report["external_website_required"], false);
|
assert_eq!(report["private_website_required"], false);
|
||||||
assert_eq!(report["self_hosted_cli_only"], true);
|
assert_eq!(report["self_hosted_cli_only"], true);
|
||||||
assert_eq!(report["project_config_written"], true);
|
assert_eq!(report["project_config_written"], true);
|
||||||
assert_eq!(report["project_init"]["command"], "project init");
|
assert_eq!(report["project_init"]["command"], "project init");
|
||||||
assert_eq!(report["project_init"]["external_website_required"], false);
|
assert_eq!(report["project_init"]["private_website_required"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
report["project_init"]["project_config"]["project"],
|
report["project_init"]["project_config"]["project"],
|
||||||
"self-hosted"
|
"self-hosted"
|
||||||
|
|
@ -2264,7 +2265,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
|
||||||
}
|
}
|
||||||
assert!(steps.iter().all(|step| {
|
assert!(steps.iter().all(|step| {
|
||||||
!step
|
!step
|
||||||
.get("external_website_required")
|
.get("private_website_required")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}));
|
}));
|
||||||
|
|
@ -2892,13 +2893,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
|
||||||
|
|
||||||
assert_eq!(status["command"], "admin status");
|
assert_eq!(status["command"], "admin status");
|
||||||
assert_eq!(status["safe_default"], "read_only");
|
assert_eq!(status["safe_default"], "read_only");
|
||||||
assert_eq!(status["external_website_required"], false);
|
assert_eq!(status["private_website_required"], false);
|
||||||
assert_eq!(status["suspended"], false);
|
assert_eq!(status["suspended"], false);
|
||||||
assert_eq!(suspended["command"], "admin suspend-tenant");
|
assert_eq!(suspended["command"], "admin suspend-tenant");
|
||||||
assert_eq!(suspended["tenant"], "tenant");
|
assert_eq!(suspended["tenant"], "tenant");
|
||||||
assert_eq!(suspended["actor_tenant"], "admin-tenant");
|
assert_eq!(suspended["actor_tenant"], "admin-tenant");
|
||||||
assert_eq!(suspended["suspended"], true);
|
assert_eq!(suspended["suspended"], true);
|
||||||
assert_eq!(suspended["external_website_required"], false);
|
assert_eq!(suspended["private_website_required"], false);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -2967,7 +2968,7 @@ fn debug_attach_reports_public_authorization() {
|
||||||
assert_eq!(report["charged_debug_read_bytes"], 1024);
|
assert_eq!(report["charged_debug_read_bytes"], 1024);
|
||||||
assert_eq!(report["used_debug_read_bytes"], 1024);
|
assert_eq!(report["used_debug_read_bytes"], 1024);
|
||||||
assert_eq!(report["debug_reads_quota_limited"], true);
|
assert_eq!(report["debug_reads_quota_limited"], true);
|
||||||
assert_eq!(report["external_website_required"], false);
|
assert_eq!(report["private_website_required"], false);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -3004,14 +3005,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
|
||||||
"restart_task",
|
"restart_task",
|
||||||
br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
|
br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
|
||||||
),
|
),
|
||||||
(
|
|
||||||
"list_task_events",
|
|
||||||
br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"list_task_events",
|
|
||||||
br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(),
|
|
||||||
),
|
|
||||||
(
|
(
|
||||||
"create_artifact_download_link",
|
"create_artifact_download_link",
|
||||||
br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
|
br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
|
||||||
|
|
@ -3116,26 +3109,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
|
||||||
Some(&session),
|
Some(&session),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let logs = logs_report_with_session(
|
artifact_download_report_with_session(
|
||||||
LogsArgs {
|
|
||||||
scope: scope.clone(),
|
|
||||||
process: Some("vp".to_owned()),
|
|
||||||
task: Some("task-a".to_owned()),
|
|
||||||
},
|
|
||||||
Some(&session),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let artifacts = artifact_list_report_with_session(
|
|
||||||
ArtifactListArgs {
|
|
||||||
scope: scope.clone(),
|
|
||||||
process: Some("vp".to_owned()),
|
|
||||||
},
|
|
||||||
Some(&session),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let download = artifact_download_report_with_session(
|
|
||||||
ArtifactDownloadArgs {
|
ArtifactDownloadArgs {
|
||||||
scope,
|
scope: coordinator_scope.clone(),
|
||||||
artifact: "app.txt".to_owned(),
|
artifact: "app.txt".to_owned(),
|
||||||
to: None,
|
to: None,
|
||||||
max_bytes: 2048,
|
max_bytes: 2048,
|
||||||
|
|
@ -3143,9 +3119,6 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
|
||||||
Some(&session),
|
Some(&session),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n");
|
|
||||||
assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt");
|
|
||||||
assert_eq!(download["coordinator"], session.coordinator);
|
|
||||||
debug_attach_report_with_dap_and_session(
|
debug_attach_report_with_dap_and_session(
|
||||||
DebugAttachArgs {
|
DebugAttachArgs {
|
||||||
scope: coordinator_scope,
|
scope: coordinator_scope,
|
||||||
|
|
@ -3338,7 +3311,7 @@ fn project_init_uses_public_create_before_writing_local_config() {
|
||||||
created["safe_defaults"]["browser_interaction_required"],
|
created["safe_defaults"]["browser_interaction_required"],
|
||||||
false
|
false
|
||||||
);
|
);
|
||||||
assert_eq!(created["external_website_required"], false);
|
assert_eq!(created["private_website_required"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
read_project_config(temp_success.path())
|
read_project_config(temp_success.path())
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -3438,7 +3411,7 @@ fn project_list_and_select_use_public_api_without_website() {
|
||||||
assert_eq!(list["source"], "public_coordinator_api");
|
assert_eq!(list["source"], "public_coordinator_api");
|
||||||
assert_eq!(list["project_count"], 1);
|
assert_eq!(list["project_count"], 1);
|
||||||
assert_eq!(list["projects"][0]["id"], "project-a");
|
assert_eq!(list["projects"][0]["id"], "project-a");
|
||||||
assert_eq!(list["external_website_required"], false);
|
assert_eq!(list["private_website_required"], false);
|
||||||
assert_eq!(list["coordinator_session_requests"], 1);
|
assert_eq!(list["coordinator_session_requests"], 1);
|
||||||
|
|
||||||
let selected = project_select_report(
|
let selected = project_select_report(
|
||||||
|
|
@ -3453,7 +3426,7 @@ fn project_list_and_select_use_public_api_without_website() {
|
||||||
assert_eq!(selected["source"], "public_coordinator_api");
|
assert_eq!(selected["source"], "public_coordinator_api");
|
||||||
assert_eq!(selected["selected_project"]["id"], "project-a");
|
assert_eq!(selected["selected_project"]["id"], "project-a");
|
||||||
assert_eq!(selected["project_config_written"], true);
|
assert_eq!(selected["project_config_written"], true);
|
||||||
assert_eq!(selected["external_website_required"], false);
|
assert_eq!(selected["private_website_required"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
read_project_config(temp.path()).unwrap().unwrap().project,
|
read_project_config(temp.path()).unwrap().unwrap().project,
|
||||||
"project-a"
|
"project-a"
|
||||||
|
|
@ -4482,7 +4455,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||||
.join("../..")
|
.join("../..")
|
||||||
.join("examples/hello-build");
|
.join("tests/fixtures/runtime-conformance");
|
||||||
let output = temp.path().join("bundle");
|
let output = temp.path().join("bundle");
|
||||||
|
|
||||||
let report = build_report(
|
let report = build_report(
|
||||||
|
|
@ -4500,8 +4473,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
|
||||||
assert_eq!(report["command"], "build");
|
assert_eq!(report["command"], "build");
|
||||||
assert_eq!(report["content_addressed"], true);
|
assert_eq!(report["content_addressed"], true);
|
||||||
assert_eq!(report["contains_full_repository_upload"], false);
|
assert_eq!(report["contains_full_repository_upload"], false);
|
||||||
assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2);
|
assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 12);
|
||||||
assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1);
|
assert_eq!(report["bundle_artifact"]["entrypoint_count"], 6);
|
||||||
assert!(output.join("module.wasm").is_file());
|
assert!(output.join("module.wasm").is_file());
|
||||||
assert!(output.join("manifest.json").is_file());
|
assert!(output.join("manifest.json").is_file());
|
||||||
assert!(output.join("task-descriptors.json").is_file());
|
assert!(output.join("task-descriptors.json").is_file());
|
||||||
|
|
@ -4512,9 +4485,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.iter()
|
.iter()
|
||||||
.any(|descriptor| {
|
.any(|descriptor| {
|
||||||
descriptor["name"] == "compile"
|
descriptor["name"] == "task_add_one"
|
||||||
&& descriptor["argument_schema"] == "source : SourceSnapshot"
|
&& descriptor["argument_schema"] == "input : i32"
|
||||||
&& descriptor["result_schema"] == "Result < Artifact >"
|
&& descriptor["result_schema"] == "i32"
|
||||||
&& descriptor["restart_compatibility_hash"]
|
&& descriptor["restart_compatibility_hash"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
|
@ -4675,7 +4648,7 @@ fn node_enroll_reports_short_lived_public_api_grant() {
|
||||||
|
|
||||||
assert_eq!(report["command"], "node enroll");
|
assert_eq!(report["command"], "node enroll");
|
||||||
assert_eq!(report["status"], "created");
|
assert_eq!(report["status"], "created");
|
||||||
assert_eq!(report["external_website_required"], false);
|
assert_eq!(report["private_website_required"], false);
|
||||||
assert_eq!(report["tenant"], "tenant");
|
assert_eq!(report["tenant"], "tenant");
|
||||||
assert_eq!(report["project"], "project");
|
assert_eq!(report["project"], "project");
|
||||||
assert_eq!(report["user"], "user");
|
assert_eq!(report["user"], "user");
|
||||||
|
|
@ -4818,7 +4791,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() {
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(enroll["status"], "requires_coordinator");
|
assert_eq!(enroll["status"], "requires_coordinator");
|
||||||
assert_eq!(enroll["external_website_required"], false);
|
assert_eq!(enroll["private_website_required"], false);
|
||||||
assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
|
assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
|
||||||
assert_eq!(enroll["requested_ttl_seconds"], 60);
|
assert_eq!(enroll["requested_ttl_seconds"], 60);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
[package]
|
|
||||||
name = "clusterflux-client"
|
|
||||||
version = "0.1.0"
|
|
||||||
edition.workspace = true
|
|
||||||
license.workspace = true
|
|
||||||
repository.workspace = true
|
|
||||||
|
|
||||||
[dependencies]
|
|
||||||
base64.workspace = true
|
|
||||||
clusterflux-control = { path = "../clusterflux-control" }
|
|
||||||
clusterflux-core = { path = "../clusterflux-core" }
|
|
||||||
serde.workspace = true
|
|
||||||
serde_json.workspace = true
|
|
||||||
thiserror.workspace = true
|
|
||||||
tokio = { workspace = true, features = ["sync", "time"] }
|
|
||||||
|
|
||||||
[dev-dependencies]
|
|
||||||
clusterflux-coordinator = { path = "../clusterflux-coordinator" }
|
|
||||||
|
|
@ -1,68 +0,0 @@
|
||||||
# clusterflux-client
|
|
||||||
|
|
||||||
`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web
|
|
||||||
backend. It talks to the same versioned control and login endpoints as the CLI.
|
|
||||||
Callers do not construct JSON envelopes or place session secrets into request
|
|
||||||
objects.
|
|
||||||
|
|
||||||
The crate intentionally contains no coordinator implementation, persistence
|
|
||||||
types, scheduler policy, or website-specific business rules. A web backend only
|
|
||||||
needs this crate to authenticate and work with account status, projects, Agent
|
|
||||||
keys, node enrollment and liveness, current/recent processes, task attempts,
|
|
||||||
recent logs, artifacts, quota state, process control, task recovery, and Debug
|
|
||||||
Epoch state. It does not expose workflow launch or whole-process replay.
|
|
||||||
|
|
||||||
```rust,no_run
|
|
||||||
use clusterflux_client::{ClusterfluxClient, SessionCredential};
|
|
||||||
|
|
||||||
# async fn example() -> Result<(), clusterflux_client::ClientError> {
|
|
||||||
let credential = SessionCredential::from_secret(
|
|
||||||
std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"),
|
|
||||||
);
|
|
||||||
let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")?
|
|
||||||
.with_session_credential(&credential);
|
|
||||||
|
|
||||||
let account = client.account_status().await?;
|
|
||||||
let nodes = client.list_nodes().await?;
|
|
||||||
let processes = client.list_processes(None, 20).await?;
|
|
||||||
# let _ = (account, nodes, processes);
|
|
||||||
# Ok(())
|
|
||||||
# }
|
|
||||||
```
|
|
||||||
|
|
||||||
## Login boundary
|
|
||||||
|
|
||||||
`begin_browser_login` starts the hosted Authentik flow. The identity callback
|
|
||||||
returns only to the fixed configured website URL with a short-lived handoff.
|
|
||||||
The website backend calls `exchange_browser_login_handoff` once and stores the
|
|
||||||
returned `SessionCredential` only in encrypted server-side session storage.
|
|
||||||
The credential has redacted `Debug` output and deliberately does not implement
|
|
||||||
Serde serialization. Logout calls the existing session-revocation operation.
|
|
||||||
|
|
||||||
The browser must not receive or persist the Clusterflux session credential,
|
|
||||||
provider tokens, authorization code, PKCE verifier, or CLI polling secret.
|
|
||||||
|
|
||||||
## Errors, bounds, and transport
|
|
||||||
|
|
||||||
API failures are returned as `ClientError::Api(ApiError)`. Decisions should use
|
|
||||||
the stable `code`, `category`, `retryable`, and `request_id` fields, while
|
|
||||||
`message` is for people. The client rejects an error whose request ID does not
|
|
||||||
match the originating request.
|
|
||||||
|
|
||||||
Paginated list methods require an explicit bounded page size. Process pages are
|
|
||||||
limited to 100 entries; node, artifact, and recent-log pages to 200.
|
|
||||||
`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes
|
|
||||||
its cursor. Recent logs are ephemeral and can report truncation or sequence
|
|
||||||
loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates
|
|
||||||
offsets and decodes bounded chunks without materializing the complete artifact.
|
|
||||||
|
|
||||||
`ControlTransport` has bounded connect and I/O timeouts and performs blocking
|
|
||||||
network work outside the async executor. Dropping a request future cancels the
|
|
||||||
caller’s wait; any already-started blocking I/O remains bounded by its timeout.
|
|
||||||
`MockTransport` supplies deterministic responses and records exact envelopes
|
|
||||||
for application tests.
|
|
||||||
|
|
||||||
`CLIENT_API_VERSION` is the one supported control protocol version. The
|
|
||||||
checked-in `web_operations.json` contract fixture records every website
|
|
||||||
operation and its stable error shape, and the crate’s contract suite checks the
|
|
||||||
fixture.
|
|
||||||
|
|
@ -1,924 +0,0 @@
|
||||||
mod protocol;
|
|
||||||
mod transport;
|
|
||||||
mod types;
|
|
||||||
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
|
|
||||||
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
|
||||||
use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
|
|
||||||
use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION};
|
|
||||||
use protocol::{AuthenticatedRequest, LoginRequest, WireResponse};
|
|
||||||
use serde_json::json;
|
|
||||||
use thiserror::Error;
|
|
||||||
use transport::{ClientTransport, TransportRequest};
|
|
||||||
|
|
||||||
pub use clusterflux_core::{
|
|
||||||
AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind,
|
|
||||||
Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId,
|
|
||||||
ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId,
|
|
||||||
UserId, VfsPath,
|
|
||||||
};
|
|
||||||
pub use transport::{
|
|
||||||
ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse,
|
|
||||||
};
|
|
||||||
pub use types::*;
|
|
||||||
|
|
||||||
pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION;
|
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum ClientError {
|
|
||||||
#[error("Clusterflux API error: {0}")]
|
|
||||||
Api(ApiError),
|
|
||||||
#[error(transparent)]
|
|
||||||
Transport(#[from] ClientTransportError),
|
|
||||||
#[error("Clusterflux client protocol error: {0}")]
|
|
||||||
Protocol(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone)]
|
|
||||||
pub struct ClusterfluxClient {
|
|
||||||
transport: Arc<dyn ClientTransport>,
|
|
||||||
session_secret: Arc<Mutex<Option<String>>>,
|
|
||||||
next_request: Arc<AtomicU64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ClusterfluxClient {
|
|
||||||
pub fn connect(endpoint: impl Into<String>) -> Result<Self, ClientError> {
|
|
||||||
Ok(Self::with_transport(ControlTransport::new(endpoint)?))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_transport(transport: impl ClientTransport) -> Self {
|
|
||||||
Self {
|
|
||||||
transport: Arc::new(transport),
|
|
||||||
session_secret: Arc::new(Mutex::new(None)),
|
|
||||||
next_request: Arc::new(AtomicU64::new(1)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self {
|
|
||||||
self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone())));
|
|
||||||
self
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn is_session_configured(&self) -> bool {
|
|
||||||
self.session_secret
|
|
||||||
.lock()
|
|
||||||
.map(|secret| secret.is_some())
|
|
||||||
.unwrap_or(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn begin_browser_login(&self) -> Result<BrowserLoginStart, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_login(LoginRequest::BeginWebBrowserLogin {})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::WebBrowserLoginStarted {
|
|
||||||
transaction_id,
|
|
||||||
authorization_url,
|
|
||||||
expires_at_epoch_seconds,
|
|
||||||
} => Ok(BrowserLoginStart {
|
|
||||||
transaction_id,
|
|
||||||
authorization_url,
|
|
||||||
expires_at_epoch_seconds,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("web_browser_login_started")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn exchange_browser_login_handoff(
|
|
||||||
&self,
|
|
||||||
transaction_id: impl Into<String>,
|
|
||||||
handoff_code: impl Into<String>,
|
|
||||||
) -> Result<BrowserSession, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_login(LoginRequest::ExchangeWebLoginHandoff {
|
|
||||||
transaction_id: transaction_id.into(),
|
|
||||||
handoff_code: handoff_code.into(),
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::WebBrowserSession { session } => Ok(BrowserSession {
|
|
||||||
tenant: session.tenant,
|
|
||||||
project: session.project,
|
|
||||||
user: session.user,
|
|
||||||
credential: SessionCredential(session.session_secret),
|
|
||||||
expires_at_epoch_seconds: session.expires_at_epoch_seconds,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("web_browser_session")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn cancel_browser_login(
|
|
||||||
&self,
|
|
||||||
transaction_id: impl Into<String>,
|
|
||||||
) -> Result<(), ClientError> {
|
|
||||||
match self
|
|
||||||
.send_login(LoginRequest::CancelWebBrowserLogin {
|
|
||||||
transaction_id: transaction_id.into(),
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::WebBrowserLoginCancelled {} => Ok(()),
|
|
||||||
_ => Err(unexpected_response("web_browser_login_cancelled")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn account_status(&self) -> Result<AccountStatus, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::AuthStatus)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::AuthStatus {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
authenticated,
|
|
||||||
account_status,
|
|
||||||
suspended,
|
|
||||||
disabled,
|
|
||||||
deleted,
|
|
||||||
manual_review,
|
|
||||||
sanitized_reason,
|
|
||||||
next_actions,
|
|
||||||
} => Ok(AccountStatus {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
authenticated,
|
|
||||||
account_status,
|
|
||||||
suspended,
|
|
||||||
disabled,
|
|
||||||
deleted,
|
|
||||||
manual_review,
|
|
||||||
sanitized_reason,
|
|
||||||
next_actions,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("auth_status")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn logout(&self) -> Result<(), ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::RevokeCliSession)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::CliSessionRevoked {} => {
|
|
||||||
*self.session_secret.lock().map_err(|_| {
|
|
||||||
ClientError::Protocol("session credential lock was poisoned".to_owned())
|
|
||||||
})? = None;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
_ => Err(unexpected_response("cli_session_revoked")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_projects(&self) -> Result<Vec<Project>, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListProjects)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::Projects { projects } => Ok(projects),
|
|
||||||
_ => Err(unexpected_response("projects")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_project(
|
|
||||||
&self,
|
|
||||||
project: ProjectId,
|
|
||||||
name: impl Into<String>,
|
|
||||||
) -> Result<Project, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::CreateProject {
|
|
||||||
project,
|
|
||||||
name: name.into(),
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ProjectCreated { project } => Ok(project),
|
|
||||||
_ => Err(unexpected_response("project_created")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn select_project(&self, project: ProjectId) -> Result<Project, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::SelectProject { project })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ProjectSelected { project } => Ok(project),
|
|
||||||
_ => Err(unexpected_response("project_selected")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn register_agent_public_key(
|
|
||||||
&self,
|
|
||||||
agent: AgentId,
|
|
||||||
public_key: impl Into<String>,
|
|
||||||
) -> Result<AgentPublicKey, ClientError> {
|
|
||||||
self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey {
|
|
||||||
agent,
|
|
||||||
public_key: public_key.into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_agent_public_keys(&self) -> Result<Vec<AgentPublicKey>, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListAgentPublicKeys)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::AgentPublicKeys { records } => Ok(records),
|
|
||||||
_ => Err(unexpected_response("agent_public_keys")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn rotate_agent_public_key(
|
|
||||||
&self,
|
|
||||||
agent: AgentId,
|
|
||||||
public_key: impl Into<String>,
|
|
||||||
) -> Result<AgentPublicKey, ClientError> {
|
|
||||||
self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey {
|
|
||||||
agent,
|
|
||||||
public_key: public_key.into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn revoke_agent_public_key(
|
|
||||||
&self,
|
|
||||||
agent: AgentId,
|
|
||||||
) -> Result<AgentPublicKey, ClientError> {
|
|
||||||
self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent })
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn agent_key_mutation(
|
|
||||||
&self,
|
|
||||||
request: AuthenticatedRequest,
|
|
||||||
) -> Result<AgentPublicKey, ClientError> {
|
|
||||||
match self.send_authenticated(request).await? {
|
|
||||||
WireResponse::AgentPublicKey { record } => Ok(record),
|
|
||||||
_ => Err(unexpected_response("agent_public_key")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_node_enrollment_grant(
|
|
||||||
&self,
|
|
||||||
ttl_seconds: u64,
|
|
||||||
) -> Result<NodeEnrollmentGrant, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::NodeEnrollmentGrantCreated {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
grant,
|
|
||||||
scope,
|
|
||||||
expires_at_epoch_seconds,
|
|
||||||
} => Ok(NodeEnrollmentGrant {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
grant,
|
|
||||||
scope,
|
|
||||||
expires_at_epoch_seconds,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("node_enrollment_grant_created")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_nodes(&self) -> Result<Vec<NodeSummary>, ClientError> {
|
|
||||||
Ok(self.list_nodes_page(None, 200).await?.nodes)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_nodes_page(
|
|
||||||
&self,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<NodePage, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::NodeSummaries { nodes, next_cursor } => {
|
|
||||||
Ok(NodePage { nodes, next_cursor })
|
|
||||||
}
|
|
||||||
_ => Err(unexpected_response("node_summaries")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn revoke_node(&self, node: NodeId) -> Result<NodeRevocation, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::NodeCredentialRevoked {
|
|
||||||
node,
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
descriptor_removed,
|
|
||||||
queued_assignments_removed,
|
|
||||||
} => Ok(NodeRevocation {
|
|
||||||
node,
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
descriptor_removed,
|
|
||||||
queued_assignments_removed,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("node_credential_revoked")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_processes(
|
|
||||||
&self,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<ProcessPage, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ProcessSummaries {
|
|
||||||
processes,
|
|
||||||
next_cursor,
|
|
||||||
} => Ok(ProcessPage {
|
|
||||||
processes,
|
|
||||||
next_cursor,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("process_summaries")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn cancel_process(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
) -> Result<ProcessCancellation, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::CancelProcess { process })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ProcessCancellationRequested {
|
|
||||||
process,
|
|
||||||
cancelled_tasks,
|
|
||||||
affected_nodes,
|
|
||||||
} => Ok(ProcessCancellation {
|
|
||||||
process,
|
|
||||||
affected_tasks: cancelled_tasks,
|
|
||||||
affected_nodes,
|
|
||||||
aborted: false,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("process_cancellation_requested")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn abort_process(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
) -> Result<ProcessCancellation, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::AbortProcess {
|
|
||||||
process,
|
|
||||||
launch_attempt: None,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ProcessAborted {
|
|
||||||
process,
|
|
||||||
aborted_tasks,
|
|
||||||
affected_nodes,
|
|
||||||
} => Ok(ProcessCancellation {
|
|
||||||
process,
|
|
||||||
affected_tasks: aborted_tasks,
|
|
||||||
affected_nodes,
|
|
||||||
aborted: true,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("process_aborted")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn quota_status(&self) -> Result<QuotaStatus, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::QuotaStatus)
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::QuotaStatus {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
policy_label,
|
|
||||||
limits,
|
|
||||||
window_seconds,
|
|
||||||
usage,
|
|
||||||
window_started_epoch_seconds,
|
|
||||||
} => Ok(QuotaStatus {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor,
|
|
||||||
policy_label,
|
|
||||||
limits,
|
|
||||||
window_seconds,
|
|
||||||
usage,
|
|
||||||
window_started_epoch_seconds,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("quota_status")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_task_events(
|
|
||||||
&self,
|
|
||||||
process: Option<ProcessId>,
|
|
||||||
) -> Result<Vec<TaskCompletionEvent>, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListTaskEvents { process })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::TaskEvents { events } => Ok(events),
|
|
||||||
_ => Err(unexpected_response("task_events")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_task_snapshots(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
) -> Result<Vec<TaskAttemptSnapshot>, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::TaskSnapshots { snapshots } => Ok(snapshots),
|
|
||||||
_ => Err(unexpected_response("task_snapshots")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_recent_logs(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
task: Option<TaskInstanceId>,
|
|
||||||
after_sequence: Option<u64>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<RecentLogPage, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListRecentLogs {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
after_sequence,
|
|
||||||
limit,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::RecentLogs {
|
|
||||||
entries,
|
|
||||||
next_sequence,
|
|
||||||
history_truncated,
|
|
||||||
} => Ok(RecentLogPage {
|
|
||||||
entries,
|
|
||||||
next_sequence,
|
|
||||||
history_truncated,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("recent_logs")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn restart_task(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
replacement_bundle: Option<TaskReplacementBundle>,
|
|
||||||
) -> Result<TaskRestart, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::RestartTask {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
replacement_bundle,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::TaskRestart {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
restarted_task_instance,
|
|
||||||
restarted_attempt_id,
|
|
||||||
actor,
|
|
||||||
accepted,
|
|
||||||
clean_boundary_available,
|
|
||||||
active_task,
|
|
||||||
completed_event_observed,
|
|
||||||
requires_whole_process_restart,
|
|
||||||
message,
|
|
||||||
audit_event,
|
|
||||||
} => Ok(TaskRestart {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
restarted_task_instance,
|
|
||||||
restarted_attempt_id,
|
|
||||||
actor,
|
|
||||||
accepted,
|
|
||||||
clean_boundary_available,
|
|
||||||
active_task,
|
|
||||||
completed_event_observed,
|
|
||||||
requires_whole_process_restart,
|
|
||||||
message,
|
|
||||||
audit_event,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("task_restart")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn resolve_task_failure(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
resolution: TaskFailureResolution,
|
|
||||||
) -> Result<TaskFailureResolutionResult, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ResolveTaskFailure {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
resolution,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::TaskFailureResolved {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
attempt_id,
|
|
||||||
resolution,
|
|
||||||
} => Ok(TaskFailureResolutionResult {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
attempt_id,
|
|
||||||
resolution,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("task_failure_resolved")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn debug_attach(&self, process: ProcessId) -> Result<DebugAttach, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::DebugAttach { process })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::DebugAttach {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
authorization,
|
|
||||||
audit_event,
|
|
||||||
} => Ok(DebugAttach {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
authorization,
|
|
||||||
audit_event,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("debug_attach")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn create_debug_epoch(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
stopped_task: TaskInstanceId,
|
|
||||||
reason: impl Into<String>,
|
|
||||||
) -> Result<DebugEpochControl, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::CreateDebugEpoch {
|
|
||||||
process,
|
|
||||||
stopped_task,
|
|
||||||
reason: reason.into(),
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::DebugEpoch {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
affected_tasks,
|
|
||||||
all_stop_requested,
|
|
||||||
audit_event,
|
|
||||||
} => Ok(DebugEpochControl {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
affected_tasks,
|
|
||||||
all_stop_requested,
|
|
||||||
audit_event,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("debug_epoch")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn resume_debug_epoch(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
epoch: u64,
|
|
||||||
) -> Result<DebugEpochControl, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::DebugEpoch {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
affected_tasks,
|
|
||||||
all_stop_requested,
|
|
||||||
audit_event,
|
|
||||||
} => Ok(DebugEpochControl {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
affected_tasks,
|
|
||||||
all_stop_requested,
|
|
||||||
audit_event,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("debug_epoch")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn inspect_debug_epoch(
|
|
||||||
&self,
|
|
||||||
process: ProcessId,
|
|
||||||
epoch: u64,
|
|
||||||
) -> Result<DebugEpochStatus, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::DebugEpochStatus {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
expected_tasks,
|
|
||||||
acknowledgements,
|
|
||||||
fully_frozen,
|
|
||||||
partially_frozen,
|
|
||||||
fully_resumed,
|
|
||||||
failed,
|
|
||||||
failure_messages,
|
|
||||||
audit_event,
|
|
||||||
} => Ok(DebugEpochStatus {
|
|
||||||
process,
|
|
||||||
actor,
|
|
||||||
epoch,
|
|
||||||
command,
|
|
||||||
expected_tasks,
|
|
||||||
acknowledgements,
|
|
||||||
fully_frozen,
|
|
||||||
partially_frozen,
|
|
||||||
fully_resumed,
|
|
||||||
failed,
|
|
||||||
failure_messages,
|
|
||||||
audit_event,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("debug_epoch_status")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn list_artifacts(
|
|
||||||
&self,
|
|
||||||
process: Option<ProcessId>,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<ArtifactPage, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::ListArtifacts {
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::Artifacts {
|
|
||||||
artifacts,
|
|
||||||
next_cursor,
|
|
||||||
} => Ok(ArtifactPage {
|
|
||||||
artifacts,
|
|
||||||
next_cursor,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("artifacts")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn get_artifact(&self, artifact: ArtifactId) -> Result<ArtifactSummary, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::GetArtifact { artifact })
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::Artifact { artifact } => Ok(artifact),
|
|
||||||
_ => Err(unexpected_response("artifact")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn begin_artifact_download(
|
|
||||||
&self,
|
|
||||||
artifact: ArtifactId,
|
|
||||||
max_bytes: u64,
|
|
||||||
ttl_seconds: u64,
|
|
||||||
chunk_bytes: u64,
|
|
||||||
) -> Result<ArtifactDownload, ClientError> {
|
|
||||||
match self
|
|
||||||
.send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink {
|
|
||||||
artifact: artifact.clone(),
|
|
||||||
max_bytes,
|
|
||||||
ttl_seconds,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload {
|
|
||||||
client: self.clone(),
|
|
||||||
artifact,
|
|
||||||
max_bytes,
|
|
||||||
chunk_bytes,
|
|
||||||
token_digest: link.scoped_token_digest.clone(),
|
|
||||||
link,
|
|
||||||
expected_offset: 0,
|
|
||||||
finished: false,
|
|
||||||
}),
|
|
||||||
_ => Err(unexpected_response("artifact_download_link")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_authenticated(
|
|
||||||
&self,
|
|
||||||
request: AuthenticatedRequest,
|
|
||||||
) -> Result<WireResponse, ClientError> {
|
|
||||||
let session_secret = self
|
|
||||||
.session_secret
|
|
||||||
.lock()
|
|
||||||
.map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))?
|
|
||||||
.clone()
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ClientError::Api(ApiError::from_message(
|
|
||||||
"client",
|
|
||||||
"no authenticated Clusterflux session is configured",
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
self.send(
|
|
||||||
CONTROL_API_PATH,
|
|
||||||
json!({
|
|
||||||
"type": "authenticated",
|
|
||||||
"session_secret": session_secret,
|
|
||||||
"request": request,
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send_login(&self, request: LoginRequest) -> Result<WireResponse, ClientError> {
|
|
||||||
let payload = serde_json::to_value(request)
|
|
||||||
.map_err(|error| ClientError::Protocol(error.to_string()))?;
|
|
||||||
self.send(LOGIN_API_PATH, payload).await
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn send(
|
|
||||||
&self,
|
|
||||||
api_path: &str,
|
|
||||||
payload: serde_json::Value,
|
|
||||||
) -> Result<WireResponse, ClientError> {
|
|
||||||
let request_number = self.next_request.fetch_add(1, Ordering::Relaxed);
|
|
||||||
let request_id = format!("client-{request_number}");
|
|
||||||
let envelope = coordinator_wire_request(&request_id, payload);
|
|
||||||
let body = serde_json::to_vec(&envelope)
|
|
||||||
.map_err(|error| ClientError::Protocol(error.to_string()))?;
|
|
||||||
let response = self
|
|
||||||
.transport
|
|
||||||
.send(TransportRequest {
|
|
||||||
api_path: api_path.to_owned(),
|
|
||||||
body,
|
|
||||||
})
|
|
||||||
.await?;
|
|
||||||
let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| {
|
|
||||||
ClientError::Protocol(format!("decode typed API response: {error}"))
|
|
||||||
})?;
|
|
||||||
match response {
|
|
||||||
WireResponse::Error {
|
|
||||||
code,
|
|
||||||
category,
|
|
||||||
message,
|
|
||||||
retryable,
|
|
||||||
request_id: response_request_id,
|
|
||||||
} => {
|
|
||||||
if response_request_id != request_id {
|
|
||||||
return Err(ClientError::Protocol(format!(
|
|
||||||
"error response request_id {response_request_id} does not match {request_id}"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
Err(ClientError::Api(ApiError::new(
|
|
||||||
code,
|
|
||||||
category,
|
|
||||||
message,
|
|
||||||
retryable,
|
|
||||||
response_request_id,
|
|
||||||
)))
|
|
||||||
}
|
|
||||||
response => Ok(response),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct ArtifactDownload {
|
|
||||||
client: ClusterfluxClient,
|
|
||||||
artifact: ArtifactId,
|
|
||||||
max_bytes: u64,
|
|
||||||
chunk_bytes: u64,
|
|
||||||
token_digest: Digest,
|
|
||||||
link: clusterflux_core::DownloadLink,
|
|
||||||
expected_offset: u64,
|
|
||||||
finished: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ArtifactDownload {
|
|
||||||
pub fn link(&self) -> &clusterflux_core::DownloadLink {
|
|
||||||
&self.link
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn next_chunk(&mut self) -> Result<ArtifactDownloadPoll, ClientError> {
|
|
||||||
if self.finished {
|
|
||||||
return Ok(ArtifactDownloadPoll::Chunk {
|
|
||||||
offset: self.expected_offset,
|
|
||||||
bytes: Vec::new(),
|
|
||||||
eof: true,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
match self
|
|
||||||
.client
|
|
||||||
.send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream {
|
|
||||||
artifact: self.artifact.clone(),
|
|
||||||
max_bytes: self.max_bytes,
|
|
||||||
token_digest: self.token_digest.clone(),
|
|
||||||
chunk_bytes: self.chunk_bytes,
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ArtifactDownloadStream {
|
|
||||||
content_bytes_available,
|
|
||||||
content_offset,
|
|
||||||
content_eof,
|
|
||||||
content_base64,
|
|
||||||
..
|
|
||||||
} => {
|
|
||||||
if !content_bytes_available {
|
|
||||||
return Ok(ArtifactDownloadPoll::Pending);
|
|
||||||
}
|
|
||||||
let offset = content_offset.ok_or_else(|| {
|
|
||||||
ClientError::Protocol(
|
|
||||||
"artifact response contained bytes without an offset".to_owned(),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
if offset != self.expected_offset {
|
|
||||||
return Err(ClientError::Protocol(format!(
|
|
||||||
"artifact chunk offset {offset} does not match expected {}",
|
|
||||||
self.expected_offset
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let bytes = BASE64_STANDARD
|
|
||||||
.decode(content_base64.unwrap_or_default())
|
|
||||||
.map_err(|error| {
|
|
||||||
ClientError::Protocol(format!(
|
|
||||||
"artifact response contains invalid base64: {error}"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64);
|
|
||||||
self.finished = content_eof;
|
|
||||||
Ok(ArtifactDownloadPoll::Chunk {
|
|
||||||
offset,
|
|
||||||
bytes,
|
|
||||||
eof: content_eof,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_ => Err(unexpected_response("artifact_download_stream")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn cancel(mut self) -> Result<(), ClientError> {
|
|
||||||
if self.finished {
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
match self
|
|
||||||
.client
|
|
||||||
.send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink {
|
|
||||||
artifact: self.artifact.clone(),
|
|
||||||
token_digest: self.token_digest.clone(),
|
|
||||||
})
|
|
||||||
.await?
|
|
||||||
{
|
|
||||||
WireResponse::ArtifactDownloadLinkRevoked {} => {
|
|
||||||
self.finished = true;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
_ => Err(unexpected_response("artifact_download_link_revoked")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unexpected_response(expected: &str) -> ClientError {
|
|
||||||
ClientError::Protocol(format!(
|
|
||||||
"expected typed response {expected}, received another response variant"
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
@ -1,403 +0,0 @@
|
||||||
use std::collections::BTreeMap;
|
|
||||||
|
|
||||||
use clusterflux_core::{
|
|
||||||
AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink,
|
|
||||||
LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId,
|
|
||||||
};
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
|
|
||||||
use crate::types::*;
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
|
||||||
pub(crate) enum AuthenticatedRequest {
|
|
||||||
AuthStatus,
|
|
||||||
RevokeCliSession,
|
|
||||||
CreateProject {
|
|
||||||
project: ProjectId,
|
|
||||||
name: String,
|
|
||||||
},
|
|
||||||
SelectProject {
|
|
||||||
project: ProjectId,
|
|
||||||
},
|
|
||||||
ListProjects,
|
|
||||||
RegisterAgentPublicKey {
|
|
||||||
agent: AgentId,
|
|
||||||
public_key: String,
|
|
||||||
},
|
|
||||||
ListAgentPublicKeys,
|
|
||||||
RotateAgentPublicKey {
|
|
||||||
agent: AgentId,
|
|
||||||
public_key: String,
|
|
||||||
},
|
|
||||||
RevokeAgentPublicKey {
|
|
||||||
agent: AgentId,
|
|
||||||
},
|
|
||||||
CreateNodeEnrollmentGrant {
|
|
||||||
ttl_seconds: u64,
|
|
||||||
},
|
|
||||||
ListNodeSummaries {
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
RevokeNodeCredential {
|
|
||||||
node: NodeId,
|
|
||||||
},
|
|
||||||
ListProcessSummaries {
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
CancelProcess {
|
|
||||||
process: ProcessId,
|
|
||||||
},
|
|
||||||
AbortProcess {
|
|
||||||
process: ProcessId,
|
|
||||||
launch_attempt: Option<String>,
|
|
||||||
},
|
|
||||||
QuotaStatus,
|
|
||||||
ListTaskEvents {
|
|
||||||
process: Option<ProcessId>,
|
|
||||||
},
|
|
||||||
ListTaskSnapshots {
|
|
||||||
process: ProcessId,
|
|
||||||
},
|
|
||||||
ListRecentLogs {
|
|
||||||
process: ProcessId,
|
|
||||||
task: Option<TaskInstanceId>,
|
|
||||||
after_sequence: Option<u64>,
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
RestartTask {
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
replacement_bundle: Option<TaskReplacementBundle>,
|
|
||||||
},
|
|
||||||
ResolveTaskFailure {
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
resolution: TaskFailureResolution,
|
|
||||||
},
|
|
||||||
DebugAttach {
|
|
||||||
process: ProcessId,
|
|
||||||
},
|
|
||||||
CreateDebugEpoch {
|
|
||||||
process: ProcessId,
|
|
||||||
stopped_task: TaskInstanceId,
|
|
||||||
reason: String,
|
|
||||||
},
|
|
||||||
ResumeDebugEpoch {
|
|
||||||
process: ProcessId,
|
|
||||||
epoch: u64,
|
|
||||||
},
|
|
||||||
InspectDebugEpoch {
|
|
||||||
process: ProcessId,
|
|
||||||
epoch: u64,
|
|
||||||
},
|
|
||||||
ListArtifacts {
|
|
||||||
process: Option<ProcessId>,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
GetArtifact {
|
|
||||||
artifact: ArtifactId,
|
|
||||||
},
|
|
||||||
CreateArtifactDownloadLink {
|
|
||||||
artifact: ArtifactId,
|
|
||||||
max_bytes: u64,
|
|
||||||
ttl_seconds: u64,
|
|
||||||
},
|
|
||||||
OpenArtifactDownloadStream {
|
|
||||||
artifact: ArtifactId,
|
|
||||||
max_bytes: u64,
|
|
||||||
token_digest: Digest,
|
|
||||||
chunk_bytes: u64,
|
|
||||||
},
|
|
||||||
RevokeArtifactDownloadLink {
|
|
||||||
artifact: ArtifactId,
|
|
||||||
token_digest: Digest,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Serialize, Deserialize)]
|
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
|
||||||
pub(crate) enum LoginRequest {
|
|
||||||
BeginWebBrowserLogin {},
|
|
||||||
CancelWebBrowserLogin {
|
|
||||||
transaction_id: String,
|
|
||||||
},
|
|
||||||
ExchangeWebLoginHandoff {
|
|
||||||
transaction_id: String,
|
|
||||||
handoff_code: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Deserialize)]
|
|
||||||
pub(crate) struct WireBrowserSession {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub user: UserId,
|
|
||||||
pub session_secret: String,
|
|
||||||
pub expires_at_epoch_seconds: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::large_enum_variant)]
|
|
||||||
#[derive(Clone, Deserialize)]
|
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
|
||||||
pub(crate) enum WireResponse {
|
|
||||||
AuthStatus {
|
|
||||||
tenant: TenantId,
|
|
||||||
project: ProjectId,
|
|
||||||
actor: UserId,
|
|
||||||
authenticated: bool,
|
|
||||||
account_status: String,
|
|
||||||
suspended: bool,
|
|
||||||
disabled: bool,
|
|
||||||
deleted: bool,
|
|
||||||
manual_review: bool,
|
|
||||||
sanitized_reason: Option<String>,
|
|
||||||
next_actions: Vec<String>,
|
|
||||||
},
|
|
||||||
CliSessionRevoked {},
|
|
||||||
ProjectCreated {
|
|
||||||
project: Project,
|
|
||||||
},
|
|
||||||
ProjectSelected {
|
|
||||||
project: Project,
|
|
||||||
},
|
|
||||||
Projects {
|
|
||||||
projects: Vec<Project>,
|
|
||||||
},
|
|
||||||
AgentPublicKey {
|
|
||||||
record: AgentPublicKey,
|
|
||||||
},
|
|
||||||
AgentPublicKeys {
|
|
||||||
records: Vec<AgentPublicKey>,
|
|
||||||
},
|
|
||||||
NodeEnrollmentGrantCreated {
|
|
||||||
tenant: TenantId,
|
|
||||||
project: ProjectId,
|
|
||||||
grant: String,
|
|
||||||
scope: String,
|
|
||||||
expires_at_epoch_seconds: u64,
|
|
||||||
},
|
|
||||||
NodeSummaries {
|
|
||||||
nodes: Vec<NodeSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
},
|
|
||||||
NodeCredentialRevoked {
|
|
||||||
node: NodeId,
|
|
||||||
tenant: TenantId,
|
|
||||||
project: ProjectId,
|
|
||||||
actor: UserId,
|
|
||||||
descriptor_removed: bool,
|
|
||||||
queued_assignments_removed: usize,
|
|
||||||
},
|
|
||||||
ProcessSummaries {
|
|
||||||
processes: Vec<ProcessSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
},
|
|
||||||
ProcessCancellationRequested {
|
|
||||||
process: ProcessId,
|
|
||||||
cancelled_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
affected_nodes: Vec<NodeId>,
|
|
||||||
},
|
|
||||||
ProcessAborted {
|
|
||||||
process: ProcessId,
|
|
||||||
aborted_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
affected_nodes: Vec<NodeId>,
|
|
||||||
},
|
|
||||||
QuotaStatus {
|
|
||||||
tenant: TenantId,
|
|
||||||
project: ProjectId,
|
|
||||||
actor: UserId,
|
|
||||||
policy_label: Option<String>,
|
|
||||||
limits: ResourceLimits,
|
|
||||||
window_seconds: BTreeMap<LimitKind, u64>,
|
|
||||||
usage: BTreeMap<LimitKind, u64>,
|
|
||||||
window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
|
|
||||||
},
|
|
||||||
TaskEvents {
|
|
||||||
events: Vec<TaskCompletionEvent>,
|
|
||||||
},
|
|
||||||
TaskSnapshots {
|
|
||||||
snapshots: Vec<TaskAttemptSnapshot>,
|
|
||||||
},
|
|
||||||
RecentLogs {
|
|
||||||
entries: Vec<RecentLogEntry>,
|
|
||||||
next_sequence: Option<u64>,
|
|
||||||
history_truncated: bool,
|
|
||||||
},
|
|
||||||
TaskRestart {
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
restarted_task_instance: Option<TaskInstanceId>,
|
|
||||||
restarted_attempt_id: Option<String>,
|
|
||||||
actor: UserId,
|
|
||||||
accepted: bool,
|
|
||||||
clean_boundary_available: bool,
|
|
||||||
active_task: bool,
|
|
||||||
completed_event_observed: bool,
|
|
||||||
requires_whole_process_restart: bool,
|
|
||||||
message: String,
|
|
||||||
audit_event: DebugAuditEvent,
|
|
||||||
},
|
|
||||||
TaskFailureResolved {
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
attempt_id: String,
|
|
||||||
resolution: TaskFailureResolution,
|
|
||||||
},
|
|
||||||
DebugAttach {
|
|
||||||
process: ProcessId,
|
|
||||||
actor: UserId,
|
|
||||||
authorization: Authorization,
|
|
||||||
audit_event: DebugAuditEvent,
|
|
||||||
},
|
|
||||||
DebugEpoch {
|
|
||||||
process: ProcessId,
|
|
||||||
actor: UserId,
|
|
||||||
epoch: u64,
|
|
||||||
command: String,
|
|
||||||
affected_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
all_stop_requested: bool,
|
|
||||||
audit_event: DebugAuditEvent,
|
|
||||||
},
|
|
||||||
DebugEpochStatus {
|
|
||||||
process: ProcessId,
|
|
||||||
actor: UserId,
|
|
||||||
epoch: u64,
|
|
||||||
command: String,
|
|
||||||
expected_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
acknowledgements: Vec<DebugParticipantAcknowledgement>,
|
|
||||||
fully_frozen: bool,
|
|
||||||
partially_frozen: bool,
|
|
||||||
fully_resumed: bool,
|
|
||||||
failed: bool,
|
|
||||||
failure_messages: Vec<String>,
|
|
||||||
audit_event: DebugAuditEvent,
|
|
||||||
},
|
|
||||||
Artifacts {
|
|
||||||
artifacts: Vec<ArtifactSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
},
|
|
||||||
Artifact {
|
|
||||||
artifact: ArtifactSummary,
|
|
||||||
},
|
|
||||||
ArtifactDownloadLink {
|
|
||||||
link: DownloadLink,
|
|
||||||
},
|
|
||||||
ArtifactDownloadLinkRevoked {},
|
|
||||||
ArtifactDownloadStream {
|
|
||||||
#[serde(rename = "link")]
|
|
||||||
_link: DownloadLink,
|
|
||||||
content_bytes_available: bool,
|
|
||||||
content_offset: Option<u64>,
|
|
||||||
content_eof: bool,
|
|
||||||
content_base64: Option<String>,
|
|
||||||
},
|
|
||||||
WebBrowserLoginStarted {
|
|
||||||
transaction_id: String,
|
|
||||||
authorization_url: String,
|
|
||||||
expires_at_epoch_seconds: u64,
|
|
||||||
},
|
|
||||||
WebBrowserLoginCancelled {},
|
|
||||||
WebBrowserSession {
|
|
||||||
session: WireBrowserSession,
|
|
||||||
},
|
|
||||||
Error {
|
|
||||||
code: ApiErrorCode,
|
|
||||||
category: ApiErrorCategory,
|
|
||||||
message: String,
|
|
||||||
retryable: bool,
|
|
||||||
request_id: String,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use std::collections::BTreeSet;
|
|
||||||
|
|
||||||
use serde::Deserialize;
|
|
||||||
use serde_json::Value;
|
|
||||||
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct OperationFixture {
|
|
||||||
operation: String,
|
|
||||||
boundary: String,
|
|
||||||
request: Value,
|
|
||||||
response: Value,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn website_operation_contract_fixtures_cover_every_typed_request() {
|
|
||||||
let fixtures: Vec<OperationFixture> =
|
|
||||||
serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap();
|
|
||||||
let expected = BTreeSet::from([
|
|
||||||
"abort_process",
|
|
||||||
"auth_status",
|
|
||||||
"begin_web_browser_login",
|
|
||||||
"cancel_web_browser_login",
|
|
||||||
"cancel_process",
|
|
||||||
"create_artifact_download_link",
|
|
||||||
"create_debug_epoch",
|
|
||||||
"create_node_enrollment_grant",
|
|
||||||
"create_project",
|
|
||||||
"debug_attach",
|
|
||||||
"exchange_web_login_handoff",
|
|
||||||
"get_artifact",
|
|
||||||
"inspect_debug_epoch",
|
|
||||||
"list_agent_public_keys",
|
|
||||||
"list_artifacts",
|
|
||||||
"list_node_summaries",
|
|
||||||
"list_process_summaries",
|
|
||||||
"list_projects",
|
|
||||||
"list_recent_logs",
|
|
||||||
"list_task_events",
|
|
||||||
"list_task_snapshots",
|
|
||||||
"open_artifact_download_stream",
|
|
||||||
"quota_status",
|
|
||||||
"register_agent_public_key",
|
|
||||||
"resolve_task_failure",
|
|
||||||
"restart_task",
|
|
||||||
"resume_debug_epoch",
|
|
||||||
"revoke_agent_public_key",
|
|
||||||
"revoke_artifact_download_link",
|
|
||||||
"revoke_cli_session",
|
|
||||||
"revoke_node_credential",
|
|
||||||
"rotate_agent_public_key",
|
|
||||||
"select_project",
|
|
||||||
])
|
|
||||||
.into_iter()
|
|
||||||
.map(str::to_owned)
|
|
||||||
.collect();
|
|
||||||
let mut observed = BTreeSet::new();
|
|
||||||
|
|
||||||
for fixture in fixtures {
|
|
||||||
assert_eq!(fixture.request["type"], fixture.operation);
|
|
||||||
let round_trip = match fixture.boundary.as_str() {
|
|
||||||
"control" => {
|
|
||||||
let typed: AuthenticatedRequest =
|
|
||||||
serde_json::from_value(fixture.request.clone()).unwrap();
|
|
||||||
serde_json::to_value(typed).unwrap()
|
|
||||||
}
|
|
||||||
"login" => {
|
|
||||||
let typed: LoginRequest =
|
|
||||||
serde_json::from_value(fixture.request.clone()).unwrap();
|
|
||||||
serde_json::to_value(typed).unwrap()
|
|
||||||
}
|
|
||||||
boundary => panic!("unknown fixture boundary {boundary}"),
|
|
||||||
};
|
|
||||||
assert_eq!(round_trip, fixture.request);
|
|
||||||
let response: WireResponse = serde_json::from_value(fixture.response).unwrap();
|
|
||||||
assert!(
|
|
||||||
!matches!(response, WireResponse::Error { .. }),
|
|
||||||
"{} must carry its operation-specific success response fixture",
|
|
||||||
fixture.operation
|
|
||||||
);
|
|
||||||
assert!(observed.insert(fixture.operation));
|
|
||||||
}
|
|
||||||
assert_eq!(observed, expected);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,250 +0,0 @@
|
||||||
use std::collections::{BTreeMap, VecDeque};
|
|
||||||
use std::future::Future;
|
|
||||||
use std::pin::Pin;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use clusterflux_control::{endpoint_identity, ControlSession};
|
|
||||||
use thiserror::Error;
|
|
||||||
|
|
||||||
pub type TransportFuture =
|
|
||||||
Pin<Box<dyn Future<Output = Result<TransportResponse, ClientTransportError>> + Send + 'static>>;
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub struct TransportRequest {
|
|
||||||
pub api_path: String,
|
|
||||||
pub body: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub struct TransportResponse {
|
|
||||||
pub body: Vec<u8>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, Error, PartialEq, Eq)]
|
|
||||||
pub enum ClientTransportError {
|
|
||||||
#[error("client transport failed: {0}")]
|
|
||||||
Failed(String),
|
|
||||||
#[error("client transport task failed: {0}")]
|
|
||||||
Task(String),
|
|
||||||
}
|
|
||||||
|
|
||||||
pub trait ClientTransport: Send + Sync + 'static {
|
|
||||||
fn send(&self, request: TransportRequest) -> TransportFuture;
|
|
||||||
}
|
|
||||||
|
|
||||||
type ControlSessionPool = Arc<Vec<Mutex<Option<ControlSession>>>>;
|
|
||||||
|
|
||||||
pub struct ControlTransport {
|
|
||||||
endpoint: String,
|
|
||||||
connect_timeout: Duration,
|
|
||||||
io_timeout: Duration,
|
|
||||||
sessions: Arc<Mutex<BTreeMap<String, ControlSessionPool>>>,
|
|
||||||
next_session: Arc<AtomicU64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
// A small pool allows independent HTMX reads to progress concurrently while
|
|
||||||
// keeping connection and blocking-worker use strictly bounded.
|
|
||||||
const SESSIONS_PER_API_PATH: usize = 4;
|
|
||||||
|
|
||||||
impl ControlTransport {
|
|
||||||
pub fn new(endpoint: impl Into<String>) -> Result<Self, ClientTransportError> {
|
|
||||||
Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn with_timeouts(
|
|
||||||
endpoint: impl Into<String>,
|
|
||||||
connect_timeout: Duration,
|
|
||||||
io_timeout: Duration,
|
|
||||||
) -> Result<Self, ClientTransportError> {
|
|
||||||
let endpoint = endpoint.into();
|
|
||||||
endpoint_identity(&endpoint)
|
|
||||||
.map_err(|error| ClientTransportError::Failed(error.to_string()))?;
|
|
||||||
Ok(Self {
|
|
||||||
endpoint,
|
|
||||||
connect_timeout,
|
|
||||||
io_timeout,
|
|
||||||
sessions: Arc::new(Mutex::new(BTreeMap::new())),
|
|
||||||
next_session: Arc::new(AtomicU64::new(0)),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ClientTransport for ControlTransport {
|
|
||||||
fn send(&self, request: TransportRequest) -> TransportFuture {
|
|
||||||
let endpoint = self.endpoint.clone();
|
|
||||||
let connect_timeout = self.connect_timeout;
|
|
||||||
let io_timeout = self.io_timeout;
|
|
||||||
let sessions = Arc::clone(&self.sessions);
|
|
||||||
let next_session = Arc::clone(&self.next_session);
|
|
||||||
Box::pin(async move {
|
|
||||||
let pool = {
|
|
||||||
let mut sessions = sessions.lock().map_err(|_| {
|
|
||||||
ClientTransportError::Failed(
|
|
||||||
"client transport pool lock was poisoned".to_owned(),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| {
|
|
||||||
Arc::new(
|
|
||||||
(0..SESSIONS_PER_API_PATH)
|
|
||||||
.map(|_| Mutex::new(None))
|
|
||||||
.collect(),
|
|
||||||
)
|
|
||||||
}))
|
|
||||||
};
|
|
||||||
let slot_index =
|
|
||||||
next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH;
|
|
||||||
tokio::task::spawn_blocking(move || {
|
|
||||||
let value = serde_json::from_slice(&request.body)
|
|
||||||
.map_err(|error| ClientTransportError::Failed(error.to_string()))?;
|
|
||||||
let mut selected = None;
|
|
||||||
for offset in 0..SESSIONS_PER_API_PATH {
|
|
||||||
let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
|
|
||||||
match pool[index].try_lock() {
|
|
||||||
Ok(session) if session.is_some() => {
|
|
||||||
selected = Some(session);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {}
|
|
||||||
Err(std::sync::TryLockError::Poisoned(_)) => {
|
|
||||||
return Err(ClientTransportError::Failed(
|
|
||||||
"client transport session lock was poisoned".to_owned(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if selected.is_none() {
|
|
||||||
for offset in 0..SESSIONS_PER_API_PATH {
|
|
||||||
let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
|
|
||||||
match pool[index].try_lock() {
|
|
||||||
Ok(session) => {
|
|
||||||
selected = Some(session);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(std::sync::TryLockError::WouldBlock) => {}
|
|
||||||
Err(std::sync::TryLockError::Poisoned(_)) => {
|
|
||||||
return Err(ClientTransportError::Failed(
|
|
||||||
"client transport session lock was poisoned".to_owned(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let mut session = match selected {
|
|
||||||
Some(session) => session,
|
|
||||||
None => pool[slot_index].lock().map_err(|_| {
|
|
||||||
ClientTransportError::Failed(
|
|
||||||
"client transport session lock was poisoned".to_owned(),
|
|
||||||
)
|
|
||||||
})?,
|
|
||||||
};
|
|
||||||
if session.is_none() {
|
|
||||||
*session = Some(
|
|
||||||
ControlSession::connect_to_api_path_with_timeouts(
|
|
||||||
&endpoint,
|
|
||||||
&request.api_path,
|
|
||||||
connect_timeout,
|
|
||||||
io_timeout,
|
|
||||||
)
|
|
||||||
.map_err(|error| ClientTransportError::Failed(error.to_string()))?,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let response = session
|
|
||||||
.as_mut()
|
|
||||||
.expect("session was initialized for the requested API path")
|
|
||||||
.request(&value);
|
|
||||||
match response {
|
|
||||||
Ok(response) => serde_json::to_vec(&response)
|
|
||||||
.map(|body| TransportResponse { body })
|
|
||||||
.map_err(|error| ClientTransportError::Failed(error.to_string())),
|
|
||||||
Err(error) => {
|
|
||||||
*session = None;
|
|
||||||
Err(ClientTransportError::Failed(error.to_string()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|error| ClientTransportError::Task(error.to_string()))?
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
|
||||||
pub struct MockTransport {
|
|
||||||
state: Arc<Mutex<MockTransportState>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct MockTransportState {
|
|
||||||
responses: VecDeque<Result<Vec<u8>, ClientTransportError>>,
|
|
||||||
requests: Vec<TransportRequest>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MockTransport {
|
|
||||||
pub fn from_json_responses(responses: impl IntoIterator<Item = impl Into<String>>) -> Self {
|
|
||||||
let responses = responses
|
|
||||||
.into_iter()
|
|
||||||
.map(|response| Ok(response.into().into_bytes()))
|
|
||||||
.collect();
|
|
||||||
Self {
|
|
||||||
state: Arc::new(Mutex::new(MockTransportState {
|
|
||||||
responses,
|
|
||||||
requests: Vec::new(),
|
|
||||||
})),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn push_json_response(&self, response: impl Into<String>) {
|
|
||||||
self.state
|
|
||||||
.lock()
|
|
||||||
.expect("mock transport lock is not poisoned")
|
|
||||||
.responses
|
|
||||||
.push_back(Ok(response.into().into_bytes()));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn push_error(&self, message: impl Into<String>) {
|
|
||||||
self.state
|
|
||||||
.lock()
|
|
||||||
.expect("mock transport lock is not poisoned")
|
|
||||||
.responses
|
|
||||||
.push_back(Err(ClientTransportError::Failed(message.into())));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn request_bodies(&self) -> Vec<String> {
|
|
||||||
self.state
|
|
||||||
.lock()
|
|
||||||
.expect("mock transport lock is not poisoned")
|
|
||||||
.requests
|
|
||||||
.iter()
|
|
||||||
.map(|request| String::from_utf8_lossy(&request.body).into_owned())
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn requests(&self) -> Vec<TransportRequest> {
|
|
||||||
self.state
|
|
||||||
.lock()
|
|
||||||
.expect("mock transport lock is not poisoned")
|
|
||||||
.requests
|
|
||||||
.clone()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl ClientTransport for MockTransport {
|
|
||||||
fn send(&self, request: TransportRequest) -> TransportFuture {
|
|
||||||
let state = Arc::clone(&self.state);
|
|
||||||
Box::pin(async move {
|
|
||||||
let mut state = state.lock().map_err(|_| {
|
|
||||||
ClientTransportError::Failed("mock transport lock was poisoned".to_owned())
|
|
||||||
})?;
|
|
||||||
state.requests.push(request);
|
|
||||||
state
|
|
||||||
.responses
|
|
||||||
.pop_front()
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ClientTransportError::Failed("mock transport has no queued response".to_owned())
|
|
||||||
})?
|
|
||||||
.map(|body| TransportResponse { body })
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -1,468 +0,0 @@
|
||||||
use std::collections::BTreeMap;
|
|
||||||
|
|
||||||
use clusterflux_core::{
|
|
||||||
AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement,
|
|
||||||
ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy,
|
|
||||||
TaskInstanceId, TenantId, UserId, VfsPath,
|
|
||||||
};
|
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use std::fmt;
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct AccountStatus {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub authenticated: bool,
|
|
||||||
pub account_status: String,
|
|
||||||
pub suspended: bool,
|
|
||||||
pub disabled: bool,
|
|
||||||
pub deleted: bool,
|
|
||||||
pub manual_review: bool,
|
|
||||||
pub sanitized_reason: Option<String>,
|
|
||||||
pub next_actions: Vec<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct Project {
|
|
||||||
pub id: ProjectId,
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub name: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct AgentPublicKey {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub user: UserId,
|
|
||||||
pub agent: AgentId,
|
|
||||||
pub public_key: String,
|
|
||||||
pub public_key_fingerprint: Digest,
|
|
||||||
pub version: u64,
|
|
||||||
pub revoked: bool,
|
|
||||||
pub scopes: Vec<String>,
|
|
||||||
pub human_account_creation_privilege: bool,
|
|
||||||
pub browser_interaction_required_each_run: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct NodeEnrollmentGrant {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub grant: String,
|
|
||||||
pub scope: String,
|
|
||||||
pub expires_at_epoch_seconds: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct NodeSummary {
|
|
||||||
pub id: NodeId,
|
|
||||||
pub display_name: String,
|
|
||||||
pub online: bool,
|
|
||||||
pub stale: bool,
|
|
||||||
pub last_seen_epoch_seconds: Option<u64>,
|
|
||||||
pub capabilities: NodeCapabilities,
|
|
||||||
pub direct_connectivity: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct NodePage {
|
|
||||||
pub nodes: Vec<NodeSummary>,
|
|
||||||
pub next_cursor: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct NodeRevocation {
|
|
||||||
pub node: NodeId,
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub descriptor_removed: bool,
|
|
||||||
pub queued_assignments_removed: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessLifecycleState {
|
|
||||||
Active,
|
|
||||||
RecentTerminal,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessActivityState {
|
|
||||||
Running,
|
|
||||||
WaitingForNode,
|
|
||||||
WaitingForTask,
|
|
||||||
AwaitingAction,
|
|
||||||
DebugEpochPartial,
|
|
||||||
Cancelling,
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessFinalResult {
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugEpochSummary {
|
|
||||||
pub epoch: u64,
|
|
||||||
pub command: String,
|
|
||||||
pub fully_frozen: bool,
|
|
||||||
pub partially_frozen: bool,
|
|
||||||
pub fully_resumed: bool,
|
|
||||||
pub failed: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ProcessSummary {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub lifecycle: ProcessLifecycleState,
|
|
||||||
pub activity: ProcessActivityState,
|
|
||||||
pub main_wait_state: Option<String>,
|
|
||||||
pub started_at_epoch_seconds: u64,
|
|
||||||
pub ended_at_epoch_seconds: Option<u64>,
|
|
||||||
pub final_result: Option<ProcessFinalResult>,
|
|
||||||
pub connected_nodes: Vec<NodeId>,
|
|
||||||
pub current_debug_epoch: Option<DebugEpochSummary>,
|
|
||||||
pub order_cursor: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ProcessPage {
|
|
||||||
pub processes: Vec<ProcessSummary>,
|
|
||||||
pub next_cursor: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskTerminalState {
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskExecutor {
|
|
||||||
CoordinatorMain,
|
|
||||||
Node,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskCompletionEvent {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub node: NodeId,
|
|
||||||
pub executor: TaskExecutor,
|
|
||||||
pub task_definition: TaskDefinitionId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub attempt_id: Option<String>,
|
|
||||||
pub placement: Option<Placement>,
|
|
||||||
pub terminal_state: TaskTerminalState,
|
|
||||||
pub status_code: Option<i32>,
|
|
||||||
pub stdout_bytes: u64,
|
|
||||||
pub stderr_bytes: u64,
|
|
||||||
pub stdout_tail: String,
|
|
||||||
pub stderr_tail: String,
|
|
||||||
pub stdout_truncated: bool,
|
|
||||||
pub stderr_truncated: bool,
|
|
||||||
pub artifact_path: Option<VfsPath>,
|
|
||||||
pub artifact_digest: Option<Digest>,
|
|
||||||
pub artifact_size_bytes: Option<u64>,
|
|
||||||
pub result: Option<TaskBoundaryValue>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskAttemptState {
|
|
||||||
Queued,
|
|
||||||
Running,
|
|
||||||
FailedAwaitingAction,
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskAttemptSnapshot {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub attempt_id: String,
|
|
||||||
pub attempt_number: u32,
|
|
||||||
pub task_definition: TaskDefinitionId,
|
|
||||||
pub display_name: String,
|
|
||||||
pub state: TaskAttemptState,
|
|
||||||
pub current: bool,
|
|
||||||
pub node: Option<NodeId>,
|
|
||||||
pub environment_id: Option<String>,
|
|
||||||
pub environment_digest: Option<Digest>,
|
|
||||||
pub argument_summary: Vec<String>,
|
|
||||||
pub handle_summary: Vec<String>,
|
|
||||||
pub command_state: Option<String>,
|
|
||||||
pub vfs_checkpoint: String,
|
|
||||||
pub probe_symbol: Option<String>,
|
|
||||||
pub source_path: Option<String>,
|
|
||||||
pub source_line: Option<u32>,
|
|
||||||
pub restart_compatible: bool,
|
|
||||||
pub failure_policy: TaskFailurePolicy,
|
|
||||||
pub artifact_path: Option<VfsPath>,
|
|
||||||
pub artifact_digest: Option<Digest>,
|
|
||||||
pub artifact_size_bytes: Option<u64>,
|
|
||||||
pub status_code: Option<i32>,
|
|
||||||
pub error: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskLogStream {
|
|
||||||
Stdout,
|
|
||||||
Stderr,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RecentLogEntry {
|
|
||||||
pub sequence: u64,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub stream: TaskLogStream,
|
|
||||||
pub text: String,
|
|
||||||
pub server_timestamp_epoch_seconds: u64,
|
|
||||||
pub truncated: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RecentLogPage {
|
|
||||||
pub entries: Vec<RecentLogEntry>,
|
|
||||||
pub next_sequence: Option<u64>,
|
|
||||||
pub history_truncated: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ArtifactAvailability {
|
|
||||||
Available,
|
|
||||||
NodeOffline,
|
|
||||||
Unavailable,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ArtifactRetentionState {
|
|
||||||
NodeRetained,
|
|
||||||
ExplicitStorage,
|
|
||||||
Lost,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ArtifactSummary {
|
|
||||||
pub id: ArtifactId,
|
|
||||||
pub display_path: String,
|
|
||||||
pub display_name: String,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub producer_task: TaskInstanceId,
|
|
||||||
pub safe_node: Option<NodeId>,
|
|
||||||
pub digest: Digest,
|
|
||||||
pub size_bytes: u64,
|
|
||||||
pub availability: ArtifactAvailability,
|
|
||||||
pub downloadable_now: bool,
|
|
||||||
pub retention_state: ArtifactRetentionState,
|
|
||||||
pub explicit_storage: bool,
|
|
||||||
pub order_cursor: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ArtifactPage {
|
|
||||||
pub artifacts: Vec<ArtifactSummary>,
|
|
||||||
pub next_cursor: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct QuotaStatus {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub policy_label: Option<String>,
|
|
||||||
pub limits: ResourceLimits,
|
|
||||||
pub window_seconds: BTreeMap<LimitKind, u64>,
|
|
||||||
pub usage: BTreeMap<LimitKind, u64>,
|
|
||||||
pub window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskCancellationTarget {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub node: NodeId,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ProcessCancellation {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub affected_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
pub affected_nodes: Vec<NodeId>,
|
|
||||||
pub aborted: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugAuditEvent {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: Option<TaskInstanceId>,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub operation: String,
|
|
||||||
pub allowed: bool,
|
|
||||||
pub reason: String,
|
|
||||||
pub charged_debug_read_bytes: u64,
|
|
||||||
pub used_debug_read_bytes: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugAttach {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub authorization: Authorization,
|
|
||||||
pub audit_event: DebugAuditEvent,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugEpochControl {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub epoch: u64,
|
|
||||||
pub command: String,
|
|
||||||
pub affected_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
pub all_stop_requested: bool,
|
|
||||||
pub audit_event: DebugAuditEvent,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum DebugAcknowledgementState {
|
|
||||||
Frozen,
|
|
||||||
Running,
|
|
||||||
Failed,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugParticipantAcknowledgement {
|
|
||||||
pub node: NodeId,
|
|
||||||
pub task_definition: TaskDefinitionId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub epoch: u64,
|
|
||||||
pub state: DebugAcknowledgementState,
|
|
||||||
pub stack_frames: Vec<String>,
|
|
||||||
pub local_values: Vec<(String, String)>,
|
|
||||||
pub task_args: Vec<(String, String)>,
|
|
||||||
pub handles: Vec<(String, String)>,
|
|
||||||
pub command_status: Option<String>,
|
|
||||||
pub recent_output: Vec<String>,
|
|
||||||
pub message: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugEpochStatus {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub epoch: u64,
|
|
||||||
pub command: String,
|
|
||||||
pub expected_tasks: Vec<TaskCancellationTarget>,
|
|
||||||
pub acknowledgements: Vec<DebugParticipantAcknowledgement>,
|
|
||||||
pub fully_frozen: bool,
|
|
||||||
pub partially_frozen: bool,
|
|
||||||
pub fully_resumed: bool,
|
|
||||||
pub failed: bool,
|
|
||||||
pub failure_messages: Vec<String>,
|
|
||||||
pub audit_event: DebugAuditEvent,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskRestart {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub restarted_task_instance: Option<TaskInstanceId>,
|
|
||||||
pub restarted_attempt_id: Option<String>,
|
|
||||||
pub actor: UserId,
|
|
||||||
pub accepted: bool,
|
|
||||||
pub clean_boundary_available: bool,
|
|
||||||
pub active_task: bool,
|
|
||||||
pub completed_event_observed: bool,
|
|
||||||
pub requires_whole_process_restart: bool,
|
|
||||||
pub message: String,
|
|
||||||
pub audit_event: DebugAuditEvent,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskFailureResolutionResult {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub attempt_id: String,
|
|
||||||
pub resolution: TaskFailureResolution,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskFailureResolution {
|
|
||||||
AcceptFailure,
|
|
||||||
Cancel,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct TaskReplacementBundle {
|
|
||||||
pub bundle_digest: Digest,
|
|
||||||
pub wasm_module_base64: String,
|
|
||||||
pub source_snapshot: Option<Digest>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct BrowserLoginStart {
|
|
||||||
pub transaction_id: String,
|
|
||||||
pub authorization_url: String,
|
|
||||||
pub expires_at_epoch_seconds: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, PartialEq, Eq)]
|
|
||||||
pub struct SessionCredential(pub(crate) String);
|
|
||||||
|
|
||||||
impl SessionCredential {
|
|
||||||
pub fn from_secret(secret: impl Into<String>) -> Self {
|
|
||||||
Self(secret.into())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn expose_secret(&self) -> &str {
|
|
||||||
&self.0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Debug for SessionCredential {
|
|
||||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
formatter.write_str("SessionCredential([REDACTED])")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub struct BrowserSession {
|
|
||||||
pub tenant: TenantId,
|
|
||||||
pub project: ProjectId,
|
|
||||||
pub user: UserId,
|
|
||||||
pub credential: SessionCredential,
|
|
||||||
pub expires_at_epoch_seconds: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub enum ArtifactDownloadPoll {
|
|
||||||
Pending,
|
|
||||||
Chunk {
|
|
||||||
offset: u64,
|
|
||||||
bytes: Vec<u8>,
|
|
||||||
eof: bool,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
@ -1,254 +0,0 @@
|
||||||
use std::net::TcpListener;
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
use clusterflux_client::{
|
|
||||||
ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError,
|
|
||||||
ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId,
|
|
||||||
UserId, CLIENT_API_VERSION,
|
|
||||||
};
|
|
||||||
use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
|
|
||||||
use clusterflux_coordinator::service::CoordinatorService;
|
|
||||||
use serde_json::{json, Value};
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn mock_transport_exercises_typed_envelope_and_session_plumbing() {
|
|
||||||
let transport = MockTransport::from_json_responses([json!({
|
|
||||||
"type": "projects",
|
|
||||||
"projects": [{
|
|
||||||
"id": "project-one",
|
|
||||||
"tenant": "tenant-one",
|
|
||||||
"name": "Project one"
|
|
||||||
}],
|
|
||||||
"actor": "user-one"
|
|
||||||
})
|
|
||||||
.to_string()]);
|
|
||||||
let client = ClusterfluxClient::with_transport(transport.clone())
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
|
|
||||||
|
|
||||||
let projects = client.list_projects().await.unwrap();
|
|
||||||
assert_eq!(projects.len(), 1);
|
|
||||||
assert_eq!(projects[0].id, ProjectId::from("project-one"));
|
|
||||||
|
|
||||||
let requests = transport.requests();
|
|
||||||
assert_eq!(requests.len(), 1);
|
|
||||||
assert_eq!(requests[0].api_path, CONTROL_API_PATH);
|
|
||||||
let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
|
|
||||||
assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION);
|
|
||||||
assert_eq!(envelope["request_id"], "client-1");
|
|
||||||
assert_eq!(envelope["operation"], "authenticated");
|
|
||||||
assert_eq!(envelope["payload"]["type"], "authenticated");
|
|
||||||
assert_eq!(envelope["payload"]["request"]["type"], "list_projects");
|
|
||||||
assert_eq!(envelope["payload"]["session_secret"], "test-session-secret");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn structured_errors_retain_machine_fields_and_originating_request_id() {
|
|
||||||
let transport = MockTransport::from_json_responses([json!({
|
|
||||||
"type": "error",
|
|
||||||
"code": "account_suspended",
|
|
||||||
"category": "authorization",
|
|
||||||
"message": "account access is suspended",
|
|
||||||
"retryable": false,
|
|
||||||
"request_id": "client-1"
|
|
||||||
})
|
|
||||||
.to_string()]);
|
|
||||||
let client = ClusterfluxClient::with_transport(transport)
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
|
|
||||||
|
|
||||||
let ClientError::Api(error) = client.account_status().await.unwrap_err() else {
|
|
||||||
panic!("expected typed API error");
|
|
||||||
};
|
|
||||||
assert_eq!(error.code, ApiErrorCode::AccountSuspended);
|
|
||||||
assert_eq!(error.category, ApiErrorCategory::Authorization);
|
|
||||||
assert_eq!(error.request_id, "client-1");
|
|
||||||
assert!(!error.retryable);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() {
|
|
||||||
let transport = MockTransport::from_json_responses([
|
|
||||||
json!({ "type": "web_browser_login_cancelled" }).to_string(),
|
|
||||||
]);
|
|
||||||
let client = ClusterfluxClient::with_transport(transport.clone());
|
|
||||||
|
|
||||||
client
|
|
||||||
.cancel_browser_login("login-transaction")
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let requests = transport.requests();
|
|
||||||
assert_eq!(requests.len(), 1);
|
|
||||||
assert_eq!(requests[0].api_path, LOGIN_API_PATH);
|
|
||||||
let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
|
|
||||||
assert_eq!(envelope["operation"], "cancel_web_browser_login");
|
|
||||||
assert_eq!(
|
|
||||||
envelope["payload"],
|
|
||||||
json!({
|
|
||||||
"type": "cancel_web_browser_login",
|
|
||||||
"transaction_id": "login-transaction"
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() {
|
|
||||||
let transport = MockTransport::from_json_responses([json!({
|
|
||||||
"type": "error",
|
|
||||||
"code": "validation_error",
|
|
||||||
"category": "validation",
|
|
||||||
"message": "bad request",
|
|
||||||
"retryable": false,
|
|
||||||
"request_id": "another-request"
|
|
||||||
})
|
|
||||||
.to_string()]);
|
|
||||||
let client = ClusterfluxClient::with_transport(transport)
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
|
|
||||||
|
|
||||||
let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else {
|
|
||||||
panic!("expected protocol error");
|
|
||||||
};
|
|
||||||
assert!(message.contains("does not match client-1"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn session_credentials_are_redacted_from_debug_output() {
|
|
||||||
let credential = SessionCredential::from_secret("must-not-appear");
|
|
||||||
assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn artifact_download_is_a_typed_bounded_chunk_stream() {
|
|
||||||
let link = json!({
|
|
||||||
"artifact": "artifact-one",
|
|
||||||
"artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
|
||||||
"artifact_size_bytes": 5,
|
|
||||||
"source": { "RetainedNode": "node-one" },
|
|
||||||
"url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one",
|
|
||||||
"scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
|
|
||||||
"expires_at_epoch_seconds": 1000,
|
|
||||||
"tenant": "tenant-one",
|
|
||||||
"project": "project-one",
|
|
||||||
"process": "process-one",
|
|
||||||
"actor": { "User": "user-one" },
|
|
||||||
"max_bytes": 1024,
|
|
||||||
"policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
|
|
||||||
});
|
|
||||||
let transport = MockTransport::from_json_responses([
|
|
||||||
json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(),
|
|
||||||
json!({
|
|
||||||
"type": "artifact_download_stream",
|
|
||||||
"link": link.clone(),
|
|
||||||
"streamed_bytes": 0,
|
|
||||||
"charged_download_bytes": 0,
|
|
||||||
"content_bytes_available": false,
|
|
||||||
"content_eof": false
|
|
||||||
})
|
|
||||||
.to_string(),
|
|
||||||
json!({
|
|
||||||
"type": "artifact_download_stream",
|
|
||||||
"link": link,
|
|
||||||
"streamed_bytes": 5,
|
|
||||||
"charged_download_bytes": 5,
|
|
||||||
"content_bytes_available": true,
|
|
||||||
"content_offset": 0,
|
|
||||||
"content_eof": true,
|
|
||||||
"content_base64": "aGVsbG8="
|
|
||||||
})
|
|
||||||
.to_string(),
|
|
||||||
]);
|
|
||||||
let client = ClusterfluxClient::with_transport(transport)
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
|
|
||||||
let mut download = client
|
|
||||||
.begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
download.next_chunk().await.unwrap(),
|
|
||||||
ArtifactDownloadPoll::Pending
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
download.next_chunk().await.unwrap(),
|
|
||||||
ArtifactDownloadPoll::Chunk {
|
|
||||||
offset: 0,
|
|
||||||
bytes: b"hello".to_vec(),
|
|
||||||
eof: true,
|
|
||||||
}
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn typed_client_runs_against_the_real_strict_control_endpoint() {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
||||||
let address = listener.local_addr().unwrap();
|
|
||||||
let server = std::thread::spawn(move || {
|
|
||||||
let mut service = CoordinatorService::new(7);
|
|
||||||
service
|
|
||||||
.issue_cli_session(
|
|
||||||
TenantId::from("tenant-one"),
|
|
||||||
ProjectId::from("project-one"),
|
|
||||||
UserId::from("user-one"),
|
|
||||||
"real-endpoint-session",
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let (stream, _) = listener.accept().unwrap();
|
|
||||||
service.handle_stream(stream).unwrap();
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}"))
|
|
||||||
.unwrap()
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("real-endpoint-session"));
|
|
||||||
let projects = client.list_projects().await.unwrap();
|
|
||||||
assert_eq!(projects.len(), 1);
|
|
||||||
assert_eq!(projects[0].id, ProjectId::from("project-one"));
|
|
||||||
let status = client.account_status().await.unwrap();
|
|
||||||
assert!(status.authenticated);
|
|
||||||
assert_eq!(status.actor, UserId::from("user-one"));
|
|
||||||
|
|
||||||
drop(client);
|
|
||||||
server.join().unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn concurrent_requests_expand_the_bounded_pool_without_serializing() {
|
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
||||||
let address = listener.local_addr().unwrap();
|
|
||||||
let server = std::thread::spawn(move || {
|
|
||||||
let handlers = (0..2)
|
|
||||||
.map(|_| {
|
|
||||||
let (stream, _) = listener.accept().unwrap();
|
|
||||||
std::thread::spawn(move || {
|
|
||||||
let mut service = CoordinatorService::new(7);
|
|
||||||
service
|
|
||||||
.issue_cli_session(
|
|
||||||
TenantId::from("tenant-one"),
|
|
||||||
ProjectId::from("project-one"),
|
|
||||||
UserId::from("user-one"),
|
|
||||||
"concurrent-session",
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
service.handle_stream(stream).unwrap();
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
for handler in handlers {
|
|
||||||
handler.join().unwrap();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let transport = ControlTransport::with_timeouts(
|
|
||||||
format!("clusterflux+tcp://{address}"),
|
|
||||||
Duration::from_secs(2),
|
|
||||||
Duration::from_secs(5),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let client = ClusterfluxClient::with_transport(transport)
|
|
||||||
.with_session_credential(&SessionCredential::from_secret("concurrent-session"));
|
|
||||||
let (first, second) = tokio::join!(client.account_status(), client.account_status());
|
|
||||||
assert!(first.unwrap().authenticated);
|
|
||||||
assert!(second.unwrap().authenticated);
|
|
||||||
|
|
||||||
drop(client);
|
|
||||||
server.join().unwrap();
|
|
||||||
}
|
|
||||||
|
|
@ -1,200 +0,0 @@
|
||||||
[
|
|
||||||
{
|
|
||||||
"operation": "begin_web_browser_login",
|
|
||||||
"boundary": "login",
|
|
||||||
"request": { "type": "begin_web_browser_login" },
|
|
||||||
"response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "cancel_web_browser_login",
|
|
||||||
"boundary": "login",
|
|
||||||
"request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" },
|
|
||||||
"response": { "type": "web_browser_login_cancelled" }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "exchange_web_login_handoff",
|
|
||||||
"boundary": "login",
|
|
||||||
"request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" },
|
|
||||||
"response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "auth_status",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "auth_status" },
|
|
||||||
"response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "revoke_cli_session",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "revoke_cli_session" },
|
|
||||||
"response": { "type": "cli_session_revoked" }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "create_project",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "create_project", "project": "project-new", "name": "New project" },
|
|
||||||
"response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "select_project",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "select_project", "project": "project-selected" },
|
|
||||||
"response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_projects",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_projects" },
|
|
||||||
"response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "register_agent_public_key",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" },
|
|
||||||
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_agent_public_keys",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_agent_public_keys" },
|
|
||||||
"response": { "type": "agent_public_keys", "records": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "rotate_agent_public_key",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" },
|
|
||||||
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "revoke_agent_public_key",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "revoke_agent_public_key", "agent": "agent-browser" },
|
|
||||||
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "create_node_enrollment_grant",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 },
|
|
||||||
"response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_node_summaries",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_node_summaries", "cursor": null, "limit": 200 },
|
|
||||||
"response": { "type": "node_summaries", "nodes": [], "next_cursor": null }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "revoke_node_credential",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "revoke_node_credential", "node": "node-one" },
|
|
||||||
"response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_process_summaries",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 },
|
|
||||||
"response": { "type": "process_summaries", "processes": [], "next_cursor": null }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "cancel_process",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "cancel_process", "process": "process-one" },
|
|
||||||
"response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "abort_process",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "abort_process", "process": "process-one", "launch_attempt": null },
|
|
||||||
"response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "quota_status",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "quota_status" },
|
|
||||||
"response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_task_events",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_task_events", "process": "process-one" },
|
|
||||||
"response": { "type": "task_events", "events": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_task_snapshots",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_task_snapshots", "process": "process-one" },
|
|
||||||
"response": { "type": "task_snapshots", "snapshots": [] }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_recent_logs",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 },
|
|
||||||
"response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "restart_task",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null },
|
|
||||||
"response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "resolve_task_failure",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" },
|
|
||||||
"response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "debug_attach",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "debug_attach", "process": "process-one" },
|
|
||||||
"response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "create_debug_epoch",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" },
|
|
||||||
"response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "resume_debug_epoch",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 },
|
|
||||||
"response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "inspect_debug_epoch",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 },
|
|
||||||
"response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "list_artifacts",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 },
|
|
||||||
"response": { "type": "artifacts", "artifacts": [], "next_cursor": null }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "get_artifact",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "get_artifact", "artifact": "artifact-one" },
|
|
||||||
"response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "create_artifact_download_link",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 },
|
|
||||||
"response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "open_artifact_download_stream",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 },
|
|
||||||
"response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" }
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"operation": "revoke_artifact_download_link",
|
|
||||||
"boundary": "control",
|
|
||||||
"request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
|
|
||||||
"response": { "type": "artifact_download_link_revoked" }
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
@ -61,21 +61,7 @@ impl ControlSession {
|
||||||
endpoint: &str,
|
endpoint: &str,
|
||||||
api_path: &str,
|
api_path: &str,
|
||||||
) -> Result<Self, ControlTransportError> {
|
) -> Result<Self, ControlTransportError> {
|
||||||
Self::connect_to_api_path_with_timeouts(
|
let session = Self::connect(endpoint)?;
|
||||||
endpoint,
|
|
||||||
api_path,
|
|
||||||
Duration::from_secs(10),
|
|
||||||
Duration::from_secs(30),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn connect_to_api_path_with_timeouts(
|
|
||||||
endpoint: &str,
|
|
||||||
api_path: &str,
|
|
||||||
connect_timeout: Duration,
|
|
||||||
io_timeout: Duration,
|
|
||||||
) -> Result<Self, ControlTransportError> {
|
|
||||||
let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?;
|
|
||||||
#[cfg(not(target_arch = "wasm32"))]
|
#[cfg(not(target_arch = "wasm32"))]
|
||||||
{
|
{
|
||||||
let mut session = session;
|
let mut session = session;
|
||||||
|
|
|
||||||
|
|
@ -585,22 +585,6 @@ impl Coordinator {
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn tenant_count(&self) -> usize {
|
|
||||||
self.durable.tenants.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn user_count(&self) -> usize {
|
|
||||||
self.durable.users.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn project_count(&self) -> usize {
|
|
||||||
self.durable.projects.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn node_identity_count(&self) -> usize {
|
|
||||||
self.durable.node_identities.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn node_identity(
|
pub fn node_identity(
|
||||||
&self,
|
&self,
|
||||||
tenant: &TenantId,
|
tenant: &TenantId,
|
||||||
|
|
@ -1062,7 +1046,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn account_policy_state_summarizes_sensitive_admin_records_safely() {
|
fn account_policy_state_summarizes_private_admin_records_safely() {
|
||||||
let tenant = TenantId::from("tenant");
|
let tenant = TenantId::from("tenant");
|
||||||
let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
|
let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,40 +5,17 @@ use clusterflux_core::{ProjectId, TenantId, UserId};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
|
|
||||||
match raw_args.as_slice() {
|
|
||||||
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
|
|
||||||
println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION"));
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
|
|
||||||
println!(
|
|
||||||
"Clusterflux coordinator.\n\n\
|
|
||||||
Usage: clusterflux-coordinator [OPTIONS]\n\n\
|
|
||||||
Options:\n \
|
|
||||||
--listen <ADDRESS> [default: 127.0.0.1:0]\n \
|
|
||||||
--allow-local-trusted-loopback\n \
|
|
||||||
-h, --help\n \
|
|
||||||
-V, --version"
|
|
||||||
);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut listen = "127.0.0.1:0".to_owned();
|
let mut listen = "127.0.0.1:0".to_owned();
|
||||||
let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
|
let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
|
||||||
.ok()
|
.ok()
|
||||||
.as_deref()
|
.as_deref()
|
||||||
== Some("1");
|
== Some("1");
|
||||||
let mut args = raw_args.into_iter();
|
let mut args = std::env::args().skip(1);
|
||||||
while let Some(arg) = args.next() {
|
while let Some(arg) = args.next() {
|
||||||
if arg == "--listen" {
|
if arg == "--listen" {
|
||||||
listen = args.next().ok_or("--listen requires an address")?;
|
listen = args.next().ok_or("--listen requires an address")?;
|
||||||
} else if arg == "--allow-local-trusted-loopback" {
|
} else if arg == "--allow-local-trusted-loopback" {
|
||||||
allow_local_trusted = true;
|
allow_local_trusted = true;
|
||||||
} else {
|
|
||||||
return Err(format!("unknown argument: {arg}").into());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,9 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use clusterflux_core::{
|
use clusterflux_core::{
|
||||||
Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
|
Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
|
||||||
CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
|
NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
|
||||||
NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
|
RateLimit, TenantId, TransportError, UserId,
|
||||||
TenantId, TransportError, UserId,
|
|
||||||
};
|
};
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
|
@ -35,7 +34,6 @@ mod quota;
|
||||||
mod relay;
|
mod relay;
|
||||||
mod routing;
|
mod routing;
|
||||||
mod signed_nodes;
|
mod signed_nodes;
|
||||||
mod summaries;
|
|
||||||
mod tcp;
|
mod tcp;
|
||||||
mod wire_protocol;
|
mod wire_protocol;
|
||||||
use authorization::authorize_authenticated_user_operation;
|
use authorization::authorize_authenticated_user_operation;
|
||||||
|
|
@ -45,14 +43,12 @@ use keys::{
|
||||||
ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
|
ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
|
||||||
};
|
};
|
||||||
pub use protocol::{
|
pub use protocol::{
|
||||||
ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
|
ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
|
||||||
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
|
CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
|
||||||
DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
|
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
|
||||||
NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
|
TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
|
||||||
RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
|
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
|
||||||
TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
|
TaskTerminalState, VirtualProcessStatus, WorkflowActor,
|
||||||
TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
|
|
||||||
VirtualProcessStatus, WorkflowActor,
|
|
||||||
};
|
};
|
||||||
pub use quota::CoordinatorQuotaConfiguration;
|
pub use quota::CoordinatorQuotaConfiguration;
|
||||||
pub use relay::{
|
pub use relay::{
|
||||||
|
|
@ -73,15 +69,9 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
|
||||||
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
|
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
|
||||||
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
|
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
|
||||||
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
|
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
|
||||||
const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
|
const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
|
||||||
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
|
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
|
||||||
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
|
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
|
||||||
const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
|
|
||||||
const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
|
|
||||||
const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
|
|
||||||
const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
|
|
||||||
const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
|
|
||||||
const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
|
|
||||||
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
|
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
|
||||||
fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
|
fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
|
||||||
requested.clamp(1, maximum)
|
requested.clamp(1, maximum)
|
||||||
|
|
@ -121,24 +111,6 @@ pub struct CoordinatorAdmission {
|
||||||
pub max_artifact_download_ttl_seconds: u64,
|
pub max_artifact_download_ttl_seconds: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
||||||
pub struct CoordinatorOperationalMetrics {
|
|
||||||
pub tenants: usize,
|
|
||||||
pub users: usize,
|
|
||||||
pub projects: usize,
|
|
||||||
pub enrolled_nodes: usize,
|
|
||||||
pub reported_nodes: usize,
|
|
||||||
pub live_nodes: usize,
|
|
||||||
pub active_processes: usize,
|
|
||||||
pub active_coordinator_mains: usize,
|
|
||||||
pub max_active_coordinator_mains: usize,
|
|
||||||
pub active_tasks: usize,
|
|
||||||
pub queued_tasks: usize,
|
|
||||||
pub artifacts: usize,
|
|
||||||
pub retained_download_links: usize,
|
|
||||||
pub relay: ArtifactRelayUsage,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for CoordinatorAdmission {
|
impl Default for CoordinatorAdmission {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
|
|
@ -179,98 +151,6 @@ pub enum CoordinatorServiceError {
|
||||||
Durable(String),
|
Durable(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CoordinatorServiceError {
|
|
||||||
pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
|
|
||||||
let request_id = request_id.into();
|
|
||||||
let message = self.to_string();
|
|
||||||
let (code, category, retryable) = match self {
|
|
||||||
Self::Io(_) => (
|
|
||||||
ApiErrorCode::TemporaryCapacity,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Json(_)
|
|
||||||
| Self::CapabilityReport(_)
|
|
||||||
| Self::InvalidArtifactPath(_)
|
|
||||||
| Self::InvalidTaskLogTail(_) => (
|
|
||||||
ApiErrorCode::ValidationError,
|
|
||||||
ApiErrorCategory::Validation,
|
|
||||||
false,
|
|
||||||
),
|
|
||||||
Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
|
|
||||||
return ApiError::from_message(request_id, message);
|
|
||||||
}
|
|
||||||
Self::Coordinator(CoordinatorError::UnknownNode) => {
|
|
||||||
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
|
|
||||||
}
|
|
||||||
Self::Coordinator(CoordinatorError::Enrollment(_)) => (
|
|
||||||
ApiErrorCode::Unauthenticated,
|
|
||||||
ApiErrorCategory::Authentication,
|
|
||||||
false,
|
|
||||||
),
|
|
||||||
Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
|
|
||||||
(ApiErrorCode::Conflict, ApiErrorCategory::State, true)
|
|
||||||
}
|
|
||||||
Self::Download(DownloadError::NotFound) => {
|
|
||||||
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
|
|
||||||
}
|
|
||||||
Self::Download(DownloadError::Unavailable)
|
|
||||||
| Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
|
|
||||||
ApiErrorCode::ArtifactUnavailable,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Download(DownloadError::LimitExceeded { .. }) => (
|
|
||||||
ApiErrorCode::ArtifactLimitExceeded,
|
|
||||||
ApiErrorCategory::Resource,
|
|
||||||
false,
|
|
||||||
),
|
|
||||||
Self::Download(DownloadError::Unauthorized(_))
|
|
||||||
| Self::Download(DownloadError::InvalidToken)
|
|
||||||
| Self::Download(DownloadError::Expired)
|
|
||||||
| Self::Download(DownloadError::Revoked) => (
|
|
||||||
ApiErrorCode::Forbidden,
|
|
||||||
ApiErrorCategory::Authorization,
|
|
||||||
false,
|
|
||||||
),
|
|
||||||
Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
|
|
||||||
ApiErrorCode::QuotaExceeded,
|
|
||||||
ApiErrorCategory::Resource,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Scheduler(_) => (
|
|
||||||
ApiErrorCode::NoCapableNode,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Transport(_) => (
|
|
||||||
ApiErrorCode::NodeOffline,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Panel(PanelError::RateLimited) => (
|
|
||||||
ApiErrorCode::QuotaExceeded,
|
|
||||||
ApiErrorCategory::Resource,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
Self::Panel(PanelError::UnknownWidget(_)) => {
|
|
||||||
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
|
|
||||||
}
|
|
||||||
Self::Panel(_) => (
|
|
||||||
ApiErrorCode::Forbidden,
|
|
||||||
ApiErrorCategory::Authorization,
|
|
||||||
false,
|
|
||||||
),
|
|
||||||
Self::Durable(_) => (
|
|
||||||
ApiErrorCode::InternalError,
|
|
||||||
ApiErrorCategory::Internal,
|
|
||||||
true,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
ApiError::new(code, category, message, retryable, request_id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub struct CoordinatorService {
|
pub struct CoordinatorService {
|
||||||
coordinator: Coordinator,
|
coordinator: Coordinator,
|
||||||
store: RuntimeDurableStore,
|
store: RuntimeDurableStore,
|
||||||
|
|
@ -281,37 +161,6 @@ pub struct CoordinatorService {
|
||||||
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
|
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
|
||||||
task_events: VecDeque<TaskCompletionEvent>,
|
task_events: VecDeque<TaskCompletionEvent>,
|
||||||
process_scope_history: VecDeque<ProcessControlKey>,
|
process_scope_history: VecDeque<ProcessControlKey>,
|
||||||
process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
|
|
||||||
process_summary_order: VecDeque<ProcessControlKey>,
|
|
||||||
next_process_summary_order: u64,
|
|
||||||
task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
|
|
||||||
recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
|
|
||||||
recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
|
|
||||||
recent_log_accounted_bytes: BTreeMap<
|
|
||||||
(
|
|
||||||
TenantId,
|
|
||||||
ProjectId,
|
|
||||||
ProcessId,
|
|
||||||
clusterflux_core::TaskInstanceId,
|
|
||||||
String,
|
|
||||||
),
|
|
||||||
u64,
|
|
||||||
>,
|
|
||||||
recent_log_truncated_streams: BTreeSet<(
|
|
||||||
TenantId,
|
|
||||||
ProjectId,
|
|
||||||
ProcessId,
|
|
||||||
clusterflux_core::TaskInstanceId,
|
|
||||||
String,
|
|
||||||
)>,
|
|
||||||
recent_log_quota_truncated_streams: BTreeSet<(
|
|
||||||
TenantId,
|
|
||||||
ProjectId,
|
|
||||||
ProcessId,
|
|
||||||
clusterflux_core::TaskInstanceId,
|
|
||||||
String,
|
|
||||||
)>,
|
|
||||||
next_recent_log_sequence: u64,
|
|
||||||
debug_audit_events: VecDeque<DebugAuditEvent>,
|
debug_audit_events: VecDeque<DebugAuditEvent>,
|
||||||
debug_epochs: BTreeMap<ProcessControlKey, u64>,
|
debug_epochs: BTreeMap<ProcessControlKey, u64>,
|
||||||
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
|
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
|
||||||
|
|
@ -366,29 +215,6 @@ impl CoordinatorService {
|
||||||
self.artifact_relay.usage()
|
self.artifact_relay.usage()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
|
|
||||||
CoordinatorOperationalMetrics {
|
|
||||||
tenants: self.coordinator.tenant_count(),
|
|
||||||
users: self.coordinator.user_count(),
|
|
||||||
projects: self.coordinator.project_count(),
|
|
||||||
enrolled_nodes: self.coordinator.node_identity_count(),
|
|
||||||
reported_nodes: self.node_descriptors.len(),
|
|
||||||
live_nodes: self
|
|
||||||
.node_descriptors
|
|
||||||
.keys()
|
|
||||||
.filter(|scope| self.node_is_live(scope))
|
|
||||||
.count(),
|
|
||||||
active_processes: self.coordinator.active_process_count(),
|
|
||||||
active_coordinator_mains: self.main_runtime.active_main_count(),
|
|
||||||
max_active_coordinator_mains: self.main_runtime.max_active_mains(),
|
|
||||||
active_tasks: self.active_tasks.len(),
|
|
||||||
queued_tasks: self.pending_task_launches.len(),
|
|
||||||
artifacts: self.artifact_registry.artifact_count(),
|
|
||||||
retained_download_links: self.artifact_registry.retained_download_link_count(),
|
|
||||||
relay: self.artifact_relay.usage(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn commit_artifact_relay(
|
fn commit_artifact_relay(
|
||||||
&mut self,
|
&mut self,
|
||||||
candidate: relay::ArtifactRelayLedger,
|
candidate: relay::ArtifactRelayLedger,
|
||||||
|
|
@ -578,16 +404,6 @@ impl CoordinatorService {
|
||||||
enrollment_grants: BTreeMap::new(),
|
enrollment_grants: BTreeMap::new(),
|
||||||
task_events: VecDeque::new(),
|
task_events: VecDeque::new(),
|
||||||
process_scope_history: VecDeque::new(),
|
process_scope_history: VecDeque::new(),
|
||||||
process_summaries: BTreeMap::new(),
|
|
||||||
process_summary_order: VecDeque::new(),
|
|
||||||
next_process_summary_order: 1,
|
|
||||||
task_terminal_states: BTreeMap::new(),
|
|
||||||
recent_logs: BTreeMap::new(),
|
|
||||||
recent_log_dropped_through: BTreeMap::new(),
|
|
||||||
recent_log_accounted_bytes: BTreeMap::new(),
|
|
||||||
recent_log_truncated_streams: BTreeSet::new(),
|
|
||||||
recent_log_quota_truncated_streams: BTreeSet::new(),
|
|
||||||
next_recent_log_sequence: 1,
|
|
||||||
debug_audit_events: VecDeque::new(),
|
debug_audit_events: VecDeque::new(),
|
||||||
debug_epochs: BTreeMap::new(),
|
debug_epochs: BTreeMap::new(),
|
||||||
debug_epoch_runtime: BTreeMap::new(),
|
debug_epoch_runtime: BTreeMap::new(),
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@ pub(super) enum PublicUserOperation {
|
||||||
RevokeAgentPublicKey,
|
RevokeAgentPublicKey,
|
||||||
CreateNodeEnrollmentGrant,
|
CreateNodeEnrollmentGrant,
|
||||||
ListNodeDescriptors,
|
ListNodeDescriptors,
|
||||||
ListNodeSummaries,
|
|
||||||
RevokeNodeCredential,
|
RevokeNodeCredential,
|
||||||
StartProcess,
|
StartProcess,
|
||||||
ScheduleTask,
|
ScheduleTask,
|
||||||
|
|
@ -25,7 +24,6 @@ pub(super) enum PublicUserOperation {
|
||||||
CancelProcess,
|
CancelProcess,
|
||||||
AbortProcess,
|
AbortProcess,
|
||||||
ListProcesses,
|
ListProcesses,
|
||||||
ListProcessSummaries,
|
|
||||||
QuotaStatus,
|
QuotaStatus,
|
||||||
RestartTask,
|
RestartTask,
|
||||||
ResolveTaskFailure,
|
ResolveTaskFailure,
|
||||||
|
|
@ -37,10 +35,7 @@ pub(super) enum PublicUserOperation {
|
||||||
InspectDebugEpoch,
|
InspectDebugEpoch,
|
||||||
ListTaskEvents,
|
ListTaskEvents,
|
||||||
ListTaskSnapshots,
|
ListTaskSnapshots,
|
||||||
ListRecentLogs,
|
|
||||||
JoinTask,
|
JoinTask,
|
||||||
ListArtifacts,
|
|
||||||
GetArtifact,
|
|
||||||
CreateArtifactDownloadLink,
|
CreateArtifactDownloadLink,
|
||||||
OpenArtifactDownloadStream,
|
OpenArtifactDownloadStream,
|
||||||
RevokeArtifactDownloadLink,
|
RevokeArtifactDownloadLink,
|
||||||
|
|
@ -61,7 +56,6 @@ impl PublicUserOperation {
|
||||||
Self::RevokeAgentPublicKey => "revoke_agent_public_key",
|
Self::RevokeAgentPublicKey => "revoke_agent_public_key",
|
||||||
Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
|
Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
|
||||||
Self::ListNodeDescriptors => "list_node_descriptors",
|
Self::ListNodeDescriptors => "list_node_descriptors",
|
||||||
Self::ListNodeSummaries => "list_node_summaries",
|
|
||||||
Self::RevokeNodeCredential => "revoke_node_credential",
|
Self::RevokeNodeCredential => "revoke_node_credential",
|
||||||
Self::StartProcess => "start_process",
|
Self::StartProcess => "start_process",
|
||||||
Self::ScheduleTask => "schedule_task",
|
Self::ScheduleTask => "schedule_task",
|
||||||
|
|
@ -69,7 +63,6 @@ impl PublicUserOperation {
|
||||||
Self::CancelProcess => "cancel_process",
|
Self::CancelProcess => "cancel_process",
|
||||||
Self::AbortProcess => "abort_process",
|
Self::AbortProcess => "abort_process",
|
||||||
Self::ListProcesses => "list_processes",
|
Self::ListProcesses => "list_processes",
|
||||||
Self::ListProcessSummaries => "list_process_summaries",
|
|
||||||
Self::QuotaStatus => "quota_status",
|
Self::QuotaStatus => "quota_status",
|
||||||
Self::RestartTask => "restart_task",
|
Self::RestartTask => "restart_task",
|
||||||
Self::ResolveTaskFailure => "resolve_task_failure",
|
Self::ResolveTaskFailure => "resolve_task_failure",
|
||||||
|
|
@ -81,10 +74,7 @@ impl PublicUserOperation {
|
||||||
Self::InspectDebugEpoch => "inspect_debug_epoch",
|
Self::InspectDebugEpoch => "inspect_debug_epoch",
|
||||||
Self::ListTaskEvents => "list_task_events",
|
Self::ListTaskEvents => "list_task_events",
|
||||||
Self::ListTaskSnapshots => "list_task_snapshots",
|
Self::ListTaskSnapshots => "list_task_snapshots",
|
||||||
Self::ListRecentLogs => "list_recent_logs",
|
|
||||||
Self::JoinTask => "join_task",
|
Self::JoinTask => "join_task",
|
||||||
Self::ListArtifacts => "list_artifacts",
|
|
||||||
Self::GetArtifact => "get_artifact",
|
|
||||||
Self::CreateArtifactDownloadLink => "create_artifact_download_link",
|
Self::CreateArtifactDownloadLink => "create_artifact_download_link",
|
||||||
Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
|
Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
|
||||||
Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
|
Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
|
||||||
|
|
@ -115,7 +105,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
|
||||||
Self::CreateNodeEnrollmentGrant
|
Self::CreateNodeEnrollmentGrant
|
||||||
}
|
}
|
||||||
AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
|
AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
|
||||||
AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries,
|
|
||||||
AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
|
AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
|
||||||
Self::RevokeNodeCredential
|
Self::RevokeNodeCredential
|
||||||
}
|
}
|
||||||
|
|
@ -125,9 +114,6 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
|
||||||
AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
|
AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
|
||||||
AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
|
AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
|
||||||
AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
|
AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
|
||||||
AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => {
|
|
||||||
Self::ListProcessSummaries
|
|
||||||
}
|
|
||||||
AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
|
AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
|
||||||
AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
|
AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
|
||||||
AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
|
AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
|
||||||
|
|
@ -143,10 +129,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
|
||||||
AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
|
AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
|
||||||
AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
|
AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
|
||||||
AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
|
AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
|
||||||
AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs,
|
|
||||||
AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
|
AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
|
||||||
AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts,
|
|
||||||
AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact,
|
|
||||||
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
|
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
|
||||||
Self::CreateArtifactDownloadLink
|
Self::CreateArtifactDownloadLink
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,7 @@ use super::keys::{process_control_key, task_control_key, task_restart_key};
|
||||||
use super::protocol::TaskAttemptState;
|
use super::protocol::TaskAttemptState;
|
||||||
use super::{
|
use super::{
|
||||||
artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
|
artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
|
||||||
RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState,
|
TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
|
||||||
MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES,
|
|
||||||
MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT,
|
|
||||||
MAX_TASK_LOG_TAIL_BYTES,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
impl CoordinatorService {
|
impl CoordinatorService {
|
||||||
|
|
@ -39,44 +36,23 @@ impl CoordinatorService {
|
||||||
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
|
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
|
||||||
validate_task_log_tail("stdout_tail", &stdout_tail)?;
|
validate_task_log_tail("stdout_tail", &stdout_tail)?;
|
||||||
validate_task_log_tail("stderr_tail", &stderr_tail)?;
|
validate_task_log_tail("stderr_tail", &stderr_tail)?;
|
||||||
|
let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
|
||||||
let now_epoch_seconds = self.current_epoch_seconds()?;
|
let now_epoch_seconds = self.current_epoch_seconds()?;
|
||||||
let stdout_retained = self.accept_final_log_stream(
|
self.quota
|
||||||
&tenant,
|
.can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
|
||||||
&project,
|
self.quota
|
||||||
&process,
|
.charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
|
||||||
&task,
|
|
||||||
TaskLogStream::Stdout,
|
|
||||||
stdout_bytes,
|
|
||||||
&stdout_tail,
|
|
||||||
stdout_truncated,
|
|
||||||
now_epoch_seconds,
|
|
||||||
)?;
|
|
||||||
let stderr_retained = self.accept_final_log_stream(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&process,
|
|
||||||
&task,
|
|
||||||
TaskLogStream::Stderr,
|
|
||||||
stderr_bytes,
|
|
||||||
&stderr_tail,
|
|
||||||
stderr_truncated,
|
|
||||||
now_epoch_seconds,
|
|
||||||
)?;
|
|
||||||
Ok(CoordinatorResponse::TaskLogRecorded {
|
Ok(CoordinatorResponse::TaskLogRecorded {
|
||||||
process,
|
process,
|
||||||
task,
|
task,
|
||||||
stdout_bytes,
|
stdout_bytes,
|
||||||
stderr_bytes,
|
stderr_bytes,
|
||||||
stdout_tail: if !stdout_retained {
|
stdout_tail: if stdout_truncated {
|
||||||
"[log output truncated at project log quota]".to_owned()
|
|
||||||
} else if stdout_truncated {
|
|
||||||
format!("{stdout_tail}\n... truncated")
|
format!("{stdout_tail}\n... truncated")
|
||||||
} else {
|
} else {
|
||||||
stdout_tail
|
stdout_tail
|
||||||
},
|
},
|
||||||
stderr_tail: if !stderr_retained {
|
stderr_tail: if stderr_truncated {
|
||||||
"[log output truncated at project log quota]".to_owned()
|
|
||||||
} else if stderr_truncated {
|
|
||||||
format!("{stderr_tail}\n... truncated")
|
format!("{stderr_tail}\n... truncated")
|
||||||
} else {
|
} else {
|
||||||
stderr_tail
|
stderr_tail
|
||||||
|
|
@ -85,158 +61,6 @@ impl CoordinatorService {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
pub(super) fn handle_report_task_log_chunk(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
process: String,
|
|
||||||
node: String,
|
|
||||||
task: String,
|
|
||||||
stream: TaskLogStream,
|
|
||||||
offset: u64,
|
|
||||||
source_bytes: u64,
|
|
||||||
text: String,
|
|
||||||
truncated: bool,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
|
|
||||||
return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
|
|
||||||
"live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}",
|
|
||||||
text.len()
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
if source_bytes == 0 && !text.is_empty() && !truncated {
|
|
||||||
return Err(CoordinatorServiceError::Protocol(
|
|
||||||
"live log chunk source_bytes must describe non-empty text".to_owned(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) {
|
|
||||||
return Err(CoordinatorServiceError::Protocol(
|
|
||||||
"live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let process = ProcessId::new(process);
|
|
||||||
let node = NodeId::new(node);
|
|
||||||
let task = TaskInstanceId::new(task);
|
|
||||||
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
|
|
||||||
let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
|
|
||||||
let expected = self
|
|
||||||
.recent_log_accounted_bytes
|
|
||||||
.get(&key)
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(0);
|
|
||||||
let end = offset.checked_add(source_bytes).ok_or_else(|| {
|
|
||||||
CoordinatorServiceError::Protocol(
|
|
||||||
"live log chunk offset exceeds the supported range".to_owned(),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
let state_marker = source_bytes == 0 && truncated;
|
|
||||||
if end < expected || (end == expected && !state_marker) {
|
|
||||||
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
sequence: None,
|
|
||||||
next_offset: expected,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let now_epoch_seconds = self.current_epoch_seconds()?;
|
|
||||||
if self.recent_log_quota_truncated_streams.contains(&key) {
|
|
||||||
if end > expected {
|
|
||||||
self.recent_log_accounted_bytes.insert(key, end);
|
|
||||||
}
|
|
||||||
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
sequence: None,
|
|
||||||
next_offset: end.max(expected),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
let newly_accounted = end.saturating_sub(expected);
|
|
||||||
if self
|
|
||||||
.quota
|
|
||||||
.charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
if end > expected {
|
|
||||||
self.recent_log_accounted_bytes.insert(key.clone(), end);
|
|
||||||
}
|
|
||||||
let sequence = self.mark_log_quota_truncated(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&process,
|
|
||||||
&task,
|
|
||||||
&stream,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
sequence,
|
|
||||||
next_offset: end.max(expected),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if offset > expected {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
format!("[log output lost: {} bytes]", offset - expected),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
} else if offset < expected {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
format!(
|
|
||||||
"[log output overlap omitted: {} new source bytes]",
|
|
||||||
end - expected
|
|
||||||
),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let marker_is_new = if truncated {
|
|
||||||
self.recent_log_truncated_streams.insert(key.clone())
|
|
||||||
} else {
|
|
||||||
false
|
|
||||||
};
|
|
||||||
let text = if state_marker && text.is_empty() {
|
|
||||||
"[log output truncated at source]".to_owned()
|
|
||||||
} else {
|
|
||||||
text
|
|
||||||
};
|
|
||||||
let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
|
|
||||||
.then(|| {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream,
|
|
||||||
text,
|
|
||||||
truncated,
|
|
||||||
now_epoch_seconds,
|
|
||||||
)
|
|
||||||
});
|
|
||||||
if end > expected {
|
|
||||||
self.recent_log_accounted_bytes.insert(key, end);
|
|
||||||
}
|
|
||||||
Ok(CoordinatorResponse::TaskLogChunkRecorded {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
sequence,
|
|
||||||
next_offset: end,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_report_vfs_metadata(
|
pub(super) fn handle_report_vfs_metadata(
|
||||||
&mut self,
|
&mut self,
|
||||||
tenant: String,
|
tenant: String,
|
||||||
|
|
@ -355,37 +179,20 @@ impl CoordinatorService {
|
||||||
artifact_size_bytes,
|
artifact_size_bytes,
|
||||||
result,
|
result,
|
||||||
};
|
};
|
||||||
|
let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
|
||||||
let now_epoch_seconds = self.current_epoch_seconds()?;
|
let now_epoch_seconds = self.current_epoch_seconds()?;
|
||||||
let stdout_retained = self.accept_final_log_stream(
|
self.quota.can_charge_log_bytes(
|
||||||
&event.tenant,
|
&event.tenant,
|
||||||
&event.project,
|
&event.project,
|
||||||
&event.process,
|
reported_bytes,
|
||||||
&event.task,
|
|
||||||
TaskLogStream::Stdout,
|
|
||||||
event.stdout_bytes,
|
|
||||||
&event.stdout_tail,
|
|
||||||
event.stdout_truncated,
|
|
||||||
now_epoch_seconds,
|
now_epoch_seconds,
|
||||||
)?;
|
)?;
|
||||||
let stderr_retained = self.accept_final_log_stream(
|
self.quota.charge_log_bytes(
|
||||||
&event.tenant,
|
&event.tenant,
|
||||||
&event.project,
|
&event.project,
|
||||||
&event.process,
|
reported_bytes,
|
||||||
&event.task,
|
|
||||||
TaskLogStream::Stderr,
|
|
||||||
event.stderr_bytes,
|
|
||||||
&event.stderr_tail,
|
|
||||||
event.stderr_truncated,
|
|
||||||
now_epoch_seconds,
|
now_epoch_seconds,
|
||||||
)?;
|
)?;
|
||||||
if !stdout_retained {
|
|
||||||
event.stdout_tail = "[log output truncated at project log quota]".to_owned();
|
|
||||||
event.stdout_truncated = true;
|
|
||||||
}
|
|
||||||
if !stderr_retained {
|
|
||||||
event.stderr_tail = "[log output truncated at project log quota]".to_owned();
|
|
||||||
event.stderr_truncated = true;
|
|
||||||
}
|
|
||||||
let task_key = task_control_key(
|
let task_key = task_control_key(
|
||||||
&event.tenant,
|
&event.tenant,
|
||||||
&event.project,
|
&event.project,
|
||||||
|
|
@ -414,12 +221,6 @@ impl CoordinatorService {
|
||||||
self.task_aborts.remove(&task_key);
|
self.task_aborts.remove(&task_key);
|
||||||
self.debug_commands.remove(&task_key);
|
self.debug_commands.remove(&task_key);
|
||||||
self.active_tasks.remove(&task_key);
|
self.active_tasks.remove(&task_key);
|
||||||
self.clear_recent_log_offsets_for_task(
|
|
||||||
&event.tenant,
|
|
||||||
&event.project,
|
|
||||||
&event.process,
|
|
||||||
&event.task,
|
|
||||||
);
|
|
||||||
self.task_assignments.retain(|_, assignments| {
|
self.task_assignments.retain(|_, assignments| {
|
||||||
assignments.retain(|assignment| {
|
assignments.retain(|assignment| {
|
||||||
assignment.tenant != event.tenant
|
assignment.tenant != event.tenant
|
||||||
|
|
@ -447,19 +248,10 @@ impl CoordinatorService {
|
||||||
self.notify_coordinator_main_waiters(&event);
|
self.notify_coordinator_main_waiters(&event);
|
||||||
}
|
}
|
||||||
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
|
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
|
||||||
let events_recorded = self
|
|
||||||
.task_events
|
|
||||||
.iter()
|
|
||||||
.filter(|recorded| {
|
|
||||||
recorded.tenant == event.tenant
|
|
||||||
&& recorded.project == event.project
|
|
||||||
&& recorded.process == event.process
|
|
||||||
})
|
|
||||||
.count();
|
|
||||||
Ok(CoordinatorResponse::TaskRecorded {
|
Ok(CoordinatorResponse::TaskRecorded {
|
||||||
process: event.process,
|
process: event.process,
|
||||||
task: event.task,
|
task: event.task,
|
||||||
events_recorded,
|
events_recorded: self.task_events.len(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -519,7 +311,7 @@ impl CoordinatorService {
|
||||||
Ok(CoordinatorResponse::TaskSnapshots { snapshots })
|
Ok(CoordinatorResponse::TaskSnapshots { snapshots })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn authorize_task_event_process_scope(
|
fn authorize_task_event_process_scope(
|
||||||
&self,
|
&self,
|
||||||
tenant: &TenantId,
|
tenant: &TenantId,
|
||||||
project: &ProjectId,
|
project: &ProjectId,
|
||||||
|
|
@ -559,47 +351,6 @@ impl CoordinatorService {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn handle_list_recent_logs(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
process: String,
|
|
||||||
task: Option<String>,
|
|
||||||
after_sequence: Option<u64>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let _actor = UserId::new(actor_user);
|
|
||||||
let process = ProcessId::new(process);
|
|
||||||
let task = task.map(TaskInstanceId::new);
|
|
||||||
self.authorize_task_event_process_scope(&tenant, &project, &process)?;
|
|
||||||
let after_sequence = after_sequence.unwrap_or(0);
|
|
||||||
let retained = self.recent_logs.get(&(tenant.clone(), project.clone()));
|
|
||||||
let history_truncated = self
|
|
||||||
.recent_log_dropped_through
|
|
||||||
.get(&process_control_key(&tenant, &project, &process))
|
|
||||||
.is_some_and(|dropped_through| *dropped_through > after_sequence);
|
|
||||||
let entries = retained
|
|
||||||
.into_iter()
|
|
||||||
.flatten()
|
|
||||||
.filter(|entry| {
|
|
||||||
entry.process == process
|
|
||||||
&& entry.sequence > after_sequence
|
|
||||||
&& task.as_ref().is_none_or(|task| &entry.task == task)
|
|
||||||
})
|
|
||||||
.take(limit as usize)
|
|
||||||
.cloned()
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let next_sequence = entries.last().map(|entry| entry.sequence);
|
|
||||||
Ok(CoordinatorResponse::RecentLogs {
|
|
||||||
entries,
|
|
||||||
next_sequence,
|
|
||||||
history_truncated,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_join_task(
|
pub(super) fn handle_join_task(
|
||||||
&mut self,
|
&mut self,
|
||||||
tenant: String,
|
tenant: String,
|
||||||
|
|
@ -718,22 +469,6 @@ impl CoordinatorService {
|
||||||
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
|
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
|
||||||
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
|
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
|
||||||
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
|
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
|
||||||
match event.executor {
|
|
||||||
super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
|
|
||||||
&event.tenant,
|
|
||||||
&event.project,
|
|
||||||
&event.process,
|
|
||||||
event.task_definition.clone(),
|
|
||||||
event.task.clone(),
|
|
||||||
event.terminal_state.clone(),
|
|
||||||
),
|
|
||||||
super::TaskExecutor::Node => {
|
|
||||||
self.task_terminal_states.insert(
|
|
||||||
task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
|
|
||||||
event.terminal_state.clone(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let process_scope = (
|
let process_scope = (
|
||||||
event.tenant.clone(),
|
event.tenant.clone(),
|
||||||
event.project.clone(),
|
event.project.clone(),
|
||||||
|
|
@ -771,321 +506,6 @@ impl CoordinatorService {
|
||||||
self.task_events.push_back(event);
|
self.task_events.push_back(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn record_recent_log(
|
|
||||||
&mut self,
|
|
||||||
tenant: TenantId,
|
|
||||||
project: ProjectId,
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
stream: TaskLogStream,
|
|
||||||
mut text: String,
|
|
||||||
mut truncated: bool,
|
|
||||||
server_timestamp_epoch_seconds: u64,
|
|
||||||
) -> u64 {
|
|
||||||
if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
|
|
||||||
let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES;
|
|
||||||
while !text.is_char_boundary(boundary) {
|
|
||||||
boundary -= 1;
|
|
||||||
}
|
|
||||||
text.truncate(boundary);
|
|
||||||
truncated = true;
|
|
||||||
}
|
|
||||||
let sequence = self.next_recent_log_sequence;
|
|
||||||
self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1);
|
|
||||||
let logs = self
|
|
||||||
.recent_logs
|
|
||||||
.entry((tenant.clone(), project.clone()))
|
|
||||||
.or_default();
|
|
||||||
let mut dropped = Vec::new();
|
|
||||||
while logs.iter().filter(|entry| entry.process == process).count()
|
|
||||||
>= MAX_RECENT_LOG_ENTRIES_PER_PROCESS
|
|
||||||
{
|
|
||||||
let Some(index) = logs.iter().position(|entry| entry.process == process) else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
if let Some(entry) = logs.remove(index) {
|
|
||||||
dropped.push(entry);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT
|
|
||||||
|| logs
|
|
||||||
.iter()
|
|
||||||
.map(|entry| entry.text.len())
|
|
||||||
.sum::<usize>()
|
|
||||||
.saturating_add(text.len())
|
|
||||||
> MAX_RECENT_LOG_BYTES_PER_PROJECT
|
|
||||||
{
|
|
||||||
match logs.pop_front() {
|
|
||||||
Some(entry) => dropped.push(entry),
|
|
||||||
None => break,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
logs.push_back(RecentLogEntry {
|
|
||||||
sequence,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
stream,
|
|
||||||
text,
|
|
||||||
server_timestamp_epoch_seconds,
|
|
||||||
truncated,
|
|
||||||
});
|
|
||||||
for entry in dropped {
|
|
||||||
let key = process_control_key(&tenant, &project, &entry.process);
|
|
||||||
self.recent_log_dropped_through
|
|
||||||
.entry(key)
|
|
||||||
.and_modify(|dropped_through| {
|
|
||||||
*dropped_through = (*dropped_through).max(entry.sequence);
|
|
||||||
})
|
|
||||||
.or_insert(entry.sequence);
|
|
||||||
}
|
|
||||||
sequence
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn accept_final_log_stream(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
task: &TaskInstanceId,
|
|
||||||
stream: TaskLogStream,
|
|
||||||
total_source_bytes: u64,
|
|
||||||
final_tail: &str,
|
|
||||||
source_truncated: bool,
|
|
||||||
now_epoch_seconds: u64,
|
|
||||||
) -> Result<bool, CoordinatorServiceError> {
|
|
||||||
let key = recent_log_offset_key(tenant, project, process, task, &stream);
|
|
||||||
let accounted = self
|
|
||||||
.recent_log_accounted_bytes
|
|
||||||
.get(&key)
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(0);
|
|
||||||
let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| {
|
|
||||||
let stream_name = match stream {
|
|
||||||
TaskLogStream::Stdout => "stdout",
|
|
||||||
TaskLogStream::Stderr => "stderr",
|
|
||||||
};
|
|
||||||
CoordinatorServiceError::Protocol(format!(
|
|
||||||
"final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
if self.recent_log_quota_truncated_streams.contains(&key) {
|
|
||||||
self.recent_log_accounted_bytes
|
|
||||||
.insert(key, total_source_bytes);
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
if self
|
|
||||||
.quota
|
|
||||||
.charge_log_bytes(tenant, project, remaining, now_epoch_seconds)
|
|
||||||
.is_err()
|
|
||||||
{
|
|
||||||
self.recent_log_accounted_bytes
|
|
||||||
.insert(key, total_source_bytes);
|
|
||||||
self.mark_log_quota_truncated(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
&stream,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
return Ok(false);
|
|
||||||
}
|
|
||||||
self.reconcile_final_log_stream(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
stream,
|
|
||||||
total_source_bytes,
|
|
||||||
final_tail,
|
|
||||||
source_truncated,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
Ok(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn mark_log_quota_truncated(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
task: &TaskInstanceId,
|
|
||||||
stream: &TaskLogStream,
|
|
||||||
now_epoch_seconds: u64,
|
|
||||||
) -> Option<u64> {
|
|
||||||
let key = recent_log_offset_key(tenant, project, process, task, stream);
|
|
||||||
if !self.recent_log_quota_truncated_streams.insert(key.clone()) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
self.recent_log_truncated_streams.insert(key);
|
|
||||||
Some(self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
"[log output truncated at project log quota]".to_owned(),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn reconcile_final_log_stream(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
task: &TaskInstanceId,
|
|
||||||
stream: TaskLogStream,
|
|
||||||
total_source_bytes: u64,
|
|
||||||
final_tail: &str,
|
|
||||||
source_truncated: bool,
|
|
||||||
now_epoch_seconds: u64,
|
|
||||||
) {
|
|
||||||
let key = recent_log_offset_key(tenant, project, process, task, &stream);
|
|
||||||
let accounted = self
|
|
||||||
.recent_log_accounted_bytes
|
|
||||||
.get(&key)
|
|
||||||
.copied()
|
|
||||||
.unwrap_or(0);
|
|
||||||
let mut visible_truncation = false;
|
|
||||||
if total_source_bytes > accounted {
|
|
||||||
let missing = total_source_bytes - accounted;
|
|
||||||
if final_tail.is_empty() {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
format!("[log output unavailable: {missing} source bytes]"),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
visible_truncation = true;
|
|
||||||
} else if (final_tail.len() as u64) <= total_source_bytes {
|
|
||||||
let tail_source_start = total_source_bytes - final_tail.len() as u64;
|
|
||||||
if accounted < tail_source_start {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
format!(
|
|
||||||
"[log output lost before final tail: {} source bytes]",
|
|
||||||
tail_source_start - accounted
|
|
||||||
),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
visible_truncation = true;
|
|
||||||
}
|
|
||||||
let source_start = accounted.max(tail_source_start) - tail_source_start;
|
|
||||||
let mut byte_start = usize::try_from(source_start)
|
|
||||||
.unwrap_or(final_tail.len())
|
|
||||||
.min(final_tail.len());
|
|
||||||
while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
|
|
||||||
byte_start += 1;
|
|
||||||
}
|
|
||||||
let suffix = &final_tail[byte_start..];
|
|
||||||
if !suffix.is_empty() {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
suffix.to_owned(),
|
|
||||||
source_truncated || visible_truncation,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
visible_truncation |= source_truncated;
|
|
||||||
}
|
|
||||||
} else if accounted == 0 {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
final_tail.to_owned(),
|
|
||||||
source_truncated,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
visible_truncation |= source_truncated;
|
|
||||||
} else {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream.clone(),
|
|
||||||
format!(
|
|
||||||
"[{missing} additional source bytes could not be merged without duplicating redacted output]"
|
|
||||||
),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
visible_truncation = true;
|
|
||||||
}
|
|
||||||
self.recent_log_accounted_bytes
|
|
||||||
.insert(key.clone(), total_source_bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
let marker_is_new = (source_truncated || visible_truncation)
|
|
||||||
&& self.recent_log_truncated_streams.insert(key);
|
|
||||||
if marker_is_new && !visible_truncation {
|
|
||||||
self.record_recent_log(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
stream,
|
|
||||||
"[log output truncated at source]".to_owned(),
|
|
||||||
true,
|
|
||||||
now_epoch_seconds,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn clear_recent_log_offsets_for_task(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
task: &TaskInstanceId,
|
|
||||||
) {
|
|
||||||
self.recent_log_accounted_bytes.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, entry_task, _), _| {
|
|
||||||
entry_tenant != tenant
|
|
||||||
|| entry_project != project
|
|
||||||
|| entry_process != process
|
|
||||||
|| entry_task != task
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.recent_log_truncated_streams.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, entry_task, _)| {
|
|
||||||
entry_tenant != tenant
|
|
||||||
|| entry_project != project
|
|
||||||
|| entry_process != process
|
|
||||||
|| entry_task != task
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.recent_log_quota_truncated_streams.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, entry_task, _)| {
|
|
||||||
entry_tenant != tenant
|
|
||||||
|| entry_project != project
|
|
||||||
|| entry_process != process
|
|
||||||
|| entry_task != task
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
|
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
|
||||||
let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
|
let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
|
||||||
let Some(attempt) = self
|
let Some(attempt) = self
|
||||||
|
|
@ -1169,11 +589,13 @@ impl CoordinatorService {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
let main_completed = self
|
let main_completed = self.task_events.iter().rev().any(|event| {
|
||||||
.process_summaries
|
&event.tenant == tenant
|
||||||
.get(&process_key)
|
&& &event.project == project
|
||||||
.and_then(|summary| summary.main_terminal_state.as_ref())
|
&& &event.process == process
|
||||||
.is_some_and(|state| matches!(state, TaskTerminalState::Completed));
|
&& matches!(event.executor, super::TaskExecutor::CoordinatorMain)
|
||||||
|
&& matches!(event.terminal_state, TaskTerminalState::Completed)
|
||||||
|
});
|
||||||
let cancellation_completed = self.process_cancellations.contains(&process_key);
|
let cancellation_completed = self.process_cancellations.contains(&process_key);
|
||||||
if !main_completed && !cancellation_completed {
|
if !main_completed && !cancellation_completed {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
|
|
@ -1188,47 +610,9 @@ impl CoordinatorService {
|
||||||
{
|
{
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
let final_result = if cancellation_completed {
|
|
||||||
super::ProcessFinalResult::Cancelled
|
|
||||||
} else if self.task_terminal_states.iter().any(
|
|
||||||
|((task_tenant, task_project, task_process, _), terminal_state)| {
|
|
||||||
task_tenant == tenant
|
|
||||||
&& task_project == project
|
|
||||||
&& task_process == process
|
|
||||||
&& terminal_state == &TaskTerminalState::Failed
|
|
||||||
},
|
|
||||||
) {
|
|
||||||
super::ProcessFinalResult::Failed
|
|
||||||
} else if self.task_terminal_states.iter().any(
|
|
||||||
|((task_tenant, task_project, task_process, _), terminal_state)| {
|
|
||||||
task_tenant == tenant
|
|
||||||
&& task_project == project
|
|
||||||
&& task_process == process
|
|
||||||
&& terminal_state == &TaskTerminalState::Cancelled
|
|
||||||
},
|
|
||||||
) {
|
|
||||||
super::ProcessFinalResult::Cancelled
|
|
||||||
} else {
|
|
||||||
super::ProcessFinalResult::Completed
|
|
||||||
};
|
|
||||||
self.record_process_terminal(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
final_result,
|
|
||||||
self.current_epoch_seconds()?,
|
|
||||||
);
|
|
||||||
self.coordinator.abort_process(tenant, project, process)?;
|
self.coordinator.abort_process(tenant, project, process)?;
|
||||||
self.clear_debug_state_for_process(tenant, project, process);
|
self.clear_debug_state_for_process(tenant, project, process);
|
||||||
self.clear_operator_panel_state(tenant, project, process);
|
self.clear_operator_panel_state(tenant, project, process);
|
||||||
let (pinned, protected_processes) =
|
|
||||||
self.artifact_retention_guards_for_project(tenant, project);
|
|
||||||
self.artifact_registry.enforce_project_metadata_limit(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
&pinned,
|
|
||||||
&protected_processes,
|
|
||||||
);
|
|
||||||
Ok(true)
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1239,30 +623,8 @@ impl CoordinatorService {
|
||||||
let now_epoch_seconds = self.current_epoch_seconds()?;
|
let now_epoch_seconds = self.current_epoch_seconds()?;
|
||||||
self.artifact_registry
|
self.artifact_registry
|
||||||
.expire_download_links(now_epoch_seconds);
|
.expire_download_links(now_epoch_seconds);
|
||||||
let tenant = flush.tenant.clone();
|
|
||||||
let project = flush.project.clone();
|
|
||||||
let (pinned, protected_processes) =
|
|
||||||
self.artifact_retention_guards_for_project(&tenant, &project);
|
|
||||||
self.artifact_registry
|
|
||||||
.flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
|
|
||||||
.map(|_| ())
|
|
||||||
.map_err(CoordinatorServiceError::Protocol)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn artifact_retention_guards_for_project(
|
|
||||||
&self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
) -> (
|
|
||||||
std::collections::BTreeSet<ArtifactScopeKey>,
|
|
||||||
std::collections::BTreeSet<ProcessId>,
|
|
||||||
) {
|
|
||||||
let mut pinned = std::collections::BTreeSet::new();
|
let mut pinned = std::collections::BTreeSet::new();
|
||||||
for checkpoint in self.task_restart_checkpoints.values() {
|
for checkpoint in self.task_restart_checkpoints.values() {
|
||||||
if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
|
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
|
||||||
pinned.insert(ArtifactScopeKey::from_refs(
|
pinned.insert(ArtifactScopeKey::from_refs(
|
||||||
&checkpoint.assignment.tenant,
|
&checkpoint.assignment.tenant,
|
||||||
|
|
@ -1272,9 +634,6 @@ impl CoordinatorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for pending in &self.pending_task_launches {
|
for pending in &self.pending_task_launches {
|
||||||
if &pending.tenant != tenant || &pending.project != project {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for artifact in &pending.task_spec.required_artifacts {
|
for artifact in &pending.task_spec.required_artifacts {
|
||||||
pinned.insert(ArtifactScopeKey::from_refs(
|
pinned.insert(ArtifactScopeKey::from_refs(
|
||||||
&pending.tenant,
|
&pending.tenant,
|
||||||
|
|
@ -1283,13 +642,10 @@ impl CoordinatorService {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let protected_processes = self
|
self.artifact_registry
|
||||||
.coordinator
|
.flush_metadata_bounded(flush, &pinned)
|
||||||
.active_processes_for_project(tenant, project)
|
.map(|_| ())
|
||||||
.into_iter()
|
.map_err(CoordinatorServiceError::Protocol)
|
||||||
.map(|process| process.id)
|
|
||||||
.collect();
|
|
||||||
(pinned, protected_processes)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn task_is_known_or_active(
|
fn task_is_known_or_active(
|
||||||
|
|
@ -1324,24 +680,15 @@ impl CoordinatorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn recent_log_offset_key(
|
fn checked_reported_log_bytes(
|
||||||
tenant: &TenantId,
|
stdout_bytes: u64,
|
||||||
project: &ProjectId,
|
stderr_bytes: u64,
|
||||||
process: &ProcessId,
|
) -> Result<u64, CoordinatorServiceError> {
|
||||||
task: &TaskInstanceId,
|
stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
|
||||||
stream: &TaskLogStream,
|
CoordinatorServiceError::Protocol(
|
||||||
) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) {
|
"reported task log byte counts exceed the supported range".to_owned(),
|
||||||
(
|
)
|
||||||
tenant.clone(),
|
})
|
||||||
project.clone(),
|
|
||||||
process.clone(),
|
|
||||||
task.clone(),
|
|
||||||
match stream {
|
|
||||||
TaskLogStream::Stdout => "stdout",
|
|
||||||
TaskLogStream::Stderr => "stderr",
|
|
||||||
}
|
|
||||||
.to_owned(),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
|
fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
|
||||||
|
|
@ -1358,11 +705,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
|
||||||
if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
|
if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES;
|
let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
|
||||||
while boundary < value.len() && !value.is_char_boundary(boundary) {
|
while !value.is_char_boundary(boundary) {
|
||||||
boundary += 1;
|
boundary -= 1;
|
||||||
}
|
}
|
||||||
value.drain(..boundary);
|
value.truncate(boundary);
|
||||||
*truncated = true;
|
*truncated = true;
|
||||||
value
|
value
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -109,14 +109,6 @@ impl Default for CoordinatorMainRuntime {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CoordinatorMainRuntime {
|
impl CoordinatorMainRuntime {
|
||||||
pub(super) fn active_main_count(&self) -> usize {
|
|
||||||
self.controls.len()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn max_active_mains(&self) -> usize {
|
|
||||||
self.max_active_mains
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn configure(
|
pub(super) fn configure(
|
||||||
&mut self,
|
&mut self,
|
||||||
configuration: super::CoordinatorMainRuntimeConfiguration,
|
configuration: super::CoordinatorMainRuntimeConfiguration,
|
||||||
|
|
@ -737,13 +729,6 @@ impl CoordinatorService {
|
||||||
&process,
|
&process,
|
||||||
"coordinator main launch failed admission or validation",
|
"coordinator main launch failed admission or validation",
|
||||||
);
|
);
|
||||||
self.record_process_terminal(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&process,
|
|
||||||
super::ProcessFinalResult::Failed,
|
|
||||||
self.liveness_now_epoch_seconds(),
|
|
||||||
);
|
|
||||||
let _ = self.coordinator.abort_process(&tenant, &project, &process);
|
let _ = self.coordinator.abort_process(&tenant, &project, &process);
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
|
|
@ -1007,13 +992,6 @@ impl CoordinatorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.process_aborts.insert(process_key.clone());
|
self.process_aborts.insert(process_key.clone());
|
||||||
self.record_process_terminal(
|
|
||||||
&scope.tenant,
|
|
||||||
&scope.project,
|
|
||||||
&scope.process,
|
|
||||||
super::ProcessFinalResult::Failed,
|
|
||||||
self.liveness_now_epoch_seconds(),
|
|
||||||
);
|
|
||||||
let _ = self
|
let _ = self
|
||||||
.coordinator
|
.coordinator
|
||||||
.abort_process(&scope.tenant, &scope.project, &scope.process);
|
.abort_process(&scope.tenant, &scope.project, &scope.process);
|
||||||
|
|
|
||||||
|
|
@ -300,8 +300,8 @@ impl CoordinatorService {
|
||||||
node_scope,
|
node_scope,
|
||||||
NodeDescriptor {
|
NodeDescriptor {
|
||||||
id: node.clone(),
|
id: node.clone(),
|
||||||
tenant: tenant.clone(),
|
tenant,
|
||||||
project: project.clone(),
|
project,
|
||||||
capabilities,
|
capabilities,
|
||||||
cached_environments: cached_environment_digests.into_iter().collect(),
|
cached_environments: cached_environment_digests.into_iter().collect(),
|
||||||
dependency_caches: dependency_cache_digests.into_iter().collect(),
|
dependency_caches: dependency_cache_digests.into_iter().collect(),
|
||||||
|
|
@ -311,14 +311,9 @@ impl CoordinatorService {
|
||||||
online,
|
online,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
let node_descriptors = self
|
|
||||||
.node_descriptors
|
|
||||||
.values()
|
|
||||||
.filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
|
|
||||||
.count();
|
|
||||||
Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
|
Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
|
||||||
node,
|
node,
|
||||||
node_descriptors,
|
node_descriptors: self.node_descriptors.len(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -545,22 +545,13 @@ impl CoordinatorService {
|
||||||
task_spec,
|
task_spec,
|
||||||
wasm_module_base64,
|
wasm_module_base64,
|
||||||
});
|
});
|
||||||
let queued_tasks = self
|
|
||||||
.pending_task_launches
|
|
||||||
.iter()
|
|
||||||
.filter(|pending| {
|
|
||||||
pending.tenant == tenant
|
|
||||||
&& pending.project == project
|
|
||||||
&& pending.process == process
|
|
||||||
})
|
|
||||||
.count();
|
|
||||||
return Ok(CoordinatorResponse::TaskQueued {
|
return Ok(CoordinatorResponse::TaskQueued {
|
||||||
process,
|
process,
|
||||||
task,
|
task,
|
||||||
actor,
|
actor,
|
||||||
reason,
|
reason,
|
||||||
charged_spawns,
|
charged_spawns,
|
||||||
queued_tasks,
|
queued_tasks: self.pending_task_launches.len(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
Err(err) => return Err(err.into()),
|
Err(err) => return Err(err.into()),
|
||||||
|
|
@ -671,9 +662,7 @@ impl CoordinatorService {
|
||||||
));
|
));
|
||||||
};
|
};
|
||||||
self.task_attempts.remove(&removable);
|
self.task_attempts.remove(&removable);
|
||||||
self.task_terminal_states.remove(&removable);
|
|
||||||
}
|
}
|
||||||
self.task_terminal_states.remove(&key);
|
|
||||||
let attempts = self.task_attempts.entry(key).or_default();
|
let attempts = self.task_attempts.entry(key).or_default();
|
||||||
for attempt in attempts.iter_mut() {
|
for attempt in attempts.iter_mut() {
|
||||||
attempt.current = false;
|
attempt.current = false;
|
||||||
|
|
|
||||||
|
|
@ -382,10 +382,6 @@ impl CoordinatorService {
|
||||||
|| attempt_project != &project
|
|| attempt_project != &project
|
||||||
|| attempt_process != &process
|
|| attempt_process != &process
|
||||||
});
|
});
|
||||||
self.task_terminal_states
|
|
||||||
.retain(|(task_tenant, task_project, task_process, _), _| {
|
|
||||||
task_tenant != &tenant || task_project != &project || task_process != &process
|
|
||||||
});
|
|
||||||
self.restart_launches
|
self.restart_launches
|
||||||
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
|
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
|
||||||
attempt_tenant != &tenant
|
attempt_tenant != &tenant
|
||||||
|
|
@ -396,12 +392,11 @@ impl CoordinatorService {
|
||||||
event.tenant != tenant || event.project != project || event.process != process
|
event.tenant != tenant || event.project != project || event.process != process
|
||||||
});
|
});
|
||||||
let active = self.coordinator.start_process_for_launch_attempt(
|
let active = self.coordinator.start_process_for_launch_attempt(
|
||||||
tenant.clone(),
|
tenant,
|
||||||
project.clone(),
|
project,
|
||||||
process.clone(),
|
process.clone(),
|
||||||
launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
|
launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
|
||||||
);
|
);
|
||||||
self.record_process_started(&tenant, &project, &process, now_epoch_seconds);
|
|
||||||
Ok(CoordinatorResponse::ProcessStarted {
|
Ok(CoordinatorResponse::ProcessStarted {
|
||||||
process,
|
process,
|
||||||
launch_attempt: active
|
launch_attempt: active
|
||||||
|
|
@ -518,13 +513,6 @@ impl CoordinatorService {
|
||||||
}
|
}
|
||||||
let process_key = process_control_key(&tenant, &project, &process);
|
let process_key = process_control_key(&tenant, &project, &process);
|
||||||
if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
|
if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
|
||||||
self.record_process_terminal(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&process,
|
|
||||||
super::ProcessFinalResult::Cancelled,
|
|
||||||
self.current_epoch_seconds()?,
|
|
||||||
);
|
|
||||||
self.coordinator
|
self.coordinator
|
||||||
.abort_process(&tenant, &project, &process)?;
|
.abort_process(&tenant, &project, &process)?;
|
||||||
self.clear_operator_panel_state(&tenant, &project, &process);
|
self.clear_operator_panel_state(&tenant, &project, &process);
|
||||||
|
|
@ -610,13 +598,6 @@ impl CoordinatorService {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
self.record_process_terminal(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&process,
|
|
||||||
super::ProcessFinalResult::Cancelled,
|
|
||||||
self.current_epoch_seconds()?,
|
|
||||||
);
|
|
||||||
if let Some(launch_attempt) = launch_attempt.as_ref() {
|
if let Some(launch_attempt) = launch_attempt.as_ref() {
|
||||||
self.coordinator.abort_process_for_launch_attempt(
|
self.coordinator.abort_process_for_launch_attempt(
|
||||||
&tenant,
|
&tenant,
|
||||||
|
|
@ -675,18 +656,10 @@ impl CoordinatorService {
|
||||||
.map(|active| {
|
.map(|active| {
|
||||||
let process_key = process_control_key(&active.tenant, &active.project, &active.id);
|
let process_key = process_control_key(&active.tenant, &active.project, &active.id);
|
||||||
let main = self.main_runtime.controls.get(&process_key);
|
let main = self.main_runtime.controls.get(&process_key);
|
||||||
let stored = self.process_summaries.get(&process_key);
|
|
||||||
let stored_main_state = stored
|
|
||||||
.and_then(|summary| summary.main_terminal_state.as_ref())
|
|
||||||
.map(|state| match state {
|
|
||||||
super::TaskTerminalState::Completed => "completed",
|
|
||||||
super::TaskTerminalState::Failed => "failed",
|
|
||||||
super::TaskTerminalState::Cancelled => "cancelled",
|
|
||||||
});
|
|
||||||
let state = if self.process_cancellations.contains(&process_key) {
|
let state = if self.process_cancellations.contains(&process_key) {
|
||||||
"cancelling"
|
"cancelling"
|
||||||
} else {
|
} else {
|
||||||
"running"
|
main.map_or("running", |main| main.state.as_str())
|
||||||
};
|
};
|
||||||
let main_wait_state = main.and_then(|main| {
|
let main_wait_state = main.and_then(|main| {
|
||||||
if main.state != "running" {
|
if main.state != "running" {
|
||||||
|
|
@ -711,15 +684,9 @@ impl CoordinatorService {
|
||||||
VirtualProcessStatus {
|
VirtualProcessStatus {
|
||||||
process: active.id,
|
process: active.id,
|
||||||
state: state.to_owned(),
|
state: state.to_owned(),
|
||||||
main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
|
main_task_definition: main.map(|main| main.task_definition.clone()),
|
||||||
|| stored.and_then(|summary| summary.main_task_definition.clone()),
|
main_task_instance: main.map(|main| main.task_instance.clone()),
|
||||||
),
|
main_state: main.map(|main| main.state.clone()),
|
||||||
main_task_instance: main
|
|
||||||
.map(|main| main.task_instance.clone())
|
|
||||||
.or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
|
|
||||||
main_state: main
|
|
||||||
.map(|main| main.state.clone())
|
|
||||||
.or_else(|| stored_main_state.map(str::to_owned)),
|
|
||||||
main_wait_state,
|
main_wait_state,
|
||||||
main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
|
main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
|
||||||
connected_nodes: active.connected_nodes.into_iter().collect(),
|
connected_nodes: active.connected_nodes.into_iter().collect(),
|
||||||
|
|
|
||||||
|
|
@ -150,15 +150,6 @@ pub enum CoordinatorRequest {
|
||||||
project: String,
|
project: String,
|
||||||
actor_user: String,
|
actor_user: String,
|
||||||
},
|
},
|
||||||
ListNodeSummaries {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
RevokeNodeCredential {
|
RevokeNodeCredential {
|
||||||
tenant: String,
|
tenant: String,
|
||||||
project: String,
|
project: String,
|
||||||
|
|
@ -312,15 +303,6 @@ pub enum CoordinatorRequest {
|
||||||
project: String,
|
project: String,
|
||||||
actor_user: String,
|
actor_user: String,
|
||||||
},
|
},
|
||||||
ListProcessSummaries {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
QuotaStatus {
|
QuotaStatus {
|
||||||
tenant: String,
|
tenant: String,
|
||||||
project: String,
|
project: String,
|
||||||
|
|
@ -447,19 +429,6 @@ pub enum CoordinatorRequest {
|
||||||
stderr_truncated: bool,
|
stderr_truncated: bool,
|
||||||
backpressured: bool,
|
backpressured: bool,
|
||||||
},
|
},
|
||||||
ReportTaskLogChunk {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
process: String,
|
|
||||||
node: String,
|
|
||||||
task: String,
|
|
||||||
stream: TaskLogStream,
|
|
||||||
offset: u64,
|
|
||||||
source_bytes: u64,
|
|
||||||
text: String,
|
|
||||||
#[serde(default)]
|
|
||||||
truncated: bool,
|
|
||||||
},
|
|
||||||
ReportVfsMetadata {
|
ReportVfsMetadata {
|
||||||
tenant: String,
|
tenant: String,
|
||||||
project: String,
|
project: String,
|
||||||
|
|
@ -509,18 +478,6 @@ pub enum CoordinatorRequest {
|
||||||
actor_user: String,
|
actor_user: String,
|
||||||
process: String,
|
process: String,
|
||||||
},
|
},
|
||||||
ListRecentLogs {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
process: String,
|
|
||||||
#[serde(default)]
|
|
||||||
task: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
after_sequence: Option<u64>,
|
|
||||||
#[serde(default = "default_log_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
JoinTask {
|
JoinTask {
|
||||||
tenant: String,
|
tenant: String,
|
||||||
project: String,
|
project: String,
|
||||||
|
|
@ -553,23 +510,6 @@ pub enum CoordinatorRequest {
|
||||||
#[serde(default = "default_download_ttl_seconds")]
|
#[serde(default = "default_download_ttl_seconds")]
|
||||||
ttl_seconds: u64,
|
ttl_seconds: u64,
|
||||||
},
|
},
|
||||||
ListArtifacts {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
#[serde(default)]
|
|
||||||
process: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
GetArtifact {
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
artifact: String,
|
|
||||||
},
|
|
||||||
OpenArtifactDownloadStream {
|
OpenArtifactDownloadStream {
|
||||||
tenant: String,
|
tenant: String,
|
||||||
project: String,
|
project: String,
|
||||||
|
|
@ -726,18 +666,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
validate_tenant_project(tenant, project, path)?;
|
validate_tenant_project(tenant, project, path)?;
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))
|
validate_user(actor_user, &format!("{path}.actor_user"))
|
||||||
}
|
}
|
||||||
CoordinatorRequest::ListNodeSummaries {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => {
|
|
||||||
validate_tenant_project(tenant, project, path)?;
|
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
CoordinatorRequest::ExchangeNodeEnrollmentGrant {
|
CoordinatorRequest::ExchangeNodeEnrollmentGrant {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -973,14 +901,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
task,
|
task,
|
||||||
..
|
..
|
||||||
}
|
}
|
||||||
| CoordinatorRequest::ReportTaskLogChunk {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
node,
|
|
||||||
task,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
| CoordinatorRequest::ReportVfsMetadata {
|
| CoordinatorRequest::ReportVfsMetadata {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -1059,23 +979,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
||||||
validate_process(process, &format!("{path}.process"))
|
validate_process(process, &format!("{path}.process"))
|
||||||
}
|
}
|
||||||
CoordinatorRequest::ListRecentLogs {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
limit,
|
|
||||||
..
|
|
||||||
} => {
|
|
||||||
validate_tenant_project(tenant, project, path)?;
|
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
|
||||||
validate_process(process, &format!("{path}.process"))?;
|
|
||||||
if let Some(task) = task {
|
|
||||||
validate_task_instance(task, &format!("{path}.task"))?;
|
|
||||||
}
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
CoordinatorRequest::AbortProcess {
|
CoordinatorRequest::AbortProcess {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -1104,18 +1007,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
validate_tenant_project(tenant, project, path)?;
|
validate_tenant_project(tenant, project, path)?;
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))
|
validate_user(actor_user, &format!("{path}.actor_user"))
|
||||||
}
|
}
|
||||||
CoordinatorRequest::ListProcessSummaries {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => {
|
|
||||||
validate_tenant_project(tenant, project, path)?;
|
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 100)
|
|
||||||
}
|
|
||||||
CoordinatorRequest::RestartTask {
|
CoordinatorRequest::RestartTask {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -1189,20 +1080,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
validate_process(process, &format!("{path}.process"))?;
|
validate_process(process, &format!("{path}.process"))?;
|
||||||
validate_external_token(widget_id, &format!("{path}.widget_id"), 256)
|
validate_external_token(widget_id, &format!("{path}.widget_id"), 256)
|
||||||
}
|
}
|
||||||
CoordinatorRequest::ListArtifacts {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => {
|
|
||||||
validate_tenant_project(tenant, project, path)?;
|
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
|
||||||
validate_optional_process(process.as_deref(), &format!("{path}.process"))?;
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
CoordinatorRequest::CreateArtifactDownloadLink {
|
CoordinatorRequest::CreateArtifactDownloadLink {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -1223,12 +1100,6 @@ fn validate_coordinator_request(request: &CoordinatorRequest, path: &str) -> Res
|
||||||
actor_user,
|
actor_user,
|
||||||
artifact,
|
artifact,
|
||||||
..
|
..
|
||||||
}
|
|
||||||
| CoordinatorRequest::GetArtifact {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
artifact,
|
|
||||||
} => {
|
} => {
|
||||||
validate_tenant_project(tenant, project, path)?;
|
validate_tenant_project(tenant, project, path)?;
|
||||||
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
validate_user(actor_user, &format!("{path}.actor_user"))?;
|
||||||
|
|
@ -1262,14 +1133,6 @@ fn validate_authenticated_request(
|
||||||
| AuthenticatedCoordinatorRequest::ListNodeDescriptors
|
| AuthenticatedCoordinatorRequest::ListNodeDescriptors
|
||||||
| AuthenticatedCoordinatorRequest::ListProcesses
|
| AuthenticatedCoordinatorRequest::ListProcesses
|
||||||
| AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()),
|
| AuthenticatedCoordinatorRequest::QuotaStatus => Ok(()),
|
||||||
AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => {
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => {
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 100)
|
|
||||||
}
|
|
||||||
AuthenticatedCoordinatorRequest::CreateProject { project, .. }
|
AuthenticatedCoordinatorRequest::CreateProject { project, .. }
|
||||||
| AuthenticatedCoordinatorRequest::SelectProject { project } => {
|
| AuthenticatedCoordinatorRequest::SelectProject { project } => {
|
||||||
validate_project(project, &format!("{path}.project"))
|
validate_project(project, &format!("{path}.project"))
|
||||||
|
|
@ -1325,18 +1188,6 @@ fn validate_authenticated_request(
|
||||||
| AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => {
|
| AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => {
|
||||||
validate_process(process, &format!("{path}.process"))
|
validate_process(process, &format!("{path}.process"))
|
||||||
}
|
}
|
||||||
AuthenticatedCoordinatorRequest::ListRecentLogs {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
limit,
|
|
||||||
..
|
|
||||||
} => {
|
|
||||||
validate_process(process, &format!("{path}.process"))?;
|
|
||||||
if let Some(task) = task {
|
|
||||||
validate_task_instance(task, &format!("{path}.task"))?;
|
|
||||||
}
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
AuthenticatedCoordinatorRequest::RestartTask { process, task, .. }
|
AuthenticatedCoordinatorRequest::RestartTask { process, task, .. }
|
||||||
| AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. }
|
| AuthenticatedCoordinatorRequest::ResolveTaskFailure { process, task, .. }
|
||||||
| AuthenticatedCoordinatorRequest::JoinTask { process, task } => {
|
| AuthenticatedCoordinatorRequest::JoinTask { process, task } => {
|
||||||
|
|
@ -1354,19 +1205,9 @@ fn validate_authenticated_request(
|
||||||
AuthenticatedCoordinatorRequest::ListTaskEvents { process } => {
|
AuthenticatedCoordinatorRequest::ListTaskEvents { process } => {
|
||||||
validate_optional_process(process.as_deref(), &format!("{path}.process"))
|
validate_optional_process(process.as_deref(), &format!("{path}.process"))
|
||||||
}
|
}
|
||||||
AuthenticatedCoordinatorRequest::ListArtifacts {
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => {
|
|
||||||
validate_optional_process(process.as_deref(), &format!("{path}.process"))?;
|
|
||||||
validate_optional_cursor(cursor.as_deref(), &format!("{path}.cursor"))?;
|
|
||||||
validate_page_limit(*limit, &format!("{path}.limit"), 200)
|
|
||||||
}
|
|
||||||
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. }
|
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { artifact, .. }
|
||||||
| AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. }
|
| AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { artifact, .. }
|
||||||
| AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. }
|
| AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { artifact, .. } => {
|
||||||
| AuthenticatedCoordinatorRequest::GetArtifact { artifact } => {
|
|
||||||
validate_artifact(artifact, &format!("{path}.artifact"))
|
validate_artifact(artifact, &format!("{path}.artifact"))
|
||||||
}
|
}
|
||||||
AuthenticatedCoordinatorRequest::ExportArtifactToNode {
|
AuthenticatedCoordinatorRequest::ExportArtifactToNode {
|
||||||
|
|
@ -1521,19 +1362,6 @@ fn validate_optional_process(value: Option<&str>, path: &str) -> Result<(), Stri
|
||||||
value.map_or(Ok(()), |value| validate_process(value, path))
|
value.map_or(Ok(()), |value| validate_process(value, path))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_optional_cursor(value: Option<&str>, path: &str) -> Result<(), String> {
|
|
||||||
value.map_or(Ok(()), |value| validate_external_token(value, path, 256))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate_page_limit(value: u32, path: &str, maximum: u32) -> Result<(), String> {
|
|
||||||
if value == 0 || value > maximum {
|
|
||||||
return Err(format!(
|
|
||||||
"malformed pagination limit {path}: expected 1 through {maximum}, received {value}"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> {
|
fn validate_optional_launch_attempt(value: Option<&str>, path: &str) -> Result<(), String> {
|
||||||
value.map_or(Ok(()), |value| validate_launch_attempt(value, path))
|
value.map_or(Ok(()), |value| validate_launch_attempt(value, path))
|
||||||
}
|
}
|
||||||
|
|
@ -1858,12 +1686,6 @@ pub enum AuthenticatedCoordinatorRequest {
|
||||||
ttl_seconds: u64,
|
ttl_seconds: u64,
|
||||||
},
|
},
|
||||||
ListNodeDescriptors,
|
ListNodeDescriptors,
|
||||||
ListNodeSummaries {
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
RevokeNodeCredential {
|
RevokeNodeCredential {
|
||||||
node: String,
|
node: String,
|
||||||
},
|
},
|
||||||
|
|
@ -1900,12 +1722,6 @@ pub enum AuthenticatedCoordinatorRequest {
|
||||||
launch_attempt: Option<String>,
|
launch_attempt: Option<String>,
|
||||||
},
|
},
|
||||||
ListProcesses,
|
ListProcesses,
|
||||||
ListProcessSummaries {
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
QuotaStatus,
|
QuotaStatus,
|
||||||
RestartTask {
|
RestartTask {
|
||||||
process: String,
|
process: String,
|
||||||
|
|
@ -1950,15 +1766,6 @@ pub enum AuthenticatedCoordinatorRequest {
|
||||||
ListTaskSnapshots {
|
ListTaskSnapshots {
|
||||||
process: String,
|
process: String,
|
||||||
},
|
},
|
||||||
ListRecentLogs {
|
|
||||||
process: String,
|
|
||||||
#[serde(default)]
|
|
||||||
task: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
after_sequence: Option<u64>,
|
|
||||||
#[serde(default = "default_log_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
JoinTask {
|
JoinTask {
|
||||||
process: String,
|
process: String,
|
||||||
task: String,
|
task: String,
|
||||||
|
|
@ -1969,17 +1776,6 @@ pub enum AuthenticatedCoordinatorRequest {
|
||||||
#[serde(default = "default_download_ttl_seconds")]
|
#[serde(default = "default_download_ttl_seconds")]
|
||||||
ttl_seconds: u64,
|
ttl_seconds: u64,
|
||||||
},
|
},
|
||||||
ListArtifacts {
|
|
||||||
#[serde(default)]
|
|
||||||
process: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
cursor: Option<String>,
|
|
||||||
#[serde(default = "default_page_limit")]
|
|
||||||
limit: u32,
|
|
||||||
},
|
|
||||||
GetArtifact {
|
|
||||||
artifact: String,
|
|
||||||
},
|
|
||||||
OpenArtifactDownloadStream {
|
OpenArtifactDownloadStream {
|
||||||
artifact: String,
|
artifact: String,
|
||||||
max_bytes: u64,
|
max_bytes: u64,
|
||||||
|
|
@ -2002,14 +1798,6 @@ fn default_download_ttl_seconds() -> u64 {
|
||||||
900
|
900
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_page_limit() -> u32 {
|
|
||||||
50
|
|
||||||
}
|
|
||||||
|
|
||||||
fn default_log_page_limit() -> u32 {
|
|
||||||
100
|
|
||||||
}
|
|
||||||
|
|
||||||
fn default_node_enrollment_ttl_seconds() -> u64 {
|
fn default_node_enrollment_ttl_seconds() -> u64 {
|
||||||
900
|
900
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -183,121 +183,6 @@ pub struct VirtualProcessStatus {
|
||||||
pub coordinator_epoch: u64,
|
pub coordinator_epoch: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct NodeSummary {
|
|
||||||
pub id: NodeId,
|
|
||||||
pub display_name: String,
|
|
||||||
pub online: bool,
|
|
||||||
pub stale: bool,
|
|
||||||
pub last_seen_epoch_seconds: Option<u64>,
|
|
||||||
pub capabilities: NodeCapabilities,
|
|
||||||
pub direct_connectivity: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessLifecycleState {
|
|
||||||
Active,
|
|
||||||
RecentTerminal,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessActivityState {
|
|
||||||
Running,
|
|
||||||
WaitingForNode,
|
|
||||||
WaitingForTask,
|
|
||||||
AwaitingAction,
|
|
||||||
DebugEpochPartial,
|
|
||||||
Cancelling,
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ProcessFinalResult {
|
|
||||||
Completed,
|
|
||||||
Failed,
|
|
||||||
Cancelled,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct DebugEpochSummary {
|
|
||||||
pub epoch: u64,
|
|
||||||
pub command: String,
|
|
||||||
pub fully_frozen: bool,
|
|
||||||
pub partially_frozen: bool,
|
|
||||||
pub fully_resumed: bool,
|
|
||||||
pub failed: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ProcessSummary {
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub lifecycle: ProcessLifecycleState,
|
|
||||||
pub activity: ProcessActivityState,
|
|
||||||
pub main_wait_state: Option<String>,
|
|
||||||
pub started_at_epoch_seconds: u64,
|
|
||||||
pub ended_at_epoch_seconds: Option<u64>,
|
|
||||||
pub final_result: Option<ProcessFinalResult>,
|
|
||||||
pub connected_nodes: Vec<NodeId>,
|
|
||||||
pub current_debug_epoch: Option<DebugEpochSummary>,
|
|
||||||
pub order_cursor: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ArtifactAvailability {
|
|
||||||
Available,
|
|
||||||
NodeOffline,
|
|
||||||
Unavailable,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ArtifactRetentionState {
|
|
||||||
NodeRetained,
|
|
||||||
ExplicitStorage,
|
|
||||||
Lost,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct ArtifactSummary {
|
|
||||||
pub id: ArtifactId,
|
|
||||||
pub display_path: String,
|
|
||||||
pub display_name: String,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub producer_task: TaskInstanceId,
|
|
||||||
pub safe_node: Option<NodeId>,
|
|
||||||
pub digest: Digest,
|
|
||||||
pub size_bytes: u64,
|
|
||||||
pub availability: ArtifactAvailability,
|
|
||||||
pub downloadable_now: bool,
|
|
||||||
pub retention_state: ArtifactRetentionState,
|
|
||||||
pub explicit_storage: bool,
|
|
||||||
pub order_cursor: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum TaskLogStream {
|
|
||||||
Stdout,
|
|
||||||
Stderr,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
pub struct RecentLogEntry {
|
|
||||||
pub sequence: u64,
|
|
||||||
pub process: ProcessId,
|
|
||||||
pub task: TaskInstanceId,
|
|
||||||
pub stream: TaskLogStream,
|
|
||||||
pub text: String,
|
|
||||||
pub server_timestamp_epoch_seconds: u64,
|
|
||||||
pub truncated: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub enum SourcePreparationDisposition {
|
pub enum SourcePreparationDisposition {
|
||||||
Pending { reason: String },
|
Pending { reason: String },
|
||||||
|
|
@ -328,7 +213,7 @@ pub enum CoordinatorResponse {
|
||||||
manual_review: bool,
|
manual_review: bool,
|
||||||
sanitized_reason: Option<String>,
|
sanitized_reason: Option<String>,
|
||||||
next_actions: Vec<String>,
|
next_actions: Vec<String>,
|
||||||
sensitive_moderation_details_exposed: bool,
|
private_moderation_details_exposed: bool,
|
||||||
signup_failure_details_exposed: bool,
|
signup_failure_details_exposed: bool,
|
||||||
},
|
},
|
||||||
AdminStatus {
|
AdminStatus {
|
||||||
|
|
@ -397,11 +282,6 @@ pub enum CoordinatorResponse {
|
||||||
descriptors: Vec<NodeDescriptor>,
|
descriptors: Vec<NodeDescriptor>,
|
||||||
actor: UserId,
|
actor: UserId,
|
||||||
},
|
},
|
||||||
NodeSummaries {
|
|
||||||
nodes: Vec<NodeSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
actor: UserId,
|
|
||||||
},
|
|
||||||
NodeCredentialRevoked {
|
NodeCredentialRevoked {
|
||||||
node: NodeId,
|
node: NodeId,
|
||||||
tenant: TenantId,
|
tenant: TenantId,
|
||||||
|
|
@ -493,11 +373,6 @@ pub enum CoordinatorResponse {
|
||||||
processes: Vec<VirtualProcessStatus>,
|
processes: Vec<VirtualProcessStatus>,
|
||||||
actor: UserId,
|
actor: UserId,
|
||||||
},
|
},
|
||||||
ProcessSummaries {
|
|
||||||
processes: Vec<ProcessSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
actor: UserId,
|
|
||||||
},
|
|
||||||
QuotaStatus {
|
QuotaStatus {
|
||||||
tenant: TenantId,
|
tenant: TenantId,
|
||||||
project: ProjectId,
|
project: ProjectId,
|
||||||
|
|
@ -608,17 +483,6 @@ pub enum CoordinatorResponse {
|
||||||
stderr_tail: String,
|
stderr_tail: String,
|
||||||
backpressured: bool,
|
backpressured: bool,
|
||||||
},
|
},
|
||||||
TaskLogChunkRecorded {
|
|
||||||
process: ProcessId,
|
|
||||||
task: TaskInstanceId,
|
|
||||||
sequence: Option<u64>,
|
|
||||||
next_offset: u64,
|
|
||||||
},
|
|
||||||
RecentLogs {
|
|
||||||
entries: Vec<RecentLogEntry>,
|
|
||||||
next_sequence: Option<u64>,
|
|
||||||
history_truncated: bool,
|
|
||||||
},
|
|
||||||
VfsMetadataRecorded {
|
VfsMetadataRecorded {
|
||||||
process: ProcessId,
|
process: ProcessId,
|
||||||
task: TaskInstanceId,
|
task: TaskInstanceId,
|
||||||
|
|
@ -655,13 +519,6 @@ pub enum CoordinatorResponse {
|
||||||
ArtifactDownloadLink {
|
ArtifactDownloadLink {
|
||||||
link: DownloadLink,
|
link: DownloadLink,
|
||||||
},
|
},
|
||||||
Artifacts {
|
|
||||||
artifacts: Vec<ArtifactSummary>,
|
|
||||||
next_cursor: Option<String>,
|
|
||||||
},
|
|
||||||
Artifact {
|
|
||||||
artifact: ArtifactSummary,
|
|
||||||
},
|
|
||||||
ArtifactDownloadLinkRevoked {
|
ArtifactDownloadLinkRevoked {
|
||||||
link: DownloadLink,
|
link: DownloadLink,
|
||||||
},
|
},
|
||||||
|
|
@ -686,24 +543,6 @@ pub enum CoordinatorResponse {
|
||||||
artifact_size_bytes: u64,
|
artifact_size_bytes: u64,
|
||||||
},
|
},
|
||||||
Error {
|
Error {
|
||||||
#[serde(flatten)]
|
message: String,
|
||||||
error: clusterflux_core::ApiError,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CoordinatorResponse {
|
|
||||||
pub fn error(request_id: impl Into<String>, message: impl Into<String>) -> Self {
|
|
||||||
Self::Error {
|
|
||||||
error: clusterflux_core::ApiError::from_message(request_id, message),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn service_error(
|
|
||||||
request_id: impl Into<String>,
|
|
||||||
error: &crate::service::CoordinatorServiceError,
|
|
||||||
) -> Self {
|
|
||||||
Self::Error {
|
|
||||||
error: error.api_error(request_id),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -260,6 +260,22 @@ impl CoordinatorQuota {
|
||||||
self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
|
self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn can_charge_log_bytes(
|
||||||
|
&self,
|
||||||
|
tenant: &TenantId,
|
||||||
|
project: &ProjectId,
|
||||||
|
bytes: u64,
|
||||||
|
now_epoch_seconds: u64,
|
||||||
|
) -> Result<(), LimitError> {
|
||||||
|
self.can_charge(
|
||||||
|
tenant,
|
||||||
|
project,
|
||||||
|
LimitKind::LogBytes,
|
||||||
|
bytes,
|
||||||
|
now_epoch_seconds,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn charge_log_bytes(
|
pub(super) fn charge_log_bytes(
|
||||||
&mut self,
|
&mut self,
|
||||||
tenant: &TenantId,
|
tenant: &TenantId,
|
||||||
|
|
|
||||||
|
|
@ -65,16 +65,6 @@ pub struct ArtifactRelayUsage {
|
||||||
pub egress_bytes: u64,
|
pub egress_bytes: u64,
|
||||||
pub abandoned_or_failed_bytes: u64,
|
pub abandoned_or_failed_bytes: u64,
|
||||||
pub reserved_bytes: u64,
|
pub reserved_bytes: u64,
|
||||||
pub lifetime_ingress_bytes: u64,
|
|
||||||
pub lifetime_egress_bytes: u64,
|
|
||||||
pub lifetime_abandoned_or_failed_bytes: u64,
|
|
||||||
pub completed_transfers: u64,
|
|
||||||
pub failed_transfers: u64,
|
|
||||||
pub cancelled_transfers: u64,
|
|
||||||
pub expired_transfers: u64,
|
|
||||||
pub tracked_scopes: usize,
|
|
||||||
pub max_active_global: usize,
|
|
||||||
pub max_tracked_scopes: usize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
|
@ -87,20 +77,6 @@ pub struct ArtifactRelayDurableState {
|
||||||
pub ingress_used: u64,
|
pub ingress_used: u64,
|
||||||
pub egress_used: u64,
|
pub egress_used: u64,
|
||||||
pub abandoned_or_failed_used: u64,
|
pub abandoned_or_failed_used: u64,
|
||||||
#[serde(default)]
|
|
||||||
pub lifetime_ingress_bytes: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub lifetime_egress_bytes: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub lifetime_abandoned_or_failed_bytes: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub completed_transfers: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub failed_transfers: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub cancelled_transfers: u64,
|
|
||||||
#[serde(default)]
|
|
||||||
pub expired_transfers: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
|
@ -169,13 +145,6 @@ pub(super) struct ArtifactRelayLedger {
|
||||||
ingress_used: u64,
|
ingress_used: u64,
|
||||||
egress_used: u64,
|
egress_used: u64,
|
||||||
abandoned_or_failed_used: u64,
|
abandoned_or_failed_used: u64,
|
||||||
lifetime_ingress_bytes: u64,
|
|
||||||
lifetime_egress_bytes: u64,
|
|
||||||
lifetime_abandoned_or_failed_bytes: u64,
|
|
||||||
completed_transfers: u64,
|
|
||||||
failed_transfers: u64,
|
|
||||||
cancelled_transfers: u64,
|
|
||||||
expired_transfers: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ArtifactRelayLedger {
|
impl Default for ArtifactRelayLedger {
|
||||||
|
|
@ -196,13 +165,6 @@ impl ArtifactRelayLedger {
|
||||||
ingress_used: 0,
|
ingress_used: 0,
|
||||||
egress_used: 0,
|
egress_used: 0,
|
||||||
abandoned_or_failed_used: 0,
|
abandoned_or_failed_used: 0,
|
||||||
lifetime_ingress_bytes: 0,
|
|
||||||
lifetime_egress_bytes: 0,
|
|
||||||
lifetime_abandoned_or_failed_bytes: 0,
|
|
||||||
completed_transfers: 0,
|
|
||||||
failed_transfers: 0,
|
|
||||||
cancelled_transfers: 0,
|
|
||||||
expired_transfers: 0,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -256,13 +218,6 @@ impl ArtifactRelayLedger {
|
||||||
ingress_used: state.ingress_used,
|
ingress_used: state.ingress_used,
|
||||||
egress_used: state.egress_used,
|
egress_used: state.egress_used,
|
||||||
abandoned_or_failed_used: state.abandoned_or_failed_used,
|
abandoned_or_failed_used: state.abandoned_or_failed_used,
|
||||||
lifetime_ingress_bytes: state.lifetime_ingress_bytes,
|
|
||||||
lifetime_egress_bytes: state.lifetime_egress_bytes,
|
|
||||||
lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
|
|
||||||
completed_transfers: state.completed_transfers,
|
|
||||||
failed_transfers: state.failed_transfers,
|
|
||||||
cancelled_transfers: state.cancelled_transfers,
|
|
||||||
expired_transfers: state.expired_transfers,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -305,13 +260,6 @@ impl ArtifactRelayLedger {
|
||||||
ingress_used: self.ingress_used,
|
ingress_used: self.ingress_used,
|
||||||
egress_used: self.egress_used,
|
egress_used: self.egress_used,
|
||||||
abandoned_or_failed_used: self.abandoned_or_failed_used,
|
abandoned_or_failed_used: self.abandoned_or_failed_used,
|
||||||
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
|
|
||||||
lifetime_egress_bytes: self.lifetime_egress_bytes,
|
|
||||||
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
|
|
||||||
completed_transfers: self.completed_transfers,
|
|
||||||
failed_transfers: self.failed_transfers,
|
|
||||||
cancelled_transfers: self.cancelled_transfers,
|
|
||||||
expired_transfers: self.expired_transfers,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -543,11 +491,9 @@ impl ArtifactRelayLedger {
|
||||||
if ingress {
|
if ingress {
|
||||||
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
|
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
|
||||||
self.ingress_used = self.ingress_used.saturating_add(bytes);
|
self.ingress_used = self.ingress_used.saturating_add(bytes);
|
||||||
self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
|
|
||||||
} else {
|
} else {
|
||||||
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
|
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
|
||||||
self.egress_used = self.egress_used.saturating_add(bytes);
|
self.egress_used = self.egress_used.saturating_add(bytes);
|
||||||
self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
|
|
||||||
}
|
}
|
||||||
let project_key = (
|
let project_key = (
|
||||||
reservation.scope.tenant.clone(),
|
reservation.scope.tenant.clone(),
|
||||||
|
|
@ -608,29 +554,10 @@ impl ArtifactRelayLedger {
|
||||||
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
|
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
|
||||||
if let Some(reservation) = self.reservations.remove(key) {
|
if let Some(reservation) = self.reservations.remove(key) {
|
||||||
if reason != RelayFinishReason::Completed {
|
if reason != RelayFinishReason::Completed {
|
||||||
let abandoned_or_failed_bytes = reservation
|
|
||||||
.ingress_bytes
|
|
||||||
.saturating_add(reservation.egress_bytes);
|
|
||||||
self.abandoned_or_failed_used = self
|
self.abandoned_or_failed_used = self
|
||||||
.abandoned_or_failed_used
|
.abandoned_or_failed_used
|
||||||
.saturating_add(abandoned_or_failed_bytes);
|
.saturating_add(reservation.ingress_bytes)
|
||||||
self.lifetime_abandoned_or_failed_bytes = self
|
.saturating_add(reservation.egress_bytes);
|
||||||
.lifetime_abandoned_or_failed_bytes
|
|
||||||
.saturating_add(abandoned_or_failed_bytes);
|
|
||||||
}
|
|
||||||
match reason {
|
|
||||||
RelayFinishReason::Completed => {
|
|
||||||
self.completed_transfers = self.completed_transfers.saturating_add(1);
|
|
||||||
}
|
|
||||||
RelayFinishReason::Failed => {
|
|
||||||
self.failed_transfers = self.failed_transfers.saturating_add(1);
|
|
||||||
}
|
|
||||||
RelayFinishReason::Cancelled => {
|
|
||||||
self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
|
|
||||||
}
|
|
||||||
RelayFinishReason::Expired => {
|
|
||||||
self.expired_transfers = self.expired_transfers.saturating_add(1);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -653,18 +580,6 @@ impl ArtifactRelayLedger {
|
||||||
ingress_bytes: self.ingress_used,
|
ingress_bytes: self.ingress_used,
|
||||||
egress_bytes: self.egress_used,
|
egress_bytes: self.egress_used,
|
||||||
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
|
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
|
||||||
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
|
|
||||||
lifetime_egress_bytes: self.lifetime_egress_bytes,
|
|
||||||
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
|
|
||||||
completed_transfers: self.completed_transfers,
|
|
||||||
failed_transfers: self.failed_transfers,
|
|
||||||
cancelled_transfers: self.cancelled_transfers,
|
|
||||||
expired_transfers: self.expired_transfers,
|
|
||||||
tracked_scopes: self.project_used.len()
|
|
||||||
+ self.tenant_used.len()
|
|
||||||
+ self.account_used.len(),
|
|
||||||
max_active_global: self.configuration.max_active_global,
|
|
||||||
max_tracked_scopes: self.configuration.max_tracked_scopes,
|
|
||||||
reserved_bytes: self
|
reserved_bytes: self
|
||||||
.reservations
|
.reservations
|
||||||
.values()
|
.values()
|
||||||
|
|
@ -673,68 +588,3 @@ impl ArtifactRelayLedger {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
|
|
||||||
let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
|
|
||||||
ledger
|
|
||||||
.reserve(
|
|
||||||
"transfer".to_owned(),
|
|
||||||
TenantId::from("tenant"),
|
|
||||||
ProjectId::from("project"),
|
|
||||||
UserId::from("user"),
|
|
||||||
16,
|
|
||||||
16,
|
|
||||||
100,
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
ledger.charge_ingress("transfer", 24, 1).unwrap();
|
|
||||||
ledger.charge_egress("transfer", 24, 1).unwrap();
|
|
||||||
ledger.finish("transfer", RelayFinishReason::Completed);
|
|
||||||
|
|
||||||
let usage = ledger.usage();
|
|
||||||
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
||||||
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
||||||
assert_eq!(usage.completed_transfers, 1);
|
|
||||||
|
|
||||||
ledger.prepare_period(u64::MAX);
|
|
||||||
let usage = ledger.usage();
|
|
||||||
assert_eq!(usage.ingress_bytes, 0);
|
|
||||||
assert_eq!(usage.egress_bytes, 0);
|
|
||||||
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
||||||
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
||||||
|
|
||||||
let restored = ArtifactRelayLedger::from_durable(
|
|
||||||
ArtifactRelayConfiguration::unlimited(),
|
|
||||||
ledger.durable_state(),
|
|
||||||
);
|
|
||||||
let usage = restored.usage();
|
|
||||||
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
||||||
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
||||||
assert_eq!(usage.completed_transfers, 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn older_durable_relay_state_defaults_new_metrics() {
|
|
||||||
let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
|
|
||||||
"reservations": {},
|
|
||||||
"period": 1,
|
|
||||||
"project_used": [],
|
|
||||||
"tenant_used": [],
|
|
||||||
"account_used": [],
|
|
||||||
"ingress_used": 3,
|
|
||||||
"egress_used": 4,
|
|
||||||
"abandoned_or_failed_used": 2
|
|
||||||
}))
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
assert_eq!(state.lifetime_ingress_bytes, 0);
|
|
||||||
assert_eq!(state.lifetime_egress_bytes, 0);
|
|
||||||
assert_eq!(state.completed_transfers, 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,7 @@ impl CoordinatorService {
|
||||||
manual_review: account_state.manual_review,
|
manual_review: account_state.manual_review,
|
||||||
sanitized_reason: account_state.sanitized_reason,
|
sanitized_reason: account_state.sanitized_reason,
|
||||||
next_actions: account_state.next_actions,
|
next_actions: account_state.next_actions,
|
||||||
sensitive_moderation_details_exposed: false,
|
private_moderation_details_exposed: false,
|
||||||
signup_failure_details_exposed: false,
|
signup_failure_details_exposed: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -298,13 +298,6 @@ impl CoordinatorService {
|
||||||
project,
|
project,
|
||||||
actor_user,
|
actor_user,
|
||||||
} => self.handle_list_node_descriptors(tenant, project, actor_user),
|
} => self.handle_list_node_descriptors(tenant, project, actor_user),
|
||||||
CoordinatorRequest::ListNodeSummaries {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit),
|
|
||||||
CoordinatorRequest::RevokeNodeCredential {
|
CoordinatorRequest::RevokeNodeCredential {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -428,13 +421,6 @@ impl CoordinatorService {
|
||||||
project,
|
project,
|
||||||
actor_user,
|
actor_user,
|
||||||
} => self.handle_list_processes(tenant, project, actor_user),
|
} => self.handle_list_processes(tenant, project, actor_user),
|
||||||
CoordinatorRequest::ListProcessSummaries {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit),
|
|
||||||
CoordinatorRequest::QuotaStatus {
|
CoordinatorRequest::QuotaStatus {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -480,8 +466,7 @@ impl CoordinatorService {
|
||||||
CoordinatorRequest::PollDebugCommand { .. }
|
CoordinatorRequest::PollDebugCommand { .. }
|
||||||
| CoordinatorRequest::ReportDebugState { .. }
|
| CoordinatorRequest::ReportDebugState { .. }
|
||||||
| CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
|
| CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
|
||||||
CoordinatorRequest::ReportTaskLog { .. }
|
CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
|
||||||
| CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(),
|
|
||||||
CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
|
CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
|
||||||
CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
|
CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
|
||||||
CoordinatorRequest::ListTaskEvents {
|
CoordinatorRequest::ListTaskEvents {
|
||||||
|
|
@ -496,23 +481,6 @@ impl CoordinatorService {
|
||||||
actor_user,
|
actor_user,
|
||||||
process,
|
process,
|
||||||
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
|
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
|
||||||
CoordinatorRequest::ListRecentLogs {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
after_sequence,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_recent_logs(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
after_sequence,
|
|
||||||
limit,
|
|
||||||
),
|
|
||||||
CoordinatorRequest::JoinTask {
|
CoordinatorRequest::JoinTask {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -559,20 +527,6 @@ impl CoordinatorService {
|
||||||
max_bytes,
|
max_bytes,
|
||||||
ttl_seconds,
|
ttl_seconds,
|
||||||
),
|
),
|
||||||
CoordinatorRequest::ListArtifacts {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit),
|
|
||||||
CoordinatorRequest::GetArtifact {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
actor_user,
|
|
||||||
artifact,
|
|
||||||
} => self.handle_get_artifact(tenant, project, actor_user, artifact),
|
|
||||||
CoordinatorRequest::OpenArtifactDownloadStream {
|
CoordinatorRequest::OpenArtifactDownloadStream {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -655,7 +609,7 @@ impl CoordinatorService {
|
||||||
manual_review: account_state.manual_review,
|
manual_review: account_state.manual_review,
|
||||||
sanitized_reason: account_state.sanitized_reason,
|
sanitized_reason: account_state.sanitized_reason,
|
||||||
next_actions: account_state.next_actions,
|
next_actions: account_state.next_actions,
|
||||||
sensitive_moderation_details_exposed: false,
|
private_moderation_details_exposed: false,
|
||||||
signup_failure_details_exposed: false,
|
signup_failure_details_exposed: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
@ -742,14 +696,6 @@ impl CoordinatorService {
|
||||||
context.project.as_str().to_owned(),
|
context.project.as_str().to_owned(),
|
||||||
actor.as_str().to_owned(),
|
actor.as_str().to_owned(),
|
||||||
),
|
),
|
||||||
AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self
|
|
||||||
.handle_list_node_summaries(
|
|
||||||
context.tenant.as_str().to_owned(),
|
|
||||||
context.project.as_str().to_owned(),
|
|
||||||
actor.as_str().to_owned(),
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
),
|
|
||||||
AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
|
AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
|
||||||
.handle_revoke_node_credential(
|
.handle_revoke_node_credential(
|
||||||
context.tenant.as_str().to_owned(),
|
context.tenant.as_str().to_owned(),
|
||||||
|
|
@ -801,14 +747,6 @@ impl CoordinatorService {
|
||||||
context.project.as_str().to_owned(),
|
context.project.as_str().to_owned(),
|
||||||
actor.as_str().to_owned(),
|
actor.as_str().to_owned(),
|
||||||
),
|
),
|
||||||
AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self
|
|
||||||
.handle_list_process_summaries(
|
|
||||||
context.tenant.as_str().to_owned(),
|
|
||||||
context.project.as_str().to_owned(),
|
|
||||||
actor.as_str().to_owned(),
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
),
|
|
||||||
AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
|
AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
|
||||||
context.tenant.as_str().to_owned(),
|
context.tenant.as_str().to_owned(),
|
||||||
context.project.as_str().to_owned(),
|
context.project.as_str().to_owned(),
|
||||||
|
|
@ -864,20 +802,6 @@ impl CoordinatorService {
|
||||||
actor.as_str().to_owned(),
|
actor.as_str().to_owned(),
|
||||||
process,
|
process,
|
||||||
),
|
),
|
||||||
AuthenticatedCoordinatorRequest::ListRecentLogs {
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
after_sequence,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_recent_logs(
|
|
||||||
context.tenant.as_str().to_owned(),
|
|
||||||
context.project.as_str().to_owned(),
|
|
||||||
actor.as_str().to_owned(),
|
|
||||||
process,
|
|
||||||
task,
|
|
||||||
after_sequence,
|
|
||||||
limit,
|
|
||||||
),
|
|
||||||
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
|
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
|
||||||
context.tenant.as_str().to_owned(),
|
context.tenant.as_str().to_owned(),
|
||||||
context.project.as_str().to_owned(),
|
context.project.as_str().to_owned(),
|
||||||
|
|
@ -897,24 +821,6 @@ impl CoordinatorService {
|
||||||
max_bytes,
|
max_bytes,
|
||||||
ttl_seconds,
|
ttl_seconds,
|
||||||
),
|
),
|
||||||
AuthenticatedCoordinatorRequest::ListArtifacts {
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
} => self.handle_list_artifacts(
|
|
||||||
context.tenant.as_str().to_owned(),
|
|
||||||
context.project.as_str().to_owned(),
|
|
||||||
actor.as_str().to_owned(),
|
|
||||||
process,
|
|
||||||
cursor,
|
|
||||||
limit,
|
|
||||||
),
|
|
||||||
AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact(
|
|
||||||
context.tenant.as_str().to_owned(),
|
|
||||||
context.project.as_str().to_owned(),
|
|
||||||
actor.as_str().to_owned(),
|
|
||||||
artifact,
|
|
||||||
),
|
|
||||||
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
|
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
|
||||||
artifact,
|
artifact,
|
||||||
max_bytes,
|
max_bytes,
|
||||||
|
|
|
||||||
|
|
@ -253,29 +253,6 @@ impl CoordinatorService {
|
||||||
stderr_truncated,
|
stderr_truncated,
|
||||||
backpressured,
|
backpressured,
|
||||||
),
|
),
|
||||||
CoordinatorRequest::ReportTaskLogChunk {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
node,
|
|
||||||
task,
|
|
||||||
stream,
|
|
||||||
offset,
|
|
||||||
source_bytes,
|
|
||||||
text,
|
|
||||||
truncated,
|
|
||||||
} => self.handle_report_task_log_chunk(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
node,
|
|
||||||
task,
|
|
||||||
stream,
|
|
||||||
offset,
|
|
||||||
source_bytes,
|
|
||||||
text,
|
|
||||||
truncated,
|
|
||||||
),
|
|
||||||
CoordinatorRequest::ReportVfsMetadata {
|
CoordinatorRequest::ReportVfsMetadata {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
@ -369,7 +346,6 @@ fn signed_node_request_kind(
|
||||||
CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
|
CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
|
||||||
CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
|
CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
|
||||||
CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
|
CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
|
||||||
CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"),
|
|
||||||
CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
|
CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
|
||||||
CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
|
CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
|
||||||
_ => Err(CoordinatorError::Unauthorized(
|
_ => Err(CoordinatorError::Unauthorized(
|
||||||
|
|
@ -465,12 +441,6 @@ fn signed_node_request_scope(
|
||||||
node,
|
node,
|
||||||
..
|
..
|
||||||
}
|
}
|
||||||
| CoordinatorRequest::ReportTaskLogChunk {
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
node,
|
|
||||||
..
|
|
||||||
}
|
|
||||||
| CoordinatorRequest::ReportVfsMetadata {
|
| CoordinatorRequest::ReportVfsMetadata {
|
||||||
tenant,
|
tenant,
|
||||||
project,
|
project,
|
||||||
|
|
|
||||||
|
|
@ -1,565 +0,0 @@
|
||||||
use std::collections::BTreeSet;
|
|
||||||
use std::time::Instant;
|
|
||||||
|
|
||||||
use clusterflux_core::{
|
|
||||||
ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
|
|
||||||
TenantId, UserId,
|
|
||||||
};
|
|
||||||
|
|
||||||
use super::keys::{process_control_key, ProcessControlKey};
|
|
||||||
use super::{
|
|
||||||
ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse,
|
|
||||||
CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary,
|
|
||||||
NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
|
|
||||||
TaskAttemptState,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
||||||
pub(super) struct StoredProcessSummary {
|
|
||||||
pub(super) started_at_epoch_seconds: u64,
|
|
||||||
pub(super) ended_at_epoch_seconds: Option<u64>,
|
|
||||||
pub(super) final_result: Option<ProcessFinalResult>,
|
|
||||||
pub(super) connected_nodes: Vec<NodeId>,
|
|
||||||
pub(super) main_task_definition: Option<TaskDefinitionId>,
|
|
||||||
pub(super) main_task_instance: Option<TaskInstanceId>,
|
|
||||||
pub(super) main_terminal_state: Option<super::TaskTerminalState>,
|
|
||||||
pub(super) order: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl CoordinatorService {
|
|
||||||
pub(super) fn handle_list_node_summaries(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let actor = UserId::new(actor_user);
|
|
||||||
let cursor = cursor.as_deref();
|
|
||||||
let mut nodes = self
|
|
||||||
.node_descriptors
|
|
||||||
.iter()
|
|
||||||
.filter(|(scope, descriptor)| {
|
|
||||||
scope.tenant == tenant
|
|
||||||
&& scope.project == project
|
|
||||||
&& cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor)
|
|
||||||
})
|
|
||||||
.map(|(scope, descriptor)| {
|
|
||||||
let online = self.node_is_live(scope);
|
|
||||||
NodeSummary {
|
|
||||||
id: descriptor.id.clone(),
|
|
||||||
display_name: descriptor.id.as_str().to_owned(),
|
|
||||||
online,
|
|
||||||
stale: !online,
|
|
||||||
last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(),
|
|
||||||
capabilities: descriptor.capabilities.clone(),
|
|
||||||
direct_connectivity: descriptor.direct_connectivity,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
nodes.sort_by(|left, right| left.id.cmp(&right.id));
|
|
||||||
let has_more = nodes.len() > limit as usize;
|
|
||||||
nodes.truncate(limit as usize);
|
|
||||||
let next_cursor = has_more
|
|
||||||
.then(|| nodes.last().map(|node| node.id.as_str().to_owned()))
|
|
||||||
.flatten();
|
|
||||||
Ok(CoordinatorResponse::NodeSummaries {
|
|
||||||
nodes,
|
|
||||||
next_cursor,
|
|
||||||
actor,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn record_process_started(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
now_epoch_seconds: u64,
|
|
||||||
) {
|
|
||||||
let key = process_control_key(tenant, project, process);
|
|
||||||
if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) {
|
|
||||||
logs.retain(|entry| &entry.process != process);
|
|
||||||
if logs.is_empty() {
|
|
||||||
self.recent_logs.remove(&(tenant.clone(), project.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.recent_log_dropped_through.remove(&key);
|
|
||||||
self.recent_log_accounted_bytes.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, _, _), _| {
|
|
||||||
entry_tenant != tenant || entry_project != project || entry_process != process
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.recent_log_truncated_streams.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, _, _)| {
|
|
||||||
entry_tenant != tenant || entry_project != project || entry_process != process
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.recent_log_quota_truncated_streams.retain(
|
|
||||||
|(entry_tenant, entry_project, entry_process, _, _)| {
|
|
||||||
entry_tenant != tenant || entry_project != project || entry_process != process
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.process_summary_order
|
|
||||||
.retain(|retained| retained != &key);
|
|
||||||
self.evict_process_summaries_for_project(tenant, project);
|
|
||||||
self.evict_process_summaries_total();
|
|
||||||
let order = self.next_process_summary_order;
|
|
||||||
self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
|
|
||||||
self.process_summaries.insert(
|
|
||||||
key.clone(),
|
|
||||||
StoredProcessSummary {
|
|
||||||
started_at_epoch_seconds: now_epoch_seconds,
|
|
||||||
ended_at_epoch_seconds: None,
|
|
||||||
final_result: None,
|
|
||||||
connected_nodes: Vec::new(),
|
|
||||||
main_task_definition: None,
|
|
||||||
main_task_instance: None,
|
|
||||||
main_terminal_state: None,
|
|
||||||
order,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
self.process_summary_order.push_back(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn record_process_terminal(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
final_result: ProcessFinalResult,
|
|
||||||
now_epoch_seconds: u64,
|
|
||||||
) {
|
|
||||||
let key = process_control_key(tenant, project, process);
|
|
||||||
let connected_nodes = self
|
|
||||||
.coordinator
|
|
||||||
.active_process(tenant, project, process)
|
|
||||||
.map(|active| active.connected_nodes.iter().cloned().collect())
|
|
||||||
.unwrap_or_default();
|
|
||||||
let order = self.next_process_summary_order;
|
|
||||||
let entry = self
|
|
||||||
.process_summaries
|
|
||||||
.entry(key.clone())
|
|
||||||
.or_insert_with(|| {
|
|
||||||
self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
|
|
||||||
self.process_summary_order.push_back(key.clone());
|
|
||||||
StoredProcessSummary {
|
|
||||||
started_at_epoch_seconds: now_epoch_seconds,
|
|
||||||
ended_at_epoch_seconds: None,
|
|
||||||
final_result: None,
|
|
||||||
connected_nodes: Vec::new(),
|
|
||||||
main_task_definition: None,
|
|
||||||
main_task_instance: None,
|
|
||||||
main_terminal_state: None,
|
|
||||||
order,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
entry.ended_at_epoch_seconds = Some(now_epoch_seconds);
|
|
||||||
entry.final_result = Some(final_result);
|
|
||||||
entry.connected_nodes = connected_nodes;
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn record_main_terminal_state(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
process: &ProcessId,
|
|
||||||
task_definition: TaskDefinitionId,
|
|
||||||
task_instance: TaskInstanceId,
|
|
||||||
terminal_state: super::TaskTerminalState,
|
|
||||||
) {
|
|
||||||
let key = process_control_key(tenant, project, process);
|
|
||||||
if !self.process_summaries.contains_key(&key) {
|
|
||||||
self.record_process_started(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
process,
|
|
||||||
self.liveness_now_epoch_seconds(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if let Some(summary) = self.process_summaries.get_mut(&key) {
|
|
||||||
summary.main_task_definition = Some(task_definition);
|
|
||||||
summary.main_task_instance = Some(task_instance);
|
|
||||||
summary.main_terminal_state = Some(terminal_state);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_list_process_summaries(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let actor = UserId::new(actor_user);
|
|
||||||
let cursor = parse_order_cursor(cursor.as_deref(), "process")?;
|
|
||||||
let mut stored = self
|
|
||||||
.process_summaries
|
|
||||||
.iter()
|
|
||||||
.filter(|((entry_tenant, entry_project, _), summary)| {
|
|
||||||
entry_tenant == &tenant
|
|
||||||
&& entry_project == &project
|
|
||||||
&& cursor.is_none_or(|cursor| summary.order < cursor)
|
|
||||||
})
|
|
||||||
.map(|(key, summary)| (key.clone(), summary.clone()))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order));
|
|
||||||
let has_more = stored.len() > limit as usize;
|
|
||||||
stored.truncate(limit as usize);
|
|
||||||
let processes = stored
|
|
||||||
.into_iter()
|
|
||||||
.map(|(key, stored)| self.process_summary_from_stored(&key, stored))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let next_cursor = has_more
|
|
||||||
.then(|| processes.last().map(|process| process.order_cursor.clone()))
|
|
||||||
.flatten();
|
|
||||||
Ok(CoordinatorResponse::ProcessSummaries {
|
|
||||||
processes,
|
|
||||||
next_cursor,
|
|
||||||
actor,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn process_summary_from_stored(
|
|
||||||
&self,
|
|
||||||
key: &ProcessControlKey,
|
|
||||||
stored: StoredProcessSummary,
|
|
||||||
) -> ProcessSummary {
|
|
||||||
let (tenant, project, process) = key;
|
|
||||||
let active = self.coordinator.active_process(tenant, project, process);
|
|
||||||
let process_key = process_control_key(tenant, project, process);
|
|
||||||
let main_wait_state = active.and_then(|_| {
|
|
||||||
if self.pending_task_launches.iter().any(|pending| {
|
|
||||||
&pending.tenant == tenant
|
|
||||||
&& &pending.project == project
|
|
||||||
&& &pending.process == process
|
|
||||||
}) {
|
|
||||||
Some("waiting_for_node".to_owned())
|
|
||||||
} else if self
|
|
||||||
.main_runtime
|
|
||||||
.is_waiting_for_task(tenant, project, process)
|
|
||||||
{
|
|
||||||
Some("waiting_for_task".to_owned())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let current_debug_epoch = self.debug_epoch_summary(&process_key);
|
|
||||||
let awaiting_action = self.task_attempts.iter().any(
|
|
||||||
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
|
|
||||||
attempt_tenant == tenant
|
|
||||||
&& attempt_project == project
|
|
||||||
&& attempt_process == process
|
|
||||||
&& attempts.iter().any(|attempt| {
|
|
||||||
attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction
|
|
||||||
})
|
|
||||||
},
|
|
||||||
);
|
|
||||||
let activity = if let Some(result) = &stored.final_result {
|
|
||||||
match result {
|
|
||||||
ProcessFinalResult::Completed => ProcessActivityState::Completed,
|
|
||||||
ProcessFinalResult::Failed => ProcessActivityState::Failed,
|
|
||||||
ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled,
|
|
||||||
}
|
|
||||||
} else if self.process_cancellations.contains(&process_key) {
|
|
||||||
ProcessActivityState::Cancelling
|
|
||||||
} else if current_debug_epoch
|
|
||||||
.as_ref()
|
|
||||||
.is_some_and(|epoch| epoch.partially_frozen)
|
|
||||||
{
|
|
||||||
ProcessActivityState::DebugEpochPartial
|
|
||||||
} else if awaiting_action {
|
|
||||||
ProcessActivityState::AwaitingAction
|
|
||||||
} else {
|
|
||||||
match main_wait_state.as_deref() {
|
|
||||||
Some("waiting_for_node") => ProcessActivityState::WaitingForNode,
|
|
||||||
Some("waiting_for_task") => ProcessActivityState::WaitingForTask,
|
|
||||||
_ => ProcessActivityState::Running,
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let connected_nodes = active
|
|
||||||
.map(|active| active.connected_nodes.iter().cloned().collect())
|
|
||||||
.unwrap_or(stored.connected_nodes);
|
|
||||||
ProcessSummary {
|
|
||||||
process: process.clone(),
|
|
||||||
lifecycle: if active.is_some() {
|
|
||||||
ProcessLifecycleState::Active
|
|
||||||
} else {
|
|
||||||
ProcessLifecycleState::RecentTerminal
|
|
||||||
},
|
|
||||||
activity,
|
|
||||||
main_wait_state,
|
|
||||||
started_at_epoch_seconds: stored.started_at_epoch_seconds,
|
|
||||||
ended_at_epoch_seconds: stored.ended_at_epoch_seconds,
|
|
||||||
final_result: stored.final_result,
|
|
||||||
connected_nodes,
|
|
||||||
current_debug_epoch,
|
|
||||||
order_cursor: format!("process:{}", stored.order),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option<DebugEpochSummary> {
|
|
||||||
let runtime = self.debug_epoch_runtime.get(key)?;
|
|
||||||
let acknowledgements = runtime.acknowledgements.values().collect::<Vec<_>>();
|
|
||||||
let all_acknowledged = !runtime.expected.is_empty()
|
|
||||||
&& runtime
|
|
||||||
.expected
|
|
||||||
.iter()
|
|
||||||
.all(|participant| runtime.acknowledgements.contains_key(participant));
|
|
||||||
let fully_frozen = runtime.command == "freeze"
|
|
||||||
&& all_acknowledged
|
|
||||||
&& acknowledgements
|
|
||||||
.iter()
|
|
||||||
.all(|ack| ack.state == DebugAcknowledgementState::Frozen);
|
|
||||||
let freeze_deadline_elapsed =
|
|
||||||
runtime.command == "freeze" && Instant::now() >= runtime.deadline;
|
|
||||||
let frozen_count = acknowledgements
|
|
||||||
.iter()
|
|
||||||
.filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
|
|
||||||
.count();
|
|
||||||
let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen;
|
|
||||||
let fully_resumed = runtime.command == "resume"
|
|
||||||
&& all_acknowledged
|
|
||||||
&& acknowledgements
|
|
||||||
.iter()
|
|
||||||
.all(|ack| ack.state == DebugAcknowledgementState::Running);
|
|
||||||
let failed = acknowledgements
|
|
||||||
.iter()
|
|
||||||
.any(|ack| ack.state == DebugAcknowledgementState::Failed)
|
|
||||||
|| (freeze_deadline_elapsed
|
|
||||||
&& runtime
|
|
||||||
.expected
|
|
||||||
.iter()
|
|
||||||
.any(|participant| !runtime.acknowledgements.contains_key(participant)));
|
|
||||||
Some(DebugEpochSummary {
|
|
||||||
epoch: runtime.epoch,
|
|
||||||
command: runtime.command.clone(),
|
|
||||||
fully_frozen,
|
|
||||||
partially_frozen,
|
|
||||||
fully_resumed,
|
|
||||||
failed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_list_artifacts(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
process: Option<String>,
|
|
||||||
cursor: Option<String>,
|
|
||||||
limit: u32,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let _actor = UserId::new(actor_user);
|
|
||||||
let process = process.map(ProcessId::new);
|
|
||||||
if let Some(process) = &process {
|
|
||||||
self.authorize_task_event_process_scope(&tenant, &project, process)?;
|
|
||||||
}
|
|
||||||
let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?;
|
|
||||||
let mut metadata = self
|
|
||||||
.artifact_registry
|
|
||||||
.metadata_for_project(&tenant, &project)
|
|
||||||
.filter(|metadata| {
|
|
||||||
process
|
|
||||||
.as_ref()
|
|
||||||
.is_none_or(|process| &metadata.process == process)
|
|
||||||
&& cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor)
|
|
||||||
})
|
|
||||||
.cloned()
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch));
|
|
||||||
let has_more = metadata.len() > limit as usize;
|
|
||||||
metadata.truncate(limit as usize);
|
|
||||||
let artifacts = metadata
|
|
||||||
.into_iter()
|
|
||||||
.map(|metadata| self.artifact_summary(metadata))
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let next_cursor = has_more
|
|
||||||
.then(|| {
|
|
||||||
artifacts
|
|
||||||
.last()
|
|
||||||
.map(|artifact| artifact.order_cursor.clone())
|
|
||||||
})
|
|
||||||
.flatten();
|
|
||||||
Ok(CoordinatorResponse::Artifacts {
|
|
||||||
artifacts,
|
|
||||||
next_cursor,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn handle_get_artifact(
|
|
||||||
&mut self,
|
|
||||||
tenant: String,
|
|
||||||
project: String,
|
|
||||||
actor_user: String,
|
|
||||||
artifact: String,
|
|
||||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
|
||||||
let tenant = TenantId::new(tenant);
|
|
||||||
let project = ProjectId::new(project);
|
|
||||||
let _actor = UserId::new(actor_user);
|
|
||||||
let artifact = ArtifactId::new(artifact);
|
|
||||||
let metadata = self
|
|
||||||
.artifact_registry
|
|
||||||
.metadata(&tenant, &project, &artifact)
|
|
||||||
.cloned()
|
|
||||||
.ok_or(clusterflux_core::DownloadError::NotFound)?;
|
|
||||||
Ok(CoordinatorResponse::Artifact {
|
|
||||||
artifact: self.artifact_summary(metadata),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary {
|
|
||||||
let live_retaining_nodes = metadata
|
|
||||||
.retaining_nodes
|
|
||||||
.iter()
|
|
||||||
.filter(|node| {
|
|
||||||
self.node_is_live(&crate::NodeScopeKey::from_refs(
|
|
||||||
&metadata.tenant,
|
|
||||||
&metadata.project,
|
|
||||||
node,
|
|
||||||
))
|
|
||||||
})
|
|
||||||
.cloned()
|
|
||||||
.collect::<BTreeSet<_>>();
|
|
||||||
let safe_node = live_retaining_nodes
|
|
||||||
.iter()
|
|
||||||
.next()
|
|
||||||
.cloned()
|
|
||||||
.or_else(|| metadata.retaining_nodes.iter().next().cloned());
|
|
||||||
let explicit_storage = !metadata.explicit_locations.is_empty();
|
|
||||||
let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage;
|
|
||||||
let availability = if downloadable_now {
|
|
||||||
ArtifactAvailability::Available
|
|
||||||
} else if !metadata.retaining_nodes.is_empty() {
|
|
||||||
ArtifactAvailability::NodeOffline
|
|
||||||
} else {
|
|
||||||
ArtifactAvailability::Unavailable
|
|
||||||
};
|
|
||||||
let retention_state = if explicit_storage {
|
|
||||||
ArtifactRetentionState::ExplicitStorage
|
|
||||||
} else if !metadata.retaining_nodes.is_empty() {
|
|
||||||
ArtifactRetentionState::NodeRetained
|
|
||||||
} else {
|
|
||||||
ArtifactRetentionState::Lost
|
|
||||||
};
|
|
||||||
let display_suffix = metadata.id.as_str().replace(':', "/");
|
|
||||||
let display_path = format!("/vfs/artifacts/{display_suffix}");
|
|
||||||
let display_name = display_suffix
|
|
||||||
.rsplit('/')
|
|
||||||
.next()
|
|
||||||
.unwrap_or(metadata.id.as_str())
|
|
||||||
.to_owned();
|
|
||||||
ArtifactSummary {
|
|
||||||
id: metadata.id,
|
|
||||||
display_path,
|
|
||||||
display_name,
|
|
||||||
process: metadata.process,
|
|
||||||
producer_task: metadata.producer_task,
|
|
||||||
safe_node,
|
|
||||||
digest: metadata.digest,
|
|
||||||
size_bytes: metadata.size,
|
|
||||||
availability,
|
|
||||||
downloadable_now,
|
|
||||||
retention_state,
|
|
||||||
explicit_storage,
|
|
||||||
order_cursor: format!("artifact:{}", metadata.flushed_epoch),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) {
|
|
||||||
while self
|
|
||||||
.process_summaries
|
|
||||||
.keys()
|
|
||||||
.filter(|(entry_tenant, entry_project, _)| {
|
|
||||||
entry_tenant == tenant && entry_project == project
|
|
||||||
})
|
|
||||||
.count()
|
|
||||||
>= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT
|
|
||||||
{
|
|
||||||
let candidate = self.process_summary_order.iter().find(|key| {
|
|
||||||
&key.0 == tenant
|
|
||||||
&& &key.1 == project
|
|
||||||
&& self
|
|
||||||
.process_summaries
|
|
||||||
.get(*key)
|
|
||||||
.is_some_and(|summary| summary.final_result.is_some())
|
|
||||||
});
|
|
||||||
let Some(candidate) = candidate.cloned() else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
self.remove_process_summary_state(&candidate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn evict_process_summaries_total(&mut self) {
|
|
||||||
while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL {
|
|
||||||
let candidate = self.process_summary_order.iter().find(|key| {
|
|
||||||
self.process_summaries
|
|
||||||
.get(*key)
|
|
||||||
.is_some_and(|summary| summary.final_result.is_some())
|
|
||||||
});
|
|
||||||
let Some(candidate) = candidate.cloned() else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
self.remove_process_summary_state(&candidate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
|
|
||||||
self.process_summaries.remove(key);
|
|
||||||
self.recent_log_dropped_through.remove(key);
|
|
||||||
if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
|
|
||||||
logs.retain(|entry| entry.process != key.2);
|
|
||||||
if logs.is_empty() {
|
|
||||||
self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.recent_log_accounted_bytes
|
|
||||||
.retain(|(tenant, project, process, _, _), _| {
|
|
||||||
tenant != &key.0 || project != &key.1 || process != &key.2
|
|
||||||
});
|
|
||||||
self.recent_log_truncated_streams
|
|
||||||
.retain(|(tenant, project, process, _, _)| {
|
|
||||||
tenant != &key.0 || project != &key.1 || process != &key.2
|
|
||||||
});
|
|
||||||
self.recent_log_quota_truncated_streams
|
|
||||||
.retain(|(tenant, project, process, _, _)| {
|
|
||||||
tenant != &key.0 || project != &key.1 || process != &key.2
|
|
||||||
});
|
|
||||||
self.process_summary_order
|
|
||||||
.retain(|retained| retained != key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_order_cursor(
|
|
||||||
cursor: Option<&str>,
|
|
||||||
expected_kind: &str,
|
|
||||||
) -> Result<Option<u64>, CoordinatorServiceError> {
|
|
||||||
cursor
|
|
||||||
.map(|cursor| {
|
|
||||||
let (kind, order) = cursor.split_once(':').ok_or_else(|| {
|
|
||||||
CoordinatorServiceError::Protocol(format!(
|
|
||||||
"invalid {expected_kind} pagination cursor"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
if kind != expected_kind {
|
|
||||||
return Err(CoordinatorServiceError::Protocol(format!(
|
|
||||||
"invalid {expected_kind} pagination cursor"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
order.parse::<u64>().map_err(|_| {
|
|
||||||
CoordinatorServiceError::Protocol(format!(
|
|
||||||
"invalid {expected_kind} pagination cursor"
|
|
||||||
))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.transpose()
|
|
||||||
}
|
|
||||||
|
|
@ -79,15 +79,17 @@ impl CoordinatorService {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let response = match decode_wire_request(&line) {
|
let response = match decode_wire_request(&line) {
|
||||||
Ok((request_id, request)) => {
|
Ok(request) => match authorize_client_request(&request, authority_mode)
|
||||||
match authorize_client_request(&request, authority_mode)
|
.and_then(|()| self.handle_request(request))
|
||||||
.and_then(|()| self.handle_request(request))
|
{
|
||||||
{
|
Ok(response) => response,
|
||||||
Ok(response) => response,
|
Err(err) => CoordinatorResponse::Error {
|
||||||
Err(err) => CoordinatorResponse::service_error(request_id, &err),
|
message: err.to_string(),
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
|
Err(err) => CoordinatorResponse::Error {
|
||||||
|
message: err.to_string(),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
serde_json::to_writer(&mut writer, &response)?;
|
serde_json::to_writer(&mut writer, &response)?;
|
||||||
writer.write_all(b"\n")?;
|
writer.write_all(b"\n")?;
|
||||||
|
|
@ -112,19 +114,25 @@ fn handle_shared_stream(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let response = match decode_wire_request(&line) {
|
let response = match decode_wire_request(&line) {
|
||||||
Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) {
|
Ok(request) => match authorize_client_request(&request, authority_mode) {
|
||||||
Ok(()) => match service.lock() {
|
Ok(()) => match service.lock() {
|
||||||
Ok(mut service) => match service.handle_request(request) {
|
Ok(mut service) => match service.handle_request(request) {
|
||||||
Ok(response) => response,
|
Ok(response) => response,
|
||||||
Err(err) => CoordinatorResponse::service_error(request_id, &err),
|
Err(err) => CoordinatorResponse::Error {
|
||||||
|
message: err.to_string(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Err(_) => CoordinatorResponse::Error {
|
||||||
|
message: "coordinator service lock poisoned".to_owned(),
|
||||||
},
|
},
|
||||||
Err(_) => {
|
|
||||||
CoordinatorResponse::error(request_id, "coordinator service lock poisoned")
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
Err(err) => CoordinatorResponse::service_error(request_id, &err),
|
Err(err) => CoordinatorResponse::Error {
|
||||||
|
message: err.to_string(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Err(err) => CoordinatorResponse::Error {
|
||||||
|
message: err.to_string(),
|
||||||
},
|
},
|
||||||
Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
|
|
||||||
};
|
};
|
||||||
serde_json::to_writer(&mut writer, &response)?;
|
serde_json::to_writer(&mut writer, &response)?;
|
||||||
writer.write_all(b"\n")?;
|
writer.write_all(b"\n")?;
|
||||||
|
|
@ -138,27 +146,12 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato
|
||||||
Ok((listener, addr))
|
Ok((listener, addr))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn decode_wire_request(
|
fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
|
||||||
line: &str,
|
|
||||||
) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> {
|
|
||||||
serde_json::from_str::<super::CoordinatorWireRequest>(line)?
|
serde_json::from_str::<super::CoordinatorWireRequest>(line)?
|
||||||
.into_parts()
|
.into_request()
|
||||||
.map_err(CoordinatorServiceError::Protocol)
|
.map_err(CoordinatorServiceError::Protocol)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn wire_request_id_hint(line: &str) -> String {
|
|
||||||
serde_json::from_str::<serde_json::Value>(line)
|
|
||||||
.ok()
|
|
||||||
.and_then(|value| {
|
|
||||||
value
|
|
||||||
.get("request_id")
|
|
||||||
.and_then(|value| value.as_str())
|
|
||||||
.map(str::to_owned)
|
|
||||||
})
|
|
||||||
.filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok())
|
|
||||||
.unwrap_or_else(|| "unavailable".to_owned())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn authorize_client_request(
|
fn authorize_client_request(
|
||||||
request: &CoordinatorRequest,
|
request: &CoordinatorRequest,
|
||||||
authority_mode: ClientAuthorityMode,
|
authority_mode: ClientAuthorityMode,
|
||||||
|
|
@ -187,11 +180,10 @@ fn authorize_client_request(
|
||||||
}
|
}
|
||||||
| CoordinatorRequest::AdminStatus { .. }
|
| CoordinatorRequest::AdminStatus { .. }
|
||||||
| CoordinatorRequest::SuspendTenant { .. } => Ok(()),
|
| CoordinatorRequest::SuspendTenant { .. } => Ok(()),
|
||||||
_ => Err(crate::CoordinatorError::Unauthorized(
|
_ => Err(CoordinatorServiceError::Protocol(
|
||||||
"strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
|
"strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
)
|
)),
|
||||||
.into()),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -261,19 +253,16 @@ mod transport_boundary_tests {
|
||||||
|
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
reader.read_line(&mut line).unwrap();
|
reader.read_line(&mut line).unwrap();
|
||||||
let CoordinatorResponse::Error { error } =
|
let CoordinatorResponse::Error { message } =
|
||||||
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
|
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
|
||||||
else {
|
else {
|
||||||
panic!("malformed identifier request unexpectedly succeeded");
|
panic!("malformed identifier request unexpectedly succeeded");
|
||||||
};
|
};
|
||||||
assert!(
|
assert!(
|
||||||
error.message.contains("malformed external identifier")
|
message.contains("malformed external identifier")
|
||||||
&& error.message.contains("request.request.process"),
|
&& message.contains("request.request.process"),
|
||||||
"unexpected malformed identifier response: {}",
|
"unexpected malformed identifier response: {message}"
|
||||||
error.message
|
|
||||||
);
|
);
|
||||||
assert_eq!(error.request_id, format!("malformed-{index}"));
|
|
||||||
assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
|
|
||||||
|
|
||||||
let valid = coordinator_wire_request(
|
let valid = coordinator_wire_request(
|
||||||
format!("healthy-{index}"),
|
format!("healthy-{index}"),
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -12,12 +12,8 @@ pub enum CoordinatorWireRequest {
|
||||||
|
|
||||||
impl CoordinatorWireRequest {
|
impl CoordinatorWireRequest {
|
||||||
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
|
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
|
||||||
self.into_parts().map(|(_, request)| request)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
|
|
||||||
match self {
|
match self {
|
||||||
Self::Envelope(envelope) => envelope.into_parts(),
|
Self::Envelope(envelope) => envelope.into_request(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -36,10 +32,6 @@ pub struct CoordinatorRequestEnvelope {
|
||||||
|
|
||||||
impl CoordinatorRequestEnvelope {
|
impl CoordinatorRequestEnvelope {
|
||||||
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
|
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
|
||||||
self.into_parts().map(|(_, request)| request)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
|
|
||||||
if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
|
if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
|
||||||
return Err(format!(
|
return Err(format!(
|
||||||
"unsupported coordinator wire request type {}; expected {}",
|
"unsupported coordinator wire request type {}; expected {}",
|
||||||
|
|
@ -62,6 +54,6 @@ impl CoordinatorRequestEnvelope {
|
||||||
self.operation, payload_operation
|
self.operation, payload_operation
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
Ok((self.request_id, self.payload))
|
Ok(self.payload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,296 +0,0 @@
|
||||||
use serde::{Deserialize, Serialize};
|
|
||||||
use std::fmt;
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ApiErrorCode {
|
|
||||||
Unauthenticated,
|
|
||||||
SessionExpired,
|
|
||||||
AccountSuspended,
|
|
||||||
Forbidden,
|
|
||||||
ValidationError,
|
|
||||||
NotFound,
|
|
||||||
Conflict,
|
|
||||||
ActiveProcessExists,
|
|
||||||
NodeOffline,
|
|
||||||
NoCapableNode,
|
|
||||||
TaskNotRestartable,
|
|
||||||
ArtifactUnavailable,
|
|
||||||
ArtifactLimitExceeded,
|
|
||||||
QuotaExceeded,
|
|
||||||
TemporaryCapacity,
|
|
||||||
DebugEpochPartial,
|
|
||||||
InternalError,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "snake_case")]
|
|
||||||
pub enum ApiErrorCategory {
|
|
||||||
Authentication,
|
|
||||||
Authorization,
|
|
||||||
Validation,
|
|
||||||
State,
|
|
||||||
Availability,
|
|
||||||
Resource,
|
|
||||||
Internal,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
||||||
#[serde(deny_unknown_fields)]
|
|
||||||
pub struct ApiError {
|
|
||||||
pub code: ApiErrorCode,
|
|
||||||
pub category: ApiErrorCategory,
|
|
||||||
pub message: String,
|
|
||||||
pub retryable: bool,
|
|
||||||
pub request_id: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl fmt::Display for ApiError {
|
|
||||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
||||||
write!(
|
|
||||||
formatter,
|
|
||||||
"{} (code {:?}, request {})",
|
|
||||||
self.message, self.code, self.request_id
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::error::Error for ApiError {}
|
|
||||||
|
|
||||||
impl ApiError {
|
|
||||||
pub fn new(
|
|
||||||
code: ApiErrorCode,
|
|
||||||
category: ApiErrorCategory,
|
|
||||||
message: impl Into<String>,
|
|
||||||
retryable: bool,
|
|
||||||
request_id: impl Into<String>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
|
||||||
code,
|
|
||||||
category,
|
|
||||||
message: message.into(),
|
|
||||||
retryable,
|
|
||||||
request_id: request_id.into(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn from_message(request_id: impl Into<String>, message: impl Into<String>) -> Self {
|
|
||||||
let request_id = request_id.into();
|
|
||||||
let message = message.into();
|
|
||||||
let normalized = message.to_ascii_lowercase();
|
|
||||||
let (code, category, retryable) = if normalized.contains("session credential has expired")
|
|
||||||
|| normalized.contains("session expired")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::SessionExpired,
|
|
||||||
ApiErrorCategory::Authentication,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("tenant is suspended")
|
|
||||||
|| normalized.contains("account is suspended")
|
|
||||||
|| normalized.contains("suspended by hosted")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::AccountSuspended,
|
|
||||||
ApiErrorCategory::Authorization,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("session credential")
|
|
||||||
|| normalized.contains("no authenticated")
|
|
||||||
|| normalized.contains("not authenticated")
|
|
||||||
|| normalized.contains("credential is not")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::Unauthenticated,
|
|
||||||
ApiErrorCategory::Authentication,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("project already has active virtual process") {
|
|
||||||
(
|
|
||||||
ApiErrorCode::ActiveProcessExists,
|
|
||||||
ApiErrorCategory::State,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("no capable node") {
|
|
||||||
(
|
|
||||||
ApiErrorCode::NoCapableNode,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("node offline")
|
|
||||||
|| normalized.contains("node is not live")
|
|
||||||
|| normalized.contains("source node is not connected")
|
|
||||||
|| normalized.contains("direct connectivity unavailable")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::NodeOffline,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("restart")
|
|
||||||
&& (normalized.contains("not restartable")
|
|
||||||
|| normalized.contains("requires whole")
|
|
||||||
|| normalized.contains("clean boundary"))
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::TaskNotRestartable,
|
|
||||||
ApiErrorCategory::State,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("artifact")
|
|
||||||
&& (normalized.contains("exceeds download limit")
|
|
||||||
|| normalized.contains("download session limit")
|
|
||||||
|| normalized.contains("artifact limit"))
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::ArtifactLimitExceeded,
|
|
||||||
ApiErrorCategory::Resource,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("artifact")
|
|
||||||
&& (normalized.contains("does not exist")
|
|
||||||
|| normalized.contains("not found")
|
|
||||||
|| normalized.contains("unknown artifact"))
|
|
||||||
{
|
|
||||||
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
|
|
||||||
} else if normalized.contains("artifact")
|
|
||||||
&& (normalized.contains("unavailable") || normalized.contains("retention"))
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::ArtifactUnavailable,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("resource limit")
|
|
||||||
|| normalized.contains("quota")
|
|
||||||
|| normalized.contains("limit exceeded")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::QuotaExceeded,
|
|
||||||
ApiErrorCategory::Resource,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("capacity")
|
|
||||||
|| normalized.contains("temporarily full")
|
|
||||||
|| normalized.contains("replay window is full")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::TemporaryCapacity,
|
|
||||||
ApiErrorCategory::Availability,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("partial debug epoch")
|
|
||||||
|| normalized.contains("debug epoch is partially")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::DebugEpochPartial,
|
|
||||||
ApiErrorCategory::State,
|
|
||||||
true,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("malformed")
|
|
||||||
|| normalized.contains("invalid ")
|
|
||||||
|| normalized.contains("protocol")
|
|
||||||
|| normalized.contains("unknown field")
|
|
||||||
|| normalized.contains("missing field")
|
|
||||||
|| normalized.contains("must ")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::ValidationError,
|
|
||||||
ApiErrorCategory::Validation,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("outside")
|
|
||||||
|| normalized.contains("unauthorized")
|
|
||||||
|| normalized.contains("denied")
|
|
||||||
|| normalized.contains("requires an authenticated")
|
|
||||||
|| normalized.contains("may only")
|
|
||||||
{
|
|
||||||
(
|
|
||||||
ApiErrorCode::Forbidden,
|
|
||||||
ApiErrorCategory::Authorization,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
} else if normalized.contains("not found")
|
|
||||||
|| normalized.contains("does not exist")
|
|
||||||
|| normalized.contains("unknown ")
|
|
||||||
{
|
|
||||||
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
|
|
||||||
} else if normalized.contains("already")
|
|
||||||
|| normalized.contains("conflict")
|
|
||||||
|| normalized.contains("requires an active")
|
|
||||||
{
|
|
||||||
(ApiErrorCode::Conflict, ApiErrorCategory::State, false)
|
|
||||||
} else {
|
|
||||||
(
|
|
||||||
ApiErrorCode::InternalError,
|
|
||||||
ApiErrorCategory::Internal,
|
|
||||||
false,
|
|
||||||
)
|
|
||||||
};
|
|
||||||
Self::new(code, category, message, retryable, request_id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn required_machine_codes_are_stably_serialized() {
|
|
||||||
let required = [
|
|
||||||
ApiErrorCode::Unauthenticated,
|
|
||||||
ApiErrorCode::SessionExpired,
|
|
||||||
ApiErrorCode::AccountSuspended,
|
|
||||||
ApiErrorCode::Forbidden,
|
|
||||||
ApiErrorCode::ValidationError,
|
|
||||||
ApiErrorCode::NotFound,
|
|
||||||
ApiErrorCode::Conflict,
|
|
||||||
ApiErrorCode::ActiveProcessExists,
|
|
||||||
ApiErrorCode::NodeOffline,
|
|
||||||
ApiErrorCode::NoCapableNode,
|
|
||||||
ApiErrorCode::TaskNotRestartable,
|
|
||||||
ApiErrorCode::ArtifactUnavailable,
|
|
||||||
ApiErrorCode::ArtifactLimitExceeded,
|
|
||||||
ApiErrorCode::QuotaExceeded,
|
|
||||||
ApiErrorCode::TemporaryCapacity,
|
|
||||||
ApiErrorCode::DebugEpochPartial,
|
|
||||||
];
|
|
||||||
let serialized = required
|
|
||||||
.into_iter()
|
|
||||||
.map(|code| serde_json::to_value(code).unwrap())
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
assert_eq!(
|
|
||||||
serialized,
|
|
||||||
vec![
|
|
||||||
"unauthenticated",
|
|
||||||
"session_expired",
|
|
||||||
"account_suspended",
|
|
||||||
"forbidden",
|
|
||||||
"validation_error",
|
|
||||||
"not_found",
|
|
||||||
"conflict",
|
|
||||||
"active_process_exists",
|
|
||||||
"node_offline",
|
|
||||||
"no_capable_node",
|
|
||||||
"task_not_restartable",
|
|
||||||
"artifact_unavailable",
|
|
||||||
"artifact_limit_exceeded",
|
|
||||||
"quota_exceeded",
|
|
||||||
"temporary_capacity",
|
|
||||||
"debug_epoch_partial",
|
|
||||||
]
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn message_classification_keeps_request_identity() {
|
|
||||||
let error = ApiError::from_message(
|
|
||||||
"request-17",
|
|
||||||
"CLI session credential has expired; run login again",
|
|
||||||
);
|
|
||||||
assert_eq!(error.code, ApiErrorCode::SessionExpired);
|
|
||||||
assert_eq!(error.category, ApiErrorCategory::Authentication);
|
|
||||||
assert_eq!(error.request_id, "request-17");
|
|
||||||
assert!(!error.retryable);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -10,7 +10,7 @@ use crate::{
|
||||||
|
|
||||||
const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
|
const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
|
||||||
const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
|
const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
|
||||||
const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
|
const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
|
|
@ -184,45 +184,54 @@ impl ArtifactRegistry {
|
||||||
&mut self,
|
&mut self,
|
||||||
flush: ArtifactFlush,
|
flush: ArtifactFlush,
|
||||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
pinned: &BTreeSet<ArtifactScopeKey>,
|
||||||
) -> Result<ArtifactMetadata, String> {
|
|
||||||
self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn flush_metadata_with_protected_processes(
|
|
||||||
&mut self,
|
|
||||||
flush: ArtifactFlush,
|
|
||||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
|
||||||
protected_processes: &BTreeSet<ProcessId>,
|
|
||||||
) -> Result<ArtifactMetadata, String> {
|
) -> Result<ArtifactMetadata, String> {
|
||||||
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
|
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
|
||||||
let replacing = self.artifacts.contains_key(&key);
|
let replacing_existing = self.artifacts.contains_key(&key);
|
||||||
let retained_for_project = self
|
while !replacing_existing
|
||||||
.artifacts
|
&& self
|
||||||
.values()
|
.artifacts
|
||||||
.filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
|
.values()
|
||||||
.count();
|
.filter(|metadata| {
|
||||||
let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
|
metadata.tenant == flush.tenant
|
||||||
None
|
&& metadata.project == flush.project
|
||||||
} else {
|
&& metadata.process == flush.process
|
||||||
let mut protected_processes = protected_processes.clone();
|
})
|
||||||
protected_processes.insert(flush.process.clone());
|
.count()
|
||||||
Some(
|
>= MAX_ARTIFACT_METADATA_PER_PROCESS
|
||||||
self.project_metadata_eviction_candidate(
|
{
|
||||||
&flush.tenant,
|
let candidate = self
|
||||||
&flush.project,
|
.artifacts
|
||||||
pinned,
|
.values()
|
||||||
&protected_processes,
|
.filter(|metadata| {
|
||||||
)
|
metadata.tenant == flush.tenant
|
||||||
.ok_or_else(|| {
|
&& metadata.project == flush.project
|
||||||
format!(
|
&& metadata.process == flush.process
|
||||||
"artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
|
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
||||||
exhausted by active or retained artifacts"
|
&metadata.tenant,
|
||||||
|
&metadata.project,
|
||||||
|
&metadata.id,
|
||||||
|
))
|
||||||
|
&& !self.issued_download_links.values().any(|issued| {
|
||||||
|
issued.link.tenant == metadata.tenant
|
||||||
|
&& issued.link.project == metadata.project
|
||||||
|
&& issued.link.artifact == metadata.id
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.min_by_key(|metadata| metadata.flushed_epoch)
|
||||||
|
.map(|metadata| {
|
||||||
|
ArtifactScopeKey::from_refs(
|
||||||
|
&metadata.tenant,
|
||||||
|
&metadata.project,
|
||||||
|
&metadata.id,
|
||||||
)
|
)
|
||||||
})?,
|
})
|
||||||
)
|
.ok_or_else(|| {
|
||||||
};
|
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
|
||||||
|
.to_owned()
|
||||||
self.next_epoch = self.next_epoch.saturating_add(1);
|
})?;
|
||||||
|
self.artifacts.remove(&candidate);
|
||||||
|
}
|
||||||
|
self.next_epoch += 1;
|
||||||
let metadata = ArtifactMetadata {
|
let metadata = ArtifactMetadata {
|
||||||
id: flush.id.clone(),
|
id: flush.id.clone(),
|
||||||
tenant: flush.tenant,
|
tenant: flush.tenant,
|
||||||
|
|
@ -237,75 +246,10 @@ impl ArtifactRegistry {
|
||||||
explicit_locations: Vec::new(),
|
explicit_locations: Vec::new(),
|
||||||
coordinator_has_large_bytes: false,
|
coordinator_has_large_bytes: false,
|
||||||
};
|
};
|
||||||
if let Some(eviction) = eviction {
|
|
||||||
self.artifacts.remove(&eviction);
|
|
||||||
}
|
|
||||||
self.artifacts.insert(key, metadata.clone());
|
self.artifacts.insert(key, metadata.clone());
|
||||||
Ok(metadata)
|
Ok(metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn enforce_project_metadata_limit(
|
|
||||||
&mut self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
|
||||||
protected_processes: &BTreeSet<ProcessId>,
|
|
||||||
) -> usize {
|
|
||||||
let mut evicted = 0;
|
|
||||||
while self
|
|
||||||
.artifacts
|
|
||||||
.values()
|
|
||||||
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
|
|
||||||
.count()
|
|
||||||
> MAX_ARTIFACT_METADATA_PER_PROJECT
|
|
||||||
{
|
|
||||||
let candidate = self.project_metadata_eviction_candidate(
|
|
||||||
tenant,
|
|
||||||
project,
|
|
||||||
pinned,
|
|
||||||
protected_processes,
|
|
||||||
);
|
|
||||||
let Some(candidate) = candidate else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
self.artifacts.remove(&candidate);
|
|
||||||
evicted += 1;
|
|
||||||
}
|
|
||||||
evicted
|
|
||||||
}
|
|
||||||
|
|
||||||
fn project_metadata_eviction_candidate(
|
|
||||||
&self,
|
|
||||||
tenant: &TenantId,
|
|
||||||
project: &ProjectId,
|
|
||||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
|
||||||
protected_processes: &BTreeSet<ProcessId>,
|
|
||||||
) -> Option<ArtifactScopeKey> {
|
|
||||||
self.artifacts
|
|
||||||
.values()
|
|
||||||
.filter(|metadata| {
|
|
||||||
&metadata.tenant == tenant
|
|
||||||
&& &metadata.project == project
|
|
||||||
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
|
||||||
&metadata.tenant,
|
|
||||||
&metadata.project,
|
|
||||||
&metadata.id,
|
|
||||||
))
|
|
||||||
&& !protected_processes.contains(&metadata.process)
|
|
||||||
&& metadata.explicit_locations.is_empty()
|
|
||||||
&& !self.issued_download_links.values().any(|issued| {
|
|
||||||
!issued.revoked
|
|
||||||
&& issued.link.tenant == metadata.tenant
|
|
||||||
&& issued.link.project == metadata.project
|
|
||||||
&& issued.link.artifact == metadata.id
|
|
||||||
})
|
|
||||||
})
|
|
||||||
.min_by_key(|metadata| metadata.flushed_epoch)
|
|
||||||
.map(|metadata| {
|
|
||||||
ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn sync_to_explicit_store(
|
pub fn sync_to_explicit_store(
|
||||||
&mut self,
|
&mut self,
|
||||||
tenant: &TenantId,
|
tenant: &TenantId,
|
||||||
|
|
@ -360,16 +304,6 @@ impl ArtifactRegistry {
|
||||||
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
|
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn metadata_for_project<'a>(
|
|
||||||
&'a self,
|
|
||||||
tenant: &'a TenantId,
|
|
||||||
project: &'a ProjectId,
|
|
||||||
) -> impl Iterator<Item = &'a ArtifactMetadata> + 'a {
|
|
||||||
self.artifacts
|
|
||||||
.values()
|
|
||||||
.filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn download_action(
|
pub fn download_action(
|
||||||
&self,
|
&self,
|
||||||
context: &AuthContext,
|
context: &AuthContext,
|
||||||
|
|
@ -1406,10 +1340,8 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
|
fn artifact_metadata_is_bounded_without_evicting_pins() {
|
||||||
let mut registry = ArtifactRegistry::default();
|
let mut registry = ArtifactRegistry::default();
|
||||||
let tenant = TenantId::from("tenant");
|
|
||||||
let project = ProjectId::from("project");
|
|
||||||
registry.flush_metadata(ArtifactFlush {
|
registry.flush_metadata(ArtifactFlush {
|
||||||
id: ArtifactId::from("artifact-0"),
|
id: ArtifactId::from("artifact-0"),
|
||||||
tenant: TenantId::from("other-tenant"),
|
tenant: TenantId::from("other-tenant"),
|
||||||
|
|
@ -1420,92 +1352,72 @@ mod tests {
|
||||||
digest: Digest::sha256("other-content"),
|
digest: Digest::sha256("other-content"),
|
||||||
size: 1,
|
size: 1,
|
||||||
});
|
});
|
||||||
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
|
let mut pinned = BTreeSet::new();
|
||||||
|
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
|
||||||
let id = ArtifactId::new(format!("artifact-{index}"));
|
let id = ArtifactId::new(format!("artifact-{index}"));
|
||||||
registry.flush_metadata(ArtifactFlush {
|
pinned.insert(ArtifactScopeKey::new(
|
||||||
id,
|
TenantId::from("tenant"),
|
||||||
tenant: tenant.clone(),
|
ProjectId::from("project"),
|
||||||
project: project.clone(),
|
id.clone(),
|
||||||
process: ProcessId::new(if index == 3 {
|
));
|
||||||
"active-process-3".to_owned()
|
registry
|
||||||
} else {
|
.flush_metadata_bounded(
|
||||||
format!("completed-process-{index}")
|
ArtifactFlush {
|
||||||
}),
|
id,
|
||||||
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
tenant: TenantId::from("tenant"),
|
||||||
retaining_node: NodeId::from("node"),
|
project: ProjectId::from("project"),
|
||||||
digest: Digest::sha256(format!("content-{index}")),
|
process: ProcessId::from("process"),
|
||||||
size: 1,
|
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
||||||
});
|
retaining_node: NodeId::from("node"),
|
||||||
|
digest: Digest::sha256(format!("content-{index}")),
|
||||||
|
size: 1,
|
||||||
|
},
|
||||||
|
&pinned,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
registry.artifact_count(),
|
registry.artifact_count(),
|
||||||
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
|
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
|
||||||
);
|
);
|
||||||
let pinned = BTreeSet::from([ArtifactScopeKey::new(
|
|
||||||
tenant.clone(),
|
|
||||||
project.clone(),
|
|
||||||
ArtifactId::from("artifact-0"),
|
|
||||||
)]);
|
|
||||||
registry
|
|
||||||
.sync_to_explicit_store(
|
|
||||||
&tenant,
|
|
||||||
&project,
|
|
||||||
&ArtifactId::from("artifact-1"),
|
|
||||||
"store://retained-export",
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let context = AuthContext {
|
|
||||||
tenant: tenant.clone(),
|
|
||||||
project: project.clone(),
|
|
||||||
actor: Actor::User(UserId::from("user")),
|
|
||||||
};
|
|
||||||
registry
|
|
||||||
.create_download_link(
|
|
||||||
&context,
|
|
||||||
&ArtifactId::from("artifact-2"),
|
|
||||||
&DownloadPolicy { max_bytes: 1 },
|
|
||||||
"active-download",
|
|
||||||
10,
|
|
||||||
60,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let protected_processes = BTreeSet::from([
|
|
||||||
ProcessId::from("active-process-3"),
|
|
||||||
ProcessId::from("active-process"),
|
|
||||||
]);
|
|
||||||
let next = ArtifactFlush {
|
let next = ArtifactFlush {
|
||||||
id: ArtifactId::from("artifact-next"),
|
id: ArtifactId::from("artifact-next"),
|
||||||
tenant: tenant.clone(),
|
tenant: TenantId::from("tenant"),
|
||||||
project: project.clone(),
|
project: ProjectId::from("project"),
|
||||||
process: ProcessId::from("active-process"),
|
process: ProcessId::from("process"),
|
||||||
producer_task: TaskInstanceId::from("task-next"),
|
producer_task: TaskInstanceId::from("task-next"),
|
||||||
retaining_node: NodeId::from("node"),
|
retaining_node: NodeId::from("node"),
|
||||||
digest: Digest::sha256("next"),
|
digest: Digest::sha256("next"),
|
||||||
size: 1,
|
size: 1,
|
||||||
};
|
};
|
||||||
registry
|
assert!(registry
|
||||||
.flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
|
.flush_metadata_bounded(next.clone(), &pinned)
|
||||||
.unwrap();
|
.unwrap_err()
|
||||||
|
.contains("pinned"));
|
||||||
|
|
||||||
|
pinned.remove(&ArtifactScopeKey::new(
|
||||||
|
TenantId::from("tenant"),
|
||||||
|
ProjectId::from("project"),
|
||||||
|
ArtifactId::from("artifact-0"),
|
||||||
|
));
|
||||||
|
registry.flush_metadata_bounded(next, &pinned).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
registry.artifact_count(),
|
registry.artifact_count(),
|
||||||
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
|
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
|
||||||
);
|
|
||||||
for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
|
|
||||||
assert!(
|
|
||||||
registry
|
|
||||||
.metadata(&tenant, &project, &ArtifactId::from(id))
|
|
||||||
.is_some(),
|
|
||||||
"{id} was evicted despite being pinned, exported, downloaded, or live"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert!(
|
|
||||||
registry
|
|
||||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
|
|
||||||
.is_none(),
|
|
||||||
"the oldest completed unprotected metadata should be evicted"
|
|
||||||
);
|
);
|
||||||
assert!(registry
|
assert!(registry
|
||||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
|
.metadata(
|
||||||
|
&TenantId::from("tenant"),
|
||||||
|
&ProjectId::from("project"),
|
||||||
|
&ArtifactId::from("artifact-0"),
|
||||||
|
)
|
||||||
|
.is_none());
|
||||||
|
assert!(registry
|
||||||
|
.metadata(
|
||||||
|
&TenantId::from("tenant"),
|
||||||
|
&ProjectId::from("project"),
|
||||||
|
&ArtifactId::from("artifact-next"),
|
||||||
|
)
|
||||||
.is_some());
|
.is_some());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
registry
|
registry
|
||||||
|
|
@ -1521,87 +1433,6 @@ mod tests {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
|
|
||||||
let mut registry = ArtifactRegistry::default();
|
|
||||||
let tenant = TenantId::from("tenant");
|
|
||||||
let project = ProjectId::from("project");
|
|
||||||
let active_process = ProcessId::from("active-process");
|
|
||||||
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
|
|
||||||
registry.flush_metadata(ArtifactFlush {
|
|
||||||
id: ArtifactId::new(format!("artifact-{index}")),
|
|
||||||
tenant: tenant.clone(),
|
|
||||||
project: project.clone(),
|
|
||||||
process: active_process.clone(),
|
|
||||||
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
|
||||||
retaining_node: NodeId::from("node"),
|
|
||||||
digest: Digest::sha256(format!("content-{index}")),
|
|
||||||
size: 1,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
let replacement = ArtifactFlush {
|
|
||||||
id: ArtifactId::from("artifact-0"),
|
|
||||||
tenant: tenant.clone(),
|
|
||||||
project: project.clone(),
|
|
||||||
process: active_process.clone(),
|
|
||||||
producer_task: TaskInstanceId::from("replacement-task"),
|
|
||||||
retaining_node: NodeId::from("node"),
|
|
||||||
digest: Digest::sha256("replacement"),
|
|
||||||
size: 2,
|
|
||||||
};
|
|
||||||
registry
|
|
||||||
.flush_metadata_with_protected_processes(
|
|
||||||
replacement,
|
|
||||||
&BTreeSet::new(),
|
|
||||||
&BTreeSet::from([active_process.clone()]),
|
|
||||||
)
|
|
||||||
.expect("replacement must remain possible at the metadata bound");
|
|
||||||
assert_eq!(
|
|
||||||
registry
|
|
||||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
|
|
||||||
.unwrap()
|
|
||||||
.digest,
|
|
||||||
Digest::sha256("replacement")
|
|
||||||
);
|
|
||||||
|
|
||||||
let epoch_before_rejection = registry.next_epoch;
|
|
||||||
let result = registry.flush_metadata_with_protected_processes(
|
|
||||||
ArtifactFlush {
|
|
||||||
id: ArtifactId::from("artifact-rejected"),
|
|
||||||
tenant: tenant.clone(),
|
|
||||||
project: project.clone(),
|
|
||||||
process: active_process.clone(),
|
|
||||||
producer_task: TaskInstanceId::from("rejected-task"),
|
|
||||||
retaining_node: NodeId::from("node"),
|
|
||||||
digest: Digest::sha256("rejected"),
|
|
||||||
size: 3,
|
|
||||||
},
|
|
||||||
&BTreeSet::new(),
|
|
||||||
&BTreeSet::from([active_process]),
|
|
||||||
);
|
|
||||||
|
|
||||||
assert!(result
|
|
||||||
.unwrap_err()
|
|
||||||
.contains("exhausted by active or retained artifacts"));
|
|
||||||
assert_eq!(registry.next_epoch, epoch_before_rejection);
|
|
||||||
assert_eq!(
|
|
||||||
registry.metadata_for_project(&tenant, &project).count(),
|
|
||||||
MAX_ARTIFACT_METADATA_PER_PROJECT
|
|
||||||
);
|
|
||||||
assert!(registry
|
|
||||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
|
|
||||||
.is_none());
|
|
||||||
assert_eq!(
|
|
||||||
registry
|
|
||||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
|
|
||||||
.unwrap()
|
|
||||||
.digest,
|
|
||||||
Digest::sha256("replacement"),
|
|
||||||
"rejection must not modify existing metadata"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn download_stream_accounts_usage_before_and_during_streaming() {
|
fn download_stream_accounts_usage_before_and_during_streaming() {
|
||||||
let mut registry = registry_with_artifact();
|
let mut registry = registry_with_artifact();
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
mod api_error;
|
|
||||||
pub mod artifact;
|
pub mod artifact;
|
||||||
pub mod auth;
|
pub mod auth;
|
||||||
pub mod bundle;
|
pub mod bundle;
|
||||||
|
|
@ -19,7 +18,6 @@ pub mod transport;
|
||||||
pub mod vfs;
|
pub mod vfs;
|
||||||
pub mod wire;
|
pub mod wire;
|
||||||
|
|
||||||
pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode};
|
|
||||||
pub use artifact::{
|
pub use artifact::{
|
||||||
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
|
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
|
||||||
ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
|
ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
|
||||||
|
|
|
||||||
|
|
@ -1208,12 +1208,10 @@ fn emit_runtime_outcome(
|
||||||
)?;
|
)?;
|
||||||
}
|
}
|
||||||
RuntimeContinuationOutcome::Terminal(record) => {
|
RuntimeContinuationOutcome::Terminal(record) => {
|
||||||
let exit_code = record.status_code.unwrap_or(1);
|
|
||||||
apply_runtime_record_with_thread_events(writer, state, record)?;
|
apply_runtime_record_with_thread_events(writer, state, record)?;
|
||||||
if state.last_task_failed {
|
if state.last_task_failed {
|
||||||
writer.output("stderr", format!("{}\n", state.command_status))?;
|
writer.output("stderr", format!("{}\n", state.command_status))?;
|
||||||
}
|
}
|
||||||
writer.event("exited", json!({ "exitCode": exit_code }))?;
|
|
||||||
writer.event("terminated", json!({}))?;
|
writer.event("terminated", json!({}))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ use breakpoints::{
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use dap_protocol::{initialize_capabilities, read_message};
|
use dap_protocol::{initialize_capabilities, read_message};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code};
|
use runtime_client::{client_user_request, parse_task_restart_response};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use variables::variables_response;
|
use variables::variables_response;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|
@ -36,26 +36,6 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
|
||||||
use virtual_model::{process_id, RuntimeBackend};
|
use virtual_model::{process_id, RuntimeBackend};
|
||||||
|
|
||||||
fn main() -> Result<()> {
|
fn main() -> Result<()> {
|
||||||
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
|
|
||||||
match raw_args.as_slice() {
|
|
||||||
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
|
|
||||||
println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION"));
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
|
|
||||||
println!(
|
|
||||||
"Clusterflux Debug Adapter Protocol server.\n\n\
|
|
||||||
Usage: clusterflux-debug-dap\n\n\
|
|
||||||
The adapter communicates over standard input and output.\n\n\
|
|
||||||
Options:\n \
|
|
||||||
-h, --help\n \
|
|
||||||
-V, --version"
|
|
||||||
);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
[] => {}
|
|
||||||
[argument, ..] => anyhow::bail!("unknown argument: {argument}"),
|
|
||||||
}
|
|
||||||
adapter::run_adapter()
|
adapter::run_adapter()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -742,16 +742,14 @@ fn fetch_current_process_status(
|
||||||
client_user_request(
|
client_user_request(
|
||||||
state,
|
state,
|
||||||
json!({
|
json!({
|
||||||
"type": "list_process_summaries",
|
"type": "list_processes",
|
||||||
"tenant": state.tenant,
|
"tenant": state.tenant,
|
||||||
"project": state.project_id,
|
"project": state.project_id,
|
||||||
"actor_user": state.actor_user,
|
"actor_user": state.actor_user,
|
||||||
"cursor": null,
|
|
||||||
"limit": 100,
|
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
)?;
|
)?;
|
||||||
let summary = statuses
|
let current = statuses
|
||||||
.get("processes")
|
.get("processes")
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
.and_then(|processes| {
|
.and_then(|processes| {
|
||||||
|
|
@ -760,9 +758,6 @@ fn fetch_current_process_status(
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.cloned();
|
.cloned();
|
||||||
let current = merge_active_process_status(state, summary, |request| {
|
|
||||||
coordinator_request(coordinator, request)
|
|
||||||
})?;
|
|
||||||
Ok((statuses, current))
|
Ok((statuses, current))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -773,15 +768,13 @@ fn fetch_current_process_status_in(
|
||||||
let statuses = session.request(client_user_request(
|
let statuses = session.request(client_user_request(
|
||||||
state,
|
state,
|
||||||
json!({
|
json!({
|
||||||
"type": "list_process_summaries",
|
"type": "list_processes",
|
||||||
"tenant": state.tenant,
|
"tenant": state.tenant,
|
||||||
"project": state.project_id,
|
"project": state.project_id,
|
||||||
"actor_user": state.actor_user,
|
"actor_user": state.actor_user,
|
||||||
"cursor": null,
|
|
||||||
"limit": 100,
|
|
||||||
}),
|
}),
|
||||||
))?;
|
))?;
|
||||||
let summary = statuses
|
let current = statuses
|
||||||
.get("processes")
|
.get("processes")
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)
|
||||||
.and_then(|processes| {
|
.and_then(|processes| {
|
||||||
|
|
@ -790,51 +783,9 @@ fn fetch_current_process_status_in(
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.cloned();
|
.cloned();
|
||||||
let current = merge_active_process_status(state, summary, |request| session.request(request))?;
|
|
||||||
Ok((statuses, current))
|
Ok((statuses, current))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_active_process_status<F>(
|
|
||||||
state: &AdapterState,
|
|
||||||
summary: Option<Value>,
|
|
||||||
mut request: F,
|
|
||||||
) -> Result<Option<Value>>
|
|
||||||
where
|
|
||||||
F: FnMut(Value) -> Result<Value>,
|
|
||||||
{
|
|
||||||
let Some(mut summary) = summary else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
|
||||||
if summary.get("lifecycle").and_then(Value::as_str) != Some("active") {
|
|
||||||
return Ok(Some(summary));
|
|
||||||
}
|
|
||||||
let active_statuses = request(client_user_request(
|
|
||||||
state,
|
|
||||||
json!({
|
|
||||||
"type": "list_processes",
|
|
||||||
"tenant": state.tenant,
|
|
||||||
"project": state.project_id,
|
|
||||||
"actor_user": state.actor_user,
|
|
||||||
}),
|
|
||||||
))?;
|
|
||||||
let active = active_statuses
|
|
||||||
.get("processes")
|
|
||||||
.and_then(Value::as_array)
|
|
||||||
.and_then(|processes| {
|
|
||||||
processes.iter().find(|process| {
|
|
||||||
process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
|
|
||||||
})
|
|
||||||
});
|
|
||||||
if let (Some(summary), Some(active)) =
|
|
||||||
(summary.as_object_mut(), active.and_then(Value::as_object))
|
|
||||||
{
|
|
||||||
for (key, value) in active {
|
|
||||||
summary.entry(key.clone()).or_insert_with(|| value.clone());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(Some(summary))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> {
|
pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> {
|
||||||
let coordinator =
|
let coordinator =
|
||||||
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
|
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
|
||||||
|
|
@ -908,7 +859,27 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result<RuntimeLau
|
||||||
.count()
|
.count()
|
||||||
})
|
})
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
let (process_statuses, process_status) = fetch_current_process_status(&coordinator, state)?;
|
let process_statuses = coordinator_request(
|
||||||
|
&coordinator,
|
||||||
|
client_user_request(
|
||||||
|
state,
|
||||||
|
json!({
|
||||||
|
"type": "list_processes",
|
||||||
|
"tenant": state.tenant,
|
||||||
|
"project": state.project_id,
|
||||||
|
"actor_user": state.actor_user,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
)?;
|
||||||
|
let process_status = process_statuses
|
||||||
|
.get("processes")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.and_then(|processes| {
|
||||||
|
processes.iter().find(|process| {
|
||||||
|
process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.cloned();
|
||||||
let node = process_status
|
let node = process_status
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|status| status.get("connected_nodes"))
|
.and_then(|status| status.get("connected_nodes"))
|
||||||
|
|
@ -1108,8 +1079,9 @@ pub(crate) fn observe_services_runtime(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Events remain useful for output and attempt details, but terminal
|
// Terminal task events remain inspectable after the active process slot is
|
||||||
// authority comes from the durable process summary read below.
|
// released. Read them before active-process debug state so a fast main exit
|
||||||
|
// cannot turn a successful continuation into an authorization error.
|
||||||
let current_session = session.as_mut().expect("observer session connected");
|
let current_session = session.as_mut().expect("observer session connected");
|
||||||
let events = match current_session.request(client_user_request(
|
let events = match current_session.request(client_user_request(
|
||||||
state,
|
state,
|
||||||
|
|
@ -1146,6 +1118,63 @@ pub(crate) fn observe_services_runtime(
|
||||||
std::thread::sleep(reconnect_delay);
|
std::thread::sleep(reconnect_delay);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
|
||||||
|
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
|
||||||
|
snapshot_failure_injected = true;
|
||||||
|
Err(anyhow!("injected task snapshot transport failure"))
|
||||||
|
} else {
|
||||||
|
fetch_task_snapshots_in(current_session, state)
|
||||||
|
};
|
||||||
|
let task_snapshots = match snapshot_request {
|
||||||
|
Ok(snapshots) => snapshots,
|
||||||
|
Err(error) => {
|
||||||
|
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
|
||||||
|
"runtime terminal snapshot observation failed: {error:#}; reconnecting in {} ms",
|
||||||
|
reconnect_delay.as_millis()
|
||||||
|
))) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
session = None;
|
||||||
|
std::thread::sleep(reconnect_delay);
|
||||||
|
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let process_status_request =
|
||||||
|
if inject_process_status_failure && !process_status_failure_injected {
|
||||||
|
process_status_failure_injected = true;
|
||||||
|
Err(anyhow!("injected process-status transport failure"))
|
||||||
|
} else {
|
||||||
|
fetch_current_process_status_in(current_session, state)
|
||||||
|
};
|
||||||
|
let (process_statuses, process_status) = match process_status_request {
|
||||||
|
Ok(status) => status,
|
||||||
|
Err(error) => {
|
||||||
|
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
|
||||||
|
"runtime terminal process observation failed: {error:#}; reconnecting in {} ms",
|
||||||
|
reconnect_delay.as_millis()
|
||||||
|
))) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
session = None;
|
||||||
|
std::thread::sleep(reconnect_delay);
|
||||||
|
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !has_current_runtime_task(&task_snapshots) {
|
||||||
|
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
|
||||||
|
record.node_report = json!({
|
||||||
|
"terminal_event": record.node_report.get("terminal_event"),
|
||||||
|
"task_snapshots": task_snapshots,
|
||||||
|
"process_status": process_status,
|
||||||
|
"process_statuses": process_statuses,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
emit(outcome);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
|
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
|
||||||
snapshot_failure_injected = true;
|
snapshot_failure_injected = true;
|
||||||
Err(anyhow!("injected task snapshot transport failure"))
|
Err(anyhow!("injected task snapshot transport failure"))
|
||||||
|
|
@ -1190,22 +1219,6 @@ pub(crate) fn observe_services_runtime(
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
reconnect_delay = Duration::from_millis(100);
|
reconnect_delay = Duration::from_millis(100);
|
||||||
if !has_current_runtime_task(&task_snapshots) {
|
|
||||||
if let Some(mut outcome) =
|
|
||||||
terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
|
|
||||||
{
|
|
||||||
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
|
|
||||||
record.node_report = json!({
|
|
||||||
"terminal_event": record.node_report.get("terminal_event"),
|
|
||||||
"task_snapshots": task_snapshots,
|
|
||||||
"process_status": process_status,
|
|
||||||
"process_statuses": process_statuses,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
emit(outcome);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
|
if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
|
||||||
let failed_task = failed_task.to_owned();
|
let failed_task = failed_task.to_owned();
|
||||||
let failed_event = events
|
let failed_event = events
|
||||||
|
|
@ -1337,9 +1350,7 @@ pub(crate) fn observe_services_runtime(
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if let Some(mut outcome) =
|
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
|
||||||
terminal_runtime_outcome(&coordinator, state, &events, process_status.as_ref())
|
|
||||||
{
|
|
||||||
let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
|
let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
|
||||||
Ok(snapshots) => snapshots,
|
Ok(snapshots) => snapshots,
|
||||||
Err(snapshot_error) => {
|
Err(snapshot_error) => {
|
||||||
|
|
@ -1532,111 +1543,68 @@ fn has_current_runtime_task(task_snapshots: &Value) -> bool {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) fn whole_process_status_code(
|
|
||||||
main_status_code: Option<i32>,
|
|
||||||
task_snapshots: &Value,
|
|
||||||
) -> Option<i32> {
|
|
||||||
let main_status_code = main_status_code?;
|
|
||||||
if main_status_code != 0 {
|
|
||||||
return Some(main_status_code);
|
|
||||||
}
|
|
||||||
|
|
||||||
let snapshots = task_snapshots.get("snapshots").and_then(Value::as_array)?;
|
|
||||||
for snapshot in snapshots
|
|
||||||
.iter()
|
|
||||||
.filter(|snapshot| snapshot.get("current").and_then(Value::as_bool) == Some(true))
|
|
||||||
{
|
|
||||||
match snapshot.get("state").and_then(Value::as_str) {
|
|
||||||
Some("completed") => {}
|
|
||||||
Some("failed" | "cancelled" | "failed_awaiting_action") => {
|
|
||||||
return Some(
|
|
||||||
snapshot
|
|
||||||
.get("status_code")
|
|
||||||
.and_then(Value::as_i64)
|
|
||||||
.and_then(|status| i32::try_from(status).ok())
|
|
||||||
.filter(|status| *status != 0)
|
|
||||||
.unwrap_or(1),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Some("queued" | "running") => return None,
|
|
||||||
_ => return Some(1),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some(0)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn terminal_runtime_outcome(
|
fn terminal_runtime_outcome(
|
||||||
coordinator: &str,
|
coordinator: &str,
|
||||||
_state: &AdapterState,
|
state: &AdapterState,
|
||||||
events: &Value,
|
events: &Value,
|
||||||
process_status: Option<&Value>,
|
|
||||||
) -> Option<RuntimeContinuationOutcome> {
|
) -> Option<RuntimeContinuationOutcome> {
|
||||||
let final_result = process_status?
|
|
||||||
.get("final_result")
|
|
||||||
.and_then(Value::as_str)?;
|
|
||||||
let status_code = match final_result {
|
|
||||||
"completed" => 0,
|
|
||||||
"failed" | "cancelled" => 1,
|
|
||||||
_ => return None,
|
|
||||||
};
|
|
||||||
let event = events
|
let event = events
|
||||||
.get("events")
|
.get("events")
|
||||||
.and_then(Value::as_array)
|
.and_then(Value::as_array)?
|
||||||
.and_then(|events| {
|
.get(state.runtime_event_count..)?
|
||||||
events.iter().rev().find(|event| {
|
.iter()
|
||||||
event.get("executor").and_then(Value::as_str) == Some("coordinator_main")
|
.rev()
|
||||||
})
|
.find(|event| event.get("executor").and_then(Value::as_str) == Some("coordinator_main"))?;
|
||||||
});
|
let status_code = event
|
||||||
|
.get("status_code")
|
||||||
|
.and_then(Value::as_i64)
|
||||||
|
.map(|status| status as i32)
|
||||||
|
.or_else(
|
||||||
|
|| match event.get("terminal_state").and_then(Value::as_str) {
|
||||||
|
Some("completed") => Some(0),
|
||||||
|
Some("failed" | "cancelled") => Some(1),
|
||||||
|
_ => None,
|
||||||
|
},
|
||||||
|
);
|
||||||
Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord {
|
Some(RuntimeContinuationOutcome::Terminal(RuntimeLaunchRecord {
|
||||||
coordinator: coordinator.to_owned(),
|
coordinator: coordinator.to_owned(),
|
||||||
node: event
|
node: event
|
||||||
.and_then(|event| event.get("node"))
|
.get("node")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.or_else(|| {
|
|
||||||
process_status
|
|
||||||
.and_then(|status| status.get("connected_nodes"))
|
|
||||||
.and_then(Value::as_array)
|
|
||||||
.and_then(|nodes| nodes.first())
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
})
|
|
||||||
.unwrap_or("coordinator-main")
|
.unwrap_or("coordinator-main")
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
node_report: json!({
|
node_report: json!({ "terminal_event": event }),
|
||||||
"terminal_event": event,
|
|
||||||
"process_status": process_status,
|
|
||||||
}),
|
|
||||||
task_events: events.clone(),
|
task_events: events.clone(),
|
||||||
placed_task_launched: true,
|
placed_task_launched: true,
|
||||||
status_code: Some(status_code),
|
status_code,
|
||||||
stdout_bytes: event
|
stdout_bytes: event
|
||||||
.and_then(|event| event.get("stdout_bytes"))
|
.get("stdout_bytes")
|
||||||
.and_then(Value::as_u64)
|
.and_then(Value::as_u64)
|
||||||
.unwrap_or(0),
|
.unwrap_or(0),
|
||||||
stderr_bytes: event
|
stderr_bytes: event
|
||||||
.and_then(|event| event.get("stderr_bytes"))
|
.get("stderr_bytes")
|
||||||
.and_then(Value::as_u64)
|
.and_then(Value::as_u64)
|
||||||
.unwrap_or(0),
|
.unwrap_or(0),
|
||||||
stdout_tail: event
|
stdout_tail: event
|
||||||
.and_then(|event| event.get("stdout_tail"))
|
.get("stdout_tail")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
stderr_tail: event
|
stderr_tail: event
|
||||||
.and_then(|event| event.get("stderr_tail"))
|
.get("stderr_tail")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_owned(),
|
.to_owned(),
|
||||||
stdout_truncated: event
|
stdout_truncated: event
|
||||||
.and_then(|event| event.get("stdout_truncated"))
|
.get("stdout_truncated")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
stderr_truncated: event
|
stderr_truncated: event
|
||||||
.and_then(|event| event.get("stderr_truncated"))
|
.get("stderr_truncated")
|
||||||
.and_then(Value::as_bool)
|
.and_then(Value::as_bool)
|
||||||
.unwrap_or(false),
|
.unwrap_or(false),
|
||||||
artifact_path: event
|
artifact_path: event
|
||||||
.and_then(|event| event.get("artifact_path"))
|
.get("artifact_path")
|
||||||
.and_then(Value::as_str)
|
.and_then(Value::as_str)
|
||||||
.map(str::to_owned),
|
.map(str::to_owned),
|
||||||
event_count: events
|
event_count: events
|
||||||
|
|
@ -1722,53 +1690,6 @@ mod transactional_launch_tests {
|
||||||
assert!(error.contains("no virtual process was created"));
|
assert!(error.contains("no virtual process was created"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn durable_process_summary_is_terminal_authority_after_event_rotation() {
|
|
||||||
let state = AdapterState {
|
|
||||||
runtime_event_count: 10_000,
|
|
||||||
..AdapterState::default()
|
|
||||||
};
|
|
||||||
let completed = terminal_runtime_outcome(
|
|
||||||
"127.0.0.1:1",
|
|
||||||
&state,
|
|
||||||
&json!({ "events": [] }),
|
|
||||||
Some(&json!({
|
|
||||||
"process": state.process.as_str(),
|
|
||||||
"lifecycle": "recent_terminal",
|
|
||||||
"final_result": "completed",
|
|
||||||
"connected_nodes": []
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.expect("the durable summary should terminate observation");
|
|
||||||
let RuntimeContinuationOutcome::Terminal(completed) = completed else {
|
|
||||||
panic!("expected terminal outcome");
|
|
||||||
};
|
|
||||||
assert_eq!(completed.status_code, Some(0));
|
|
||||||
|
|
||||||
let failed = terminal_runtime_outcome(
|
|
||||||
"127.0.0.1:1",
|
|
||||||
&state,
|
|
||||||
&json!({
|
|
||||||
"events": [{
|
|
||||||
"executor": "coordinator_main",
|
|
||||||
"terminal_state": "completed",
|
|
||||||
"status_code": 0
|
|
||||||
}]
|
|
||||||
}),
|
|
||||||
Some(&json!({
|
|
||||||
"process": state.process.as_str(),
|
|
||||||
"lifecycle": "recent_terminal",
|
|
||||||
"final_result": "failed",
|
|
||||||
"connected_nodes": []
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.expect("the aggregate summary should override a successful main event");
|
|
||||||
let RuntimeContinuationOutcome::Terminal(failed) = failed else {
|
|
||||||
panic!("expected terminal outcome");
|
|
||||||
};
|
|
||||||
assert_eq!(failed.status_code, Some(1));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn failed_debug_launch_reconnects_and_aborts_the_process() {
|
fn failed_debug_launch_reconnects_and_aborts_the_process() {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
|
|
||||||
|
|
@ -704,40 +704,11 @@ fn terminal_record_without_snapshots_clears_active_threads() {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
|
fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
|
||||||
let project = std::env::temp_dir().join(format!(
|
|
||||||
"clusterflux-dap-source-locals-{}",
|
|
||||||
std::process::id()
|
|
||||||
));
|
|
||||||
let src = project.join("src");
|
|
||||||
fs::create_dir_all(&src).unwrap();
|
|
||||||
fs::write(
|
|
||||||
src.join("lib.rs"),
|
|
||||||
r#"async fn run_build_workflow() {
|
|
||||||
let source = prepare_source_snapshot();
|
|
||||||
let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
|
|
||||||
.name("compile linux")
|
|
||||||
.env(linux_env())
|
|
||||||
.start()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let linux_thread = linux.virtual_thread_id();
|
|
||||||
let linux_artifact = linux.join().await.unwrap();
|
|
||||||
let package = clusterflux::spawn::async_task_with_arg(
|
|
||||||
vec![linux_artifact.clone()],
|
|
||||||
package_release,
|
|
||||||
)
|
|
||||||
.name("package artifacts")
|
|
||||||
.env(linux_env())
|
|
||||||
.start()
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
let package_artifact = package.join().await.unwrap();
|
|
||||||
}
|
|
||||||
"#,
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let mut state = AdapterState::default();
|
let mut state = AdapterState::default();
|
||||||
|
let project = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("../../tests/fixtures/runtime-conformance")
|
||||||
|
.canonicalize()
|
||||||
|
.unwrap();
|
||||||
state.project = project.to_string_lossy().into_owned();
|
state.project = project.to_string_lossy().into_owned();
|
||||||
state.source_path = "src/lib.rs".to_owned();
|
state.source_path = "src/lib.rs".to_owned();
|
||||||
let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
|
let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
|
||||||
|
|
@ -773,8 +744,6 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
|
||||||
variable["name"] == "unavailable-local-diagnostic"
|
variable["name"] == "unavailable-local-diagnostic"
|
||||||
&& variable["type"] == "unavailable-local"
|
&& variable["type"] == "unavailable-local"
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let _ = fs::remove_dir_all(project);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -894,20 +863,11 @@ fn package_release(inputs: Vec<Artifact>) -> Artifact {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
|
fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
|
||||||
let project = std::env::temp_dir().join(format!(
|
|
||||||
"clusterflux-dap-wasm-locals-{}",
|
|
||||||
std::process::id()
|
|
||||||
));
|
|
||||||
let src = project.join("src");
|
|
||||||
fs::create_dir_all(&src).unwrap();
|
|
||||||
fs::write(
|
|
||||||
src.join("lib.rs"),
|
|
||||||
"pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n input + 1\n}\n",
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let mut state = AdapterState::default();
|
let mut state = AdapterState::default();
|
||||||
state.project = project.to_string_lossy().into_owned();
|
state.project = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("../../tests/fixtures/runtime-conformance")
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned();
|
||||||
state.source_path = "src/lib.rs".to_owned();
|
state.source_path = "src/lib.rs".to_owned();
|
||||||
let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
|
let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
|
||||||
state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
|
state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
|
||||||
|
|
@ -935,8 +895,6 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
|
||||||
assert!(!locals
|
assert!(!locals
|
||||||
.iter()
|
.iter()
|
||||||
.any(|variable| variable["name"] == "wasm-local-diagnostic"));
|
.any(|variable| variable["name"] == "wasm-local-diagnostic"));
|
||||||
|
|
||||||
let _ = fs::remove_dir_all(project);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -964,98 +922,6 @@ fn detects_current_failed_attempt_awaiting_operator_action() {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn whole_process_terminal_status_covers_every_current_logical_task_attempt() {
|
|
||||||
let snapshots = |items: serde_json::Value| json!({ "snapshots": items });
|
|
||||||
|
|
||||||
assert_eq!(
|
|
||||||
whole_process_status_code(
|
|
||||||
Some(0),
|
|
||||||
&snapshots(json!([
|
|
||||||
{
|
|
||||||
"task": "child",
|
|
||||||
"current": true,
|
|
||||||
"state": "completed",
|
|
||||||
"status_code": 0
|
|
||||||
}
|
|
||||||
]))
|
|
||||||
),
|
|
||||||
Some(0),
|
|
||||||
"a successful main and final child must succeed"
|
|
||||||
);
|
|
||||||
for (state, status_code) in [("failed", 23), ("cancelled", 1)] {
|
|
||||||
assert_eq!(
|
|
||||||
whole_process_status_code(
|
|
||||||
Some(0),
|
|
||||||
&snapshots(json!([
|
|
||||||
{
|
|
||||||
"task": "child",
|
|
||||||
"current": true,
|
|
||||||
"state": state,
|
|
||||||
"status_code": status_code
|
|
||||||
}
|
|
||||||
]))
|
|
||||||
),
|
|
||||||
Some(status_code),
|
|
||||||
"a terminal child must determine the whole-process result"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
assert_eq!(
|
|
||||||
whole_process_status_code(
|
|
||||||
Some(0),
|
|
||||||
&snapshots(json!([
|
|
||||||
{
|
|
||||||
"task": "accepted-failure",
|
|
||||||
"current": true,
|
|
||||||
"state": "failed",
|
|
||||||
"command_state": "failure_accepted",
|
|
||||||
"status_code": 17
|
|
||||||
}
|
|
||||||
]))
|
|
||||||
),
|
|
||||||
Some(17),
|
|
||||||
"accepting an AwaitOperator failure releases the process but does not turn failure into success"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
whole_process_status_code(
|
|
||||||
Some(0),
|
|
||||||
&snapshots(json!([
|
|
||||||
{
|
|
||||||
"task": "restarted",
|
|
||||||
"attempt_id": "attempt-1",
|
|
||||||
"current": false,
|
|
||||||
"state": "failed",
|
|
||||||
"status_code": 7
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"task": "restarted",
|
|
||||||
"attempt_id": "attempt-2",
|
|
||||||
"current": true,
|
|
||||||
"state": "completed",
|
|
||||||
"status_code": 0
|
|
||||||
}
|
|
||||||
]))
|
|
||||||
),
|
|
||||||
Some(0),
|
|
||||||
"a successful replacement attempt is authoritative over stale failed attempts"
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
whole_process_status_code(
|
|
||||||
Some(9),
|
|
||||||
&snapshots(json!([
|
|
||||||
{
|
|
||||||
"task": "child",
|
|
||||||
"current": true,
|
|
||||||
"state": "completed",
|
|
||||||
"status_code": 0
|
|
||||||
}
|
|
||||||
]))
|
|
||||||
),
|
|
||||||
Some(9),
|
|
||||||
"a failed coordinator main cannot be masked by successful children"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
|
fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
|
||||||
let mut state = AdapterState::default();
|
let mut state = AdapterState::default();
|
||||||
|
|
|
||||||
|
|
@ -114,7 +114,7 @@ impl Default for AdapterState {
|
||||||
project,
|
project,
|
||||||
source_path: "src/lib.rs".to_owned(),
|
source_path: "src/lib.rs".to_owned(),
|
||||||
runtime_backend: RuntimeBackend::Simulated,
|
runtime_backend: RuntimeBackend::Simulated,
|
||||||
coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(),
|
coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(),
|
||||||
tenant: TenantId::from("tenant"),
|
tenant: TenantId::from("tenant"),
|
||||||
project_id: ProjectId::from("project"),
|
project_id: ProjectId::from("project"),
|
||||||
actor_user: UserId::from("dap"),
|
actor_user: UserId::from("dap"),
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
@ -46,40 +46,6 @@ use validation::{
|
||||||
resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
|
resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug)]
|
|
||||||
struct AssignmentExecutionError {
|
|
||||||
message: String,
|
|
||||||
stdout_source_bytes: u64,
|
|
||||||
stderr_source_bytes: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::fmt::Display for AssignmentExecutionError {
|
|
||||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
formatter.write_str(&self.message)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl std::error::Error for AssignmentExecutionError {}
|
|
||||||
|
|
||||||
fn execution_error_with_log_bytes(
|
|
||||||
message: impl Into<String>,
|
|
||||||
stdout_source_bytes: &AtomicU64,
|
|
||||||
stderr_source_bytes: &AtomicU64,
|
|
||||||
) -> Box<dyn std::error::Error> {
|
|
||||||
Box::new(AssignmentExecutionError {
|
|
||||||
message: message.into(),
|
|
||||||
stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed),
|
|
||||||
stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) {
|
|
||||||
error
|
|
||||||
.downcast_ref::<AssignmentExecutionError>()
|
|
||||||
.map(|error| (error.stdout_source_bytes, error.stderr_source_bytes))
|
|
||||||
.unwrap_or((0, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn run_verified_wasmtime_assignment(
|
pub(crate) fn run_verified_wasmtime_assignment(
|
||||||
args: &Args,
|
args: &Args,
|
||||||
task: &RuntimeTask,
|
task: &RuntimeTask,
|
||||||
|
|
@ -129,8 +95,6 @@ pub(crate) fn run_verified_wasmtime_assignment(
|
||||||
return Err("Wasm entrypoint assignment omitted its descriptor export".into());
|
return Err("Wasm entrypoint assignment omitted its descriptor export".into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let command_stdout_source_bytes = Arc::new(AtomicU64::new(0));
|
|
||||||
let command_stderr_source_bytes = Arc::new(AtomicU64::new(0));
|
|
||||||
let (stdout, boundary_result) = match abi {
|
let (stdout, boundary_result) = match abi {
|
||||||
WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
|
WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
|
||||||
let invocation = WasmTaskInvocation::new(
|
let invocation = WasmTaskInvocation::new(
|
||||||
|
|
@ -138,28 +102,18 @@ pub(crate) fn run_verified_wasmtime_assignment(
|
||||||
task_spec.task_instance.clone(),
|
task_spec.task_instance.clone(),
|
||||||
task_spec.args.clone(),
|
task_spec.args.clone(),
|
||||||
);
|
);
|
||||||
let result = WasmtimeTaskRuntime::new()?
|
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
|
||||||
.run_task_export_verified_with_task_host(
|
&module,
|
||||||
|
expected_bundle_digest,
|
||||||
|
export,
|
||||||
|
&invocation,
|
||||||
|
Box::new(CoordinatorWasmTaskHost::new(
|
||||||
|
args,
|
||||||
|
task,
|
||||||
|
node_private_key,
|
||||||
&module,
|
&module,
|
||||||
expected_bundle_digest,
|
)?),
|
||||||
export,
|
)?;
|
||||||
&invocation,
|
|
||||||
Box::new(CoordinatorWasmTaskHost::new(
|
|
||||||
args,
|
|
||||||
task,
|
|
||||||
node_private_key,
|
|
||||||
&module,
|
|
||||||
Arc::clone(&command_stdout_source_bytes),
|
|
||||||
Arc::clone(&command_stderr_source_bytes),
|
|
||||||
)?),
|
|
||||||
)
|
|
||||||
.map_err(|error| {
|
|
||||||
execution_error_with_log_bytes(
|
|
||||||
error.to_string(),
|
|
||||||
&command_stdout_source_bytes,
|
|
||||||
&command_stderr_source_bytes,
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
|
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"clusterflux debug control: Wasm assignment returned for task {}",
|
"clusterflux debug control: Wasm assignment returned for task {}",
|
||||||
|
|
@ -168,35 +122,17 @@ pub(crate) fn run_verified_wasmtime_assignment(
|
||||||
}
|
}
|
||||||
match result.outcome {
|
match result.outcome {
|
||||||
WasmTaskOutcome::Completed => {
|
WasmTaskOutcome::Completed => {
|
||||||
let boundary = result.result.ok_or_else(|| {
|
let boundary = result.result.ok_or("completed Wasm task omitted result")?;
|
||||||
execution_error_with_log_bytes(
|
|
||||||
"completed Wasm task omitted result",
|
|
||||||
&command_stdout_source_bytes,
|
|
||||||
&command_stderr_source_bytes,
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
(
|
(
|
||||||
format!(
|
format!("{}\n", serde_json::to_string(&boundary)?),
|
||||||
"{}\n",
|
|
||||||
serde_json::to_string(&boundary).map_err(|error| {
|
|
||||||
execution_error_with_log_bytes(
|
|
||||||
error.to_string(),
|
|
||||||
&command_stdout_source_bytes,
|
|
||||||
&command_stderr_source_bytes,
|
|
||||||
)
|
|
||||||
})?
|
|
||||||
),
|
|
||||||
Some(boundary),
|
Some(boundary),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
WasmTaskOutcome::Failed => {
|
WasmTaskOutcome::Failed => {
|
||||||
return Err(execution_error_with_log_bytes(
|
return Err(result
|
||||||
result
|
.error
|
||||||
.error
|
.unwrap_or_else(|| "Wasm task failed without an error".to_owned())
|
||||||
.unwrap_or_else(|| "Wasm task failed without an error".to_owned()),
|
.into())
|
||||||
&command_stdout_source_bytes,
|
|
||||||
&command_stderr_source_bytes,
|
|
||||||
))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -204,18 +140,12 @@ pub(crate) fn run_verified_wasmtime_assignment(
|
||||||
let task_id = TaskInstanceId::new(task.task.clone());
|
let task_id = TaskInstanceId::new(task.task.clone());
|
||||||
let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
|
let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
|
||||||
let manifest = artifacts.flush();
|
let manifest = artifacts.flush();
|
||||||
let stdout_source_bytes = command_stdout_source_bytes
|
|
||||||
.load(Ordering::Relaxed)
|
|
||||||
.saturating_add(stdout.len() as u64);
|
|
||||||
let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed);
|
|
||||||
Ok((
|
Ok((
|
||||||
CommandOutput {
|
CommandOutput {
|
||||||
virtual_thread: task_id,
|
virtual_thread: task_id,
|
||||||
status_code: Some(0),
|
status_code: Some(0),
|
||||||
stdout,
|
stdout,
|
||||||
stderr: String::new(),
|
stderr: String::new(),
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
stdout_truncated: false,
|
stdout_truncated: false,
|
||||||
stderr_truncated: false,
|
stderr_truncated: false,
|
||||||
log_backpressured: false,
|
log_backpressured: false,
|
||||||
|
|
@ -246,8 +176,6 @@ struct CoordinatorWasmTaskHost {
|
||||||
next_handle_id: u64,
|
next_handle_id: u64,
|
||||||
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
|
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
|
||||||
command_status: Arc<Mutex<Option<String>>>,
|
command_status: Arc<Mutex<Option<String>>>,
|
||||||
command_stdout_source_bytes: Arc<AtomicU64>,
|
|
||||||
command_stderr_source_bytes: Arc<AtomicU64>,
|
|
||||||
cancellation_requested: Arc<AtomicBool>,
|
cancellation_requested: Arc<AtomicBool>,
|
||||||
abort_requested: Arc<AtomicBool>,
|
abort_requested: Arc<AtomicBool>,
|
||||||
debug_control: Arc<WasmDebugControl>,
|
debug_control: Arc<WasmDebugControl>,
|
||||||
|
|
@ -260,8 +188,6 @@ impl CoordinatorWasmTaskHost {
|
||||||
parent: &RuntimeTask,
|
parent: &RuntimeTask,
|
||||||
node_private_key: &str,
|
node_private_key: &str,
|
||||||
module: &[u8],
|
module: &[u8],
|
||||||
command_stdout_source_bytes: Arc<AtomicU64>,
|
|
||||||
command_stderr_source_bytes: Arc<AtomicU64>,
|
|
||||||
) -> Result<Self, Box<dyn std::error::Error>> {
|
) -> Result<Self, Box<dyn std::error::Error>> {
|
||||||
let task_spec = parent
|
let task_spec = parent
|
||||||
.task_spec
|
.task_spec
|
||||||
|
|
@ -342,8 +268,6 @@ impl CoordinatorWasmTaskHost {
|
||||||
next_handle_id: 1,
|
next_handle_id: 1,
|
||||||
handles,
|
handles,
|
||||||
command_status,
|
command_status,
|
||||||
command_stdout_source_bytes,
|
|
||||||
command_stderr_source_bytes,
|
|
||||||
cancellation_requested,
|
cancellation_requested,
|
||||||
abort_requested,
|
abort_requested,
|
||||||
debug_control,
|
debug_control,
|
||||||
|
|
@ -369,16 +293,7 @@ impl CoordinatorWasmTaskHost {
|
||||||
runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
|
runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
|
||||||
let execution =
|
let execution =
|
||||||
run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
|
run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
|
||||||
let (
|
let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
|
||||||
terminal_state,
|
|
||||||
status_code,
|
|
||||||
stdout,
|
|
||||||
stderr,
|
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
result,
|
|
||||||
retained,
|
|
||||||
) = match execution {
|
|
||||||
Ok((output, _manifest, result)) => {
|
Ok((output, _manifest, result)) => {
|
||||||
let retained = retained_result_artifact(
|
let retained = retained_result_artifact(
|
||||||
self.args.project_root.as_deref(),
|
self.args.project_root.as_deref(),
|
||||||
|
|
@ -391,40 +306,20 @@ impl CoordinatorWasmTaskHost {
|
||||||
output.status_code,
|
output.status_code,
|
||||||
output.stdout,
|
output.stdout,
|
||||||
output.stderr,
|
output.stderr,
|
||||||
output.stdout_source_bytes,
|
|
||||||
output.stderr_source_bytes,
|
|
||||||
result,
|
result,
|
||||||
retained,
|
retained,
|
||||||
),
|
),
|
||||||
Err(error) => (
|
Err(error) => ("failed", Some(1), String::new(), error, None, None),
|
||||||
"failed",
|
|
||||||
Some(1),
|
|
||||||
String::new(),
|
|
||||||
error.clone(),
|
|
||||||
output.stdout_source_bytes,
|
|
||||||
output
|
|
||||||
.stderr_source_bytes
|
|
||||||
.saturating_add(error.len() as u64),
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => (
|
||||||
let (stdout_source_bytes, stderr_source_bytes) =
|
"failed",
|
||||||
assignment_error_log_bytes(error.as_ref());
|
Some(1),
|
||||||
let error = error.to_string();
|
String::new(),
|
||||||
(
|
error.to_string(),
|
||||||
"failed",
|
None,
|
||||||
Some(1),
|
None,
|
||||||
String::new(),
|
),
|
||||||
error.clone(),
|
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes.saturating_add(error.len() as u64),
|
|
||||||
None,
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
let artifact_path = retained
|
let artifact_path = retained
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|
@ -444,8 +339,8 @@ impl CoordinatorWasmTaskHost {
|
||||||
"task": runtime_task.task,
|
"task": runtime_task.task,
|
||||||
"terminal_state": terminal_state,
|
"terminal_state": terminal_state,
|
||||||
"status_code": status_code,
|
"status_code": status_code,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": stdout.len(),
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": stderr.len(),
|
||||||
"stdout_tail": stdout,
|
"stdout_tail": stdout,
|
||||||
"stderr_tail": stderr,
|
"stderr_tail": stderr,
|
||||||
"stdout_truncated": false,
|
"stdout_truncated": false,
|
||||||
|
|
@ -781,7 +676,6 @@ impl WasmTaskHost for CoordinatorWasmTaskHost {
|
||||||
let mut runner = CoordinatorControlledProcessRunner::new(
|
let mut runner = CoordinatorControlledProcessRunner::new(
|
||||||
self,
|
self,
|
||||||
Duration::from_millis(request.timeout_ms),
|
Duration::from_millis(request.timeout_ms),
|
||||||
configured_secrets.clone(),
|
|
||||||
);
|
);
|
||||||
let output = LinuxRootlessPodmanBackend
|
let output = LinuxRootlessPodmanBackend
|
||||||
.execute_local_checkout_task(
|
.execute_local_checkout_task(
|
||||||
|
|
|
||||||
|
|
@ -1,83 +1,4 @@
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::mpsc::{self, Receiver, SyncSender};
|
|
||||||
|
|
||||||
struct LiveLogChunk {
|
|
||||||
stream: &'static str,
|
|
||||||
offset: u64,
|
|
||||||
source_bytes: u64,
|
|
||||||
bytes: Vec<u8>,
|
|
||||||
truncated: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn append_bounded_tail(tail: &mut Vec<u8>, bytes: &[u8], maximum: usize) {
|
|
||||||
if maximum == 0 {
|
|
||||||
tail.clear();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if bytes.len() >= maximum {
|
|
||||||
tail.clear();
|
|
||||||
tail.extend_from_slice(&bytes[bytes.len() - maximum..]);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let overflow = tail
|
|
||||||
.len()
|
|
||||||
.saturating_add(bytes.len())
|
|
||||||
.saturating_sub(maximum);
|
|
||||||
if overflow > 0 {
|
|
||||||
tail.drain(..overflow);
|
|
||||||
}
|
|
||||||
tail.extend_from_slice(bytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
fn redact_safe_live_log_prefix(
|
|
||||||
pending: &[u8],
|
|
||||||
configured_secrets: &[String],
|
|
||||||
final_chunk: bool,
|
|
||||||
) -> Option<(usize, String)> {
|
|
||||||
if pending.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let secret_bytes = configured_secrets
|
|
||||||
.iter()
|
|
||||||
.filter(|secret| secret.len() >= 4)
|
|
||||||
.map(String::as_bytes)
|
|
||||||
.collect::<Vec<_>>();
|
|
||||||
let maximum_secret_bytes = secret_bytes
|
|
||||||
.iter()
|
|
||||||
.map(|secret| secret.len())
|
|
||||||
.max()
|
|
||||||
.unwrap_or(0);
|
|
||||||
if !final_chunk && pending.len() <= maximum_secret_bytes {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let mut consumed = if final_chunk {
|
|
||||||
pending.len()
|
|
||||||
} else {
|
|
||||||
pending.len() - maximum_secret_bytes
|
|
||||||
};
|
|
||||||
loop {
|
|
||||||
let previous = consumed;
|
|
||||||
for secret in &secret_bytes {
|
|
||||||
for start in 0..=pending.len().saturating_sub(secret.len()) {
|
|
||||||
let end = start + secret.len();
|
|
||||||
if start < consumed && end > consumed && &pending[start..end] == *secret {
|
|
||||||
consumed = end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if consumed == previous {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if consumed == 0 {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let text = redact_configured_values(
|
|
||||||
String::from_utf8_lossy(&pending[..consumed]).into_owned(),
|
|
||||||
configured_secrets,
|
|
||||||
);
|
|
||||||
Some((consumed, text))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) struct CoordinatorControlledProcessRunner {
|
pub(super) struct CoordinatorControlledProcessRunner {
|
||||||
pub(super) args: Args,
|
pub(super) args: Args,
|
||||||
|
|
@ -86,20 +7,13 @@ pub(super) struct CoordinatorControlledProcessRunner {
|
||||||
pub(super) node_private_key: String,
|
pub(super) node_private_key: String,
|
||||||
pub(super) debug_control: Arc<WasmDebugControl>,
|
pub(super) debug_control: Arc<WasmDebugControl>,
|
||||||
pub(super) command_status: Arc<Mutex<Option<String>>>,
|
pub(super) command_status: Arc<Mutex<Option<String>>>,
|
||||||
pub(super) stdout_source_bytes: Arc<AtomicU64>,
|
|
||||||
pub(super) stderr_source_bytes: Arc<AtomicU64>,
|
|
||||||
pub(super) timeout: Duration,
|
pub(super) timeout: Duration,
|
||||||
pub(super) configured_secrets: Vec<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CoordinatorControlledProcessRunner {
|
impl CoordinatorControlledProcessRunner {
|
||||||
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
|
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
|
||||||
|
|
||||||
pub(super) fn new(
|
pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
|
||||||
host: &CoordinatorWasmTaskHost,
|
|
||||||
timeout: Duration,
|
|
||||||
configured_secrets: Vec<String>,
|
|
||||||
) -> Self {
|
|
||||||
Self {
|
Self {
|
||||||
args: host.args.clone(),
|
args: host.args.clone(),
|
||||||
process: host.process.clone(),
|
process: host.process.clone(),
|
||||||
|
|
@ -107,10 +21,7 @@ impl CoordinatorControlledProcessRunner {
|
||||||
node_private_key: host.node_private_key.clone(),
|
node_private_key: host.node_private_key.clone(),
|
||||||
debug_control: Arc::clone(&host.debug_control),
|
debug_control: Arc::clone(&host.debug_control),
|
||||||
command_status: Arc::clone(&host.command_status),
|
command_status: Arc::clone(&host.command_status),
|
||||||
stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes),
|
|
||||||
stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes),
|
|
||||||
timeout,
|
timeout,
|
||||||
configured_secrets,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -290,18 +201,10 @@ impl CoordinatorControlledProcessRunner {
|
||||||
fn drain_bounded(
|
fn drain_bounded(
|
||||||
mut reader: impl Read + Send + 'static,
|
mut reader: impl Read + Send + 'static,
|
||||||
maximum: usize,
|
maximum: usize,
|
||||||
stream: &'static str,
|
|
||||||
sender: SyncSender<LiveLogChunk>,
|
|
||||||
configured_secrets: Vec<String>,
|
|
||||||
source_bytes_total: Arc<AtomicU64>,
|
|
||||||
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
|
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
|
||||||
thread::spawn(move || {
|
thread::spawn(move || {
|
||||||
let mut captured = Vec::new();
|
let mut captured = Vec::new();
|
||||||
let mut buffer = [0_u8; 16 * 1024];
|
let mut buffer = [0_u8; 16 * 1024];
|
||||||
let stream_base = source_bytes_total.load(Ordering::Relaxed);
|
|
||||||
let mut source_bytes_read = 0_u64;
|
|
||||||
let mut pending_offset = stream_base;
|
|
||||||
let mut pending = Vec::new();
|
|
||||||
loop {
|
loop {
|
||||||
let count = reader
|
let count = reader
|
||||||
.read(&mut buffer)
|
.read(&mut buffer)
|
||||||
|
|
@ -309,128 +212,12 @@ impl CoordinatorControlledProcessRunner {
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let _ = source_bytes_total.fetch_update(
|
let remaining = maximum.saturating_sub(captured.len());
|
||||||
Ordering::Relaxed,
|
captured.extend_from_slice(&buffer[..count.min(remaining)]);
|
||||||
Ordering::Relaxed,
|
|
||||||
|current| Some(current.saturating_add(count as u64)),
|
|
||||||
);
|
|
||||||
source_bytes_read = source_bytes_read.saturating_add(count as u64);
|
|
||||||
append_bounded_tail(&mut captured, &buffer[..count], maximum);
|
|
||||||
pending.extend_from_slice(&buffer[..count]);
|
|
||||||
if let Some((consumed, text)) =
|
|
||||||
redact_safe_live_log_prefix(&pending, &configured_secrets, false)
|
|
||||||
{
|
|
||||||
let _ = sender.try_send(LiveLogChunk {
|
|
||||||
stream,
|
|
||||||
offset: pending_offset,
|
|
||||||
source_bytes: consumed as u64,
|
|
||||||
bytes: text.into_bytes(),
|
|
||||||
truncated: false,
|
|
||||||
});
|
|
||||||
pending.drain(..consumed);
|
|
||||||
pending_offset = pending_offset.saturating_add(consumed as u64);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some((consumed, text)) =
|
|
||||||
redact_safe_live_log_prefix(&pending, &configured_secrets, true)
|
|
||||||
{
|
|
||||||
let _ = sender.try_send(LiveLogChunk {
|
|
||||||
stream,
|
|
||||||
offset: pending_offset,
|
|
||||||
source_bytes: consumed as u64,
|
|
||||||
bytes: text.into_bytes(),
|
|
||||||
truncated: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if source_bytes_read > maximum as u64 {
|
|
||||||
let _ = sender.try_send(LiveLogChunk {
|
|
||||||
stream,
|
|
||||||
offset: stream_base.saturating_add(source_bytes_read),
|
|
||||||
source_bytes: 0,
|
|
||||||
bytes: b"[log output truncated at node capture limit]".to_vec(),
|
|
||||||
truncated: true,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
Ok(captured)
|
Ok(captured)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
|
|
||||||
let args = self.args.clone();
|
|
||||||
let process = self.process.clone();
|
|
||||||
let task = self.task.clone();
|
|
||||||
let node_private_key = self.node_private_key.clone();
|
|
||||||
let configured_secrets = self.configured_secrets.clone();
|
|
||||||
let command_status = Arc::clone(&self.command_status);
|
|
||||||
thread::spawn(move || {
|
|
||||||
let mut log_session = None;
|
|
||||||
let mut delivery_available = true;
|
|
||||||
while let Ok(chunk) = receiver.recv() {
|
|
||||||
if !delivery_available {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
|
|
||||||
text = redact_configured_values(text, &configured_secrets);
|
|
||||||
let mut delivered = false;
|
|
||||||
for _ in 0..2 {
|
|
||||||
if log_session.is_none() {
|
|
||||||
log_session = CoordinatorSession::connect_with_timeouts(
|
|
||||||
&args.coordinator,
|
|
||||||
Duration::from_millis(500),
|
|
||||||
Duration::from_millis(500),
|
|
||||||
)
|
|
||||||
.ok();
|
|
||||||
}
|
|
||||||
let Some(session) = log_session.as_mut() else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
let request = signed_node_request_json(
|
|
||||||
&args,
|
|
||||||
&node_private_key,
|
|
||||||
"report_task_log_chunk",
|
|
||||||
serde_json::json!({
|
|
||||||
"type": "report_task_log_chunk",
|
|
||||||
"tenant": &args.tenant,
|
|
||||||
"project": &args.project,
|
|
||||||
"process": &process,
|
|
||||||
"node": &args.node,
|
|
||||||
"task": &task,
|
|
||||||
"stream": chunk.stream,
|
|
||||||
"offset": chunk.offset,
|
|
||||||
"source_bytes": chunk.source_bytes,
|
|
||||||
"text": &text,
|
|
||||||
"truncated": chunk.truncated,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
let result = match request {
|
|
||||||
Ok(request) => session
|
|
||||||
.request(request)
|
|
||||||
.map(|_| ())
|
|
||||||
.map_err(|error| error.to_string()),
|
|
||||||
Err(error) => Err(error.to_string()),
|
|
||||||
};
|
|
||||||
match result {
|
|
||||||
Ok(()) => {
|
|
||||||
delivered = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
log_session = None;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !delivered {
|
|
||||||
delivery_available = false;
|
|
||||||
if let Ok(mut current) = command_status.lock() {
|
|
||||||
*current = Some(
|
|
||||||
"live log delivery was interrupted; final bounded output remains available"
|
|
||||||
.to_owned(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ProcessRunner for CoordinatorControlledProcessRunner {
|
impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
|
|
@ -458,17 +245,12 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
command.program,
|
command.program,
|
||||||
command.args.join(" ")
|
command.args.join(" ")
|
||||||
));
|
));
|
||||||
let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
|
|
||||||
let stdout = Self::drain_bounded(
|
let stdout = Self::drain_bounded(
|
||||||
child
|
child
|
||||||
.stdout
|
.stdout
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
|
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
|
||||||
Self::MAX_CAPTURE_BYTES,
|
Self::MAX_CAPTURE_BYTES,
|
||||||
"stdout",
|
|
||||||
live_log_sender.clone(),
|
|
||||||
self.configured_secrets.clone(),
|
|
||||||
Arc::clone(&self.stdout_source_bytes),
|
|
||||||
);
|
);
|
||||||
let stderr = Self::drain_bounded(
|
let stderr = Self::drain_bounded(
|
||||||
child
|
child
|
||||||
|
|
@ -476,19 +258,11 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
.take()
|
.take()
|
||||||
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
|
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
|
||||||
Self::MAX_CAPTURE_BYTES,
|
Self::MAX_CAPTURE_BYTES,
|
||||||
"stderr",
|
|
||||||
live_log_sender,
|
|
||||||
self.configured_secrets.clone(),
|
|
||||||
Arc::clone(&self.stderr_source_bytes),
|
|
||||||
);
|
);
|
||||||
let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
|
|
||||||
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
|
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
|
||||||
Ok(session) => session,
|
Ok(session) => session,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||||
let _ = stdout.join();
|
|
||||||
let _ = stderr.join();
|
|
||||||
let _ = live_log_reporter.join();
|
|
||||||
return Err(BackendError::Command(format!(
|
return Err(BackendError::Command(format!(
|
||||||
"establish execution control channel: {error}"
|
"establish execution control channel: {error}"
|
||||||
)));
|
)));
|
||||||
|
|
@ -503,9 +277,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
Ok(None) => {}
|
Ok(None) => {}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||||
let _ = stdout.join();
|
|
||||||
let _ = stderr.join();
|
|
||||||
let _ = live_log_reporter.join();
|
|
||||||
return Err(BackendError::Command(error.to_string()));
|
return Err(BackendError::Command(error.to_string()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -518,7 +289,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||||
let _ = stdout.join();
|
let _ = stdout.join();
|
||||||
let _ = stderr.join();
|
let _ = stderr.join();
|
||||||
let _ = live_log_reporter.join();
|
|
||||||
return Err(BackendError::Command(format!(
|
return Err(BackendError::Command(format!(
|
||||||
"native command exceeded wall-clock timeout of {} ms",
|
"native command exceeded wall-clock timeout of {} ms",
|
||||||
self.timeout.as_millis()
|
self.timeout.as_millis()
|
||||||
|
|
@ -533,7 +303,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||||
let _ = stdout.join();
|
let _ = stdout.join();
|
||||||
let _ = stderr.join();
|
let _ = stderr.join();
|
||||||
let _ = live_log_reporter.join();
|
|
||||||
return Err(BackendError::Cancelled(
|
return Err(BackendError::Cancelled(
|
||||||
"coordinator requested cancellation or abort".to_owned(),
|
"coordinator requested cancellation or abort".to_owned(),
|
||||||
));
|
));
|
||||||
|
|
@ -543,7 +312,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||||
let _ = stdout.join();
|
let _ = stdout.join();
|
||||||
let _ = stderr.join();
|
let _ = stderr.join();
|
||||||
let _ = live_log_reporter.join();
|
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -592,9 +360,6 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
.join()
|
.join()
|
||||||
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
|
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
|
||||||
.map_err(BackendError::Command)?;
|
.map_err(BackendError::Command)?;
|
||||||
live_log_reporter
|
|
||||||
.join()
|
|
||||||
.map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
|
|
||||||
self.set_command_status(format!(
|
self.set_command_status(format!(
|
||||||
"native command exited with status {:?}",
|
"native command exited with status {:?}",
|
||||||
status.code()
|
status.code()
|
||||||
|
|
@ -606,52 +371,3 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner};
|
|
||||||
use std::io::Cursor;
|
|
||||||
use std::sync::atomic::{AtomicU64, Ordering};
|
|
||||||
use std::sync::{mpsc, Arc};
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
|
|
||||||
let secrets = vec!["correct-horse".to_owned()];
|
|
||||||
let mut pending = b"prefix correct-".to_vec();
|
|
||||||
let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
|
|
||||||
pending.drain(..consumed);
|
|
||||||
pending.extend_from_slice(b"horse suffix");
|
|
||||||
let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
|
|
||||||
|
|
||||||
let combined = format!("{first}{second}");
|
|
||||||
assert_eq!(combined, "prefix [REDACTED] suffix");
|
|
||||||
assert!(!combined.contains("correct-"));
|
|
||||||
assert!(!combined.contains("horse"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() {
|
|
||||||
let source_bytes = Arc::new(AtomicU64::new(5));
|
|
||||||
let (sender, receiver) = mpsc::sync_channel(8);
|
|
||||||
let reader = CoordinatorControlledProcessRunner::drain_bounded(
|
|
||||||
Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()),
|
|
||||||
8,
|
|
||||||
"stdout",
|
|
||||||
sender,
|
|
||||||
Vec::new(),
|
|
||||||
Arc::clone(&source_bytes),
|
|
||||||
);
|
|
||||||
let captured = reader.join().unwrap().unwrap();
|
|
||||||
let chunks = receiver.into_iter().collect::<Vec<_>>();
|
|
||||||
|
|
||||||
assert_eq!(captured, b"opqrstuv");
|
|
||||||
assert_eq!(source_bytes.load(Ordering::Relaxed), 37);
|
|
||||||
assert_eq!(
|
|
||||||
chunks.iter().map(|chunk| chunk.source_bytes).sum::<u64>(),
|
|
||||||
32
|
|
||||||
);
|
|
||||||
assert_eq!(chunks[0].offset, 5);
|
|
||||||
assert_eq!(chunks.last().unwrap().offset, 37);
|
|
||||||
assert!(chunks.iter().any(|chunk| chunk.truncated));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -48,10 +48,7 @@ fn test_controlled_runner(
|
||||||
),
|
),
|
||||||
debug_control: Arc::new(WasmDebugControl::default()),
|
debug_control: Arc::new(WasmDebugControl::default()),
|
||||||
command_status: Arc::new(Mutex::new(None)),
|
command_status: Arc::new(Mutex::new(None)),
|
||||||
stdout_source_bytes: Arc::new(AtomicU64::new(0)),
|
|
||||||
stderr_source_bytes: Arc::new(AtomicU64::new(0)),
|
|
||||||
timeout,
|
timeout,
|
||||||
configured_secrets: Vec::new(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -92,10 +89,7 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
|
||||||
),
|
),
|
||||||
debug_control: Arc::new(WasmDebugControl::default()),
|
debug_control: Arc::new(WasmDebugControl::default()),
|
||||||
command_status: Arc::new(Mutex::new(None)),
|
command_status: Arc::new(Mutex::new(None)),
|
||||||
stdout_source_bytes: Arc::new(AtomicU64::new(0)),
|
|
||||||
stderr_source_bytes: Arc::new(AtomicU64::new(0)),
|
|
||||||
timeout: Duration::from_secs(30),
|
timeout: Duration::from_secs(30),
|
||||||
configured_secrets: Vec::new(),
|
|
||||||
};
|
};
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
let error = runner
|
let error = runner
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
use clusterflux_core::{
|
use clusterflux_core::{
|
||||||
CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay,
|
CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
|
||||||
VfsPath,
|
VfsOverlay, VfsPath,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
|
@ -42,8 +42,6 @@ pub struct CommandOutput {
|
||||||
pub status_code: Option<i32>,
|
pub status_code: Option<i32>,
|
||||||
pub stdout: String,
|
pub stdout: String,
|
||||||
pub stderr: String,
|
pub stderr: String,
|
||||||
pub stdout_source_bytes: u64,
|
|
||||||
pub stderr_source_bytes: u64,
|
|
||||||
pub stdout_truncated: bool,
|
pub stdout_truncated: bool,
|
||||||
pub stderr_truncated: bool,
|
pub stderr_truncated: bool,
|
||||||
pub log_backpressured: bool,
|
pub log_backpressured: bool,
|
||||||
|
|
@ -85,8 +83,6 @@ impl LocalCommandExecutor {
|
||||||
&output.stderr,
|
&output.stderr,
|
||||||
max_log_bytes,
|
max_log_bytes,
|
||||||
);
|
);
|
||||||
let stdout_source_bytes = output.stdout.len() as u64;
|
|
||||||
let stderr_source_bytes = output.stderr.len() as u64;
|
|
||||||
let staged_artifact = if let Some(path) = command.stage_stdout_as {
|
let staged_artifact = if let Some(path) = command.stage_stdout_as {
|
||||||
Some(overlay.write(
|
Some(overlay.write(
|
||||||
path,
|
path,
|
||||||
|
|
@ -102,8 +98,6 @@ impl LocalCommandExecutor {
|
||||||
status_code: output.status.code(),
|
status_code: output.status.code(),
|
||||||
stdout: logs.stdout,
|
stdout: logs.stdout,
|
||||||
stderr: logs.stderr,
|
stderr: logs.stderr,
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
stdout_truncated: logs.stdout_truncated,
|
stdout_truncated: logs.stdout_truncated,
|
||||||
stderr_truncated: logs.stderr_truncated,
|
stderr_truncated: logs.stderr_truncated,
|
||||||
log_backpressured: logs.backpressured,
|
log_backpressured: logs.backpressured,
|
||||||
|
|
@ -113,20 +107,24 @@ impl LocalCommandExecutor {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn capture_command_logs(
|
pub(super) fn capture_command_logs(
|
||||||
_task: &TaskInstanceId,
|
task: &TaskInstanceId,
|
||||||
stdout: &[u8],
|
stdout: &[u8],
|
||||||
stderr: &[u8],
|
stderr: &[u8],
|
||||||
max_log_bytes: usize,
|
max_log_bytes: usize,
|
||||||
) -> CapturedCommandLogs {
|
) -> CapturedCommandLogs {
|
||||||
let stdout_truncated = stdout.len() > max_log_bytes;
|
let mut logs = LogBuffer::new(max_log_bytes);
|
||||||
let stderr_truncated = stderr.len() > max_log_bytes;
|
logs.push(task.clone(), stdout);
|
||||||
let stdout_start = stdout.len().saturating_sub(max_log_bytes);
|
logs.push(task.clone(), stderr);
|
||||||
let stderr_start = stderr.len().saturating_sub(max_log_bytes);
|
let records = logs.records();
|
||||||
|
let stdout_record = &records[0];
|
||||||
|
let stderr_record = &records[1];
|
||||||
|
debug_assert_eq!(&stdout_record.task, task);
|
||||||
|
debug_assert_eq!(&stderr_record.task, task);
|
||||||
CapturedCommandLogs {
|
CapturedCommandLogs {
|
||||||
stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(),
|
stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
|
||||||
stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(),
|
stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
|
||||||
stdout_truncated,
|
stdout_truncated: stdout_record.truncated,
|
||||||
stderr_truncated,
|
stderr_truncated: stderr_record.truncated,
|
||||||
backpressured: stdout_truncated || stderr_truncated,
|
backpressured: logs.backpressured(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ use clusterflux_control::endpoint_identity;
|
||||||
use clusterflux_control::ControlSession;
|
use clusterflux_control::ControlSession;
|
||||||
use clusterflux_core::coordinator_wire_request;
|
use clusterflux_core::coordinator_wire_request;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
pub(crate) struct CoordinatorSession {
|
pub(crate) struct CoordinatorSession {
|
||||||
inner: ControlSession,
|
inner: ControlSession,
|
||||||
|
|
@ -16,16 +15,6 @@ impl CoordinatorSession {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn connect_with_timeouts(
|
|
||||||
addr: &str,
|
|
||||||
connect_timeout: Duration,
|
|
||||||
io_timeout: Duration,
|
|
||||||
) -> Result<Self, Box<dyn std::error::Error>> {
|
|
||||||
Ok(Self {
|
|
||||||
inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
|
pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
|
||||||
let request_id = format!("node-{}", self.inner.requests() + 1);
|
let request_id = format!("node-{}", self.inner.requests() + 1);
|
||||||
let wire_request = coordinator_wire_request(request_id, value);
|
let wire_request = coordinator_wire_request(request_id, value);
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ use clusterflux_core::{
|
||||||
};
|
};
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment};
|
use crate::assignment_runner::run_verified_wasmtime_assignment;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::coordinator_session::control_endpoint_identity;
|
use crate::coordinator_session::control_endpoint_identity;
|
||||||
use crate::coordinator_session::CoordinatorSession;
|
use crate::coordinator_session::CoordinatorSession;
|
||||||
|
|
@ -464,8 +464,6 @@ fn run_runtime_task(
|
||||||
capability_report,
|
capability_report,
|
||||||
debug_command,
|
debug_command,
|
||||||
node_private_key,
|
node_private_key,
|
||||||
0,
|
|
||||||
0,
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -500,13 +498,9 @@ fn run_runtime_task(
|
||||||
debug_command,
|
debug_command,
|
||||||
node_private_key,
|
node_private_key,
|
||||||
&error,
|
&error,
|
||||||
output.stdout_source_bytes,
|
|
||||||
output.stderr_source_bytes,
|
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
let (stdout_source_bytes, stderr_source_bytes) =
|
|
||||||
assignment_error_log_bytes(error.as_ref());
|
|
||||||
let error = error.to_string();
|
let error = error.to_string();
|
||||||
if error.contains("task execution cancelled:") {
|
if error.contains("task execution cancelled:") {
|
||||||
record_cancelled_task(
|
record_cancelled_task(
|
||||||
|
|
@ -518,8 +512,6 @@ fn run_runtime_task(
|
||||||
capability_report,
|
capability_report,
|
||||||
debug_command,
|
debug_command,
|
||||||
node_private_key,
|
node_private_key,
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
record_failed_task(
|
record_failed_task(
|
||||||
|
|
@ -532,8 +524,6 @@ fn run_runtime_task(
|
||||||
debug_command,
|
debug_command,
|
||||||
node_private_key,
|
node_private_key,
|
||||||
&error,
|
&error,
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -644,12 +634,13 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn hosted_url_remains_an_https_control_endpoint() {
|
fn hosted_url_remains_an_https_control_endpoint() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(),
|
control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
|
||||||
"https://clusterflux.lesstuff.com/api/v1/control"
|
"https://clusterflux.michelpaulissen.com/api/v1/control"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
|
control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
|
||||||
"https://clusterflux.lesstuff.com/api/v1/control"
|
.unwrap(),
|
||||||
|
"https://clusterflux.michelpaulissen.com/api/v1/control"
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
control_endpoint_identity("127.0.0.1:7999").unwrap(),
|
control_endpoint_identity("127.0.0.1:7999").unwrap(),
|
||||||
|
|
|
||||||
|
|
@ -741,7 +741,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() {
|
fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() {
|
||||||
let invocation = CommandInvocation {
|
let invocation = CommandInvocation {
|
||||||
program: "cargo".to_owned(),
|
program: "cargo".to_owned(),
|
||||||
args: vec!["build".to_owned()],
|
args: vec!["build".to_owned()],
|
||||||
|
|
@ -774,7 +774,7 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
|
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
|
||||||
assert_eq!(output.stdout, "cdef");
|
assert_eq!(output.stdout, "abcd");
|
||||||
assert!(output.stdout_truncated);
|
assert!(output.stdout_truncated);
|
||||||
assert!(output.log_backpressured);
|
assert!(output.log_backpressured);
|
||||||
assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
|
assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
|
||||||
|
|
@ -992,7 +992,7 @@ mod tests {
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[test]
|
#[test]
|
||||||
fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() {
|
fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() {
|
||||||
let executor = LocalCommandExecutor {
|
let executor = LocalCommandExecutor {
|
||||||
node: clusterflux_core::NodeId::from("node"),
|
node: clusterflux_core::NodeId::from("node"),
|
||||||
hosted_control_plane: false,
|
hosted_control_plane: false,
|
||||||
|
|
@ -1021,15 +1021,15 @@ mod tests {
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
|
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
|
||||||
assert_eq!(output.stdout, "cdef");
|
assert_eq!(output.stdout, "abcd");
|
||||||
assert_eq!(output.stderr, "err");
|
assert_eq!(output.stderr, "");
|
||||||
assert!(output.stdout_truncated);
|
assert!(output.stdout_truncated);
|
||||||
assert!(!output.stderr_truncated);
|
assert!(output.stderr_truncated);
|
||||||
assert!(output.log_backpressured);
|
assert!(output.log_backpressured);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn public_node_crate_does_not_require_hosted_service_types() {
|
fn public_node_crate_does_not_require_hosted_private_types() {
|
||||||
let _tenant = TenantId::from("tenant");
|
let _tenant = TenantId::from("tenant");
|
||||||
let _project = ProjectId::from("project");
|
let _project = ProjectId::from("project");
|
||||||
let _backend = LinuxRootlessPodmanBackend;
|
let _backend = LinuxRootlessPodmanBackend;
|
||||||
|
|
|
||||||
|
|
@ -8,34 +8,5 @@ mod task_artifacts;
|
||||||
mod task_reports;
|
mod task_reports;
|
||||||
|
|
||||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
|
|
||||||
match raw_args.as_slice() {
|
|
||||||
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
|
|
||||||
println!("clusterflux-node {}", env!("CARGO_PKG_VERSION"));
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
|
|
||||||
println!(
|
|
||||||
"Clusterflux node worker.\n\n\
|
|
||||||
Usage: clusterflux-node --coordinator <URL> [OPTIONS]\n\n\
|
|
||||||
Options:\n \
|
|
||||||
--coordinator <URL>\n \
|
|
||||||
--tenant <TENANT> [default: tenant]\n \
|
|
||||||
--project-id <PROJECT> [default: project]\n \
|
|
||||||
--node <NODE> [default: node]\n \
|
|
||||||
--project-root <PATH>\n \
|
|
||||||
--enrollment-grant <GRANT>\n \
|
|
||||||
--public-key <KEY>\n \
|
|
||||||
--worker\n \
|
|
||||||
--emit-ready\n \
|
|
||||||
--control-poll-ms <MILLISECONDS>\n \
|
|
||||||
--assignment-poll-ms <MILLISECONDS> [default: 500]\n \
|
|
||||||
-h, --help\n \
|
|
||||||
-V, --version"
|
|
||||||
);
|
|
||||||
return Ok(());
|
|
||||||
}
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
daemon::run()
|
daemon::run()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -47,8 +47,8 @@ pub(crate) fn record_completed_task(
|
||||||
"process": &task.process,
|
"process": &task.process,
|
||||||
"node": &args.node,
|
"node": &args.node,
|
||||||
"task": &task.task,
|
"task": &task.task,
|
||||||
"stdout_bytes": output.stdout_source_bytes,
|
"stdout_bytes": output.stdout.len(),
|
||||||
"stderr_bytes": output.stderr_source_bytes,
|
"stderr_bytes": output.stderr.len(),
|
||||||
"stdout_tail": &output.stdout,
|
"stdout_tail": &output.stdout,
|
||||||
"stderr_tail": &output.stderr,
|
"stderr_tail": &output.stderr,
|
||||||
"stdout_truncated": output.stdout_truncated,
|
"stdout_truncated": output.stdout_truncated,
|
||||||
|
|
@ -85,8 +85,8 @@ pub(crate) fn record_completed_task(
|
||||||
"node": &args.node,
|
"node": &args.node,
|
||||||
"task": &task.task,
|
"task": &task.task,
|
||||||
"status_code": output.status_code,
|
"status_code": output.status_code,
|
||||||
"stdout_bytes": output.stdout_source_bytes,
|
"stdout_bytes": output.stdout.len(),
|
||||||
"stderr_bytes": output.stderr_source_bytes,
|
"stderr_bytes": output.stderr.len(),
|
||||||
"stdout_tail": &output.stdout,
|
"stdout_tail": &output.stdout,
|
||||||
"stderr_tail": &output.stderr,
|
"stderr_tail": &output.stderr,
|
||||||
"stdout_truncated": output.stdout_truncated,
|
"stdout_truncated": output.stdout_truncated,
|
||||||
|
|
@ -123,11 +123,8 @@ pub(crate) fn record_failed_task(
|
||||||
debug_command: Value,
|
debug_command: Value,
|
||||||
node_private_key: &str,
|
node_private_key: &str,
|
||||||
error: &str,
|
error: &str,
|
||||||
stdout_source_bytes: u64,
|
|
||||||
command_stderr_source_bytes: u64,
|
|
||||||
) -> Result<Value, Box<dyn std::error::Error>> {
|
) -> Result<Value, Box<dyn std::error::Error>> {
|
||||||
let error = bounded_runtime_error(error);
|
let error = bounded_runtime_error(error);
|
||||||
let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64);
|
|
||||||
let log_event = session.request(signed_node_request_json(
|
let log_event = session.request(signed_node_request_json(
|
||||||
args,
|
args,
|
||||||
node_private_key,
|
node_private_key,
|
||||||
|
|
@ -139,8 +136,8 @@ pub(crate) fn record_failed_task(
|
||||||
"process": &task.process,
|
"process": &task.process,
|
||||||
"node": &args.node,
|
"node": &args.node,
|
||||||
"task": &task.task,
|
"task": &task.task,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": 0,
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": error.len(),
|
||||||
"stdout_tail": "",
|
"stdout_tail": "",
|
||||||
"stderr_tail": &error,
|
"stderr_tail": &error,
|
||||||
"stdout_truncated": false,
|
"stdout_truncated": false,
|
||||||
|
|
@ -178,8 +175,8 @@ pub(crate) fn record_failed_task(
|
||||||
"task": &task.task,
|
"task": &task.task,
|
||||||
"terminal_state": "failed",
|
"terminal_state": "failed",
|
||||||
"status_code": -1,
|
"status_code": -1,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": 0,
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": error.len(),
|
||||||
"stdout_tail": "",
|
"stdout_tail": "",
|
||||||
"stderr_tail": &error,
|
"stderr_tail": &error,
|
||||||
"stdout_truncated": false,
|
"stdout_truncated": false,
|
||||||
|
|
@ -192,8 +189,6 @@ pub(crate) fn record_failed_task(
|
||||||
Ok(failed_node_report(
|
Ok(failed_node_report(
|
||||||
task,
|
task,
|
||||||
&error,
|
&error,
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
registration,
|
registration,
|
||||||
heartbeat,
|
heartbeat,
|
||||||
capability_report,
|
capability_report,
|
||||||
|
|
@ -215,8 +210,6 @@ pub(crate) fn record_cancelled_task(
|
||||||
capability_report: Value,
|
capability_report: Value,
|
||||||
debug_command: Value,
|
debug_command: Value,
|
||||||
node_private_key: &str,
|
node_private_key: &str,
|
||||||
stdout_source_bytes: u64,
|
|
||||||
stderr_source_bytes: u64,
|
|
||||||
) -> Result<Value, Box<dyn std::error::Error>> {
|
) -> Result<Value, Box<dyn std::error::Error>> {
|
||||||
let recorded = session.request(signed_node_request_json(
|
let recorded = session.request(signed_node_request_json(
|
||||||
args,
|
args,
|
||||||
|
|
@ -231,12 +224,12 @@ pub(crate) fn record_cancelled_task(
|
||||||
"task": &task.task,
|
"task": &task.task,
|
||||||
"terminal_state": "cancelled",
|
"terminal_state": "cancelled",
|
||||||
"status_code": null,
|
"status_code": null,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": 0,
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": 0,
|
||||||
"stdout_tail": "",
|
"stdout_tail": "",
|
||||||
"stderr_tail": "",
|
"stderr_tail": "",
|
||||||
"stdout_truncated": stdout_source_bytes > 0,
|
"stdout_truncated": false,
|
||||||
"stderr_truncated": stderr_source_bytes > 0,
|
"stderr_truncated": false,
|
||||||
"artifact_path": null,
|
"artifact_path": null,
|
||||||
"artifact_digest": null,
|
"artifact_digest": null,
|
||||||
"artifact_size_bytes": null,
|
"artifact_size_bytes": null,
|
||||||
|
|
@ -245,8 +238,6 @@ pub(crate) fn record_cancelled_task(
|
||||||
)?)?;
|
)?)?;
|
||||||
Ok(cancelled_node_report(
|
Ok(cancelled_node_report(
|
||||||
task,
|
task,
|
||||||
stdout_source_bytes,
|
|
||||||
stderr_source_bytes,
|
|
||||||
registration,
|
registration,
|
||||||
heartbeat,
|
heartbeat,
|
||||||
capability_report,
|
capability_report,
|
||||||
|
|
@ -290,8 +281,8 @@ pub(crate) fn completed_node_report(
|
||||||
"virtual_thread": output.virtual_thread,
|
"virtual_thread": output.virtual_thread,
|
||||||
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
|
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
|
||||||
"status_code": output.status_code,
|
"status_code": output.status_code,
|
||||||
"stdout_bytes": output.stdout_source_bytes,
|
"stdout_bytes": output.stdout.len(),
|
||||||
"stderr_bytes": output.stderr_source_bytes,
|
"stderr_bytes": output.stderr.len(),
|
||||||
"stdout_tail": &output.stdout,
|
"stdout_tail": &output.stdout,
|
||||||
"stderr_tail": &output.stderr,
|
"stderr_tail": &output.stderr,
|
||||||
"stdout_truncated": output.stdout_truncated,
|
"stdout_truncated": output.stdout_truncated,
|
||||||
|
|
@ -314,8 +305,6 @@ pub(crate) fn completed_node_report(
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
pub(crate) fn cancelled_node_report(
|
pub(crate) fn cancelled_node_report(
|
||||||
task: &RuntimeTask,
|
task: &RuntimeTask,
|
||||||
stdout_source_bytes: u64,
|
|
||||||
stderr_source_bytes: u64,
|
|
||||||
registration_response: Value,
|
registration_response: Value,
|
||||||
heartbeat_response: Value,
|
heartbeat_response: Value,
|
||||||
capability_response: Value,
|
capability_response: Value,
|
||||||
|
|
@ -329,12 +318,12 @@ pub(crate) fn cancelled_node_report(
|
||||||
"virtual_thread": &task.task,
|
"virtual_thread": &task.task,
|
||||||
"terminal_state": "cancelled",
|
"terminal_state": "cancelled",
|
||||||
"status_code": null,
|
"status_code": null,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": 0,
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": 0,
|
||||||
"stdout_tail": "",
|
"stdout_tail": "",
|
||||||
"stderr_tail": "",
|
"stderr_tail": "",
|
||||||
"stdout_truncated": stdout_source_bytes > 0,
|
"stdout_truncated": false,
|
||||||
"stderr_truncated": stderr_source_bytes > 0,
|
"stderr_truncated": false,
|
||||||
"log_backpressured": false,
|
"log_backpressured": false,
|
||||||
"staged_artifact": null,
|
"staged_artifact": null,
|
||||||
"large_bytes_uploaded": false,
|
"large_bytes_uploaded": false,
|
||||||
|
|
@ -354,8 +343,6 @@ pub(crate) fn cancelled_node_report(
|
||||||
pub(crate) fn failed_node_report(
|
pub(crate) fn failed_node_report(
|
||||||
task: &RuntimeTask,
|
task: &RuntimeTask,
|
||||||
error: &str,
|
error: &str,
|
||||||
stdout_source_bytes: u64,
|
|
||||||
stderr_source_bytes: u64,
|
|
||||||
registration_response: Value,
|
registration_response: Value,
|
||||||
heartbeat_response: Value,
|
heartbeat_response: Value,
|
||||||
capability_response: Value,
|
capability_response: Value,
|
||||||
|
|
@ -370,8 +357,8 @@ pub(crate) fn failed_node_report(
|
||||||
"virtual_thread": &task.task,
|
"virtual_thread": &task.task,
|
||||||
"terminal_state": "failed",
|
"terminal_state": "failed",
|
||||||
"status_code": -1,
|
"status_code": -1,
|
||||||
"stdout_bytes": stdout_source_bytes,
|
"stdout_bytes": 0,
|
||||||
"stderr_bytes": stderr_source_bytes,
|
"stderr_bytes": error.len(),
|
||||||
"stdout_tail": "",
|
"stdout_tail": "",
|
||||||
"stderr_tail": error,
|
"stderr_tail": error,
|
||||||
"stdout_truncated": false,
|
"stdout_truncated": false,
|
||||||
|
|
@ -390,40 +377,3 @@ pub(crate) fn failed_node_report(
|
||||||
"coordinator_response": coordinator_response,
|
"coordinator_response": coordinator_response,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
use clusterflux_core::TaskInstanceId;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() {
|
|
||||||
let report = completed_node_report(
|
|
||||||
CommandOutput {
|
|
||||||
virtual_thread: TaskInstanceId::from("task"),
|
|
||||||
status_code: Some(0),
|
|
||||||
stdout: "bounded tail".to_owned(),
|
|
||||||
stderr: String::new(),
|
|
||||||
stdout_source_bytes: 519,
|
|
||||||
stderr_source_bytes: 0,
|
|
||||||
stdout_truncated: true,
|
|
||||||
stderr_truncated: false,
|
|
||||||
log_backpressured: false,
|
|
||||||
staged_artifact: None,
|
|
||||||
},
|
|
||||||
false,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
Value::Null,
|
|
||||||
1,
|
|
||||||
);
|
|
||||||
|
|
||||||
assert_eq!(report["stdout_bytes"], 519);
|
|
||||||
assert_eq!(report["stdout_tail"], "bounded tail");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context.
|
||||||
The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
|
The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
|
||||||
transfers. The CLI verifies the final digest. Hosted policy may impose
|
transfers. The CLI verifies the final digest. Hosted policy may impose
|
||||||
per-project, per-tenant, and per-account size, concurrency, and period limits.
|
per-project, per-tenant, and per-account size, concurrency, and period limits.
|
||||||
There is no hosted global circuit breaker, and one tenant's period usage does
|
Operators may disable relay traffic as an emergency safety control; one tenant's
|
||||||
not consume a shared customer byte quota.
|
period usage does not consume a shared customer byte quota.
|
||||||
|
|
|
||||||
60
docs/contributing/releases.md
Normal file
60
docs/contributing/releases.md
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
# Release candidates
|
||||||
|
|
||||||
|
This is a contributor and release-engineering procedure, not an end-user setup
|
||||||
|
path. Publication is a three-stage transaction:
|
||||||
|
|
||||||
|
1. `candidate` builds immutable archives and a manifest with paths relative to
|
||||||
|
the manifest directory.
|
||||||
|
2. `live-test` downloads that exact candidate in a clean job, deploys it, and
|
||||||
|
records the full named production-shaped acceptance result plus deployment,
|
||||||
|
runtime configuration, and proxy configuration identities.
|
||||||
|
3. `final` downloads the candidate and evidence in another clean job, verifies
|
||||||
|
every binding, and publishes without rebuilding any binary.
|
||||||
|
|
||||||
|
Set `CLUSTERFLUX_RELEASE_STAGE=candidate` while creating the candidate and
|
||||||
|
`CLUSTERFLUX_RELEASE_STAGE=final` while finalizing it. The final stage requires
|
||||||
|
`CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST` and complete live evidence. There is no
|
||||||
|
incomplete-evidence publication override.
|
||||||
|
|
||||||
|
The `clusterflux-release` runner must provide `CLUSTERFLUX_DEPLOY_COMMAND` as a
|
||||||
|
protected secret. The command runs locally with
|
||||||
|
`CLUSTERFLUX_CANDIDATE_ARCHIVE`, `CLUSTERFLUX_CANDIDATE_COORDINATOR`, and their
|
||||||
|
SHA-256 identities exported. It must deploy that executable and restart the
|
||||||
|
configured service. `scripts/deploy-release-candidate.sh` then independently
|
||||||
|
compares the running `/proc/<MainPID>/exe` digest with the candidate and records
|
||||||
|
the service and proxy unit identities; a mismatch stops the release.
|
||||||
|
|
||||||
|
Clusterflux release binaries are built once. The public client/node archive and
|
||||||
|
the private-source hosted-service archive are both digest-bound to the same
|
||||||
|
candidate. The hosted archive is deployed, the strict production-shaped batch
|
||||||
|
uses the public archive against it, and finalization copies both archives
|
||||||
|
without rebuilding.
|
||||||
|
|
||||||
|
Create the candidate in a dedicated directory:
|
||||||
|
|
||||||
|
~~~bash
|
||||||
|
CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/release-candidate \
|
||||||
|
CLUSTERFLUX_RELEASE_STAGE=candidate \
|
||||||
|
./scripts/prepare-public-release.js
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Deploy `target/release-candidate/assets/clusterflux-public-binaries-*.tar.gz`.
|
||||||
|
Set `CLUSTERFLUX_PUBLIC_RELEASE_MANIFEST` to the candidate manifest while
|
||||||
|
running the strict batch. Set `CLUSTERFLUX_QUALITY_GATE_EVIDENCE_PATH` to the
|
||||||
|
JSON record from the private and public acceptance commands. The result records
|
||||||
|
the source commit, source-tree and
|
||||||
|
public-tree identities, candidate binary digests, deployment generation, and
|
||||||
|
configuration identity.
|
||||||
|
|
||||||
|
Finalize into a different directory after the strict result passes:
|
||||||
|
|
||||||
|
~~~bash
|
||||||
|
CLUSTERFLUX_PUBLIC_RELEASE_DIR=target/public-release \
|
||||||
|
CLUSTERFLUX_RELEASE_CANDIDATE_MANIFEST=target/release-candidate/public-release-manifest.json \
|
||||||
|
CLUSTERFLUX_FINAL_RESULT_PATH=target/acceptance/cli-happy-path-live.json \
|
||||||
|
./scripts/prepare-public-release.js
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Finalization rejects a changed commit, source tree, public tree, candidate
|
||||||
|
archive, binary digest set, deployment binding, or strict result. It never runs
|
||||||
|
the release binary build when a candidate manifest is supplied.
|
||||||
|
|
@ -11,9 +11,6 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node
|
||||||
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
|
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
|
||||||
~~~
|
~~~
|
||||||
|
|
||||||
On NixOS or another system with Nix, the equivalent package is available with
|
|
||||||
`nix profile install .#clusterflux-tools`.
|
|
||||||
|
|
||||||
Install rootless Podman on each Linux node that will execute container-backed
|
Install rootless Podman on each Linux node that will execute container-backed
|
||||||
environments.
|
environments.
|
||||||
|
|
||||||
|
|
@ -55,7 +52,7 @@ same stored identity:
|
||||||
|
|
||||||
~~~bash
|
~~~bash
|
||||||
clusterflux-node \
|
clusterflux-node \
|
||||||
--coordinator https://clusterflux.lesstuff.com \
|
--coordinator https://clusterflux.michelpaulissen.com \
|
||||||
--tenant "$TENANT" \
|
--tenant "$TENANT" \
|
||||||
--project-id <hosted-project-id> \
|
--project-id <hosted-project-id> \
|
||||||
--node workstation \
|
--node workstation \
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ key pair.
|
||||||
|
|
||||||
~~~bash
|
~~~bash
|
||||||
clusterflux-node \
|
clusterflux-node \
|
||||||
--coordinator https://clusterflux.lesstuff.com \
|
--coordinator https://clusterflux.michelpaulissen.com \
|
||||||
--tenant "$TENANT" \
|
--tenant "$TENANT" \
|
||||||
--project-id <project-id> \
|
--project-id <project-id> \
|
||||||
--node workstation \
|
--node workstation \
|
||||||
|
|
|
||||||
12
flake.nix
12
flake.nix
|
|
@ -9,18 +9,6 @@
|
||||||
forAllSystems = nixpkgs.lib.genAttrs systems;
|
forAllSystems = nixpkgs.lib.genAttrs systems;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
packages = forAllSystems (system:
|
|
||||||
let
|
|
||||||
pkgs = import nixpkgs { inherit system; };
|
|
||||||
publicPackages = import ./packages.nix { inherit pkgs self; };
|
|
||||||
privatePackages =
|
|
||||||
if builtins.pathExists ./web/packages.nix then
|
|
||||||
import ./web/packages.nix { inherit pkgs self; }
|
|
||||||
else
|
|
||||||
{ };
|
|
||||||
in
|
|
||||||
publicPackages // privatePackages);
|
|
||||||
|
|
||||||
devShells = forAllSystems (system:
|
devShells = forAllSystems (system:
|
||||||
let
|
let
|
||||||
pkgs = import nixpkgs { inherit system; };
|
pkgs = import nixpkgs { inherit system; };
|
||||||
|
|
|
||||||
74
packages.nix
74
packages.nix
|
|
@ -1,74 +0,0 @@
|
||||||
{ pkgs, self }:
|
|
||||||
let
|
|
||||||
clusterflux-tools = pkgs.rustPlatform.buildRustPackage {
|
|
||||||
pname = "clusterflux-tools";
|
|
||||||
version = "0.1.0";
|
|
||||||
src = self;
|
|
||||||
cargoLock.lockFile = ./Cargo.lock;
|
|
||||||
nativeBuildInputs = [
|
|
||||||
pkgs.git
|
|
||||||
pkgs.lld
|
|
||||||
pkgs.makeWrapper
|
|
||||||
];
|
|
||||||
cargoBuildFlags = [
|
|
||||||
"--package"
|
|
||||||
"clusterflux-cli"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-node"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-coordinator"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-dap"
|
|
||||||
];
|
|
||||||
cargoTestFlags = [
|
|
||||||
"--package"
|
|
||||||
"clusterflux-cli"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-node"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-coordinator"
|
|
||||||
"--package"
|
|
||||||
"clusterflux-dap"
|
|
||||||
];
|
|
||||||
NIX_BUILD_CORES = "2";
|
|
||||||
RUST_MIN_STACK = "1073741824";
|
|
||||||
postInstall = ''
|
|
||||||
test -x "$out/bin/clusterflux"
|
|
||||||
test -x "$out/bin/clusterflux-node"
|
|
||||||
test -x "$out/bin/clusterflux-coordinator"
|
|
||||||
test -x "$out/bin/clusterflux-debug-dap"
|
|
||||||
for command in \
|
|
||||||
clusterflux \
|
|
||||||
clusterflux-node \
|
|
||||||
clusterflux-coordinator \
|
|
||||||
clusterflux-debug-dap
|
|
||||||
do
|
|
||||||
${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --version >/dev/null
|
|
||||||
${pkgs.coreutils}/bin/timeout 5 "$out/bin/$command" --help >/dev/null
|
|
||||||
done
|
|
||||||
'';
|
|
||||||
postFixup =
|
|
||||||
let
|
|
||||||
runtimePath = pkgs.lib.makeBinPath [
|
|
||||||
pkgs.cargo
|
|
||||||
pkgs.git
|
|
||||||
pkgs.lld
|
|
||||||
pkgs.rustc
|
|
||||||
];
|
|
||||||
in
|
|
||||||
''
|
|
||||||
wrapProgram "$out/bin/clusterflux" --prefix PATH : ${runtimePath}
|
|
||||||
wrapProgram "$out/bin/clusterflux-node" --prefix PATH : ${runtimePath}
|
|
||||||
wrapProgram "$out/bin/clusterflux-debug-dap" --prefix PATH : ${runtimePath}
|
|
||||||
'';
|
|
||||||
meta = {
|
|
||||||
description = "Clusterflux CLI, node, coordinator, and debugger adapter";
|
|
||||||
mainProgram = "clusterflux";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
in
|
|
||||||
{
|
|
||||||
inherit clusterflux-tools;
|
|
||||||
clusterflux = clusterflux-tools;
|
|
||||||
default = clusterflux-tools;
|
|
||||||
}
|
|
||||||
27
scripts/acceptance-private.sh
Executable file
27
scripts/acceptance-private.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repo"
|
||||||
|
|
||||||
|
if [[ ! -d private/hosted-policy ]]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
scripts/check-old-name.sh
|
||||||
|
node scripts/check-docs.js
|
||||||
|
scripts/check-code-size.sh
|
||||||
|
cargo fmt --all --manifest-path private/hosted-policy/Cargo.toml --check
|
||||||
|
cargo clippy --manifest-path private/hosted-policy/Cargo.toml --all-targets -- -D warnings
|
||||||
|
cargo test --locked --manifest-path private/hosted-policy/Cargo.toml --all-targets
|
||||||
|
node private/hosted-policy/scripts/prepare-hosted-deployment.js
|
||||||
|
if command -v podman >/dev/null 2>&1; then
|
||||||
|
node private/hosted-policy/scripts/postgres-durable-smoke.js
|
||||||
|
elif command -v nix >/dev/null 2>&1; then
|
||||||
|
nix shell nixpkgs#podman --command node private/hosted-policy/scripts/postgres-durable-smoke.js
|
||||||
|
else
|
||||||
|
node private/hosted-policy/scripts/postgres-durable-smoke.js
|
||||||
|
fi
|
||||||
|
if [[ -n "${CLUSTERFLUX_PUBLIC_RELEASE_SERVICE_ADDR:-}" ]]; then
|
||||||
|
node private/hosted-policy/scripts/hosted-service-live-check.js
|
||||||
|
fi
|
||||||
54
scripts/acceptance-public.sh
Executable file
54
scripts/acceptance-public.sh
Executable file
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repo"
|
||||||
|
|
||||||
|
scripts/check-old-name.sh
|
||||||
|
node scripts/check-docs.js
|
||||||
|
scripts/check-code-size.sh
|
||||||
|
scripts/release-source-scan.sh
|
||||||
|
cargo fmt --all --check
|
||||||
|
cargo clippy --workspace --all-targets -- -D warnings
|
||||||
|
cargo test --workspace --all-targets
|
||||||
|
cargo build --workspace --all-targets
|
||||||
|
cargo build -p runtime-conformance --target wasm32-unknown-unknown
|
||||||
|
cargo build -p hello-build --target wasm32-unknown-unknown
|
||||||
|
cargo build -p recovery-build --target wasm32-unknown-unknown
|
||||||
|
node scripts/resource-metering-contract-smoke.js
|
||||||
|
node scripts/hostile-input-contract-smoke.js
|
||||||
|
node scripts/tenant-isolation-contract-smoke.js
|
||||||
|
node scripts/self-hosted-coordinator-smoke.js
|
||||||
|
node scripts/public-local-demo-matrix-smoke.js
|
||||||
|
node scripts/cli-output-mode-smoke.js
|
||||||
|
node scripts/cli-login-smoke.js
|
||||||
|
node scripts/cli-error-exit-smoke.js
|
||||||
|
node scripts/cli-browser-login-flow-smoke.js
|
||||||
|
node scripts/cli-install-smoke.js
|
||||||
|
node scripts/user-session-token-boundary-smoke.js
|
||||||
|
node scripts/sdk-spawn-runtime-smoke.js
|
||||||
|
node scripts/node-lifecycle-contract-smoke.js
|
||||||
|
node scripts/wasmtime-node-smoke.js
|
||||||
|
node scripts/wasmtime-assignment-smoke.js
|
||||||
|
if command -v podman >/dev/null 2>&1; then
|
||||||
|
node scripts/podman-backend-smoke.js
|
||||||
|
elif command -v nix >/dev/null 2>&1; then
|
||||||
|
nix shell nixpkgs#podman --command node scripts/podman-backend-smoke.js
|
||||||
|
else
|
||||||
|
node scripts/podman-backend-smoke.js
|
||||||
|
fi
|
||||||
|
node scripts/vscode-extension-smoke.js
|
||||||
|
node scripts/vscode-f5-smoke.js
|
||||||
|
node scripts/node-attach-smoke.js
|
||||||
|
node scripts/cli-local-run-smoke.js
|
||||||
|
node scripts/artifact-download-smoke.js
|
||||||
|
node scripts/artifact-export-smoke.js
|
||||||
|
node scripts/operator-panel-smoke.js
|
||||||
|
node scripts/source-preparation-smoke.js
|
||||||
|
node scripts/scheduler-placement-smoke.js
|
||||||
|
node scripts/windows-best-effort-smoke.js
|
||||||
|
node scripts/quic-smoke.js
|
||||||
|
node scripts/dap-smoke.js
|
||||||
|
node scripts/recovery-build-smoke.js
|
||||||
|
node scripts/flagship-demo-smoke.js
|
||||||
|
scripts/verify-public-split.sh
|
||||||
99
scripts/agent-signing.js
Normal file
99
scripts/agent-signing.js
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
const crypto = require("crypto");
|
||||||
|
|
||||||
|
const { nodeIdentity, signedRequestPayloadDigest } = require("./node-signing");
|
||||||
|
|
||||||
|
function agentIdentity(seedPrefix, agent) {
|
||||||
|
const identity = nodeIdentity(seedPrefix, agent);
|
||||||
|
return {
|
||||||
|
...identity,
|
||||||
|
publicKeyFingerprint: `sha256:${crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(identity.publicKey)
|
||||||
|
.digest("hex")}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function agentWorkflowSignatureMessage({
|
||||||
|
tenant,
|
||||||
|
project,
|
||||||
|
agent,
|
||||||
|
requestKind,
|
||||||
|
process: processId,
|
||||||
|
task = "",
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
issuedAtEpochSeconds,
|
||||||
|
}) {
|
||||||
|
const parts = [
|
||||||
|
"clusterflux-agent-workflow-signature:v2",
|
||||||
|
tenant,
|
||||||
|
project,
|
||||||
|
agent,
|
||||||
|
requestKind,
|
||||||
|
processId,
|
||||||
|
task,
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
String(issuedAtEpochSeconds),
|
||||||
|
];
|
||||||
|
return Buffer.concat(
|
||||||
|
parts.flatMap((part) => [
|
||||||
|
Buffer.from(`${Buffer.byteLength(part)}:`),
|
||||||
|
Buffer.from(part),
|
||||||
|
Buffer.from("\n"),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedAgentWorkflowProof(identity, request, options = {}) {
|
||||||
|
const nonce =
|
||||||
|
options.nonce ??
|
||||||
|
`${request.type}-${process.pid}-${Date.now()}-${crypto
|
||||||
|
.randomBytes(8)
|
||||||
|
.toString("hex")}`;
|
||||||
|
const issuedAtEpochSeconds =
|
||||||
|
options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000);
|
||||||
|
const processId =
|
||||||
|
request.type === "launch_task" ? request.task_spec?.process : request.process;
|
||||||
|
const task =
|
||||||
|
request.type === "launch_task" ? request.task_spec?.task_instance : request.task || "";
|
||||||
|
const signature = crypto.sign(
|
||||||
|
null,
|
||||||
|
agentWorkflowSignatureMessage({
|
||||||
|
tenant: request.tenant,
|
||||||
|
project: request.project,
|
||||||
|
agent: request.actor_agent,
|
||||||
|
requestKind: request.type,
|
||||||
|
process: processId,
|
||||||
|
task,
|
||||||
|
payloadDigest: signedRequestPayloadDigest(request),
|
||||||
|
nonce,
|
||||||
|
issuedAtEpochSeconds,
|
||||||
|
}),
|
||||||
|
identity.privateKeyObject
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
nonce,
|
||||||
|
issued_at_epoch_seconds: issuedAtEpochSeconds,
|
||||||
|
signature: `ed25519:${signature.toString("base64")}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedAgentWorkflowRequest(identity, request, options = {}) {
|
||||||
|
const unsignedRequest = {
|
||||||
|
...request,
|
||||||
|
agent_public_key_fingerprint:
|
||||||
|
options.publicKeyFingerprint || identity.publicKeyFingerprint,
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
...unsignedRequest,
|
||||||
|
agent_signature: signedAgentWorkflowProof(identity, unsignedRequest, options),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
agentIdentity,
|
||||||
|
agentWorkflowSignatureMessage,
|
||||||
|
signedAgentWorkflowProof,
|
||||||
|
signedAgentWorkflowRequest,
|
||||||
|
};
|
||||||
401
scripts/artifact-download-smoke.js
Executable file
401
scripts/artifact-download-smoke.js
Executable file
|
|
@ -0,0 +1,401 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
const { nodeIdentity, signedNodeRequest } = require("./node-signing");
|
||||||
|
const {
|
||||||
|
ensureRootlessPodman,
|
||||||
|
flagshipNodeCapabilities,
|
||||||
|
launchFlagship,
|
||||||
|
repo,
|
||||||
|
runFlagshipWorker,
|
||||||
|
send,
|
||||||
|
waitForJsonLine,
|
||||||
|
} = require("./real-flagship-harness");
|
||||||
|
|
||||||
|
const downloadNode = "node-download";
|
||||||
|
const downloadNodeIdentity = nodeIdentity("artifact-download-smoke", downloadNode);
|
||||||
|
|
||||||
|
const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||||
|
|
||||||
|
async function downloadRetainedBytes(addr, link, artifact, expectedSize) {
|
||||||
|
const chunks = [];
|
||||||
|
let offset = 0;
|
||||||
|
for (let attempt = 0; attempt < 500; attempt += 1) {
|
||||||
|
const response = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
chunk_bytes: 256 * 1024,
|
||||||
|
});
|
||||||
|
assert.strictEqual(response.type, "artifact_download_stream");
|
||||||
|
if (!response.content_bytes_available) {
|
||||||
|
assert.strictEqual(response.content_source, "retaining_node_reverse_stream_pending");
|
||||||
|
await delay(10);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
assert.strictEqual(response.content_source, "retaining_node_reverse_stream");
|
||||||
|
assert.strictEqual(response.content_offset, offset);
|
||||||
|
const bytes = Buffer.from(response.content_base64, "base64");
|
||||||
|
assert.strictEqual(response.streamed_bytes, bytes.length);
|
||||||
|
chunks.push(bytes);
|
||||||
|
offset += bytes.length;
|
||||||
|
if (response.content_eof) {
|
||||||
|
const content = Buffer.concat(chunks);
|
||||||
|
assert.strictEqual(content.length, expectedSize);
|
||||||
|
return { content, response };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new Error("timed out waiting for retained artifact reverse stream");
|
||||||
|
}
|
||||||
|
|
||||||
|
function downloadNodeCapabilities() {
|
||||||
|
return flagshipNodeCapabilities();
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
ensureRootlessPodman();
|
||||||
|
const coordinator = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--",
|
||||||
|
"--listen",
|
||||||
|
"127.0.0.1:0",
|
||||||
|
"--allow-local-trusted-loopback",
|
||||||
|
],
|
||||||
|
{ cwd: repo }
|
||||||
|
);
|
||||||
|
|
||||||
|
let worker;
|
||||||
|
try {
|
||||||
|
const ready = await waitForJsonLine(coordinator);
|
||||||
|
const [host, portText] = ready.listen.split(":");
|
||||||
|
const addr = { host, port: Number(portText) };
|
||||||
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||||
|
|
||||||
|
worker = await runFlagshipWorker(addr, downloadNode, downloadNodeIdentity);
|
||||||
|
const workerReady = await worker.ready;
|
||||||
|
assert.strictEqual(workerReady.node_status, "ready");
|
||||||
|
assert.strictEqual(workerReady.mode, "worker");
|
||||||
|
const { compileEvent, packageEvent, process: virtualProcess } = await launchFlagship(addr);
|
||||||
|
assert.strictEqual(compileEvent.status_code, 0);
|
||||||
|
assert.strictEqual(packageEvent.status_code, 0);
|
||||||
|
assert.deepStrictEqual(compileEvent.result, {
|
||||||
|
Artifact: {
|
||||||
|
id: compileEvent.artifact_path.slice("/vfs/artifacts/".length),
|
||||||
|
digest: compileEvent.artifact_digest,
|
||||||
|
size_bytes: compileEvent.artifact_size_bytes,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.deepStrictEqual(packageEvent.result, {
|
||||||
|
Artifact: {
|
||||||
|
id: packageEvent.artifact_path.slice("/vfs/artifacts/".length),
|
||||||
|
digest: packageEvent.artifact_digest,
|
||||||
|
size_bytes: packageEvent.artifact_size_bytes,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.ok(compileEvent.artifact_size_bytes > 0);
|
||||||
|
assert.strictEqual(packageEvent.artifact_size_bytes, compileEvent.artifact_size_bytes);
|
||||||
|
assert.match(compileEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.match(packageEvent.artifact_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
const artifactPath = packageEvent.artifact_path;
|
||||||
|
assert.match(
|
||||||
|
artifactPath,
|
||||||
|
/^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/
|
||||||
|
);
|
||||||
|
const artifact = artifactPath.slice("/vfs/artifacts/".length);
|
||||||
|
|
||||||
|
const disconnectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
||||||
|
type: "report_node_capabilities",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node: downloadNode,
|
||||||
|
capabilities: downloadNodeCapabilities(),
|
||||||
|
cached_environment_digests: [],
|
||||||
|
dependency_cache_digests: [],
|
||||||
|
source_snapshots: [],
|
||||||
|
artifact_locations: [artifact],
|
||||||
|
direct_connectivity: false,
|
||||||
|
online: true,
|
||||||
|
}));
|
||||||
|
assert.strictEqual(disconnectedReport.type, "node_capabilities_recorded");
|
||||||
|
|
||||||
|
const disconnectedLink = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60,
|
||||||
|
});
|
||||||
|
assert.strictEqual(disconnectedLink.type, "artifact_download_link");
|
||||||
|
|
||||||
|
const connectedReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
||||||
|
type: "report_node_capabilities",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node: downloadNode,
|
||||||
|
capabilities: downloadNodeCapabilities(),
|
||||||
|
cached_environment_digests: [],
|
||||||
|
dependency_cache_digests: [],
|
||||||
|
source_snapshots: [],
|
||||||
|
artifact_locations: [artifact],
|
||||||
|
direct_connectivity: true,
|
||||||
|
online: true,
|
||||||
|
}));
|
||||||
|
assert.strictEqual(connectedReport.type, "node_capabilities_recorded");
|
||||||
|
|
||||||
|
const link = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60,
|
||||||
|
});
|
||||||
|
assert.strictEqual(link.type, "artifact_download_link");
|
||||||
|
assert.strictEqual(link.link.tenant, "tenant");
|
||||||
|
assert.strictEqual(link.link.project, "project");
|
||||||
|
assert.strictEqual(link.link.process, virtualProcess);
|
||||||
|
assert.deepStrictEqual(link.link.actor, { User: "user" });
|
||||||
|
assert.match(link.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.ok(link.link.expires_at_epoch_seconds > Math.floor(Date.now() / 1000));
|
||||||
|
assert.ok(link.link.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 60);
|
||||||
|
assert.ok(
|
||||||
|
link.link.url_path.endsWith(
|
||||||
|
`/artifacts/tenant/project/${virtualProcess}/${artifact}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(link.link.source, { RetainedNode: "node-download" });
|
||||||
|
|
||||||
|
const crossTenant = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "other",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossTenant.type, "error");
|
||||||
|
assert.match(crossTenant.message, /artifact does not exist/);
|
||||||
|
|
||||||
|
const crossProject = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "other-project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossProject.type, "error");
|
||||||
|
assert.match(crossProject.message, /artifact does not exist/);
|
||||||
|
|
||||||
|
const crossTenantOpen = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "other",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
chunk_bytes: 1,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossTenantOpen.type, "error");
|
||||||
|
assert.match(crossTenantOpen.message, /artifact does not exist/);
|
||||||
|
|
||||||
|
const crossProjectOpen = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "other-project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
chunk_bytes: 1,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossProjectOpen.type, "error");
|
||||||
|
assert.match(crossProjectOpen.message, /artifact does not exist/);
|
||||||
|
|
||||||
|
const guessed = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: "sha256:guessed",
|
||||||
|
chunk_bytes: 1,
|
||||||
|
});
|
||||||
|
assert.strictEqual(guessed.type, "error");
|
||||||
|
assert.match(guessed.message, /token is invalid/);
|
||||||
|
|
||||||
|
const crossActorOpen = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "other-user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
chunk_bytes: 1,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossActorOpen.type, "error");
|
||||||
|
assert.match(crossActorOpen.message, /token is invalid/);
|
||||||
|
|
||||||
|
const downloaded = await downloadRetainedBytes(
|
||||||
|
addr,
|
||||||
|
link,
|
||||||
|
artifact,
|
||||||
|
packageEvent.artifact_size_bytes,
|
||||||
|
);
|
||||||
|
const downloadedDigest = `sha256:${crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(downloaded.content)
|
||||||
|
.digest("hex")}`;
|
||||||
|
assert.strictEqual(downloadedDigest, packageEvent.artifact_digest);
|
||||||
|
const inspect = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-release-"));
|
||||||
|
try {
|
||||||
|
const executable = path.join(inspect, "hello-clusterflux");
|
||||||
|
fs.writeFileSync(executable, downloaded.content);
|
||||||
|
fs.chmodSync(executable, 0o755);
|
||||||
|
assert.strictEqual(
|
||||||
|
cp.execFileSync(executable, { encoding: "utf8" }),
|
||||||
|
"hello from a real Clusterflux build\n"
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(inspect, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
const cliDownloadDirectory = fs.mkdtempSync(
|
||||||
|
path.join(os.tmpdir(), "clusterflux-cli-download-")
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const cliDownloadPath = path.join(cliDownloadDirectory, "hello-clusterflux");
|
||||||
|
const cliDownload = JSON.parse(
|
||||||
|
cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--",
|
||||||
|
"artifact", "download", artifact,
|
||||||
|
"--to", cliDownloadPath,
|
||||||
|
"--max-bytes", "1048576",
|
||||||
|
"--coordinator", `clusterflux+tcp://${addr.host}:${addr.port}`,
|
||||||
|
"--tenant", "tenant",
|
||||||
|
"--project-id", "project",
|
||||||
|
"--user", "user",
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{ cwd: repo, env: process.env, encoding: "utf8" }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert.strictEqual(cliDownload.command, "artifact download");
|
||||||
|
assert.strictEqual(cliDownload.local_download.status, "local_bytes_written");
|
||||||
|
assert.strictEqual(
|
||||||
|
cliDownload.local_download.verified_digest,
|
||||||
|
packageEvent.artifact_digest
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
`sha256:${crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(fs.readFileSync(cliDownloadPath))
|
||||||
|
.digest("hex")}`,
|
||||||
|
packageEvent.artifact_digest
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(cliDownloadDirectory, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
assert.strictEqual(downloaded.response.content_eof, true);
|
||||||
|
assert.strictEqual(
|
||||||
|
downloaded.response.charged_download_bytes,
|
||||||
|
packageEvent.artifact_size_bytes
|
||||||
|
);
|
||||||
|
assert.strictEqual(downloaded.response.link.artifact, artifact);
|
||||||
|
|
||||||
|
const crossActorRevoke = await send(addr, {
|
||||||
|
type: "revoke_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "other-user",
|
||||||
|
artifact,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossActorRevoke.type, "error");
|
||||||
|
assert.match(crossActorRevoke.message, /token is invalid/);
|
||||||
|
|
||||||
|
const revoked = await send(addr, {
|
||||||
|
type: "revoke_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
});
|
||||||
|
assert.strictEqual(revoked.type, "artifact_download_link_revoked");
|
||||||
|
assert.strictEqual(revoked.link.scoped_token_digest, link.link.scoped_token_digest);
|
||||||
|
|
||||||
|
const revokedOpen = await send(addr, {
|
||||||
|
type: "open_artifact_download_stream",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
token_digest: link.link.scoped_token_digest,
|
||||||
|
chunk_bytes: 1,
|
||||||
|
});
|
||||||
|
assert.strictEqual(revokedOpen.type, "error");
|
||||||
|
assert.match(revokedOpen.message, /revoked/);
|
||||||
|
|
||||||
|
const gcReport = await send(addr, signedNodeRequest(downloadNode, downloadNodeIdentity, "report_node_capabilities", {
|
||||||
|
type: "report_node_capabilities",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node: downloadNode,
|
||||||
|
capabilities: downloadNodeCapabilities(),
|
||||||
|
cached_environment_digests: [],
|
||||||
|
dependency_cache_digests: [],
|
||||||
|
source_snapshots: [],
|
||||||
|
artifact_locations: [],
|
||||||
|
direct_connectivity: false,
|
||||||
|
online: true,
|
||||||
|
}));
|
||||||
|
assert.strictEqual(gcReport.type, "node_capabilities_recorded");
|
||||||
|
|
||||||
|
const collectedLink = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60,
|
||||||
|
});
|
||||||
|
assert.strictEqual(collectedLink.type, "error");
|
||||||
|
assert.match(collectedLink.message, /unavailable from current retention/);
|
||||||
|
} finally {
|
||||||
|
worker?.child.kill("SIGTERM");
|
||||||
|
coordinator.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Artifact download smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
275
scripts/artifact-export-smoke.js
Normal file
275
scripts/artifact-export-smoke.js
Normal file
|
|
@ -0,0 +1,275 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
const { nodeIdentity, signedNodeRequest } = require("./node-signing");
|
||||||
|
const {
|
||||||
|
ensureRootlessPodman,
|
||||||
|
flagshipNodeCapabilities,
|
||||||
|
launchFlagship,
|
||||||
|
repo,
|
||||||
|
runFlagshipWorker,
|
||||||
|
send,
|
||||||
|
waitForJsonLine,
|
||||||
|
waitForNodeStatus,
|
||||||
|
} = require("./real-flagship-harness");
|
||||||
|
|
||||||
|
const sourceNode = "node-export-source";
|
||||||
|
const sourceIdentity = nodeIdentity("artifact-export-smoke", sourceNode);
|
||||||
|
|
||||||
|
function nodeCapabilities() {
|
||||||
|
return flagshipNodeCapabilities();
|
||||||
|
}
|
||||||
|
|
||||||
|
function runJson(command, args, options = {}) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = cp.spawn(command, args, { cwd: repo, ...options });
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
stdout += chunk.toString();
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (chunk) => {
|
||||||
|
stderr += chunk.toString();
|
||||||
|
});
|
||||||
|
child.once("error", reject);
|
||||||
|
child.once("exit", (code) => {
|
||||||
|
if (code !== 0) {
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`${command} ${args.join(" ")} failed with code ${code}\n${stderr}\n${stdout}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(stdout));
|
||||||
|
} catch (error) {
|
||||||
|
reject(new Error(`${command} did not return JSON\n${stdout}\n${error.message}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function reportNode(
|
||||||
|
addr,
|
||||||
|
node,
|
||||||
|
identity,
|
||||||
|
{ directConnectivity = true, online = true, artifacts = [] } = {}
|
||||||
|
) {
|
||||||
|
const response = await send(addr, signedNodeRequest(node, identity, "report_node_capabilities", {
|
||||||
|
type: "report_node_capabilities",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node,
|
||||||
|
capabilities: nodeCapabilities(),
|
||||||
|
cached_environment_digests: [],
|
||||||
|
dependency_cache_digests: [],
|
||||||
|
source_snapshots: [],
|
||||||
|
artifact_locations: artifacts,
|
||||||
|
direct_connectivity: directConnectivity,
|
||||||
|
online,
|
||||||
|
}));
|
||||||
|
assert.strictEqual(response.type, "node_capabilities_recorded");
|
||||||
|
assert.strictEqual(response.node, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
ensureRootlessPodman();
|
||||||
|
const coordinator = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--",
|
||||||
|
"--listen",
|
||||||
|
"127.0.0.1:0",
|
||||||
|
"--allow-local-trusted-loopback",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
env: { ...process.env, CLUSTERFLUX_NODE_STALE_AFTER_SECONDS: "1" },
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
let worker;
|
||||||
|
try {
|
||||||
|
const ready = await waitForJsonLine(coordinator);
|
||||||
|
const [host, portText] = ready.listen.split(":");
|
||||||
|
const addr = { host, port: Number(portText) };
|
||||||
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||||
|
|
||||||
|
worker = await runFlagshipWorker(addr, sourceNode, sourceIdentity);
|
||||||
|
const workerReady = await worker.ready;
|
||||||
|
assert.strictEqual(workerReady.node_status, "ready");
|
||||||
|
assert.strictEqual(workerReady.mode, "worker");
|
||||||
|
const firstNodeTaskCompletion = waitForNodeStatus(worker.child, "completed");
|
||||||
|
const { compileEvent, process: virtualProcess } = await launchFlagship(addr);
|
||||||
|
const workerCompletion = await firstNodeTaskCompletion;
|
||||||
|
assert.strictEqual(workerCompletion.node_status, "completed");
|
||||||
|
assert.strictEqual(
|
||||||
|
workerCompletion.task_assignment_response.task_spec.task_definition,
|
||||||
|
"snapshot_current_project"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
workerCompletion.virtual_thread,
|
||||||
|
workerCompletion.task_assignment_response.task_spec.task_instance
|
||||||
|
);
|
||||||
|
assert.strictEqual(compileEvent.status_code, 0);
|
||||||
|
assert.match(
|
||||||
|
compileEvent.artifact_path,
|
||||||
|
/^\/vfs\/artifacts\/hello-clusterflux-[0-9a-f]{64}$/
|
||||||
|
);
|
||||||
|
const artifact = compileEvent.artifact_path.slice("/vfs/artifacts/".length);
|
||||||
|
|
||||||
|
await reportNode(addr, sourceNode, sourceIdentity, {
|
||||||
|
artifacts: [artifact],
|
||||||
|
});
|
||||||
|
|
||||||
|
const receiverIdentity = nodeIdentity("artifact-export-smoke", "node-export-receiver");
|
||||||
|
const attachedReceiver = await send(addr, {
|
||||||
|
type: "attach_node",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node: "node-export-receiver",
|
||||||
|
public_key: receiverIdentity.publicKey,
|
||||||
|
});
|
||||||
|
assert.strictEqual(attachedReceiver.type, "node_attached");
|
||||||
|
await reportNode(addr, "node-export-receiver", receiverIdentity);
|
||||||
|
|
||||||
|
const exportPlan = await send(addr, {
|
||||||
|
type: "export_artifact_to_node",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
receiver_node: "node-export-receiver",
|
||||||
|
direct_connectivity: true,
|
||||||
|
failure_reason: "",
|
||||||
|
});
|
||||||
|
assert.strictEqual(exportPlan.type, "artifact_export_plan");
|
||||||
|
assert.strictEqual(exportPlan.source_node, "node-export-source");
|
||||||
|
assert.strictEqual(exportPlan.receiver_node, "node-export-receiver");
|
||||||
|
assert.strictEqual(exportPlan.plan.transport, "NativeQuic");
|
||||||
|
assert.strictEqual(exportPlan.plan.scope.tenant, "tenant");
|
||||||
|
assert.strictEqual(exportPlan.plan.scope.project, "project");
|
||||||
|
assert.strictEqual(exportPlan.plan.scope.process, virtualProcess);
|
||||||
|
assert.deepStrictEqual(exportPlan.plan.scope.object, { Artifact: artifact });
|
||||||
|
assert.strictEqual(exportPlan.plan.source.node, "node-export-source");
|
||||||
|
assert.strictEqual(exportPlan.plan.destination.node, "node-export-receiver");
|
||||||
|
assert.strictEqual(exportPlan.plan.coordinator_assisted_rendezvous, true);
|
||||||
|
assert.strictEqual(exportPlan.plan.coordinator_bulk_relay_allowed, false);
|
||||||
|
assert.match(exportPlan.plan.authorization_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.strictEqual(exportPlan.artifact_size_bytes, compileEvent.artifact_size_bytes);
|
||||||
|
|
||||||
|
const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-artifact-export-"));
|
||||||
|
const exportPath = path.join(temp, "hello-clusterflux");
|
||||||
|
cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
["build", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux"],
|
||||||
|
{ cwd: repo, stdio: "inherit" }
|
||||||
|
);
|
||||||
|
const cliBinary = path.join(
|
||||||
|
path.resolve(repo, process.env.CARGO_TARGET_DIR || "target"),
|
||||||
|
"debug",
|
||||||
|
process.platform === "win32" ? "clusterflux.exe" : "clusterflux"
|
||||||
|
);
|
||||||
|
await reportNode(addr, sourceNode, sourceIdentity, {
|
||||||
|
artifacts: [artifact],
|
||||||
|
});
|
||||||
|
const cliExport = await runJson(cliBinary, [
|
||||||
|
"artifact",
|
||||||
|
"export",
|
||||||
|
"--coordinator",
|
||||||
|
`${addr.host}:${addr.port}`,
|
||||||
|
"--tenant",
|
||||||
|
"tenant",
|
||||||
|
"--project-id",
|
||||||
|
"project",
|
||||||
|
"--user",
|
||||||
|
"user",
|
||||||
|
"--json",
|
||||||
|
artifact,
|
||||||
|
"--receiver-node",
|
||||||
|
"node-export-receiver",
|
||||||
|
"--to",
|
||||||
|
exportPath,
|
||||||
|
]);
|
||||||
|
assert.strictEqual(cliExport.command, "artifact export");
|
||||||
|
assert.strictEqual(cliExport.export_plan.local_bytes_written_by_cli, true);
|
||||||
|
assert.strictEqual(cliExport.export_plan.local_export_status, "local_bytes_written");
|
||||||
|
assert.strictEqual(
|
||||||
|
cliExport.export_plan.bytes_written,
|
||||||
|
compileEvent.artifact_size_bytes
|
||||||
|
);
|
||||||
|
assert.strictEqual(cliExport.local_export.stream.content_material_returned_in_report, false);
|
||||||
|
assert.strictEqual(
|
||||||
|
cliExport.local_export.verified_digest,
|
||||||
|
compileEvent.artifact_digest
|
||||||
|
);
|
||||||
|
fs.chmodSync(exportPath, 0o755);
|
||||||
|
assert.strictEqual(
|
||||||
|
cp.execFileSync(exportPath, { encoding: "utf8" }),
|
||||||
|
"hello from a real Clusterflux build\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
const crossTenant = await send(addr, {
|
||||||
|
type: "export_artifact_to_node",
|
||||||
|
tenant: "other",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
receiver_node: "node-export-receiver",
|
||||||
|
direct_connectivity: true,
|
||||||
|
failure_reason: "",
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossTenant.type, "error");
|
||||||
|
assert.match(crossTenant.message, /artifact does not exist/);
|
||||||
|
|
||||||
|
await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] });
|
||||||
|
const failedDirect = await send(addr, {
|
||||||
|
type: "export_artifact_to_node",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
receiver_node: "node-export-receiver",
|
||||||
|
direct_connectivity: false,
|
||||||
|
failure_reason: "nat traversal failed",
|
||||||
|
});
|
||||||
|
assert.strictEqual(failedDirect.type, "error");
|
||||||
|
assert.match(failedDirect.message, /nat traversal failed/);
|
||||||
|
assert.match(failedDirect.message, /coordinator bulk relay is disabled/);
|
||||||
|
|
||||||
|
await reportNode(addr, "node-export-receiver", receiverIdentity, { online: false });
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 2100));
|
||||||
|
await reportNode(addr, sourceNode, sourceIdentity, { artifacts: [artifact] });
|
||||||
|
const offlineReceiver = await send(addr, {
|
||||||
|
type: "export_artifact_to_node",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact,
|
||||||
|
receiver_node: "node-export-receiver",
|
||||||
|
direct_connectivity: true,
|
||||||
|
failure_reason: "",
|
||||||
|
});
|
||||||
|
assert.strictEqual(offlineReceiver.type, "error");
|
||||||
|
assert.match(offlineReceiver.message, /offline/);
|
||||||
|
} finally {
|
||||||
|
worker?.child.kill("SIGTERM");
|
||||||
|
coordinator.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Artifact export smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
27
scripts/check-code-size.sh
Executable file
27
scripts/check-code-size.sh
Executable file
|
|
@ -0,0 +1,27 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
maximum_lines=3000
|
||||||
|
failed=0
|
||||||
|
source_roots=(crates)
|
||||||
|
if [[ -d private/hosted-policy/src ]]; then
|
||||||
|
source_roots+=(private/hosted-policy/src)
|
||||||
|
fi
|
||||||
|
while IFS= read -r -d '' file; do
|
||||||
|
case "$file" in
|
||||||
|
*/tests.rs|*/tests/*) continue ;;
|
||||||
|
esac
|
||||||
|
lines="$(wc -l < "$file")"
|
||||||
|
if ((lines > maximum_lines)); then
|
||||||
|
printf '%s has %s lines; production file limit is %s\n' "$file" "$lines" "$maximum_lines" >&2
|
||||||
|
failed=1
|
||||||
|
fi
|
||||||
|
done < <(find "${source_roots[@]}" -type f -name '*.rs' -print0)
|
||||||
|
|
||||||
|
if ((failed)); then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf 'production source file size guard passed (%s lines maximum)\n' "$maximum_lines"
|
||||||
113
scripts/check-docs.js
Normal file
113
scripts/check-docs.js
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
const fs = require("node:fs");
|
||||||
|
const path = require("node:path");
|
||||||
|
|
||||||
|
const root = path.resolve(__dirname, "..");
|
||||||
|
const publicDocs = [
|
||||||
|
"README.md",
|
||||||
|
"SECURITY.md",
|
||||||
|
"docs/getting-started.md",
|
||||||
|
"docs/architecture.md",
|
||||||
|
"docs/nodes.md",
|
||||||
|
"docs/environments.md",
|
||||||
|
"docs/artifacts.md",
|
||||||
|
"docs/debugging.md",
|
||||||
|
"docs/task-abi.md",
|
||||||
|
"docs/self-hosting.md",
|
||||||
|
"docs/security.md",
|
||||||
|
];
|
||||||
|
const contributorDocs = ["docs/contributing/releases.md"];
|
||||||
|
const privateDocs = [
|
||||||
|
"private/docs/hosted-deployment.md",
|
||||||
|
"private/docs/authentik.md",
|
||||||
|
"private/docs/community-policy.md",
|
||||||
|
"private/docs/bandwidth-and-cost-controls.md",
|
||||||
|
"private/docs/publishing.md",
|
||||||
|
];
|
||||||
|
const internalDocs = ["internal/finish_mvp_2.md"];
|
||||||
|
const filteredPublicTree =
|
||||||
|
process.env.CLUSTERFLUX_FILTERED_PUBLIC_TREE === "1" ||
|
||||||
|
fs.existsSync(path.join(root, "CLUSTERFLUX_PUBLIC_TREE.json"));
|
||||||
|
|
||||||
|
const failures = [];
|
||||||
|
const expectExactMarkdownSet = (directory, expected) => {
|
||||||
|
const actual = fs
|
||||||
|
.readdirSync(path.join(root, directory), { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
|
||||||
|
.map((entry) => path.posix.join(directory, entry.name))
|
||||||
|
.sort();
|
||||||
|
const wanted = [...expected].sort();
|
||||||
|
if (JSON.stringify(actual) !== JSON.stringify(wanted)) {
|
||||||
|
failures.push(`${directory} markdown set is ${actual.join(", ")}; expected ${wanted.join(", ")}`);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const requiredDocs = filteredPublicTree
|
||||||
|
? [...publicDocs, ...contributorDocs]
|
||||||
|
: [...publicDocs, ...contributorDocs, ...privateDocs, ...internalDocs];
|
||||||
|
for (const file of requiredDocs) {
|
||||||
|
if (!fs.existsSync(path.join(root, file))) failures.push(`missing canonical document: ${file}`);
|
||||||
|
}
|
||||||
|
expectExactMarkdownSet("docs", publicDocs.filter((file) => file.startsWith("docs/")));
|
||||||
|
expectExactMarkdownSet("docs/contributing", contributorDocs);
|
||||||
|
if (filteredPublicTree) {
|
||||||
|
for (const directory of ["private", "internal"]) {
|
||||||
|
if (fs.existsSync(path.join(root, directory))) {
|
||||||
|
failures.push(`${directory}/ must not exist in the filtered public tree`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const forbidden = [
|
||||||
|
[/\bmvp\b/i, "internal milestone term"],
|
||||||
|
[/acceptance criteria/i, "internal gate language"],
|
||||||
|
[/release verification/i, "internal release language"],
|
||||||
|
[/public\/private source split/i, "source split narrative"],
|
||||||
|
[/founder|business decisions/i, "business planning narrative"],
|
||||||
|
[/hacker news|\bHN\b/, "launch-channel narrative"],
|
||||||
|
[/\busers can\b/i, "indirect reader wording"],
|
||||||
|
[/node\s+scripts\//i, "developer script instruction"],
|
||||||
|
[/scripts\/[^\s)]*smoke/i, "smoke script instruction"],
|
||||||
|
[/internal\/[^\s)]*/i, "internal tooling reference"],
|
||||||
|
[/private\/[^\s)]*/i, "private source reference"],
|
||||||
|
];
|
||||||
|
const topLevelCommands = new Set([
|
||||||
|
"doctor", "login", "logout", "auth", "agent", "key", "project", "inspect",
|
||||||
|
"build", "bundle", "run", "node", "process", "task", "logs", "artifact",
|
||||||
|
"dap", "debug", "quota", "admin",
|
||||||
|
]);
|
||||||
|
|
||||||
|
for (const file of publicDocs) {
|
||||||
|
const absolute = path.join(root, file);
|
||||||
|
const content = fs.readFileSync(absolute, "utf8");
|
||||||
|
for (const [pattern, description] of forbidden) {
|
||||||
|
if (pattern.test(content)) failures.push(`${file}: contains ${description}`);
|
||||||
|
}
|
||||||
|
for (const match of content.matchAll(/\]\(([^)#]+)(?:#[^)]+)?\)/g)) {
|
||||||
|
const target = match[1];
|
||||||
|
if (/^(?:https?:|mailto:)/.test(target)) continue;
|
||||||
|
const resolved = path.resolve(path.dirname(absolute), target);
|
||||||
|
if (!fs.existsSync(resolved)) failures.push(`${file}: broken link ${target}`);
|
||||||
|
}
|
||||||
|
for (const line of content.split(/\r?\n/)) {
|
||||||
|
const command = line.trim().match(/^clusterflux\s+([a-z][a-z-]*)\b/);
|
||||||
|
if (command && !topLevelCommands.has(command[1])) {
|
||||||
|
failures.push(`${file}: unknown top-level CLI command ${command[1]}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const rootMarkdown = fs
|
||||||
|
.readdirSync(root, { withFileTypes: true })
|
||||||
|
.filter((entry) => entry.isFile() && entry.name.endsWith(".md"))
|
||||||
|
.map((entry) => entry.name)
|
||||||
|
.sort();
|
||||||
|
if (JSON.stringify(rootMarkdown) !== JSON.stringify(["README.md", "SECURITY.md"])) {
|
||||||
|
failures.push(`top-level markdown set is ${rootMarkdown.join(", ")}; expected README.md, SECURITY.md`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failures.length) {
|
||||||
|
for (const failure of failures) console.error(failure);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
console.log("documentation checks passed");
|
||||||
39
scripts/check-old-name.sh
Executable file
39
scripts/check-old-name.sh
Executable file
|
|
@ -0,0 +1,39 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
cd "$repo_root"
|
||||||
|
|
||||||
|
legacy_lower="$(printf '%s%s' 'disa' 'smer')"
|
||||||
|
legacy_title="$(printf '%s%s' 'Disa' 'smer')"
|
||||||
|
legacy_upper="$(printf '%s%s' 'DISA' 'SMER')"
|
||||||
|
pattern="${legacy_lower}|${legacy_title}|${legacy_upper}"
|
||||||
|
|
||||||
|
allowed_content='^\./scripts/migrate-clusterflux-state\.sh:'
|
||||||
|
matches="$(
|
||||||
|
rg -n --hidden \
|
||||||
|
--glob '!**/.git/**' \
|
||||||
|
--glob '!**/target/**' \
|
||||||
|
--glob '!**/node_modules/**' \
|
||||||
|
--glob '!**/vendor/**' \
|
||||||
|
--glob '!**/.direnv/**' \
|
||||||
|
--glob '!**/.cache/**' \
|
||||||
|
--glob '!**/dist/**' \
|
||||||
|
--glob '!**/out/**' \
|
||||||
|
"$pattern" . 2>/dev/null | rg -v "$allowed_content" || true
|
||||||
|
)"
|
||||||
|
|
||||||
|
paths="$(
|
||||||
|
find . \
|
||||||
|
\( -type d \( -name .git -o -name target -o -name node_modules -o -name vendor -o -name .direnv -o -name .cache -o -name dist -o -name out \) -prune \) -o \
|
||||||
|
-iname "*${legacy_lower}*" -print | sort || true
|
||||||
|
)"
|
||||||
|
|
||||||
|
if [[ -n "$matches" || -n "$paths" ]]; then
|
||||||
|
[[ -z "$matches" ]] || printf '%s\n' "$matches" >&2
|
||||||
|
[[ -z "$paths" ]] || printf '%s\n' "$paths" >&2
|
||||||
|
printf 'unexpected legacy product name remains\n' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'old-name guard passed\n'
|
||||||
222
scripts/cli-browser-login-flow-smoke.js
Normal file
222
scripts/cli-browser-login-flow-smoke.js
Normal file
|
|
@ -0,0 +1,222 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const net = require("net");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const tmp = path.join(repo, "target", "acceptance", "tmp", "cli-browser-login-flow");
|
||||||
|
const project = path.join(tmp, "project");
|
||||||
|
fs.rmSync(tmp, { recursive: true, force: true });
|
||||||
|
fs.mkdirSync(project, { recursive: true });
|
||||||
|
|
||||||
|
function writeOpener() {
|
||||||
|
const opener = path.join(tmp, "browser-opener.js");
|
||||||
|
const trace = path.join(tmp, "browser-opener.log");
|
||||||
|
fs.writeFileSync(
|
||||||
|
opener,
|
||||||
|
`#!/usr/bin/env node
|
||||||
|
const fs = require("fs");
|
||||||
|
const trace = ${JSON.stringify(trace)};
|
||||||
|
const loginUrl = new URL(process.argv[2]);
|
||||||
|
const state = loginUrl.searchParams.get("state");
|
||||||
|
const nonce = loginUrl.searchParams.get("nonce");
|
||||||
|
const challenge = loginUrl.searchParams.get("code_challenge");
|
||||||
|
if (loginUrl.protocol !== "https:" || !state || !nonce || !challenge) {
|
||||||
|
fs.appendFileSync(trace, "missing server-owned OIDC parameters\\n");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
fs.appendFileSync(trace, "server-owned browser transaction\\n");
|
||||||
|
// Model a real browser/opener that remains alive after the CLI transaction.
|
||||||
|
// Its descriptors must not keep the invoking CLI process open.
|
||||||
|
setTimeout(() => process.exit(0), 5000);
|
||||||
|
`
|
||||||
|
);
|
||||||
|
fs.chmodSync(opener, 0o755);
|
||||||
|
return opener;
|
||||||
|
}
|
||||||
|
|
||||||
|
function startCoordinator() {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const requests = [];
|
||||||
|
const server = net.createServer((socket) => {
|
||||||
|
let buffered = "";
|
||||||
|
socket.on("data", (chunk) => {
|
||||||
|
buffered += chunk.toString("utf8");
|
||||||
|
while (buffered.includes("\n")) {
|
||||||
|
const newline = buffered.indexOf("\n");
|
||||||
|
const line = buffered.slice(0, newline);
|
||||||
|
buffered = buffered.slice(newline + 1);
|
||||||
|
const envelope = JSON.parse(line);
|
||||||
|
requests.push(envelope);
|
||||||
|
assert.strictEqual(envelope.type, "coordinator_request");
|
||||||
|
assert.strictEqual(envelope.protocol_version, 1);
|
||||||
|
assert.strictEqual(envelope.authentication.kind, "none");
|
||||||
|
const request = envelope.payload;
|
||||||
|
|
||||||
|
if (requests.length === 1) {
|
||||||
|
assert.strictEqual(envelope.request_id, "cli-1");
|
||||||
|
assert.strictEqual(envelope.operation, "begin_oidc_browser_login");
|
||||||
|
assert.deepStrictEqual(request, {
|
||||||
|
type: "begin_oidc_browser_login",
|
||||||
|
});
|
||||||
|
socket.write(
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "oidc_browser_login_started",
|
||||||
|
transaction_id: "login-transaction",
|
||||||
|
polling_secret: "opaque-polling-secret",
|
||||||
|
authorization_url:
|
||||||
|
"https://auth.michelpaulissen.com/application/o/authorize/?state=server-state&nonce=server-nonce&code_challenge=server-pkce&code_challenge_method=S256&redirect_uri=https%3A%2F%2Fclusterflux.michelpaulissen.com%2Fauth%2Fcallback",
|
||||||
|
expires_at_epoch_seconds: 1800000000,
|
||||||
|
})}\n`
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.strictEqual(envelope.request_id, "cli-2");
|
||||||
|
assert.strictEqual(envelope.operation, "poll_oidc_browser_login");
|
||||||
|
assert.deepStrictEqual(request, {
|
||||||
|
type: "poll_oidc_browser_login",
|
||||||
|
transaction_id: "login-transaction",
|
||||||
|
polling_secret: "opaque-polling-secret",
|
||||||
|
});
|
||||||
|
socket.end(
|
||||||
|
`${JSON.stringify({
|
||||||
|
type: "oidc_browser_session",
|
||||||
|
session: {
|
||||||
|
tenant: "tenant-smoke",
|
||||||
|
project: "project-smoke",
|
||||||
|
user: "user-smoke",
|
||||||
|
cli_session_credential_kind: "CliDeviceSession",
|
||||||
|
cli_session_secret: "scoped-cli-session-secret",
|
||||||
|
expires_at_epoch_seconds: 1800000000,
|
||||||
|
provider_tokens_sent_to_nodes: false,
|
||||||
|
},
|
||||||
|
})}\n`
|
||||||
|
);
|
||||||
|
server.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
server.once("error", reject);
|
||||||
|
server.listen(0, "127.0.0.1", () => {
|
||||||
|
const address = server.address();
|
||||||
|
resolve({
|
||||||
|
url: `${address.address}:${address.port}`,
|
||||||
|
requests,
|
||||||
|
close: () =>
|
||||||
|
new Promise((closeResolve) => {
|
||||||
|
if (!server.listening) closeResolve();
|
||||||
|
else server.close(() => closeResolve());
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function runClusterflux(args, env) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"--manifest-path",
|
||||||
|
path.join(repo, "Cargo.toml"),
|
||||||
|
"-p",
|
||||||
|
"clusterflux-cli",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux",
|
||||||
|
"--",
|
||||||
|
...args,
|
||||||
|
],
|
||||||
|
{ cwd: project, env, stdio: ["ignore", "pipe", "pipe"] }
|
||||||
|
);
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
child.stdout.on("data", (chunk) => (stdout += chunk.toString("utf8")));
|
||||||
|
child.stderr.on("data", (chunk) => (stderr += chunk.toString("utf8")));
|
||||||
|
child.once("error", reject);
|
||||||
|
child.once("close", (code) => {
|
||||||
|
if (code === 0) resolve(stdout);
|
||||||
|
else reject(new Error(`clusterflux exited ${code}\n${stderr}\n${stdout}`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const opener = writeOpener();
|
||||||
|
const coordinator = await startCoordinator();
|
||||||
|
try {
|
||||||
|
const loginStarted = Date.now();
|
||||||
|
const report = JSON.parse(
|
||||||
|
await runClusterflux(
|
||||||
|
[
|
||||||
|
"login",
|
||||||
|
"--browser",
|
||||||
|
"--json",
|
||||||
|
"--coordinator",
|
||||||
|
coordinator.url,
|
||||||
|
"--project-id",
|
||||||
|
"project-smoke",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
...process.env,
|
||||||
|
CLUSTERFLUX_BROWSER_OPEN_COMMAND: opener,
|
||||||
|
CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS: "5",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
Date.now() - loginStarted < 3000,
|
||||||
|
"a long-lived browser opener must not keep CLI output pipes or login completion open"
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.plan.coordinator, coordinator.url);
|
||||||
|
assert.strictEqual(report.boundary.cli_contacted_coordinator, true);
|
||||||
|
assert.strictEqual(report.boundary.scoped_cli_session_received, true);
|
||||||
|
assert.strictEqual(report.boundary.local_cli_session_file_written, true);
|
||||||
|
assert.strictEqual(report.boundary.provider_tokens_persisted_locally, false);
|
||||||
|
assert.strictEqual(report.boundary.provider_tokens_exposed_to_cli, false);
|
||||||
|
assert.strictEqual(report.boundary.provider_tokens_sent_to_nodes, false);
|
||||||
|
assert.strictEqual(report.boundary.coordinator_session_requests, 2);
|
||||||
|
assert.strictEqual(coordinator.requests.length, 2);
|
||||||
|
|
||||||
|
const sessionFile = path.join(project, ".clusterflux", "session.json");
|
||||||
|
const sessionText = fs.readFileSync(sessionFile, "utf8");
|
||||||
|
const session = JSON.parse(sessionText);
|
||||||
|
assert.strictEqual(session.kind, "human");
|
||||||
|
assert.strictEqual(session.coordinator, coordinator.url);
|
||||||
|
assert.strictEqual(session.tenant, "tenant-smoke");
|
||||||
|
assert.strictEqual(session.project, "project-smoke");
|
||||||
|
assert.strictEqual(session.user, "user-smoke");
|
||||||
|
assert.strictEqual(session.cli_session_credential_kind, "CliDeviceSession");
|
||||||
|
assert.strictEqual(session.token_expiry_posture, "expires_at");
|
||||||
|
assert.strictEqual(session.expires_at, "1800000000");
|
||||||
|
assert.strictEqual(session.provider_tokens_exposed_to_cli, false);
|
||||||
|
assert.strictEqual(session.provider_tokens_sent_to_nodes, false);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
sessionText,
|
||||||
|
/access_token|refresh_token|id_token|provider-secret|authorization_code|Bearer/
|
||||||
|
);
|
||||||
|
|
||||||
|
const authStatus = JSON.parse(
|
||||||
|
await runClusterflux(["auth", "status", "--json"], process.env)
|
||||||
|
);
|
||||||
|
assert.strictEqual(authStatus.active_coordinator, coordinator.url);
|
||||||
|
assert.strictEqual(authStatus.principal, "user-smoke");
|
||||||
|
assert.strictEqual(authStatus.tenant, "tenant-smoke");
|
||||||
|
assert.strictEqual(authStatus.project, "project-smoke");
|
||||||
|
assert.strictEqual(authStatus.session.kind, "human");
|
||||||
|
assert.strictEqual(authStatus.session.source, "session_file");
|
||||||
|
assert.strictEqual(authStatus.session.provider_tokens_exposed_to_cli, false);
|
||||||
|
assert.strictEqual(authStatus.session.provider_tokens_exposed_to_nodes, false);
|
||||||
|
} finally {
|
||||||
|
await coordinator.close();
|
||||||
|
}
|
||||||
|
console.log("CLI browser login flow smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
365
scripts/cli-error-exit-smoke.js
Executable file
365
scripts/cli-error-exit-smoke.js
Executable file
|
|
@ -0,0 +1,365 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const http = require("http");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const project = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-error-"));
|
||||||
|
|
||||||
|
function runClusterflux(args) {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const child = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
child.stdout.setEncoding("utf8");
|
||||||
|
child.stderr.setEncoding("utf8");
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
stdout += chunk;
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (chunk) => {
|
||||||
|
stderr += chunk;
|
||||||
|
});
|
||||||
|
child.on("close", (code, signal) => {
|
||||||
|
resolve({ code, signal, stdout, stderr });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runWithOneCoordinatorResponse(buildArgs, response) {
|
||||||
|
let request = "";
|
||||||
|
const server = http.createServer((incoming, outgoing) => {
|
||||||
|
incoming.setEncoding("utf8");
|
||||||
|
incoming.on("data", (chunk) => {
|
||||||
|
request += chunk;
|
||||||
|
});
|
||||||
|
incoming.on("end", () => {
|
||||||
|
outgoing.writeHead(200, { "content-type": "application/json" });
|
||||||
|
outgoing.end(JSON.stringify(response));
|
||||||
|
server.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const address = await new Promise((resolve) => {
|
||||||
|
server.listen(0, "127.0.0.1", () => resolve(server.address()));
|
||||||
|
});
|
||||||
|
const coordinator = `http://${address.address}:${address.port}`;
|
||||||
|
const result = await runClusterflux(buildArgs(coordinator));
|
||||||
|
return { request, result };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const environmentProject = path.join(tempRoot, "missing-env-project");
|
||||||
|
fs.mkdirSync(path.join(environmentProject, "src"), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(environmentProject, "Cargo.toml"),
|
||||||
|
"[package]\nname = \"missing-env-project\"\nversion = \"0.1.0\"\nedition = \"2021\"\n"
|
||||||
|
);
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(environmentProject, "src", "main.rs"),
|
||||||
|
"fn main() { let _target = env!(\"linux\"); }\n"
|
||||||
|
);
|
||||||
|
const environmentFailure = await runClusterflux([
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
environmentProject,
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert.strictEqual(environmentFailure.signal, null, environmentFailure.stderr);
|
||||||
|
assert.strictEqual(environmentFailure.code, 26, environmentFailure.stderr);
|
||||||
|
const environmentReport = JSON.parse(environmentFailure.stdout);
|
||||||
|
assert.strictEqual(environmentReport.status, "blocked_before_schedule");
|
||||||
|
assert.strictEqual(environmentReport.scheduled_work, false);
|
||||||
|
assert.strictEqual(environmentReport.machine_error.category, "environment");
|
||||||
|
assert.strictEqual(
|
||||||
|
environmentReport.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
environmentReport.machine_error.next_actions.includes("clusterflux inspect")
|
||||||
|
);
|
||||||
|
assert.strictEqual(environmentReport.diagnostics[0].code, "missing_environment");
|
||||||
|
|
||||||
|
const nonInteractive = await runClusterflux([
|
||||||
|
"run",
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--non-interactive",
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert.strictEqual(nonInteractive.signal, null, nonInteractive.stderr);
|
||||||
|
assert.strictEqual(nonInteractive.code, 20, nonInteractive.stderr);
|
||||||
|
assert.doesNotMatch(nonInteractive.stderr, /Opening Clusterflux browser login/);
|
||||||
|
const nonInteractiveReport = JSON.parse(nonInteractive.stdout);
|
||||||
|
assert.strictEqual(nonInteractiveReport.status, "authentication_required");
|
||||||
|
assert.strictEqual(nonInteractiveReport.non_interactive, true);
|
||||||
|
assert.strictEqual(nonInteractiveReport.browser_opened, false);
|
||||||
|
assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication");
|
||||||
|
assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20);
|
||||||
|
assert.strictEqual(
|
||||||
|
nonInteractiveReport.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
nonInteractiveReport.machine_error.next_actions.includes(
|
||||||
|
"pass --local to run against local services"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
let request = "";
|
||||||
|
const server = http.createServer((incoming, outgoing) => {
|
||||||
|
incoming.setEncoding("utf8");
|
||||||
|
incoming.on("data", (chunk) => {
|
||||||
|
request += chunk;
|
||||||
|
});
|
||||||
|
incoming.on("end", () => {
|
||||||
|
outgoing.writeHead(200, { "content-type": "application/json" });
|
||||||
|
outgoing.end(
|
||||||
|
JSON.stringify({
|
||||||
|
type: "error",
|
||||||
|
message: "quota unavailable: resource limit exceeded for api_calls",
|
||||||
|
})
|
||||||
|
);
|
||||||
|
server.close();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const address = await new Promise((resolve) => {
|
||||||
|
server.listen(0, "127.0.0.1", () => resolve(server.address()));
|
||||||
|
});
|
||||||
|
const coordinator = `http://${address.address}:${address.port}`;
|
||||||
|
|
||||||
|
const result = await runClusterflux([
|
||||||
|
"run",
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert.strictEqual(result.signal, null, result.stderr);
|
||||||
|
assert.strictEqual(result.code, 22, result.stderr);
|
||||||
|
assert.match(request, /"type":"start_process"/);
|
||||||
|
const report = JSON.parse(result.stdout);
|
||||||
|
assert.strictEqual(report.status, "coordinator_rejected");
|
||||||
|
assert.strictEqual(report.run_start.machine_error.category, "quota");
|
||||||
|
assert.strictEqual(report.run_start.machine_error.resource_category, "api_calls");
|
||||||
|
assert.strictEqual(report.run_start.machine_error.community_tier_language, true);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.run_start.machine_error.community_tier_label,
|
||||||
|
"community tier"
|
||||||
|
);
|
||||||
|
assert.doesNotMatch(result.stdout, new RegExp(["free", "tier"].join(" "), "i"));
|
||||||
|
assert.strictEqual(
|
||||||
|
report.run_start.machine_error.private_abuse_heuristics_exposed,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.run_start.machine_error.stable_exit_code, 22);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.run_start.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
report.run_start.machine_error.next_actions.includes("clusterflux quota status")
|
||||||
|
);
|
||||||
|
|
||||||
|
const capabilityFailure = await runWithOneCoordinatorResponse(
|
||||||
|
(coordinator) => [
|
||||||
|
"run",
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
type: "error",
|
||||||
|
message:
|
||||||
|
"scheduler placement failed: no capable node for placement: missing capability Command",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(capabilityFailure.result.signal, null, capabilityFailure.result.stderr);
|
||||||
|
assert.strictEqual(capabilityFailure.result.code, 24, capabilityFailure.result.stderr);
|
||||||
|
assert.match(capabilityFailure.request, /"type":"start_process"/);
|
||||||
|
const capabilityReport = JSON.parse(capabilityFailure.result.stdout);
|
||||||
|
assert.strictEqual(
|
||||||
|
capabilityReport.run_start.machine_error.category,
|
||||||
|
"capability"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
capabilityReport.run_start.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
capabilityReport.run_start.machine_error.next_actions.includes(
|
||||||
|
"attach a node with the required capabilities"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const nodePolicyFailure = await runWithOneCoordinatorResponse(
|
||||||
|
(coordinator) => [
|
||||||
|
"run",
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
type: "error",
|
||||||
|
message: "node policy denied native command execution",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(nodePolicyFailure.result.signal, null, nodePolicyFailure.result.stderr);
|
||||||
|
assert.strictEqual(nodePolicyFailure.result.code, 23, nodePolicyFailure.result.stderr);
|
||||||
|
assert.match(nodePolicyFailure.request, /"type":"start_process"/);
|
||||||
|
const nodePolicyReport = JSON.parse(nodePolicyFailure.result.stdout);
|
||||||
|
assert.strictEqual(
|
||||||
|
nodePolicyReport.run_start.machine_error.category,
|
||||||
|
"policy"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
nodePolicyReport.run_start.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
nodePolicyReport.run_start.machine_error.next_actions.includes(
|
||||||
|
"check coordinator policy for this action"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const programFailure = await runWithOneCoordinatorResponse(
|
||||||
|
(coordinator) => [
|
||||||
|
"task",
|
||||||
|
"list",
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
type: "task_events",
|
||||||
|
events: [
|
||||||
|
{
|
||||||
|
process: "vp-current",
|
||||||
|
task: "compile",
|
||||||
|
terminal_state: "failed",
|
||||||
|
environment: "linux",
|
||||||
|
node: "node-linux",
|
||||||
|
status_code: 1,
|
||||||
|
stderr_tail: "task exited with status 1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(programFailure.result.signal, null, programFailure.result.stderr);
|
||||||
|
assert.strictEqual(programFailure.result.code, 0, programFailure.result.stderr);
|
||||||
|
assert.match(programFailure.request, /"type":"list_task_events"/);
|
||||||
|
const programReport = JSON.parse(programFailure.result.stdout);
|
||||||
|
assert.strictEqual(programReport.tasks[0].machine_error.category, "program");
|
||||||
|
assert.strictEqual(programReport.tasks[0].machine_error.stable_exit_code, 27);
|
||||||
|
assert(
|
||||||
|
programReport.tasks[0].machine_error.next_actions.includes("clusterflux logs")
|
||||||
|
);
|
||||||
|
|
||||||
|
const artifactDownload = await runWithOneCoordinatorResponse(
|
||||||
|
(coordinator) => [
|
||||||
|
"artifact",
|
||||||
|
"download",
|
||||||
|
"app.txt",
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
type: "artifact_download_denied",
|
||||||
|
message: "artifact download unauthorized for project",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(artifactDownload.result.signal, null, artifactDownload.result.stderr);
|
||||||
|
assert.strictEqual(artifactDownload.result.code, 21, artifactDownload.result.stderr);
|
||||||
|
assert.match(artifactDownload.request, /"type":"create_artifact_download_link"/);
|
||||||
|
const artifactDownloadReport = JSON.parse(artifactDownload.result.stdout);
|
||||||
|
assert.strictEqual(
|
||||||
|
artifactDownloadReport.download_session.machine_error.category,
|
||||||
|
"authorization"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
artifactDownloadReport.download_session.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
const artifactExport = await runWithOneCoordinatorResponse(
|
||||||
|
(coordinator) => [
|
||||||
|
"artifact",
|
||||||
|
"export",
|
||||||
|
"app.txt",
|
||||||
|
"--to",
|
||||||
|
path.join(project, "target", "blocked-artifact.txt"),
|
||||||
|
"--coordinator",
|
||||||
|
coordinator,
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
type: "artifact_export_unavailable",
|
||||||
|
message: "direct connectivity unavailable for artifact export",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(artifactExport.result.signal, null, artifactExport.result.stderr);
|
||||||
|
assert.strictEqual(artifactExport.result.code, 25, artifactExport.result.stderr);
|
||||||
|
assert.match(artifactExport.request, /"type":"export_artifact_to_node"/);
|
||||||
|
const artifactExportReport = JSON.parse(artifactExport.result.stdout);
|
||||||
|
assert.strictEqual(
|
||||||
|
artifactExportReport.export_plan.machine_error.category,
|
||||||
|
"connectivity"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
artifactExportReport.export_plan.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
const confirmation = await runClusterflux([
|
||||||
|
"process",
|
||||||
|
"cancel",
|
||||||
|
"--coordinator",
|
||||||
|
"127.0.0.1:9",
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert.strictEqual(confirmation.signal, null, confirmation.stderr);
|
||||||
|
assert.strictEqual(confirmation.code, 23, confirmation.stderr);
|
||||||
|
const confirmationReport = JSON.parse(confirmation.stdout);
|
||||||
|
assert.strictEqual(confirmationReport.status, "confirmation_required");
|
||||||
|
assert.strictEqual(confirmationReport.coordinator_request_sent, false);
|
||||||
|
assert.strictEqual(confirmationReport.machine_error.category, "policy");
|
||||||
|
assert.strictEqual(
|
||||||
|
confirmationReport.machine_error.process_exit_code_applied,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
confirmationReport.next_actions.some((action) => action.includes("--yes"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
main()
|
||||||
|
.then(() => {
|
||||||
|
console.log("CLI error exit smoke passed");
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
7128
scripts/cli-happy-path-live-smoke.js
Normal file
7128
scripts/cli-happy-path-live-smoke.js
Normal file
File diff suppressed because it is too large
Load diff
61
scripts/cli-install-smoke.js
Executable file
61
scripts/cli-install-smoke.js
Executable file
|
|
@ -0,0 +1,61 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const temp = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-install-"));
|
||||||
|
const installRoot = path.join(temp, "install");
|
||||||
|
const targetDir =
|
||||||
|
process.env.CLUSTERFLUX_CLI_INSTALL_CARGO_TARGET_DIR ||
|
||||||
|
process.env.CARGO_TARGET_DIR ||
|
||||||
|
path.join(repo, "target");
|
||||||
|
const project = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
const binName = process.platform === "win32" ? "clusterflux.exe" : "clusterflux";
|
||||||
|
const installedBin = path.join(installRoot, "bin", binName);
|
||||||
|
|
||||||
|
try {
|
||||||
|
cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"install",
|
||||||
|
"--path",
|
||||||
|
"crates/clusterflux-cli",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux",
|
||||||
|
"--root",
|
||||||
|
installRoot,
|
||||||
|
"--debug"
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
CARGO_TARGET_DIR: targetDir
|
||||||
|
},
|
||||||
|
stdio: "inherit"
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
assert(fs.existsSync(installedBin), "installed clusterflux binary must exist");
|
||||||
|
|
||||||
|
const inspection = JSON.parse(
|
||||||
|
cp.execFileSync(
|
||||||
|
installedBin,
|
||||||
|
["bundle", "inspect", "--project", project, "--json"],
|
||||||
|
{ cwd: repo, encoding: "utf8" }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.strictEqual(inspection.project, project);
|
||||||
|
assert.strictEqual(inspection.metadata.embeds_full_container_images, false);
|
||||||
|
assert(inspection.metadata.environments.some((env) => env.name === "linux"));
|
||||||
|
assert(inspection.metadata.selected_inputs.some((input) => input.path === "src/lib.rs"));
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(temp, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("CLI install smoke passed");
|
||||||
269
scripts/cli-local-run-smoke.js
Executable file
269
scripts/cli-local-run-smoke.js
Executable file
|
|
@ -0,0 +1,269 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const net = require("net");
|
||||||
|
const path = require("path");
|
||||||
|
const { coordinatorWireRequest } = require("./coordinator-wire");
|
||||||
|
const { configurePodmanTestEnvironment } = require("./podman-test-env");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
configurePodmanTestEnvironment(repo);
|
||||||
|
if (
|
||||||
|
!process.env.CLUSTERFLUX_PODMAN_NIX_SHELL &&
|
||||||
|
cp.spawnSync("podman", ["--version"], { stdio: "ignore" }).status !== 0 &&
|
||||||
|
cp.spawnSync("nix", ["--version"], { stdio: "ignore" }).status === 0
|
||||||
|
) {
|
||||||
|
cp.execFileSync(
|
||||||
|
"nix",
|
||||||
|
["shell", "nixpkgs#podman", "--command", "node", __filename],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
CLUSTERFLUX_PODMAN_NIX_SHELL: "1",
|
||||||
|
},
|
||||||
|
stdio: "inherit",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
const project = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
|
||||||
|
cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
["build", "-q", "-p", "clusterflux-node", "--bin", "clusterflux-node"],
|
||||||
|
{ cwd: repo, stdio: "inherit" }
|
||||||
|
);
|
||||||
|
|
||||||
|
function waitForJsonLine(child) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let buffer = "";
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
buffer += chunk.toString();
|
||||||
|
const newline = buffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(buffer.slice(0, newline).trim()));
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
child.once("exit", (code) => {
|
||||||
|
reject(new Error(`process exited before JSON line with code ${code}`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function send(addr, message) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const socket = net.connect(addr.port, addr.host, () => {
|
||||||
|
socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`);
|
||||||
|
});
|
||||||
|
let buffer = "";
|
||||||
|
socket.on("data", (chunk) => {
|
||||||
|
buffer += chunk.toString();
|
||||||
|
const newline = buffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
socket.end();
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(buffer.slice(0, newline)));
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on("error", reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function runCli(args, env = {}) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
...env
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
const cliPid = child.pid;
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
stdout += chunk.toString();
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (chunk) => {
|
||||||
|
stderr += chunk.toString();
|
||||||
|
});
|
||||||
|
child.on("exit", (code) => {
|
||||||
|
if (code !== 0) {
|
||||||
|
reject(new Error(`CLI run failed with code ${code}\n${stderr}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
resolve({ pid: cliPid, report: JSON.parse(stdout) });
|
||||||
|
} catch (error) {
|
||||||
|
reject(new Error(`CLI output was not JSON: ${stdout}\n${error.stack || error.message}`));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const coordinator = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--",
|
||||||
|
"--listen",
|
||||||
|
"127.0.0.1:0",
|
||||||
|
"--allow-local-trusted-loopback"
|
||||||
|
],
|
||||||
|
{ cwd: repo }
|
||||||
|
);
|
||||||
|
assert(Number.isInteger(coordinator.pid));
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ready = await waitForJsonLine(coordinator);
|
||||||
|
const [host, portText] = ready.listen.split(":");
|
||||||
|
const addr = { host, port: Number(portText) };
|
||||||
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||||
|
|
||||||
|
const { pid: cliPid, report } = await runCli([
|
||||||
|
"run",
|
||||||
|
"--coordinator",
|
||||||
|
`${addr.host}:${addr.port}`,
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert(Number.isInteger(cliPid));
|
||||||
|
assert.notStrictEqual(cliPid, coordinator.pid);
|
||||||
|
assert.strictEqual(report.plan.entry, "build");
|
||||||
|
assert.deepStrictEqual(report.plan.session, "Anonymous");
|
||||||
|
assert.strictEqual(report.boundary.cli_process_started_node_process, true);
|
||||||
|
assert.strictEqual(report.boundary.cli_process_started_coordinator_process, false);
|
||||||
|
assert(Number.isInteger(report.boundary.spawned_node_process_id));
|
||||||
|
assert.notStrictEqual(report.boundary.spawned_node_process_id, cliPid);
|
||||||
|
assert.notStrictEqual(report.boundary.spawned_node_process_id, coordinator.pid);
|
||||||
|
assert.strictEqual(report.boundary.node_session_requests, 0);
|
||||||
|
assert.strictEqual(report.node_report.node_status, "completed");
|
||||||
|
assert.strictEqual(report.node_report.execution_substrate, "wasm");
|
||||||
|
assert.strictEqual(report.node_report.task_spawn_host_import, true);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.node_report.pre_node_process_status.processes.length,
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
report.node_report.pre_node_process_status.processes[0].connected_nodes,
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.node_report.pre_node_process_status.processes[0].main_state,
|
||||||
|
"running"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.node_report.pre_node_process_status.processes[0].main_wait_state,
|
||||||
|
"waiting_for_node",
|
||||||
|
"the coordinator must expose that the capless main is parked on placement before a node exists"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.node_report.pre_node_process_status.processes[0].main_task_instance,
|
||||||
|
report.node_report.run.task_instance
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.node_report.run.status, "main_launched");
|
||||||
|
assert.strictEqual(report.node_report.join.type, "task_joined");
|
||||||
|
const process = report.node_report.run.process;
|
||||||
|
assert.strictEqual(process, "vp-current");
|
||||||
|
|
||||||
|
const events = await send(addr, {
|
||||||
|
type: "list_task_events",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process
|
||||||
|
});
|
||||||
|
assert.strictEqual(events.type, "task_events");
|
||||||
|
assert(events.events.length >= 4);
|
||||||
|
assert(
|
||||||
|
events.events
|
||||||
|
.filter((event) => event.executor === "node")
|
||||||
|
.every((event) => event.node === "node-cli-local")
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
events.events.some(
|
||||||
|
(event) =>
|
||||||
|
event.executor === "coordinator_main" &&
|
||||||
|
event.node === "coordinator-main"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(events.events.every((event) => event.process === process));
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
new Set(events.events.map((event) => event.task_definition)),
|
||||||
|
new Set([
|
||||||
|
report.node_report.run.task_definition,
|
||||||
|
"prepare_source",
|
||||||
|
"compile_linux",
|
||||||
|
"package_release",
|
||||||
|
])
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
new Set(events.events.map((event) => event.task)).size,
|
||||||
|
events.events.length,
|
||||||
|
"every live task event must retain its unique instance identity"
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
events.events.some(
|
||||||
|
(event) => event.task === report.node_report.run.task_instance
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(events.events.some((event) => event.task.endsWith(":child:1")));
|
||||||
|
assert(events.events.some((event) => event.task.endsWith(":child:2")));
|
||||||
|
assert(events.events.some((event) => event.task.endsWith(":child:3")));
|
||||||
|
assert(events.events.some((event) => event.artifact_path));
|
||||||
|
} finally {
|
||||||
|
coordinator.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
|
||||||
|
const { pid: autoCliPid, report: autoReport } = await runCli([
|
||||||
|
"run",
|
||||||
|
"--local",
|
||||||
|
"--project",
|
||||||
|
project,
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert(Number.isInteger(autoCliPid));
|
||||||
|
assert.strictEqual(autoReport.plan.entry, "build");
|
||||||
|
assert.deepStrictEqual(autoReport.plan.coordinator, "LocalOnly");
|
||||||
|
assert.deepStrictEqual(autoReport.plan.session, "Anonymous");
|
||||||
|
assert.strictEqual(autoReport.boundary.cli_process_started_node_process, true);
|
||||||
|
assert.strictEqual(autoReport.boundary.cli_process_started_coordinator_process, true);
|
||||||
|
assert.match(autoReport.boundary.coordinator_address, /^127\.0\.0\.1:\d+$/);
|
||||||
|
assert(Number.isInteger(autoReport.boundary.coordinator_process_id));
|
||||||
|
assert(Number.isInteger(autoReport.boundary.spawned_node_process_id));
|
||||||
|
assert.notStrictEqual(autoReport.boundary.coordinator_process_id, autoCliPid);
|
||||||
|
assert.notStrictEqual(autoReport.boundary.spawned_node_process_id, autoCliPid);
|
||||||
|
assert.notStrictEqual(
|
||||||
|
autoReport.boundary.spawned_node_process_id,
|
||||||
|
autoReport.boundary.coordinator_process_id
|
||||||
|
);
|
||||||
|
assert.strictEqual(autoReport.boundary.node_session_requests, 0);
|
||||||
|
assert.strictEqual(autoReport.node_report.node_status, "completed");
|
||||||
|
assert.strictEqual(autoReport.node_report.execution_substrate, "wasm");
|
||||||
|
assert.strictEqual(autoReport.node_report.task_spawn_host_import, true);
|
||||||
|
assert.strictEqual(autoReport.node_report.run.status, "main_launched");
|
||||||
|
assert.strictEqual(autoReport.node_report.join.type, "task_joined");
|
||||||
|
|
||||||
|
console.log("CLI local run smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
72
scripts/cli-login-smoke.js
Normal file
72
scripts/cli-login-smoke.js
Normal file
|
|
@ -0,0 +1,72 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const coordinator = "https://coord.example.test";
|
||||||
|
const defaultHostedCoordinatorEndpoint = "https://clusterflux.michelpaulissen.com";
|
||||||
|
|
||||||
|
function clusterflux(args) {
|
||||||
|
return JSON.parse(
|
||||||
|
cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
|
||||||
|
{ cwd: repo, encoding: "utf8" }
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function clusterfluxRaw(args, env = {}) {
|
||||||
|
return cp.spawnSync(
|
||||||
|
"cargo",
|
||||||
|
["run", "-q", "-p", "clusterflux-cli", "--bin", "clusterflux", "--", ...args],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
encoding: "utf8",
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
...env,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const browser = clusterflux(["login", "--plan", "--coordinator", coordinator, "--json"]);
|
||||||
|
assert.strictEqual(browser.coordinator, coordinator);
|
||||||
|
assert(browser.human_flow.Browser, "browser login should be available for human users");
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.authorization_url, null);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.server_owns_state, true);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.server_owns_nonce, true);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.pkce_required, true);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.hosted_callback, true);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.cli_receives_provider_authorization_code, false);
|
||||||
|
assert.strictEqual(browser.human_flow.Browser.cli_submits_identity_claims, false);
|
||||||
|
|
||||||
|
const defaultBrowser = clusterflux(["login", "--plan", "--json"]);
|
||||||
|
assert.strictEqual(defaultBrowser.coordinator, defaultHostedCoordinatorEndpoint);
|
||||||
|
assert(defaultBrowser.human_flow.Browser);
|
||||||
|
|
||||||
|
const nonInteractiveBrowser = clusterfluxRaw(
|
||||||
|
["login", "--browser", "--non-interactive", "--coordinator", coordinator, "--json"],
|
||||||
|
{
|
||||||
|
CLUSTERFLUX_BROWSER_OPEN_COMMAND:
|
||||||
|
"node -e 'require(\"fs\").writeFileSync(\"/tmp/clusterflux-browser-should-not-open\", \"opened\")'",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert.strictEqual(nonInteractiveBrowser.status, 20, nonInteractiveBrowser.stderr);
|
||||||
|
assert.doesNotMatch(nonInteractiveBrowser.stderr, /Opening Clusterflux browser login/);
|
||||||
|
const nonInteractiveReport = JSON.parse(nonInteractiveBrowser.stdout);
|
||||||
|
assert.strictEqual(nonInteractiveReport.status, "authentication_required");
|
||||||
|
assert.strictEqual(nonInteractiveReport.non_interactive, true);
|
||||||
|
assert.strictEqual(nonInteractiveReport.browser_opened, false);
|
||||||
|
assert.strictEqual(nonInteractiveReport.machine_error.category, "authentication");
|
||||||
|
assert.strictEqual(nonInteractiveReport.machine_error.stable_exit_code, 20);
|
||||||
|
assert(
|
||||||
|
nonInteractiveReport.machine_error.next_actions.includes(
|
||||||
|
"rerun without --non-interactive to open the browser"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log("CLI login smoke passed");
|
||||||
191
scripts/cli-output-mode-smoke.js
Normal file
191
scripts/cli-output-mode-smoke.js
Normal file
|
|
@ -0,0 +1,191 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const project = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
const isolatedCwd = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-output-"));
|
||||||
|
const isolatedHome = fs.mkdtempSync(path.join(os.tmpdir(), "clusterflux-cli-home-"));
|
||||||
|
|
||||||
|
function clusterflux(args, env = {}, cwd = isolatedCwd) {
|
||||||
|
return cp.execFileSync(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"--manifest-path",
|
||||||
|
path.join(repo, "Cargo.toml"),
|
||||||
|
"-p",
|
||||||
|
"clusterflux-cli",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux",
|
||||||
|
"--",
|
||||||
|
...args,
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd,
|
||||||
|
encoding: "utf8",
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
HOME: isolatedHome,
|
||||||
|
USERPROFILE: isolatedHome,
|
||||||
|
XDG_CONFIG_HOME: path.join(isolatedHome, ".config"),
|
||||||
|
XDG_DATA_HOME: path.join(isolatedHome, ".local", "share"),
|
||||||
|
XDG_STATE_HOME: path.join(isolatedHome, ".local", "state"),
|
||||||
|
...env,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function json(args, env, cwd) {
|
||||||
|
return JSON.parse(clusterflux(args, env, cwd));
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertHuman(name, output, requiredPatterns) {
|
||||||
|
assert(
|
||||||
|
!output.trimStart().startsWith("{"),
|
||||||
|
`${name} default output should be human-readable text, not JSON`
|
||||||
|
);
|
||||||
|
for (const pattern of requiredPatterns) {
|
||||||
|
assert.match(output, pattern, `${name} human output missing ${pattern}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const helpHuman = clusterflux(["help"]);
|
||||||
|
assertHuman("help", helpHuman, [
|
||||||
|
/Primary workflow:/,
|
||||||
|
/clusterflux login --browser/,
|
||||||
|
/clusterflux project init/,
|
||||||
|
/clusterflux node attach; clusterflux-node --worker/,
|
||||||
|
/Clusterflux: Launch Virtual Process/,
|
||||||
|
/Hosted account creation happens in the browser login flow/,
|
||||||
|
/--json/,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const loginHuman = clusterflux([
|
||||||
|
"login",
|
||||||
|
"--plan",
|
||||||
|
"--coordinator",
|
||||||
|
"https://coord.example.test",
|
||||||
|
]);
|
||||||
|
assertHuman("login", loginHuman, [
|
||||||
|
/Clusterflux login/,
|
||||||
|
/flow: browser/,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const loginJson = json([
|
||||||
|
"login",
|
||||||
|
"--plan",
|
||||||
|
"--coordinator",
|
||||||
|
"https://coord.example.test",
|
||||||
|
"--json",
|
||||||
|
]);
|
||||||
|
assert.strictEqual(loginJson.coordinator, "https://coord.example.test");
|
||||||
|
assert(loginJson.human_flow.Browser);
|
||||||
|
assert.strictEqual(loginJson.human_flow.Browser.hosted_callback, true);
|
||||||
|
assert.strictEqual(loginJson.human_flow.Browser.cli_submits_identity_claims, false);
|
||||||
|
|
||||||
|
const doctorHuman = clusterflux(["doctor"]);
|
||||||
|
assertHuman("doctor", doctorHuman, [
|
||||||
|
/Clusterflux doctor/,
|
||||||
|
/coordinator reachability: not_configured/,
|
||||||
|
/dependencies:/,
|
||||||
|
/auth:/,
|
||||||
|
/node capabilities:/,
|
||||||
|
/node readiness: (ready_to_attach|local_dependencies_missing|limited_capabilities)/,
|
||||||
|
/node next:/,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const doctorJson = json(["doctor", "--json"]);
|
||||||
|
assert.strictEqual(doctorJson.coordinator_reachability.checked, false);
|
||||||
|
assert.strictEqual(doctorJson.coordinator_reachability.status, "not_configured");
|
||||||
|
assert(
|
||||||
|
["ready_to_attach", "local_dependencies_missing", "limited_capabilities"].includes(
|
||||||
|
doctorJson.node_readiness_summary.status
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert.strictEqual(doctorJson.node_readiness_summary.explicit_attach_required, true);
|
||||||
|
assert.strictEqual(
|
||||||
|
doctorJson.node_readiness_summary.command_execution_capability,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert(Array.isArray(doctorJson.node_readiness_summary.missing_local_dependencies));
|
||||||
|
assert(doctorJson.node_readiness_summary.next_actions.length >= 2);
|
||||||
|
|
||||||
|
const authJson = json(["auth", "status", "--json"], {
|
||||||
|
CLUSTERFLUX_TOKEN: "token",
|
||||||
|
CLUSTERFLUX_TOKEN_EXPIRES_AT: "2026-07-04T00:00:00Z",
|
||||||
|
}, isolatedCwd);
|
||||||
|
assert.strictEqual(authJson.session.kind, "human");
|
||||||
|
assert.strictEqual(authJson.session.token_expiry_posture, "expires_at");
|
||||||
|
assert.strictEqual(authJson.session.expires_at, "2026-07-04T00:00:00Z");
|
||||||
|
assert.strictEqual(authJson.coordinator_account_status.checked, false);
|
||||||
|
assert.strictEqual(authJson.coordinator_account_status.account_status, "unknown");
|
||||||
|
assert.strictEqual(
|
||||||
|
authJson.coordinator_account_status.private_moderation_details_exposed,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
|
||||||
|
const inspectHuman = clusterflux(["bundle", "inspect", "--project", project]);
|
||||||
|
assertHuman("bundle inspect", inspectHuman, [
|
||||||
|
/Clusterflux bundle inspect/,
|
||||||
|
/bundle: sha256:/,
|
||||||
|
/environments:/,
|
||||||
|
]);
|
||||||
|
|
||||||
|
const inspectJson = json(["bundle", "inspect", "--project", project, "--json"]);
|
||||||
|
assert.strictEqual(inspectJson.project, project);
|
||||||
|
assert.match(inspectJson.metadata.identity, /^sha256:/);
|
||||||
|
assert.match(inspectJson.metadata.wasm_code, /^sha256:/);
|
||||||
|
assert.strictEqual(inspectJson.metadata.task_metadata.default_entrypoint, "build");
|
||||||
|
assert.deepStrictEqual(inspectJson.metadata.task_metadata.entrypoints, [
|
||||||
|
"build",
|
||||||
|
"fail",
|
||||||
|
"identity",
|
||||||
|
"long-join",
|
||||||
|
"park-wake",
|
||||||
|
"restart",
|
||||||
|
]);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.source_metadata.transfer_policy.coordinator_receives_source_bytes_by_default,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.source_metadata.transfer_policy.default_full_repo_tarball,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert.strictEqual(inspectJson.metadata.debug_metadata.dap_virtual_process, true);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.large_input_policy.selected_inputs_are_content_digests,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert.strictEqual(inspectJson.metadata.large_input_policy.selected_input_bytes_included, false);
|
||||||
|
assert.strictEqual(inspectJson.metadata.large_input_policy.full_repository_bytes_included, false);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.large_input_policy.silent_task_argument_serialization,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert(inspectJson.metadata.large_input_policy.supported_handle_types.includes("SourceSnapshot"));
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.restart_compatibility.source_edits_can_restart_from_clean_task_boundary,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.restart_compatibility.requires_clean_checkpoint_boundary,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.restart_compatibility.compares_task_abi,
|
||||||
|
inspectJson.metadata.task_metadata.task_abi
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
inspectJson.metadata.restart_compatibility.incompatible_changes_require_whole_process_restart,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
console.log("CLI output mode smoke passed");
|
||||||
43
scripts/coordinator-wire.js
Normal file
43
scripts/coordinator-wire.js
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
let requestId = 0;
|
||||||
|
|
||||||
|
function coordinatorWireRequest(payload, prefix = "acceptance") {
|
||||||
|
if (payload && payload.type === "coordinator_request") return payload;
|
||||||
|
if (!payload || typeof payload.type !== "string" || !payload.type.trim()) {
|
||||||
|
throw new Error("coordinator payload must have a non-empty type");
|
||||||
|
}
|
||||||
|
requestId += 1;
|
||||||
|
return {
|
||||||
|
type: "coordinator_request",
|
||||||
|
protocol_version: 1,
|
||||||
|
request_id: `${prefix}-${process.pid}-${requestId}`,
|
||||||
|
operation: payload.type,
|
||||||
|
authentication: authenticationMetadata(payload),
|
||||||
|
payload,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function authenticationMetadata(payload) {
|
||||||
|
if (payload.type === "authenticated") {
|
||||||
|
return {
|
||||||
|
kind: "cli_session",
|
||||||
|
session: true,
|
||||||
|
request_operation: payload.request?.type || "unknown",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (payload.type === "signed_node" || payload.node_signature) {
|
||||||
|
return { kind: "node_signature", node: payload.node || null };
|
||||||
|
}
|
||||||
|
if (payload.agent_signature) {
|
||||||
|
return {
|
||||||
|
kind: "agent_signature",
|
||||||
|
agent: payload.actor_agent || null,
|
||||||
|
fingerprint: payload.agent_public_key_fingerprint || null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (payload.admin_token) {
|
||||||
|
return { kind: "admin_credential" };
|
||||||
|
}
|
||||||
|
return { kind: "none" };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { coordinatorWireRequest };
|
||||||
135
scripts/dap-client.js
Normal file
135
scripts/dap-client.js
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
const cp = require("child_process");
|
||||||
|
|
||||||
|
class DapClient {
|
||||||
|
constructor({
|
||||||
|
cwd = process.cwd(),
|
||||||
|
env = process.env,
|
||||||
|
command = "cargo",
|
||||||
|
args = [
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-dap",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-debug-dap",
|
||||||
|
],
|
||||||
|
} = {}) {
|
||||||
|
this.child = cp.spawn(
|
||||||
|
command,
|
||||||
|
args,
|
||||||
|
{ cwd, env }
|
||||||
|
);
|
||||||
|
this.seq = 1;
|
||||||
|
this.buffer = Buffer.alloc(0);
|
||||||
|
this.messages = [];
|
||||||
|
this.waiters = [];
|
||||||
|
this.stderr = "";
|
||||||
|
|
||||||
|
this.child.stdout.on("data", (chunk) => {
|
||||||
|
this.buffer = Buffer.concat([this.buffer, chunk]);
|
||||||
|
this.parse();
|
||||||
|
});
|
||||||
|
this.child.stderr.on("data", (chunk) => {
|
||||||
|
this.stderr += chunk.toString();
|
||||||
|
});
|
||||||
|
this.child.on("exit", () => this.flushWaiters());
|
||||||
|
}
|
||||||
|
|
||||||
|
send(command, args = {}) {
|
||||||
|
const seq = this.seq++;
|
||||||
|
const message = { seq, type: "request", command, arguments: args };
|
||||||
|
const payload = Buffer.from(JSON.stringify(message));
|
||||||
|
this.child.stdin.write(`Content-Length: ${payload.length}\r\n\r\n`);
|
||||||
|
this.child.stdin.write(payload);
|
||||||
|
return seq;
|
||||||
|
}
|
||||||
|
|
||||||
|
async response(seq, command) {
|
||||||
|
const message = await this.waitFor(
|
||||||
|
(item) =>
|
||||||
|
item.type === "response" &&
|
||||||
|
item.request_seq === seq &&
|
||||||
|
item.command === command
|
||||||
|
);
|
||||||
|
if (!message.success) {
|
||||||
|
throw new Error(
|
||||||
|
`DAP ${command} failed: ${message.message || JSON.stringify(message)}\nAdapter stderr:\n${this.stderr}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
async failure(seq, command) {
|
||||||
|
const message = await this.waitFor(
|
||||||
|
(item) =>
|
||||||
|
item.type === "response" &&
|
||||||
|
item.request_seq === seq &&
|
||||||
|
item.command === command
|
||||||
|
);
|
||||||
|
if (message.success) {
|
||||||
|
throw new Error(`DAP ${command} unexpectedly succeeded`);
|
||||||
|
}
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
|
||||||
|
waitFor(
|
||||||
|
predicate,
|
||||||
|
timeoutMs = Number(process.env.CLUSTERFLUX_DAP_TIMEOUT_MS || 120000)
|
||||||
|
) {
|
||||||
|
const existing = this.messages.find(predicate);
|
||||||
|
if (existing) return Promise.resolve(existing);
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
this.child.kill("SIGKILL");
|
||||||
|
const recent = this.messages
|
||||||
|
.slice(-10)
|
||||||
|
.map((message) => JSON.stringify(message))
|
||||||
|
.join("\n");
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`timed out waiting for DAP message\nRecent DAP messages:\n${recent}\nAdapter stderr:\n${this.stderr}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}, timeoutMs);
|
||||||
|
this.waiters.push({ predicate, resolve, timer });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
parse() {
|
||||||
|
while (true) {
|
||||||
|
const headerEnd = this.buffer.indexOf("\r\n\r\n");
|
||||||
|
if (headerEnd < 0) return;
|
||||||
|
const header = this.buffer.slice(0, headerEnd).toString();
|
||||||
|
const match = header.match(/Content-Length: (\d+)/i);
|
||||||
|
if (!match) throw new Error(`bad DAP header: ${header}`);
|
||||||
|
const length = Number(match[1]);
|
||||||
|
const start = headerEnd + 4;
|
||||||
|
const end = start + length;
|
||||||
|
if (this.buffer.length < end) return;
|
||||||
|
const payload = this.buffer.slice(start, end).toString();
|
||||||
|
this.buffer = this.buffer.slice(end);
|
||||||
|
this.messages.push(JSON.parse(payload));
|
||||||
|
this.flushWaiters();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
flushWaiters() {
|
||||||
|
for (const waiter of [...this.waiters]) {
|
||||||
|
const message = this.messages.find(waiter.predicate);
|
||||||
|
if (!message) continue;
|
||||||
|
clearTimeout(waiter.timer);
|
||||||
|
this.waiters.splice(this.waiters.indexOf(waiter), 1);
|
||||||
|
waiter.resolve(message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async close() {
|
||||||
|
if (this.child.exitCode !== null) return;
|
||||||
|
const seq = this.send("disconnect");
|
||||||
|
await this.response(seq, "disconnect");
|
||||||
|
this.child.stdin.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { DapClient };
|
||||||
510
scripts/dap-smoke.js
Normal file
510
scripts/dap-smoke.js
Normal file
|
|
@ -0,0 +1,510 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const { DapClient } = require("./dap-client");
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const project = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
const sourcePath = fs.realpathSync(path.join(project, "src/lib.rs"));
|
||||||
|
const sourceLines = fs.readFileSync(sourcePath, "utf8").split(/\r?\n/);
|
||||||
|
const buildMainLine =
|
||||||
|
sourceLines.findIndex((line) => line.includes("pub async fn build_main()")) + 1;
|
||||||
|
assert(buildMainLine > 0, "flagship source must contain build_main");
|
||||||
|
|
||||||
|
const hostileDapClient = new DapClient();
|
||||||
|
try {
|
||||||
|
const initialize = hostileDapClient.send("initialize", {
|
||||||
|
adapterID: "clusterflux",
|
||||||
|
linesStartAt1: true,
|
||||||
|
columnsStartAt1: true,
|
||||||
|
});
|
||||||
|
await hostileDapClient.response(initialize, "initialize");
|
||||||
|
for (const processId of [
|
||||||
|
"",
|
||||||
|
" ",
|
||||||
|
"bad\u0000process",
|
||||||
|
"bad process!",
|
||||||
|
"x".repeat(256),
|
||||||
|
]) {
|
||||||
|
const malformedLaunch = hostileDapClient.send("launch", {
|
||||||
|
entry: "build",
|
||||||
|
project,
|
||||||
|
runtimeBackend: "local-services",
|
||||||
|
processId,
|
||||||
|
});
|
||||||
|
const rejection = await hostileDapClient.failure(
|
||||||
|
malformedLaunch,
|
||||||
|
"launch"
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
rejection.message,
|
||||||
|
/invalid DAP processId: ProcessId is invalid/,
|
||||||
|
`unexpected malformed DAP identifier response: ${JSON.stringify(rejection)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const validLaunch = hostileDapClient.send("launch", {
|
||||||
|
entry: "build",
|
||||||
|
project,
|
||||||
|
runtimeBackend: "local-services",
|
||||||
|
processId: "vp-valid-after-malformed",
|
||||||
|
});
|
||||||
|
await hostileDapClient.response(validLaunch, "launch");
|
||||||
|
await hostileDapClient.waitFor(
|
||||||
|
(message) => message.type === "event" && message.event === "initialized"
|
||||||
|
);
|
||||||
|
await hostileDapClient.close();
|
||||||
|
} catch (error) {
|
||||||
|
if (hostileDapClient.child.exitCode === null) {
|
||||||
|
hostileDapClient.child.kill("SIGKILL");
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const asynchronousClient = new DapClient();
|
||||||
|
try {
|
||||||
|
const initialize = asynchronousClient.send("initialize", {
|
||||||
|
adapterID: "clusterflux",
|
||||||
|
linesStartAt1: true,
|
||||||
|
columnsStartAt1: true,
|
||||||
|
});
|
||||||
|
await asynchronousClient.response(initialize, "initialize");
|
||||||
|
const launch = asynchronousClient.send("launch", {
|
||||||
|
entry: "long-join",
|
||||||
|
project,
|
||||||
|
runtimeBackend: "local-services",
|
||||||
|
});
|
||||||
|
await asynchronousClient.response(launch, "launch");
|
||||||
|
await asynchronousClient.waitFor(
|
||||||
|
(message) => message.type === "event" && message.event === "initialized"
|
||||||
|
);
|
||||||
|
|
||||||
|
const configuredAt = Date.now();
|
||||||
|
const configurationDone = asynchronousClient.send("configurationDone");
|
||||||
|
await asynchronousClient.response(configurationDone, "configurationDone");
|
||||||
|
assert(
|
||||||
|
Date.now() - configuredAt < 2_000,
|
||||||
|
"configurationDone must not wait for bundle build or runtime completion"
|
||||||
|
);
|
||||||
|
|
||||||
|
const threadDeadline = Date.now() + 120_000;
|
||||||
|
let runningThreads = [];
|
||||||
|
while (Date.now() < threadDeadline) {
|
||||||
|
const threadsRequest = asynchronousClient.send("threads");
|
||||||
|
runningThreads = (
|
||||||
|
await asynchronousClient.response(threadsRequest, "threads")
|
||||||
|
).body.threads;
|
||||||
|
if (runningThreads.length > 0) break;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
|
}
|
||||||
|
assert(runningThreads.length > 0, "asynchronous launch never reported a live thread");
|
||||||
|
|
||||||
|
const pauseAt = Date.now();
|
||||||
|
const pause = asynchronousClient.send("pause", {
|
||||||
|
threadId: runningThreads[0].id,
|
||||||
|
});
|
||||||
|
await asynchronousClient.response(pause, "pause");
|
||||||
|
assert(Date.now() - pauseAt < 2_000, "pause response was blocked by runtime work");
|
||||||
|
const paused = await asynchronousClient.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "stopped" &&
|
||||||
|
message.body.reason === "pause"
|
||||||
|
);
|
||||||
|
|
||||||
|
const continueAt = Date.now();
|
||||||
|
const continued = asynchronousClient.send("continue", {
|
||||||
|
threadId: paused.body.threadId,
|
||||||
|
});
|
||||||
|
await asynchronousClient.response(continued, "continue");
|
||||||
|
assert(
|
||||||
|
Date.now() - continueAt < 2_000,
|
||||||
|
"continue response was blocked by runtime observation"
|
||||||
|
);
|
||||||
|
|
||||||
|
const disconnectAt = Date.now();
|
||||||
|
await asynchronousClient.close();
|
||||||
|
assert(
|
||||||
|
Date.now() - disconnectAt < 2_000,
|
||||||
|
"disconnect response was blocked by runtime observation"
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
if (asynchronousClient.child.exitCode === null) {
|
||||||
|
asynchronousClient.child.kill("SIGKILL");
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new DapClient();
|
||||||
|
try {
|
||||||
|
const initialize = client.send("initialize", {
|
||||||
|
adapterID: "clusterflux",
|
||||||
|
linesStartAt1: true,
|
||||||
|
columnsStartAt1: true,
|
||||||
|
});
|
||||||
|
await client.response(initialize, "initialize");
|
||||||
|
|
||||||
|
const launch = client.send("launch", {
|
||||||
|
entry: "build",
|
||||||
|
project,
|
||||||
|
runtimeBackend: "local-services",
|
||||||
|
});
|
||||||
|
await client.response(launch, "launch");
|
||||||
|
await client.waitFor(
|
||||||
|
(message) => message.type === "event" && message.event === "initialized"
|
||||||
|
);
|
||||||
|
|
||||||
|
const breakpoints = client.send("setBreakpoints", {
|
||||||
|
source: { path: sourcePath },
|
||||||
|
breakpoints: [{ line: buildMainLine }],
|
||||||
|
});
|
||||||
|
const breakpointResponse = await client.response(
|
||||||
|
breakpoints,
|
||||||
|
"setBreakpoints"
|
||||||
|
);
|
||||||
|
assert.strictEqual(breakpointResponse.body.breakpoints.length, 1);
|
||||||
|
assert.strictEqual(breakpointResponse.body.breakpoints[0].verified, false);
|
||||||
|
|
||||||
|
const configurationDone = client.send("configurationDone");
|
||||||
|
await client.response(configurationDone, "configurationDone");
|
||||||
|
const installedBreakpoint = await client.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "breakpoint" &&
|
||||||
|
message.body?.breakpoint?.verified === true
|
||||||
|
);
|
||||||
|
assert.strictEqual(installedBreakpoint.body.breakpoint.line, buildMainLine);
|
||||||
|
const stopped = await client.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "stopped" &&
|
||||||
|
message.body.reason === "breakpoint"
|
||||||
|
);
|
||||||
|
assert.strictEqual(stopped.body.allThreadsStopped, true);
|
||||||
|
assert.match(stopped.body.description, /confirmed by every active participant/i);
|
||||||
|
|
||||||
|
const threadsRequest = client.send("threads");
|
||||||
|
const threads = (await client.response(threadsRequest, "threads")).body
|
||||||
|
.threads;
|
||||||
|
const mainThread = threads.find((thread) =>
|
||||||
|
thread.name.includes("build coordinator main")
|
||||||
|
);
|
||||||
|
assert(mainThread, "the running Wasm entrypoint must be the DAP coordinator-main thread");
|
||||||
|
assert.strictEqual(stopped.body.threadId, mainThread.id);
|
||||||
|
|
||||||
|
const stackRequest = client.send("stackTrace", {
|
||||||
|
threadId: mainThread.id,
|
||||||
|
startFrame: 0,
|
||||||
|
levels: 1,
|
||||||
|
});
|
||||||
|
const stack = (await client.response(stackRequest, "stackTrace")).body
|
||||||
|
.stackFrames;
|
||||||
|
assert.strictEqual(stack.length, 1);
|
||||||
|
assert.strictEqual(stack[0].line, buildMainLine);
|
||||||
|
assert.strictEqual(stack[0].source.path, sourcePath);
|
||||||
|
assert.match(stack[0].name, /build_main::wasm/);
|
||||||
|
assert.doesNotMatch(stack[0].name, /podman|cmd\.exe|powershell|pid|native child/i);
|
||||||
|
|
||||||
|
const sourceRequest = client.send("source", { source: stack[0].source });
|
||||||
|
const source = (await client.response(sourceRequest, "source")).body;
|
||||||
|
assert.match(source.content, /build_main/);
|
||||||
|
assert.match(source.mimeType, /rust/);
|
||||||
|
|
||||||
|
const scopesRequest = client.send("scopes", { frameId: stack[0].id });
|
||||||
|
const scopes = (await client.response(scopesRequest, "scopes")).body.scopes;
|
||||||
|
const localsScope = scopes.find((scope) => scope.name === "Source Locals");
|
||||||
|
const wasmScope = scopes.find((scope) => scope.name === "Wasm Frame Locals");
|
||||||
|
const argsScope = scopes.find(
|
||||||
|
(scope) => scope.name === "Task Args and Handles"
|
||||||
|
);
|
||||||
|
const runtimeScope = scopes.find(
|
||||||
|
(scope) => scope.name === "Clusterflux Runtime"
|
||||||
|
);
|
||||||
|
assert(localsScope && wasmScope && argsScope && runtimeScope);
|
||||||
|
|
||||||
|
const localsRequest = client.send("variables", {
|
||||||
|
variablesReference: localsScope.variablesReference,
|
||||||
|
});
|
||||||
|
const locals = (await client.response(localsRequest, "variables")).body
|
||||||
|
.variables;
|
||||||
|
assert(
|
||||||
|
locals.some(
|
||||||
|
(variable) =>
|
||||||
|
variable.name === "unavailable-local-diagnostic" &&
|
||||||
|
String(variable.value).includes("cannot be inspected")
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const wasmRequest = client.send("variables", {
|
||||||
|
variablesReference: wasmScope.variablesReference,
|
||||||
|
});
|
||||||
|
const wasmLocals = (await client.response(wasmRequest, "variables")).body
|
||||||
|
.variables;
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
wasmLocals.map((variable) => variable.name),
|
||||||
|
["wasm-local-diagnostic"]
|
||||||
|
);
|
||||||
|
assert.match(wasmLocals[0].value, /did not report inspectable Wasm frame locals/);
|
||||||
|
|
||||||
|
const argsRequest = client.send("variables", {
|
||||||
|
variablesReference: argsScope.variablesReference,
|
||||||
|
});
|
||||||
|
const args = (await client.response(argsRequest, "variables")).body.variables;
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
args.map((variable) => variable.name),
|
||||||
|
["runtime-boundary-diagnostic"]
|
||||||
|
);
|
||||||
|
assert.match(args[0].value, /reported no task arguments or handles/);
|
||||||
|
|
||||||
|
const runtimeRequest = client.send("variables", {
|
||||||
|
variablesReference: runtimeScope.variablesReference,
|
||||||
|
});
|
||||||
|
const runtime = (await client.response(runtimeRequest, "variables")).body
|
||||||
|
.variables;
|
||||||
|
const value = (name) => runtime.find((variable) => variable.name === name)?.value;
|
||||||
|
assert.strictEqual(value("runtime_backend"), "LocalServices");
|
||||||
|
assert.strictEqual(value("state"), "Frozen");
|
||||||
|
assert.strictEqual(value("debug_epoch"), 1);
|
||||||
|
assert.strictEqual(value("coordinator_task_events"), 0);
|
||||||
|
assert.match(
|
||||||
|
String(value("command_status")),
|
||||||
|
/frozen through local services at executing Wasm probe/
|
||||||
|
);
|
||||||
|
|
||||||
|
const step = client.send("next", { threadId: mainThread.id });
|
||||||
|
const stepFailure = await client.failure(step, "next");
|
||||||
|
assert.match(stepFailure.message, /source stepping is not yet available/i);
|
||||||
|
assert.match(stepFailure.message, /synthetic step/i);
|
||||||
|
|
||||||
|
const restart = client.send("restartFrame", { frameId: stack[0].id });
|
||||||
|
const restartFailure = await client.failure(restart, "restartFrame");
|
||||||
|
assert.match(restartFailure.message, /checkpoint boundary|still active/i);
|
||||||
|
|
||||||
|
const incompatibleRestart = client.send("restartFrame", {
|
||||||
|
frameId: stack[0].id,
|
||||||
|
sourceCompatibility: "incompatible",
|
||||||
|
});
|
||||||
|
const incompatibleFailure = await client.failure(
|
||||||
|
incompatibleRestart,
|
||||||
|
"restartFrame"
|
||||||
|
);
|
||||||
|
assert.match(incompatibleFailure.message, /incompatible source edit/i);
|
||||||
|
assert.match(incompatibleFailure.message, /whole virtual-process restart/i);
|
||||||
|
|
||||||
|
await client.close();
|
||||||
|
} catch (error) {
|
||||||
|
if (client.child.exitCode === null) client.child.kill("SIGKILL");
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const failMainLine =
|
||||||
|
sourceLines.findIndex((line) => line.includes("pub async fn fail_main()")) + 1;
|
||||||
|
assert(failMainLine > 0, "flagship source must contain fail_main");
|
||||||
|
const taskTrapLine =
|
||||||
|
sourceLines.findIndex((line) => line.includes("fn task_trap(")) + 1;
|
||||||
|
assert(taskTrapLine > 0, "flagship source must contain task_trap");
|
||||||
|
const restartClient = new DapClient();
|
||||||
|
try {
|
||||||
|
const initialize = restartClient.send("initialize", {
|
||||||
|
adapterID: "clusterflux",
|
||||||
|
linesStartAt1: true,
|
||||||
|
columnsStartAt1: true,
|
||||||
|
});
|
||||||
|
await restartClient.response(initialize, "initialize");
|
||||||
|
const launch = restartClient.send("launch", {
|
||||||
|
entry: "fail",
|
||||||
|
project,
|
||||||
|
runtimeBackend: "local-services",
|
||||||
|
});
|
||||||
|
await restartClient.response(launch, "launch");
|
||||||
|
await restartClient.waitFor(
|
||||||
|
(message) => message.type === "event" && message.event === "initialized"
|
||||||
|
);
|
||||||
|
const breakpoints = restartClient.send("setBreakpoints", {
|
||||||
|
source: { path: sourcePath },
|
||||||
|
breakpoints: [{ line: failMainLine }, { line: taskTrapLine }],
|
||||||
|
});
|
||||||
|
const breakpointResponse = await restartClient.response(
|
||||||
|
breakpoints,
|
||||||
|
"setBreakpoints"
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
breakpointResponse.body.breakpoints.map((breakpoint) => breakpoint.verified),
|
||||||
|
[false, false]
|
||||||
|
);
|
||||||
|
const configurationDone = restartClient.send("configurationDone");
|
||||||
|
await restartClient.response(configurationDone, "configurationDone");
|
||||||
|
const installedLines = [];
|
||||||
|
while (installedLines.length < 2) {
|
||||||
|
const installed = await restartClient.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "breakpoint" &&
|
||||||
|
message.body?.breakpoint?.verified === true &&
|
||||||
|
!installedLines.includes(message.body.breakpoint.line)
|
||||||
|
);
|
||||||
|
installedLines.push(installed.body.breakpoint.line);
|
||||||
|
}
|
||||||
|
assert.deepStrictEqual(installedLines.sort((a, b) => a - b), [
|
||||||
|
failMainLine,
|
||||||
|
taskTrapLine,
|
||||||
|
].sort((a, b) => a - b));
|
||||||
|
const initialStop = await restartClient.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "stopped" &&
|
||||||
|
message.body.reason === "breakpoint"
|
||||||
|
);
|
||||||
|
assert.strictEqual(initialStop.body.allThreadsStopped, true);
|
||||||
|
const threadsRequest = restartClient.send("threads");
|
||||||
|
const threads = (
|
||||||
|
await restartClient.response(threadsRequest, "threads")
|
||||||
|
).body.threads;
|
||||||
|
const failThread = threads.find(
|
||||||
|
(thread) => thread.id === initialStop.body.threadId
|
||||||
|
);
|
||||||
|
assert(failThread, "failed entrypoint must remain a virtual task thread");
|
||||||
|
const stackRequest = restartClient.send("stackTrace", {
|
||||||
|
threadId: failThread.id,
|
||||||
|
startFrame: 0,
|
||||||
|
levels: 1,
|
||||||
|
});
|
||||||
|
const failedStack = (
|
||||||
|
await restartClient.response(stackRequest, "stackTrace")
|
||||||
|
).body.stackFrames;
|
||||||
|
assert.strictEqual(failedStack[0].line, failMainLine);
|
||||||
|
|
||||||
|
const continueRequest = restartClient.send("continue", {
|
||||||
|
threadId: failThread.id,
|
||||||
|
});
|
||||||
|
await restartClient.response(continueRequest, "continue");
|
||||||
|
const childStop = await restartClient.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.seq > initialStop.seq &&
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "stopped" &&
|
||||||
|
message.body.reason === "breakpoint",
|
||||||
|
70000
|
||||||
|
);
|
||||||
|
assert.strictEqual(childStop.body.allThreadsStopped, true);
|
||||||
|
|
||||||
|
const childThreadsRequest = restartClient.send("threads");
|
||||||
|
const childThreads = (
|
||||||
|
await restartClient.response(childThreadsRequest, "threads")
|
||||||
|
).body.threads;
|
||||||
|
const childThread = childThreads.find(
|
||||||
|
(thread) => thread.id === childStop.body.threadId
|
||||||
|
);
|
||||||
|
assert(childThread, "executing child task must become a DAP virtual thread");
|
||||||
|
assert.notStrictEqual(childThread.id, failThread.id);
|
||||||
|
assert.match(childThread.name, /task trap/i);
|
||||||
|
|
||||||
|
const childStackRequest = restartClient.send("stackTrace", {
|
||||||
|
threadId: childThread.id,
|
||||||
|
startFrame: 0,
|
||||||
|
levels: 1,
|
||||||
|
});
|
||||||
|
const childStack = (
|
||||||
|
await restartClient.response(childStackRequest, "stackTrace")
|
||||||
|
).body.stackFrames;
|
||||||
|
assert.strictEqual(childStack[0].line, taskTrapLine);
|
||||||
|
assert.match(childStack[0].name, /task_trap::wasm/);
|
||||||
|
|
||||||
|
const childScopesRequest = restartClient.send("scopes", {
|
||||||
|
frameId: childStack[0].id,
|
||||||
|
});
|
||||||
|
const childScopes = (
|
||||||
|
await restartClient.response(childScopesRequest, "scopes")
|
||||||
|
).body.scopes;
|
||||||
|
const childArgsScope = childScopes.find(
|
||||||
|
(scope) => scope.name === "Task Args and Handles"
|
||||||
|
);
|
||||||
|
assert(childArgsScope, "child task argument scope must be present");
|
||||||
|
const childArgsRequest = restartClient.send("variables", {
|
||||||
|
variablesReference: childArgsScope.variablesReference,
|
||||||
|
});
|
||||||
|
const childArgs = (
|
||||||
|
await restartClient.response(childArgsRequest, "variables")
|
||||||
|
).body.variables;
|
||||||
|
assert(
|
||||||
|
childArgs.some(
|
||||||
|
(variable) =>
|
||||||
|
variable.name === "arg_0" &&
|
||||||
|
/SmallJson\(Number\(0\)\)/.test(String(variable.value))
|
||||||
|
),
|
||||||
|
"child task argument must come from the frozen node participant"
|
||||||
|
);
|
||||||
|
|
||||||
|
const parentScopesRequest = restartClient.send("scopes", {
|
||||||
|
frameId: failedStack[0].id,
|
||||||
|
});
|
||||||
|
const parentScopes = (
|
||||||
|
await restartClient.response(parentScopesRequest, "scopes")
|
||||||
|
).body.scopes;
|
||||||
|
const parentArgsScope = parentScopes.find(
|
||||||
|
(scope) => scope.name === "Task Args and Handles"
|
||||||
|
);
|
||||||
|
const parentArgsRequest = restartClient.send("variables", {
|
||||||
|
variablesReference: parentArgsScope.variablesReference,
|
||||||
|
});
|
||||||
|
const parentArgs = (
|
||||||
|
await restartClient.response(parentArgsRequest, "variables")
|
||||||
|
).body.variables;
|
||||||
|
assert(
|
||||||
|
parentArgs.some(
|
||||||
|
(variable) =>
|
||||||
|
/^task_handle_\d+$/.test(variable.name) &&
|
||||||
|
/definition=task_trap instance=ti:.*:child:\d+ state=active/.test(
|
||||||
|
variable.value
|
||||||
|
) &&
|
||||||
|
variable.type === "runtime-handle"
|
||||||
|
),
|
||||||
|
"parent task handle must come from its live Wasm host registry"
|
||||||
|
);
|
||||||
|
|
||||||
|
const continueChildRequest = restartClient.send("continue", {
|
||||||
|
threadId: childThread.id,
|
||||||
|
});
|
||||||
|
await restartClient.response(continueChildRequest, "continue");
|
||||||
|
await restartClient.waitFor(
|
||||||
|
(message) =>
|
||||||
|
message.seq > childStop.seq &&
|
||||||
|
message.type === "event" &&
|
||||||
|
message.event === "terminated"
|
||||||
|
);
|
||||||
|
|
||||||
|
const terminalThreadsRequest = restartClient.send("threads");
|
||||||
|
const terminalThreads = (
|
||||||
|
await restartClient.response(terminalThreadsRequest, "threads")
|
||||||
|
).body.threads;
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
terminalThreads,
|
||||||
|
[],
|
||||||
|
"terminated processes must not retain stale virtual threads"
|
||||||
|
);
|
||||||
|
|
||||||
|
const restartRequest = restartClient.send("restartFrame", {
|
||||||
|
frameId: failedStack[0].id,
|
||||||
|
});
|
||||||
|
const restartFailure = await restartClient.failure(
|
||||||
|
restartRequest,
|
||||||
|
"restartFrame"
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
restartFailure.message,
|
||||||
|
/does not map to a virtual task/i,
|
||||||
|
"a frame from a terminated process must not restart a stale task"
|
||||||
|
);
|
||||||
|
await restartClient.close();
|
||||||
|
} catch (error) {
|
||||||
|
if (restartClient.child.exitCode === null) restartClient.child.kill("SIGKILL");
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("DAP smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
83
scripts/deploy-release-candidate.sh
Executable file
83
scripts/deploy-release-candidate.sh
Executable file
|
|
@ -0,0 +1,83 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
manifest=${1:?usage: deploy-release-candidate.sh MANIFEST}
|
||||||
|
: "${CLUSTERFLUX_DEPLOY_COMMAND:?set the documented exact-candidate deployment command}"
|
||||||
|
: "${CLUSTERFLUX_STRICT_SSH_TARGET:?set the production SSH target}"
|
||||||
|
|
||||||
|
service_unit=${CLUSTERFLUX_STRICT_SERVICE_UNIT:-clusterflux-hosted.service}
|
||||||
|
proxy_unit=${CLUSTERFLUX_STRICT_PROXY_UNIT:-nginx.service}
|
||||||
|
evidence_path=${CLUSTERFLUX_DEPLOYMENT_EVIDENCE_PATH:-target/acceptance/deployment.json}
|
||||||
|
staging=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$staging"' EXIT
|
||||||
|
|
||||||
|
IFS=$'\t' read -r archive expected_archive_sha < <(
|
||||||
|
node - "$manifest" <<'NODE'
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const manifestPath = path.resolve(process.argv[2]);
|
||||||
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||||
|
const asset = (manifest.assets || []).find((entry) => entry.name.startsWith("clusterflux-hosted-"));
|
||||||
|
if (!asset) throw new Error("candidate manifest has no hosted archive");
|
||||||
|
const file = path.isAbsolute(asset.file)
|
||||||
|
? asset.file
|
||||||
|
: path.resolve(path.dirname(manifestPath), asset.file);
|
||||||
|
process.stdout.write(`${file}\t${asset.sha256}\n`);
|
||||||
|
NODE
|
||||||
|
)
|
||||||
|
|
||||||
|
actual_archive_sha=$(sha256sum "$archive" | awk '{print $1}')
|
||||||
|
test "$actual_archive_sha" = "${expected_archive_sha#sha256:}"
|
||||||
|
tar -xzf "$archive" -C "$staging"
|
||||||
|
candidate_coordinator=$(find "$staging" -type f -name clusterflux-hosted-service -perm -u+x -print -quit)
|
||||||
|
test -n "$candidate_coordinator"
|
||||||
|
candidate_sha=$(sha256sum "$candidate_coordinator" | awk '{print $1}')
|
||||||
|
|
||||||
|
export CLUSTERFLUX_CANDIDATE_ARCHIVE="$archive"
|
||||||
|
export CLUSTERFLUX_CANDIDATE_ARCHIVE_SHA256="sha256:$actual_archive_sha"
|
||||||
|
export CLUSTERFLUX_CANDIDATE_COORDINATOR="$candidate_coordinator"
|
||||||
|
export CLUSTERFLUX_CANDIDATE_COORDINATOR_SHA256="sha256:$candidate_sha"
|
||||||
|
bash -euo pipefail -c "$CLUSTERFLUX_DEPLOY_COMMAND"
|
||||||
|
|
||||||
|
main_pid=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=MainPID --value "$service_unit")
|
||||||
|
test "$main_pid" != 0
|
||||||
|
remote_executable=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" readlink -f "/proc/$main_pid/exe")
|
||||||
|
remote_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$remote_executable" | awk '{print $1}')
|
||||||
|
test "$remote_sha" = "$candidate_sha"
|
||||||
|
service_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$service_unit")
|
||||||
|
service_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$service_fragment" | awk '{print $1}')
|
||||||
|
service_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl cat "$service_unit" | sha256sum | awk '{print $1}')
|
||||||
|
proxy_fragment=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=FragmentPath --value "$proxy_unit")
|
||||||
|
proxy_fragment_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" sha256sum "$proxy_fragment" | awk '{print $1}')
|
||||||
|
proxy_exec_start=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" systemctl show --property=ExecStart --value "$proxy_unit")
|
||||||
|
proxy_executable=$(sed -n 's/.*path=\([^ ;]*\).*/\1/p' <<<"$proxy_exec_start")
|
||||||
|
proxy_configuration=$(sed -n 's/.* argv\[\]=.* -c \([^ ;]*\).*/\1/p' <<<"$proxy_exec_start")
|
||||||
|
test -n "$proxy_executable"
|
||||||
|
test -n "$proxy_configuration"
|
||||||
|
proxy_configuration_sha=$(ssh "$CLUSTERFLUX_STRICT_SSH_TARGET" "$proxy_executable" -T -c "$proxy_configuration" 2>/dev/null | sha256sum | awk '{print $1}')
|
||||||
|
mkdir -p "$(dirname "$evidence_path")"
|
||||||
|
|
||||||
|
EVIDENCE_PATH="$evidence_path" SERVICE_UNIT="$service_unit" \
|
||||||
|
SERVICE_FRAGMENT="$service_fragment" SERVICE_FRAGMENT_SHA="$service_fragment_sha" \
|
||||||
|
SERVICE_CONFIGURATION_SHA="$service_configuration_sha" \
|
||||||
|
REMOTE_EXECUTABLE="$remote_executable" REMOTE_SHA="$remote_sha" \
|
||||||
|
PROXY_UNIT="$proxy_unit" PROXY_FRAGMENT="$proxy_fragment" \
|
||||||
|
PROXY_FRAGMENT_SHA="$proxy_fragment_sha" PROXY_EXECUTABLE="$proxy_executable" \
|
||||||
|
PROXY_CONFIGURATION="$proxy_configuration" PROXY_CONFIGURATION_SHA="$proxy_configuration_sha" node <<'NODE'
|
||||||
|
const fs = require("fs");
|
||||||
|
fs.writeFileSync(process.env.EVIDENCE_PATH, JSON.stringify({
|
||||||
|
kind: "clusterflux-exact-candidate-deployment",
|
||||||
|
service_unit: process.env.SERVICE_UNIT,
|
||||||
|
service_unit_fragment: process.env.SERVICE_FRAGMENT,
|
||||||
|
service_unit_sha256: `sha256:${process.env.SERVICE_FRAGMENT_SHA}`,
|
||||||
|
service_configuration_sha256: `sha256:${process.env.SERVICE_CONFIGURATION_SHA}`,
|
||||||
|
hosted_service_executable: process.env.REMOTE_EXECUTABLE,
|
||||||
|
hosted_service_sha256: `sha256:${process.env.REMOTE_SHA}`,
|
||||||
|
proxy_unit: process.env.PROXY_UNIT,
|
||||||
|
proxy_unit_fragment: process.env.PROXY_FRAGMENT,
|
||||||
|
proxy_unit_sha256: `sha256:${process.env.PROXY_FRAGMENT_SHA}`,
|
||||||
|
proxy_executable: process.env.PROXY_EXECUTABLE,
|
||||||
|
proxy_configuration: process.env.PROXY_CONFIGURATION,
|
||||||
|
proxy_configuration_sha256: `sha256:${process.env.PROXY_CONFIGURATION_SHA}`,
|
||||||
|
}, null, 2) + "\n");
|
||||||
|
NODE
|
||||||
80
scripts/flagship-demo-smoke.js
Normal file
80
scripts/flagship-demo-smoke.js
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const hello = path.join(repo, "examples/hello-build");
|
||||||
|
const recovery = path.join(repo, "examples/recovery-build");
|
||||||
|
const conformance = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
const source = fs.readFileSync(path.join(hello, "src/lib.rs"), "utf8");
|
||||||
|
const nonblank = source.split(/\r?\n/).filter((line) => line.trim()).length;
|
||||||
|
|
||||||
|
assert(nonblank <= 80, "hello-build source must stay below 80 nonblank lines");
|
||||||
|
for (const forbidden of [
|
||||||
|
"#[cfg",
|
||||||
|
"unsafe",
|
||||||
|
"extern \"C\"",
|
||||||
|
"no_mangle",
|
||||||
|
"sha256:",
|
||||||
|
".task_id(",
|
||||||
|
"#[test]",
|
||||||
|
"unwrap()",
|
||||||
|
"expect(",
|
||||||
|
]) {
|
||||||
|
assert(!source.includes(forbidden), "hello-build leaked implementation detail: " + forbidden);
|
||||||
|
}
|
||||||
|
assert.strictEqual((source.match(/#\[clusterflux::task/g) || []).length, 1);
|
||||||
|
assert.strictEqual((source.match(/#\[clusterflux::main/g) || []).length, 1);
|
||||||
|
assert(source.includes("source::current_project().snapshot().await?"));
|
||||||
|
assert(source.includes("clusterflux::spawn!(compile(source))"));
|
||||||
|
assert(source.includes(".on(clusterflux::env!(\"linux\"))"));
|
||||||
|
assert(source.includes(".run()"));
|
||||||
|
assert(source.includes("fs::publish"));
|
||||||
|
assert(fs.existsSync(path.join(hello, "fixture/hello-clusterflux.c")));
|
||||||
|
assert(fs.existsSync(path.join(hello, "envs/linux/Containerfile")));
|
||||||
|
|
||||||
|
const recoverySource = fs.readFileSync(path.join(recovery, "src/lib.rs"), "utf8");
|
||||||
|
assert.strictEqual((recoverySource.match(/spawn!\(build_lane/g) || []).length, 2);
|
||||||
|
assert(recoverySource.includes("TaskFailurePolicy::AwaitOperator"));
|
||||||
|
assert(recoverySource.includes("\"exit 23\""));
|
||||||
|
for (const forbidden of ["#[cfg", "unsafe", "extern \"C\"", "no_mangle", ".task_id("]) {
|
||||||
|
assert(!recoverySource.includes(forbidden), "recovery-build leaked " + forbidden);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fixtureSource = fs.readFileSync(path.join(conformance, "src/lib.rs"), "utf8");
|
||||||
|
assert(fixtureSource.includes("task_trap"));
|
||||||
|
assert(fixtureSource.includes("cooperative_cancellation_probe"));
|
||||||
|
assert(!fs.existsSync(path.join(repo, "examples", "launch-" + "build-demo")));
|
||||||
|
assert(!fs.existsSync(path.join(hello, "src/bin/sdk-product-runtime.rs")));
|
||||||
|
|
||||||
|
const args = [
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-cli",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux",
|
||||||
|
"--",
|
||||||
|
"build",
|
||||||
|
"--project",
|
||||||
|
hello,
|
||||||
|
"--json",
|
||||||
|
];
|
||||||
|
const report = JSON.parse(cp.execFileSync("cargo", args, {
|
||||||
|
cwd: repo,
|
||||||
|
env: process.env,
|
||||||
|
encoding: "utf8",
|
||||||
|
}));
|
||||||
|
assert(report.bundle_artifact);
|
||||||
|
assert(report.bundle.metadata.selected_inputs.some((input) => input.path === "src/lib.rs"));
|
||||||
|
assert(report.bundle.metadata.task_metadata.entrypoints.includes("build"));
|
||||||
|
const tasks = JSON.parse(
|
||||||
|
fs.readFileSync(
|
||||||
|
path.resolve(repo, report.bundle_artifact.directory, "task-descriptors.json"),
|
||||||
|
"utf8"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(tasks.some((task) => task.name === "compile"));
|
||||||
|
assert(tasks.some((task) => task.name === "snapshot_current_project"));
|
||||||
|
console.log("primary and recovery example smoke passed");
|
||||||
235
scripts/hostile-input-contract-smoke.js
Normal file
235
scripts/hostile-input-contract-smoke.js
Normal file
|
|
@ -0,0 +1,235 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const {
|
||||||
|
agentIdentity,
|
||||||
|
signedAgentWorkflowRequest,
|
||||||
|
} = require("./agent-signing");
|
||||||
|
const {
|
||||||
|
nodeIdentity,
|
||||||
|
signedNodeHeartbeat,
|
||||||
|
} = require("./node-signing");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
|
||||||
|
const signingInstrumentAgent = agentIdentity(
|
||||||
|
"hostile-input-contract-agent",
|
||||||
|
"agent-hostile-input-contract"
|
||||||
|
);
|
||||||
|
const explicitlyEmptyAgentNonce = signedAgentWorkflowRequest(
|
||||||
|
signingInstrumentAgent,
|
||||||
|
{
|
||||||
|
type: "start_process",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_agent: "agent-hostile-input-contract",
|
||||||
|
process: "process",
|
||||||
|
launch_attempt: "attempt",
|
||||||
|
restart: false,
|
||||||
|
},
|
||||||
|
{ nonce: "" }
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
explicitlyEmptyAgentNonce.agent_signature.nonce,
|
||||||
|
"",
|
||||||
|
"Agent signing instrument replaced an explicitly empty hostile nonce"
|
||||||
|
);
|
||||||
|
const signingInstrumentNode = nodeIdentity(
|
||||||
|
"hostile-input-contract-node",
|
||||||
|
"node-hostile-input-contract"
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
signedNodeHeartbeat(
|
||||||
|
"tenant",
|
||||||
|
"project",
|
||||||
|
"node-hostile-input-contract",
|
||||||
|
signingInstrumentNode,
|
||||||
|
{ nonce: "" }
|
||||||
|
).nonce,
|
||||||
|
"",
|
||||||
|
"Node signing instrument replaced an explicitly empty hostile nonce"
|
||||||
|
);
|
||||||
|
|
||||||
|
function read(relativePath) {
|
||||||
|
return fs.readFileSync(path.join(repo, relativePath), "utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
function maybeRead(segments) {
|
||||||
|
const fullPath = path.join(repo, ...segments);
|
||||||
|
if (!fs.existsSync(fullPath)) return null;
|
||||||
|
return fs.readFileSync(fullPath, "utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
function expect(source, name, pattern) {
|
||||||
|
assert.match(source, pattern, `missing hostile-input evidence: ${name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const coreSource = read("crates/clusterflux-core/src/source.rs");
|
||||||
|
const coreCapabilities = read("crates/clusterflux-core/src/capability.rs");
|
||||||
|
const coordinatorService = [
|
||||||
|
read("crates/clusterflux-coordinator/src/service.rs"),
|
||||||
|
read("crates/clusterflux-coordinator/src/service/routing.rs"),
|
||||||
|
read("crates/clusterflux-coordinator/src/service/signed_nodes.rs"),
|
||||||
|
read("crates/clusterflux-coordinator/src/service/logs.rs"),
|
||||||
|
read("crates/clusterflux-coordinator/src/service/tests.rs"),
|
||||||
|
].join("\n");
|
||||||
|
const artifactDownloadSmoke = read("scripts/artifact-download-smoke.js");
|
||||||
|
const operatorPanelSmoke = read("scripts/operator-panel-smoke.js");
|
||||||
|
const schedulerSmoke = read("scripts/scheduler-placement-smoke.js");
|
||||||
|
const sourcePreparationSmoke = read("scripts/source-preparation-smoke.js");
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["source manifests validate shape", /pub fn validate_public_mvp\(&self\)[\s\S]*self\.validate_shape\(\)\?/],
|
||||||
|
["source manifests reject invalid digests", /SourceManifestError::InvalidDigest/],
|
||||||
|
["source manifests reject invalid custom providers", /SourceManifestError::InvalidProviderId/],
|
||||||
|
["source manifests reject control characters", /DescriptionControlCharacter/],
|
||||||
|
["source manifests reject coordinator checkout access", /CoordinatorCheckoutAccess/],
|
||||||
|
["source manifests reject default source-byte upload", /CoordinatorReceivesSourceBytes/],
|
||||||
|
]) {
|
||||||
|
expect(coreSource, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["capability reports validate public shape", /pub fn validate_public_report\(&self\)/],
|
||||||
|
["capability reports validate architecture labels", /InvalidArchitecture/],
|
||||||
|
["capability reports validate OS labels", /InvalidOsLabel/],
|
||||||
|
["capability reports validate source providers", /InvalidSourceProvider/],
|
||||||
|
["source provider ids reject path traversal", /valid_source_provider_id/],
|
||||||
|
]) {
|
||||||
|
expect(coreCapabilities, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["coordinator task log tails are bounded", /MAX_TASK_LOG_TAIL_BYTES: usize = 256 \* 1024/],
|
||||||
|
["coordinator validates reported stdout tails", /ReportTaskLog[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/],
|
||||||
|
["coordinator validates completed task stdout tails", /TaskCompleted[\s\S]*validate_task_log_tail\("stdout_tail", &stdout_tail\)\?/],
|
||||||
|
["coordinator rejects oversized log tail in unit coverage", /"x"\.repeat\(MAX_TASK_LOG_TAIL_BYTES \+ 1\)/],
|
||||||
|
]) {
|
||||||
|
expect(coordinatorService, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["service rejects malformed node capability report", /fn service_rejects_malformed_node_capability_report\(\)/],
|
||||||
|
["capability report rejection leaves descriptors empty", /assert!\(service\.node_descriptors\.is_empty\(\)\)/],
|
||||||
|
["node capability report rejects cross-scope writes", /fn service_rejects_node_capability_report_outside_enrollment_scope\(\)/],
|
||||||
|
["task completion rejects cross-scope writes", /task completion outside node scope|outside/],
|
||||||
|
]) {
|
||||||
|
expect(coordinatorService, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, source, patterns] of [
|
||||||
|
[
|
||||||
|
"artifact download smoke",
|
||||||
|
artifactDownloadSmoke,
|
||||||
|
[
|
||||||
|
/const crossTenant = await send/,
|
||||||
|
/const crossProject = await send/,
|
||||||
|
/const guessed = await send/,
|
||||||
|
/const crossActorOpen = await send/,
|
||||||
|
/token is invalid/,
|
||||||
|
/artifact does not exist/,
|
||||||
|
],
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"operator panel smoke",
|
||||||
|
operatorPanelSmoke,
|
||||||
|
[
|
||||||
|
/render_operator_panel/,
|
||||||
|
/submit_panel_event/,
|
||||||
|
/assert\(!JSON\.stringify\(panel\)\.includes\("<script"\)\)/,
|
||||||
|
/assert\(!JSON\.stringify\(panel\)\.toLowerCase\(\)\.includes\("oauth"\)\)/,
|
||||||
|
/rate limit/i,
|
||||||
|
/exceeds download limit/,
|
||||||
|
],
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"scheduler smoke",
|
||||||
|
schedulerSmoke,
|
||||||
|
[/const crossTenantReport = await send/, /report_node_capabilities/, /tenant\\\/project scope/],
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"source preparation smoke",
|
||||||
|
sourcePreparationSmoke,
|
||||||
|
[/const crossTenantCompletion = await send/, /complete_source_preparation/, /tenant\\\/project scope/i],
|
||||||
|
],
|
||||||
|
]) {
|
||||||
|
for (const pattern of patterns) {
|
||||||
|
expect(source, name, pattern);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const hostedServiceMain = maybeRead([
|
||||||
|
"private",
|
||||||
|
"hosted-policy",
|
||||||
|
"src",
|
||||||
|
"bin",
|
||||||
|
"clusterflux-hosted-service.rs",
|
||||||
|
]);
|
||||||
|
const hostedValidation = maybeRead([
|
||||||
|
"private",
|
||||||
|
"hosted-policy",
|
||||||
|
"src",
|
||||||
|
"bin",
|
||||||
|
"clusterflux-hosted-service",
|
||||||
|
"validation.rs",
|
||||||
|
]);
|
||||||
|
const hostedWire = maybeRead([
|
||||||
|
"private",
|
||||||
|
"hosted-policy",
|
||||||
|
"src",
|
||||||
|
"bin",
|
||||||
|
"clusterflux-hosted-service",
|
||||||
|
"wire.rs",
|
||||||
|
]);
|
||||||
|
const hostedProtocol = maybeRead([
|
||||||
|
"private",
|
||||||
|
"hosted-policy",
|
||||||
|
"src",
|
||||||
|
"bin",
|
||||||
|
"clusterflux-hosted-service",
|
||||||
|
"hosted_service_protocol.rs",
|
||||||
|
]);
|
||||||
|
const hostedService =
|
||||||
|
hostedServiceMain && hostedValidation && hostedWire && hostedProtocol
|
||||||
|
? [
|
||||||
|
hostedServiceMain,
|
||||||
|
hostedValidation,
|
||||||
|
hostedWire,
|
||||||
|
hostedProtocol,
|
||||||
|
maybeRead(["crates", "clusterflux-core", "src", "ids.rs"]),
|
||||||
|
].join("\n")
|
||||||
|
: null;
|
||||||
|
const hostedTests = [
|
||||||
|
maybeRead(["private", "hosted-policy", "src", "bin", "clusterflux-hosted-service", "tests.rs"]),
|
||||||
|
maybeRead(["private", "hosted-policy", "scripts", "hosted-deployment-smoke.js"]),
|
||||||
|
maybeRead(["private", "hosted-policy", "scripts", "hosted-client-compat-smoke.js"]),
|
||||||
|
].filter(Boolean).join("\n");
|
||||||
|
|
||||||
|
if (hostedService && hostedTests) {
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["hosted service turns malformed JSON into error responses", /decode_incoming_request[\s\S]*HostedServiceResponse::Error/],
|
||||||
|
["tenant ids are validated", /fn tenant_id\(value: String\)[\s\S]*TenantId::try_new\(value\)/],
|
||||||
|
["node ids are validated", /fn node_id\(value: String\)[\s\S]*NodeId::try_new\(value\)/],
|
||||||
|
["process ids are validated", /fn process_id\(value: String\)[\s\S]*ProcessId::try_new\(value\)/],
|
||||||
|
["identifiers reject empty, oversized, control, and invalid format values", /MAX_EXTERNAL_ID_BYTES[\s\S]*trim\(\)\.is_empty\(\)[\s\S]*char::is_control[\s\S]*is_ascii_alphanumeric\(\)/],
|
||||||
|
["OIDC text fields are bounded", /fn validate_text[\s\S]*value\.len\(\) > max_bytes[\s\S]*contains unsupported characters/],
|
||||||
|
["tokens are bounded", /fn validate_token[\s\S]*value\.len\(\) > max_bytes[\s\S]*contains unsupported characters/],
|
||||||
|
["control request bodies are bounded", /MAX_CONTROL_FRAME_BYTES[\s\S]*control request too large/],
|
||||||
|
["identity protocol rejects unknown authority fields", /deny_unknown_fields/],
|
||||||
|
]) {
|
||||||
|
expect(hostedService, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["old client identity protocol is rejected", /hosted_login_protocol_rejects_client_identity_and_provider_configuration/],
|
||||||
|
["raw operator action is rejected", /rawOperatorDenied[\s\S]*hosted_operator_request envelope/],
|
||||||
|
["unsigned client identity is rejected", /const forged = await sendHostedControl[\s\S]*authenticated CLI session/],
|
||||||
|
["cross-tenant process inspection is rejected", /crossTenantTaskEventsDenied[\s\S]*scope\|denied\|unauthorized/],
|
||||||
|
]) {
|
||||||
|
expect(hostedTests, name, pattern);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Hostile input contract smoke passed");
|
||||||
22
scripts/migrate-clusterflux-state.sh
Executable file
22
scripts/migrate-clusterflux-state.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
migrate_path() {
|
||||||
|
local source="$1"
|
||||||
|
local destination="$2"
|
||||||
|
if [[ ! -e "$source" ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
if [[ -e "$destination" ]]; then
|
||||||
|
printf 'refusing to overwrite existing destination: %s\n' "$destination" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mv -- "$source" "$destination"
|
||||||
|
printf 'migrated %s -> %s\n' "$source" "$destination"
|
||||||
|
}
|
||||||
|
|
||||||
|
migrate_path "${HOME}/.disasmer" "${HOME}/.clusterflux"
|
||||||
|
migrate_path "${XDG_CONFIG_HOME:-${HOME}/.config}/disasmer" \
|
||||||
|
"${XDG_CONFIG_HOME:-${HOME}/.config}/clusterflux"
|
||||||
|
migrate_path "${PWD}/.disasmer" "${PWD}/.clusterflux"
|
||||||
|
migrate_path "${PWD}/disasmer.toml" "${PWD}/clusterflux.toml"
|
||||||
307
scripts/node-attach-smoke.js
Executable file
307
scripts/node-attach-smoke.js
Executable file
|
|
@ -0,0 +1,307 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const net = require("net");
|
||||||
|
const path = require("path");
|
||||||
|
const { coordinatorWireRequest } = require("./coordinator-wire");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const identities = new Map();
|
||||||
|
|
||||||
|
function waitForJsonLine(child) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let buffer = "";
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
buffer += chunk.toString();
|
||||||
|
const newline = buffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(buffer.slice(0, newline).trim()));
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
child.once("exit", (code) => {
|
||||||
|
reject(new Error(`process exited before JSON line with code ${code}`));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function send(addr, message) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const socket = net.connect(addr.port, addr.host, () => {
|
||||||
|
socket.write(`${JSON.stringify(coordinatorWireRequest(message))}\n`);
|
||||||
|
});
|
||||||
|
let buffer = "";
|
||||||
|
socket.on("data", (chunk) => {
|
||||||
|
buffer += chunk.toString();
|
||||||
|
const newline = buffer.indexOf("\n");
|
||||||
|
if (newline < 0) return;
|
||||||
|
socket.end();
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(buffer.slice(0, newline)));
|
||||||
|
} catch (error) {
|
||||||
|
reject(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on("error", reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeIdentity(node) {
|
||||||
|
const existing = identities.get(node);
|
||||||
|
if (existing) return existing;
|
||||||
|
const { privateKey: privateKeyObject, publicKey } =
|
||||||
|
crypto.generateKeyPairSync("ed25519");
|
||||||
|
const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" });
|
||||||
|
const publicDer = publicKey.export({
|
||||||
|
format: "der",
|
||||||
|
type: "spki",
|
||||||
|
});
|
||||||
|
const privateSeed = Buffer.from(privateDer).subarray(-32);
|
||||||
|
const identity = {
|
||||||
|
privateKey: `ed25519:${privateSeed.toString("base64")}`,
|
||||||
|
publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`,
|
||||||
|
privateKeyObject,
|
||||||
|
};
|
||||||
|
identities.set(node, identity);
|
||||||
|
return identity;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeSignatureMessage(
|
||||||
|
node,
|
||||||
|
requestKind,
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
issuedAtEpochSeconds
|
||||||
|
) {
|
||||||
|
const parts = [
|
||||||
|
"clusterflux-node-request-signature:v2",
|
||||||
|
node,
|
||||||
|
requestKind,
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
String(issuedAtEpochSeconds),
|
||||||
|
];
|
||||||
|
return Buffer.concat(
|
||||||
|
parts.flatMap((part) => [
|
||||||
|
Buffer.from(`${Buffer.byteLength(part)}:`),
|
||||||
|
Buffer.from(part),
|
||||||
|
Buffer.from("\n"),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedNodeHeartbeat(tenant, project, node, identity) {
|
||||||
|
const nonce = `node-attach-heartbeat-${process.pid}-${Date.now()}`;
|
||||||
|
const issuedAt = Math.floor(Date.now() / 1000);
|
||||||
|
const payloadDigest = `sha256:${crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(JSON.stringify({ node, project, tenant, type: "node_heartbeat" }))
|
||||||
|
.digest("hex")}`;
|
||||||
|
const signature = crypto.sign(
|
||||||
|
null,
|
||||||
|
nodeSignatureMessage(node, "node_heartbeat", payloadDigest, nonce, issuedAt),
|
||||||
|
identity.privateKeyObject
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
nonce,
|
||||||
|
issued_at_epoch_seconds: issuedAt,
|
||||||
|
signature: `ed25519:${signature.toString("base64")}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function runAttach(addr, grant) {
|
||||||
|
const identity = nodeIdentity("node-attach");
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-cli",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux",
|
||||||
|
"--",
|
||||||
|
"node",
|
||||||
|
"attach",
|
||||||
|
"--coordinator",
|
||||||
|
`${addr.host}:${addr.port}`,
|
||||||
|
"--tenant",
|
||||||
|
"tenant",
|
||||||
|
"--project-id",
|
||||||
|
"project",
|
||||||
|
"--node",
|
||||||
|
"node-attach",
|
||||||
|
"--public-key",
|
||||||
|
identity.publicKey,
|
||||||
|
"--enrollment-grant",
|
||||||
|
grant,
|
||||||
|
"--cap",
|
||||||
|
"quic-direct",
|
||||||
|
"--json",
|
||||||
|
],
|
||||||
|
{
|
||||||
|
cwd: repo,
|
||||||
|
env: {
|
||||||
|
...process.env,
|
||||||
|
CLUSTERFLUX_NODE_PRIVATE_KEY: identity.privateKey,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
stdout += chunk.toString();
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (chunk) => {
|
||||||
|
stderr += chunk.toString();
|
||||||
|
});
|
||||||
|
child.on("exit", (code) => {
|
||||||
|
if (code !== 0) {
|
||||||
|
reject(new Error(`node attach failed with code ${code}\n${stderr}`));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
resolve(JSON.parse(stdout));
|
||||||
|
} catch (error) {
|
||||||
|
reject(
|
||||||
|
new Error(`node attach output was not JSON: ${stdout}\n${error.stack || error.message}`)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const coordinator = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--",
|
||||||
|
"--listen",
|
||||||
|
"127.0.0.1:0",
|
||||||
|
"--allow-local-trusted-loopback",
|
||||||
|
],
|
||||||
|
{ cwd: repo }
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ready = await waitForJsonLine(coordinator);
|
||||||
|
const [host, portText] = ready.listen.split(":");
|
||||||
|
const addr = { host, port: Number(portText) };
|
||||||
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||||
|
|
||||||
|
const grant = await send(addr, {
|
||||||
|
type: "create_node_enrollment_grant",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "operator",
|
||||||
|
ttl_seconds: 900
|
||||||
|
});
|
||||||
|
assert.strictEqual(grant.type, "node_enrollment_grant_created");
|
||||||
|
assert.strictEqual(grant.tenant, "tenant");
|
||||||
|
assert.strictEqual(grant.project, "project");
|
||||||
|
assert.match(grant.grant, /^node_grant_[A-Za-z0-9_-]+$/);
|
||||||
|
assert.strictEqual(grant.scope, "node:attach");
|
||||||
|
assert(grant.expires_at_epoch_seconds > Math.floor(Date.now() / 1000));
|
||||||
|
assert(grant.expires_at_epoch_seconds <= Math.floor(Date.now() / 1000) + 900);
|
||||||
|
|
||||||
|
const report = await runAttach(addr, grant.grant);
|
||||||
|
assert.strictEqual(report.plan.node, "node-attach");
|
||||||
|
assert.strictEqual(report.plan.coordinator, `${addr.host}:${addr.port}`);
|
||||||
|
assert.strictEqual(report.plan.enrollment.grant, grant.grant);
|
||||||
|
assert.match(report.plan.enrollment.public_key_fingerprint, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.plan.enrollment.exchanges_short_lived_grant_for_long_lived_node_identity,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
assert.ok(report.plan.capabilities.arch.length > 0);
|
||||||
|
assert.ok(report.plan.capabilities.source_providers.includes("filesystem"));
|
||||||
|
assert.ok(report.plan.capabilities.capabilities.includes("QuicDirect"));
|
||||||
|
assert.strictEqual(report.plan.detection.auto_detected, true);
|
||||||
|
assert.strictEqual(report.plan.detection.arch, report.plan.capabilities.arch);
|
||||||
|
assert.deepStrictEqual(report.plan.detection.manual_capability_overrides, ["quic-direct"]);
|
||||||
|
assert(
|
||||||
|
report.plan.detection.recognized_capability_overrides.includes("QuicDirect")
|
||||||
|
);
|
||||||
|
assert.strictEqual(
|
||||||
|
report.plan.detection.os_arch_capabilities_require_manual_flags,
|
||||||
|
false
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.plan.detection.command_backend, "native-command");
|
||||||
|
assert.strictEqual(report.plan.detection.command_backend_available, true);
|
||||||
|
assert(
|
||||||
|
report.plan.detection.source_provider_backends.some(
|
||||||
|
(provider) => provider.provider === "filesystem" && provider.detected
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
report.grant_disclosures.length > 0,
|
||||||
|
"node attach should disclose capability grants before reporting capabilities"
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
report.grant_disclosures.every(
|
||||||
|
(disclosure) => disclosure.coordinator_policy_limited === true
|
||||||
|
),
|
||||||
|
"node attach should mark all capability grants as coordinator-policy-limited"
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
report.grant_disclosures.some(
|
||||||
|
(disclosure) => disclosure.grant === "native_command_execution"
|
||||||
|
),
|
||||||
|
"node attach should disclose native command execution when detected"
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
report.grant_disclosures.some(
|
||||||
|
(disclosure) => disclosure.grant === "source_access"
|
||||||
|
),
|
||||||
|
"node attach should disclose source access when detected"
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.boundary.cli_contacted_coordinator, true);
|
||||||
|
assert.strictEqual(report.boundary.used_enrollment_exchange, true);
|
||||||
|
assert.strictEqual(report.boundary.coordinator_session_requests, 3);
|
||||||
|
assert.strictEqual(report.coordinator_response.type, "node_enrollment_exchanged");
|
||||||
|
assert.strictEqual(report.coordinator_response.node, "node-attach");
|
||||||
|
assert.strictEqual(report.coordinator_response.credential.node, "node-attach");
|
||||||
|
assert.strictEqual(report.coordinator_response.credential.scope, "node:attach");
|
||||||
|
assert.strictEqual(report.coordinator_response.credential.credential_kind, "NodeCredential");
|
||||||
|
assert.match(
|
||||||
|
report.coordinator_response.credential.capability_policy_digest,
|
||||||
|
/^sha256:[0-9a-f]{64}$/
|
||||||
|
);
|
||||||
|
assert.strictEqual(report.heartbeat_response.type, "node_heartbeat");
|
||||||
|
assert.strictEqual(report.capability_response.type, "node_capabilities_recorded");
|
||||||
|
|
||||||
|
const heartbeat = await send(addr, {
|
||||||
|
type: "node_heartbeat",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
node: "node-attach",
|
||||||
|
node_signature: signedNodeHeartbeat(
|
||||||
|
"tenant",
|
||||||
|
"project",
|
||||||
|
"node-attach",
|
||||||
|
nodeIdentity("node-attach")
|
||||||
|
),
|
||||||
|
});
|
||||||
|
assert.strictEqual(heartbeat.type, "node_heartbeat");
|
||||||
|
assert.strictEqual(heartbeat.node, "node-attach");
|
||||||
|
|
||||||
|
} finally {
|
||||||
|
coordinator.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Node attach smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
151
scripts/node-lifecycle-contract-smoke.js
Executable file
151
scripts/node-lifecycle-contract-smoke.js
Executable file
|
|
@ -0,0 +1,151 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
|
||||||
|
function read(relativePath) {
|
||||||
|
return fs.readFileSync(path.join(repo, relativePath), "utf8");
|
||||||
|
}
|
||||||
|
|
||||||
|
function expect(source, name, pattern) {
|
||||||
|
assert.match(source, pattern, `missing node lifecycle evidence: ${name}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nodeMain = read("crates/clusterflux-node/src/daemon.rs");
|
||||||
|
const nodeIdentity = read("crates/clusterflux-node/src/node_identity.rs");
|
||||||
|
const cliNode = read("crates/clusterflux-cli/src/node.rs");
|
||||||
|
const nodeTaskReports = read("crates/clusterflux-node/src/task_reports.rs");
|
||||||
|
const nodeDebugAgent = read("crates/clusterflux-node/src/debug_agent.rs");
|
||||||
|
const nodeLib = read("crates/clusterflux-node/src/lib.rs");
|
||||||
|
const sharedWasmtimeRuntime = `${read("crates/clusterflux-wasm-runtime/src/lib.rs")}\n${read("crates/clusterflux-wasm-runtime/src/task_host_linker.rs")}`;
|
||||||
|
const nodeRuntimeSurface = `${nodeLib}\n${sharedWasmtimeRuntime}`;
|
||||||
|
const nodeLifecycleSurface = `${nodeMain}\n${nodeIdentity}\n${nodeTaskReports}\n${nodeDebugAgent}`;
|
||||||
|
const nodeAssignmentRunner = `${read("crates/clusterflux-node/src/assignment_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/control_watcher.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/process_runner.rs")}\n${read("crates/clusterflux-node/src/assignment_runner/validation.rs")}`;
|
||||||
|
const coordinatorCore = read("crates/clusterflux-coordinator/src/lib.rs");
|
||||||
|
const coordinatorService = `${read("crates/clusterflux-coordinator/src/service.rs")}\n${read("crates/clusterflux-coordinator/src/service/routing.rs")}`;
|
||||||
|
const coordinatorServiceTests = read("crates/clusterflux-coordinator/src/service/tests.rs");
|
||||||
|
const coordinatorServiceSurface = `${coordinatorService}\n${coordinatorServiceTests}`;
|
||||||
|
const cliLocalRunSmoke = read("scripts/cli-local-run-smoke.js");
|
||||||
|
const liveSmoke = read("scripts/cli-happy-path-live-smoke.js");
|
||||||
|
const wasmtimeSmoke = read("scripts/wasmtime-node-smoke.js");
|
||||||
|
const debugCore = read("crates/clusterflux-core/src/debug.rs");
|
||||||
|
|
||||||
|
assert.strictEqual(
|
||||||
|
(nodeMain.match(/CoordinatorSession::connect/g) || []).length,
|
||||||
|
1,
|
||||||
|
"node runtime should open one coordinator session in the local process-boundary runtime"
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["enrollment exchange over session", /"type": "exchange_node_enrollment_grant"/],
|
||||||
|
["persisted node identity is reused locally", /"type": "node_identity_reused"/],
|
||||||
|
["heartbeat over session", /"type": "node_heartbeat"/],
|
||||||
|
["node-originated requests use signed envelope", /"type": "signed_node"/],
|
||||||
|
["capability report over session", /"type": "report_node_capabilities"/],
|
||||||
|
["task assignment polling over session", /"type": "poll_task_assignment"/],
|
||||||
|
["process start over session", /"type": "start_process"/],
|
||||||
|
["reconnect over session", /"type": "reconnect_node"/],
|
||||||
|
["debug command polling over session", /"type": "poll_debug_command"/],
|
||||||
|
["log event over session", /"type": "report_task_log"/],
|
||||||
|
["VFS metadata over session", /"type": "report_vfs_metadata"/],
|
||||||
|
["task control polling over session", /"type": "poll_task_control"/],
|
||||||
|
["completion over session", /"type": "task_completed"/],
|
||||||
|
["cancellation uses same session", /poll_task_cancellation\(session, args, &task, node_private_key\)/],
|
||||||
|
["request count is reported", /session\.requests\(\)/],
|
||||||
|
]) {
|
||||||
|
expect(nodeLifecycleSurface, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(cliNode, "user-authorized node attach over Client session", /"type": "attach_node"/);
|
||||||
|
assert.doesNotMatch(
|
||||||
|
nodeIdentity,
|
||||||
|
/"type": "attach_node"/,
|
||||||
|
"a persisted node must authenticate with its signed identity instead of replaying Client attach"
|
||||||
|
);
|
||||||
|
|
||||||
|
const runtimeAcceptanceSurface = `${cliLocalRunSmoke}\n${liveSmoke}\n${coordinatorServiceTests}\n${nodeLib}\n${nodeAssignmentRunner}`;
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["real CLI launches a coordinator main", /node_report\.run\.status, "main_launched"/],
|
||||||
|
["real CLI observes coordinator-main task spawning", /task_spawn_host_import, true/],
|
||||||
|
["real CLI keeps child task instances distinct", /every live task event must retain its unique instance identity/],
|
||||||
|
["signed active Wasm parent spawns and joins child", /fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only\(\)/],
|
||||||
|
["controlled native process abort is exercised", /fn abort_requested[\s\S]*poll_task_control/],
|
||||||
|
["native lifecycle freeze and resume are exercised", /linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume/],
|
||||||
|
["strict live run requires a usable partial freeze", /partial_freeze\.partially_frozen/],
|
||||||
|
["strict live run requires hosted restart evidence", /serviceRestart\.executed === true/],
|
||||||
|
]) {
|
||||||
|
expect(runtimeAcceptanceSurface, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["cooperative cancellation and abort are distinct", /cancel_requested: false,[\s\S]*abort_requested: true/],
|
||||||
|
["controlled runner polls abort while command runs", /fn abort_requested[\s\S]*poll_task_control/],
|
||||||
|
["controlled runner creates a process group", /process\.process_group\(0\)/],
|
||||||
|
["controlled runner freezes the native process group", /libc::kill\(process_group, libc::SIGSTOP\)/],
|
||||||
|
["controlled runner resumes the native process group", /libc::kill\(process_group, libc::SIGCONT\)/],
|
||||||
|
["controlled runner kills the process group", /libc::kill\(process_group, libc::SIGKILL\)/],
|
||||||
|
["Wasm code can poll cooperative cancellation", /task_control_v1/],
|
||||||
|
["matched Wasm probes remain at a quiescent boundary", /TaskHostOperation::DebugProbe[\s\S]*enter_quiescent_host_boundary[\s\S]*leave_quiescent_host_boundary/],
|
||||||
|
["debug snapshots use the live Wasm task handle registry", /debug_handle_snapshot[\s\S]*task_handle_\{handle_id\}[\s\S]*state=active/],
|
||||||
|
["native command status comes from the controlled runner", /set_command_status[\s\S]*frozen command pid[\s\S]*native command exited with status/],
|
||||||
|
]) {
|
||||||
|
expect(`${coordinatorServiceSurface}\n${nodeLifecycleSurface}\n${sharedWasmtimeRuntime}\n${nodeAssignmentRunner}`, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["coordinator rejects stale process ownership", /fn node_reconnect_rejects_stale_process_epoch_after_restart\(\)/],
|
||||||
|
["reconnect preserves scoped enrolled node identity", /reconnect_node\([\s\S]*&TenantId::from\("tenant"\),[\s\S]*&ProjectId::from\("project"\),[\s\S]*&NodeId::from\("node"\),[\s\S]*None/],
|
||||||
|
["stale process epoch is rejected", /CoordinatorError::StaleProcessEpoch/],
|
||||||
|
]) {
|
||||||
|
expect(coordinatorCore, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["coordinator delivers cancellation to connected node", /fn service_delivers_cancellation_to_connected_node_and_records_terminal_state\(\)/],
|
||||||
|
["node polls task control", /CoordinatorRequest::PollTaskControl/],
|
||||||
|
["cancelled terminal state is recorded", /TaskTerminalState::Cancelled/],
|
||||||
|
]) {
|
||||||
|
expect(coordinatorServiceSurface, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["native lifecycle test exists", /fn linux_task_lifecycle_supports_cancel_and_all_stop_freeze_resume\(\)/],
|
||||||
|
["native freeze succeeds when supported", /lifecycle\.freeze_for_debug_epoch\(\)\.unwrap\(\)/],
|
||||||
|
["native resume succeeds", /lifecycle\.resume_after_debug_epoch\(\)/],
|
||||||
|
["native cancel reaches lifecycle", /lifecycle\.cancel\(\)/],
|
||||||
|
["unsupported freeze errors", /BackendError::DebugFreezeUnsupported/],
|
||||||
|
["wasmtime runtime exposes freeze resume probe", /pub fn freeze_resume_i32_export_probe/],
|
||||||
|
["wasmtime runtime captures Wasm frame locals", /debug_i32_export_snapshot[\s\S]*local_values/],
|
||||||
|
["wasmtime runtime creates Wasm debug participant", /kind: DebugParticipantKind::WasmTask/],
|
||||||
|
["wasmtime debug participant carries local values", /local_values: snapshot\.local_values\.clone\(\)/],
|
||||||
|
["wasmtime runtime resumes after freeze", /epoch\.continue_all\(\)/],
|
||||||
|
]) {
|
||||||
|
expect(nodeRuntimeSurface, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["wasmtime smoke runs debug freeze resume mode", /--debug-freeze-resume/],
|
||||||
|
["wasmtime smoke verifies frozen state", /debugReport\.frozen_state, "Frozen"/],
|
||||||
|
["wasmtime smoke verifies resumed state", /debugReport\.resumed_state, "Running"/],
|
||||||
|
["wasmtime smoke verifies frame local values", /debugReport\.local_values[\s\S]*wasm_local_0/],
|
||||||
|
["wasmtime smoke proves node runtime reached wasm task", /node_runtime_reached_wasm_task/],
|
||||||
|
["wasmtime smoke proves node captured locals", /node_runtime_captured_wasm_locals/],
|
||||||
|
]) {
|
||||||
|
expect(wasmtimeSmoke, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [name, pattern] of [
|
||||||
|
["debug model freezes wasm and command participants", /fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks\(\)/],
|
||||||
|
["debug model rejects unsupported freeze", /fn debug_epoch_reports_failure_when_no_participant_can_freeze\(\)/],
|
||||||
|
["debug model resumes frozen participants", /fn continue_resumes_every_frozen_participant\(\)/],
|
||||||
|
["debug model includes captured locals", /local_values/],
|
||||||
|
["wasm participants are modeled", /DebugParticipantKind::WasmTask/],
|
||||||
|
["controlled native command participants are modeled", /DebugParticipantKind::ControlledNativeCommand/],
|
||||||
|
]) {
|
||||||
|
expect(debugCore, name, pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Node lifecycle contract smoke passed");
|
||||||
181
scripts/node-signing.js
Normal file
181
scripts/node-signing.js
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const identities = new Map();
|
||||||
|
|
||||||
|
function nodeIdentity(identityPurpose, node) {
|
||||||
|
const identityKey = `${identityPurpose}:${node}`;
|
||||||
|
const existing = identities.get(identityKey);
|
||||||
|
if (existing) return existing;
|
||||||
|
const { privateKey: privateKeyObject, publicKey } =
|
||||||
|
crypto.generateKeyPairSync("ed25519");
|
||||||
|
const privateDer = privateKeyObject.export({ format: "der", type: "pkcs8" });
|
||||||
|
const publicDer = publicKey.export({
|
||||||
|
format: "der",
|
||||||
|
type: "spki",
|
||||||
|
});
|
||||||
|
const privateSeed = Buffer.from(privateDer).subarray(-32);
|
||||||
|
const identity = {
|
||||||
|
privateKey: `ed25519:${privateSeed.toString("base64")}`,
|
||||||
|
publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`,
|
||||||
|
privateKeyObject,
|
||||||
|
};
|
||||||
|
identities.set(identityKey, identity);
|
||||||
|
return identity;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeIdentityFromPrivateKey(privateKey) {
|
||||||
|
if (typeof privateKey !== "string" || !privateKey.startsWith("ed25519:")) {
|
||||||
|
throw new Error("node private key must use ed25519:<base64> encoding");
|
||||||
|
}
|
||||||
|
const seed = Buffer.from(privateKey.slice("ed25519:".length), "base64");
|
||||||
|
if (seed.length !== 32) throw new Error("node private key must contain 32 bytes");
|
||||||
|
const privateKeyObject = crypto.createPrivateKey({
|
||||||
|
key: Buffer.concat([
|
||||||
|
Buffer.from("302e020100300506032b657004220420", "hex"),
|
||||||
|
seed,
|
||||||
|
]),
|
||||||
|
format: "der",
|
||||||
|
type: "pkcs8",
|
||||||
|
});
|
||||||
|
const publicKeyObject = crypto.createPublicKey(privateKeyObject);
|
||||||
|
const publicDer = publicKeyObject.export({ format: "der", type: "spki" });
|
||||||
|
return {
|
||||||
|
privateKey,
|
||||||
|
publicKey: `ed25519:${Buffer.from(publicDer).subarray(-32).toString("base64")}`,
|
||||||
|
privateKeyObject,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function canonicalSignedRequest(value, topLevel = true) {
|
||||||
|
if (Array.isArray(value)) {
|
||||||
|
return value.map((entry) => canonicalSignedRequest(entry, false));
|
||||||
|
}
|
||||||
|
if (value && typeof value === "object") {
|
||||||
|
return Object.fromEntries(
|
||||||
|
Object.entries(value)
|
||||||
|
.filter(
|
||||||
|
([key, entry]) =>
|
||||||
|
entry !== null &&
|
||||||
|
(!topLevel || !["agent_signature", "node_signature"].includes(key))
|
||||||
|
)
|
||||||
|
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
||||||
|
.map(([key, entry]) => [key, canonicalSignedRequest(entry, false)])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function withWireDefaults(request) {
|
||||||
|
const value = { ...request };
|
||||||
|
if (value.type === "report_node_capabilities") {
|
||||||
|
value.dependency_cache_digests ??= [];
|
||||||
|
} else if (value.type === "launch_task" || value.type === "launch_child_task") {
|
||||||
|
value.wait_for_node ??= false;
|
||||||
|
if (value.task_spec && typeof value.task_spec === "object") {
|
||||||
|
value.task_spec = { ...value.task_spec };
|
||||||
|
value.task_spec.failure_policy ??= "fail_fast";
|
||||||
|
}
|
||||||
|
} else if (value.type === "start_process") {
|
||||||
|
value.restart ??= false;
|
||||||
|
} else if (value.type === "report_debug_state") {
|
||||||
|
value.stack_frames ??= [];
|
||||||
|
value.local_values ??= [];
|
||||||
|
value.task_args ??= [];
|
||||||
|
value.handles ??= [];
|
||||||
|
value.recent_output ??= [];
|
||||||
|
} else if (value.type === "report_task_log") {
|
||||||
|
value.stdout_tail ??= "";
|
||||||
|
value.stderr_tail ??= "";
|
||||||
|
} else if (value.type === "task_completed") {
|
||||||
|
value.stdout_tail ??= "";
|
||||||
|
value.stderr_tail ??= "";
|
||||||
|
value.stdout_truncated ??= false;
|
||||||
|
value.stderr_truncated ??= false;
|
||||||
|
}
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedRequestPayloadDigest(request) {
|
||||||
|
return `sha256:${crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(JSON.stringify(canonicalSignedRequest(withWireDefaults(request))))
|
||||||
|
.digest("hex")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeSignatureMessage(
|
||||||
|
node,
|
||||||
|
requestKind,
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
issuedAtEpochSeconds
|
||||||
|
) {
|
||||||
|
const parts = [
|
||||||
|
"clusterflux-node-request-signature:v2",
|
||||||
|
node,
|
||||||
|
requestKind,
|
||||||
|
payloadDigest,
|
||||||
|
nonce,
|
||||||
|
String(issuedAtEpochSeconds),
|
||||||
|
];
|
||||||
|
return Buffer.concat(
|
||||||
|
parts.flatMap((part) => [
|
||||||
|
Buffer.from(`${Buffer.byteLength(part)}:`),
|
||||||
|
Buffer.from(part),
|
||||||
|
Buffer.from("\n"),
|
||||||
|
])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedNodeProof(node, identity, requestKind, request, options = {}) {
|
||||||
|
const nonce =
|
||||||
|
options.nonce ??
|
||||||
|
`${requestKind}-${process.pid}-${Date.now()}-${crypto
|
||||||
|
.randomBytes(8)
|
||||||
|
.toString("hex")}`;
|
||||||
|
const issuedAt =
|
||||||
|
options.issuedAtEpochSeconds ?? Math.floor(Date.now() / 1000);
|
||||||
|
const signature = crypto.sign(
|
||||||
|
null,
|
||||||
|
nodeSignatureMessage(
|
||||||
|
node,
|
||||||
|
requestKind,
|
||||||
|
signedRequestPayloadDigest(request),
|
||||||
|
nonce,
|
||||||
|
issuedAt
|
||||||
|
),
|
||||||
|
identity.privateKeyObject
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
nonce,
|
||||||
|
issued_at_epoch_seconds: issuedAt,
|
||||||
|
signature: `ed25519:${signature.toString("base64")}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedNodeHeartbeat(tenant, project, node, identity, options = {}) {
|
||||||
|
const request = { type: "node_heartbeat", tenant, project, node };
|
||||||
|
return signedNodeProof(node, identity, "node_heartbeat", request, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
function signedNodeRequest(node, identity, requestKind, request, options = {}) {
|
||||||
|
return {
|
||||||
|
type: "signed_node",
|
||||||
|
node,
|
||||||
|
node_signature: signedNodeProof(
|
||||||
|
node,
|
||||||
|
identity,
|
||||||
|
requestKind,
|
||||||
|
request,
|
||||||
|
options
|
||||||
|
),
|
||||||
|
request,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
nodeIdentity,
|
||||||
|
nodeIdentityFromPrivateKey,
|
||||||
|
signedNodeProof,
|
||||||
|
signedRequestPayloadDigest,
|
||||||
|
signedNodeHeartbeat,
|
||||||
|
signedNodeRequest,
|
||||||
|
};
|
||||||
385
scripts/operator-panel-smoke.js
Executable file
385
scripts/operator-panel-smoke.js
Executable file
|
|
@ -0,0 +1,385 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const path = require("path");
|
||||||
|
const { nodeIdentity } = require("./node-signing");
|
||||||
|
const {
|
||||||
|
ensureRootlessPodman,
|
||||||
|
repo,
|
||||||
|
runFlagshipWorker,
|
||||||
|
send,
|
||||||
|
startFlagship,
|
||||||
|
waitForTaskEvent,
|
||||||
|
waitForJsonLine,
|
||||||
|
waitForNodeStatus,
|
||||||
|
} = require("./real-flagship-harness");
|
||||||
|
|
||||||
|
const panelNode = "panel-node";
|
||||||
|
const panelNodeIdentity = nodeIdentity("operator-panel-smoke", panelNode);
|
||||||
|
const panelProject = path.join(repo, "tests/fixtures/runtime-conformance");
|
||||||
|
|
||||||
|
function widget(panel, id) {
|
||||||
|
const item = panel.widgets[id];
|
||||||
|
assert(item, `missing panel widget ${id}`);
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
const delay = (milliseconds) =>
|
||||||
|
new Promise((resolve) => setTimeout(resolve, milliseconds));
|
||||||
|
|
||||||
|
async function waitForBreakpointHit(addr, process) {
|
||||||
|
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
||||||
|
const status = await send(addr, {
|
||||||
|
type: "inspect_debug_breakpoints",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process,
|
||||||
|
});
|
||||||
|
if (status.type !== "debug_breakpoints") {
|
||||||
|
const events = await send(addr, {
|
||||||
|
type: "list_task_events",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process,
|
||||||
|
});
|
||||||
|
throw new Error(
|
||||||
|
`breakpoint state disappeared: ${JSON.stringify({ status, events })}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (status.hit_epoch != null) return status;
|
||||||
|
await delay(25);
|
||||||
|
}
|
||||||
|
throw new Error(`timed out waiting for package breakpoint in ${process}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForDebugEpochFrozen(addr, process, epoch) {
|
||||||
|
for (let attempt = 0; attempt < 2400; attempt += 1) {
|
||||||
|
const status = await send(addr, {
|
||||||
|
type: "inspect_debug_epoch",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process,
|
||||||
|
epoch,
|
||||||
|
});
|
||||||
|
assert.strictEqual(status.type, "debug_epoch_status", JSON.stringify(status));
|
||||||
|
if (status.failed) {
|
||||||
|
throw new Error(status.failure_messages.join("; "));
|
||||||
|
}
|
||||||
|
if (status.fully_frozen) return status;
|
||||||
|
await delay(25);
|
||||||
|
}
|
||||||
|
throw new Error(`timed out waiting for debug epoch ${epoch} to freeze`);
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
ensureRootlessPodman();
|
||||||
|
const coordinator = cp.spawn(
|
||||||
|
"cargo",
|
||||||
|
[
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-coordinator",
|
||||||
|
"--",
|
||||||
|
"--listen",
|
||||||
|
"127.0.0.1:0",
|
||||||
|
"--allow-local-trusted-loopback"
|
||||||
|
],
|
||||||
|
{ cwd: repo }
|
||||||
|
);
|
||||||
|
let coordinatorStderr = "";
|
||||||
|
let worker;
|
||||||
|
coordinator.stderr.on("data", (chunk) => {
|
||||||
|
coordinatorStderr += chunk.toString();
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ready = await waitForJsonLine(coordinator);
|
||||||
|
const [host, portText] = ready.listen.split(":");
|
||||||
|
const addr = { host, port: Number(portText) };
|
||||||
|
assert.strictEqual((await send(addr, { type: "ping" })).type, "pong");
|
||||||
|
const projectCreated = await send(addr, {
|
||||||
|
type: "create_project",
|
||||||
|
tenant: "tenant",
|
||||||
|
actor_user: "user",
|
||||||
|
project: "project",
|
||||||
|
name: "Operator panel smoke",
|
||||||
|
});
|
||||||
|
assert.strictEqual(projectCreated.type, "project_created");
|
||||||
|
|
||||||
|
worker = await runFlagshipWorker(
|
||||||
|
addr,
|
||||||
|
panelNode,
|
||||||
|
panelNodeIdentity,
|
||||||
|
panelProject
|
||||||
|
);
|
||||||
|
const workerReady = await worker.ready;
|
||||||
|
assert.strictEqual(workerReady.node_status, "ready");
|
||||||
|
const workerCompletion = waitForNodeStatus(worker.child, "completed");
|
||||||
|
const flagship = startFlagship(addr, panelProject);
|
||||||
|
const configuredBreakpoints = await send(addr, {
|
||||||
|
type: "set_debug_breakpoints",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process: flagship.process,
|
||||||
|
probe_symbols: ["clusterflux.probe.package_release"],
|
||||||
|
});
|
||||||
|
assert.strictEqual(configuredBreakpoints.type, "debug_breakpoints");
|
||||||
|
await waitForTaskEvent(
|
||||||
|
addr,
|
||||||
|
flagship.process,
|
||||||
|
(event) => event.task_definition === "prepare_source",
|
||||||
|
"prepare_source after breakpoint configuration"
|
||||||
|
);
|
||||||
|
const breakpointHit = await waitForBreakpointHit(addr, flagship.process);
|
||||||
|
assert.strictEqual(
|
||||||
|
breakpointHit.hit_probe_symbol,
|
||||||
|
"clusterflux.probe.package_release"
|
||||||
|
);
|
||||||
|
const frozenEpoch = await waitForDebugEpochFrozen(
|
||||||
|
addr,
|
||||||
|
flagship.process,
|
||||||
|
breakpointHit.hit_epoch
|
||||||
|
);
|
||||||
|
assert(frozenEpoch.acknowledgements.length >= 2);
|
||||||
|
const compileEvent = await waitForTaskEvent(
|
||||||
|
addr,
|
||||||
|
flagship.process,
|
||||||
|
(event) => event.task_definition === "compile_linux",
|
||||||
|
"compile before the package breakpoint"
|
||||||
|
);
|
||||||
|
const report = await workerCompletion;
|
||||||
|
assert.strictEqual(report.node_status, "completed");
|
||||||
|
assert.strictEqual(report.coordinator_response.type, "task_recorded");
|
||||||
|
const process = flagship.process;
|
||||||
|
|
||||||
|
const rendered = await send(addr, {
|
||||||
|
type: "render_operator_panel",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
actor_user: "user",
|
||||||
|
max_download_bytes: 1024 * 1024,
|
||||||
|
stopped: false
|
||||||
|
});
|
||||||
|
assert.strictEqual(rendered.type, "operator_panel");
|
||||||
|
const panel = rendered.panel;
|
||||||
|
assert.strictEqual(panel.tenant, "tenant");
|
||||||
|
assert.strictEqual(panel.project, "project");
|
||||||
|
assert.strictEqual(panel.process, process);
|
||||||
|
assert.strictEqual(panel.program_ui_events_enabled, true);
|
||||||
|
|
||||||
|
assert.deepStrictEqual(widget(panel, "process-status").kind, {
|
||||||
|
Text: { value: "running" }
|
||||||
|
});
|
||||||
|
const taskProgress = widget(panel, "task-progress").kind.Progress;
|
||||||
|
assert(taskProgress.current >= 2);
|
||||||
|
assert.strictEqual(taskProgress.current, taskProgress.total);
|
||||||
|
const taskSummary = widget(panel, "task-summary").kind.Text.value;
|
||||||
|
assert.match(
|
||||||
|
taskSummary,
|
||||||
|
new RegExp(`compile_linux \\[${compileEvent.task}\\]:Some\\(0\\):panel-node`)
|
||||||
|
);
|
||||||
|
assert.match(widget(panel, "recent-logs").kind.Text.value, /stdout=\d+ stderr=\d+/);
|
||||||
|
const downloadWidget = widget(panel, "download-artifact").kind;
|
||||||
|
const artifact = downloadWidget.ArtifactDownload.artifact;
|
||||||
|
assert(
|
||||||
|
artifact === compileEvent.artifact_path.slice("/vfs/artifacts/".length),
|
||||||
|
"panel download must point at a real flagship artifact"
|
||||||
|
);
|
||||||
|
assert(!JSON.stringify(downloadWidget).includes("url_path"));
|
||||||
|
assert(!JSON.stringify(downloadWidget).includes("scoped_token_digest"));
|
||||||
|
assert.deepStrictEqual(widget(panel, "debug-process").kind, {
|
||||||
|
Button: { action: "debug-process" }
|
||||||
|
});
|
||||||
|
assert.deepStrictEqual(widget(panel, "cancel-process").kind, {
|
||||||
|
Button: { action: "cancel-process" }
|
||||||
|
});
|
||||||
|
assert.deepStrictEqual(widget(panel, "restart-selected-task").kind, {
|
||||||
|
Button: { action: "restart-task" }
|
||||||
|
});
|
||||||
|
assert(panel.control_plane_actions.includes("DebugProcess"));
|
||||||
|
assert(panel.control_plane_actions.includes("CancelProcess"));
|
||||||
|
const restartTarget = panel.control_plane_actions.find(
|
||||||
|
(action) => action.RestartTask
|
||||||
|
)?.RestartTask;
|
||||||
|
assert(
|
||||||
|
restartTarget && taskSummary.includes(`[${restartTarget}]`),
|
||||||
|
"panel restart must target the real flagship task instance"
|
||||||
|
);
|
||||||
|
assert(
|
||||||
|
panel.control_plane_actions.some(
|
||||||
|
(action) => action.DownloadArtifact === artifact
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert(!JSON.stringify(panel).includes("<script"));
|
||||||
|
assert(!JSON.stringify(panel).toLowerCase().includes("oauth"));
|
||||||
|
|
||||||
|
const panelLink = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact: downloadWidget.ArtifactDownload.artifact,
|
||||||
|
max_bytes: 1024 * 1024,
|
||||||
|
ttl_seconds: 60
|
||||||
|
});
|
||||||
|
assert.strictEqual(panelLink.type, "artifact_download_link");
|
||||||
|
assert.strictEqual(panelLink.link.artifact, downloadWidget.ArtifactDownload.artifact);
|
||||||
|
assert.match(panelLink.link.policy_context_digest, /^sha256:[0-9a-f]{64}$/);
|
||||||
|
assert.deepStrictEqual(panelLink.link.source, { RetainedNode: "panel-node" });
|
||||||
|
assert(panelLink.link.url_path.endsWith(`/artifacts/tenant/project/${process}/${artifact}`));
|
||||||
|
|
||||||
|
const apiTooLarge = await send(addr, {
|
||||||
|
type: "create_artifact_download_link",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
artifact: downloadWidget.ArtifactDownload.artifact,
|
||||||
|
max_bytes: 1,
|
||||||
|
ttl_seconds: 60
|
||||||
|
});
|
||||||
|
assert.strictEqual(apiTooLarge.type, "error");
|
||||||
|
assert.match(apiTooLarge.message, /exceeds download limit/);
|
||||||
|
|
||||||
|
const panelTooLarge = await send(addr, {
|
||||||
|
type: "render_operator_panel",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
actor_user: "user",
|
||||||
|
max_download_bytes: 1,
|
||||||
|
stopped: false
|
||||||
|
});
|
||||||
|
assert.strictEqual(panelTooLarge.type, "error");
|
||||||
|
assert.match(panelTooLarge.message, /exceeds download limit/);
|
||||||
|
|
||||||
|
const accepted = await send(addr, {
|
||||||
|
type: "submit_panel_event",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
widget_id: "debug-process",
|
||||||
|
kind: "ButtonClicked",
|
||||||
|
max_events: 1
|
||||||
|
});
|
||||||
|
assert.strictEqual(accepted.type, "panel_event_accepted");
|
||||||
|
assert.strictEqual(accepted.used_events, 1);
|
||||||
|
assert.strictEqual(accepted.max_events, 1);
|
||||||
|
|
||||||
|
const rateLimited = await send(addr, {
|
||||||
|
type: "submit_panel_event",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
widget_id: "debug-process",
|
||||||
|
kind: "ButtonClicked",
|
||||||
|
max_events: 1
|
||||||
|
});
|
||||||
|
assert.strictEqual(rateLimited.type, "error");
|
||||||
|
assert.match(rateLimited.message, /rate limit/i);
|
||||||
|
|
||||||
|
const stopped = await send(addr, {
|
||||||
|
type: "render_operator_panel",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
actor_user: "user",
|
||||||
|
max_download_bytes: 1024 * 1024,
|
||||||
|
stopped: true
|
||||||
|
});
|
||||||
|
assert.strictEqual(stopped.type, "operator_panel");
|
||||||
|
assert.strictEqual(stopped.panel.program_ui_events_enabled, false);
|
||||||
|
assert.deepStrictEqual(widget(stopped.panel, "process-status").kind, {
|
||||||
|
Text: { value: "stopped" }
|
||||||
|
});
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
widget(stopped.panel, "task-progress").kind,
|
||||||
|
widget(panel, "task-progress").kind
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
widget(stopped.panel, "task-summary").kind,
|
||||||
|
widget(panel, "task-summary").kind
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
widget(stopped.panel, "recent-logs").kind,
|
||||||
|
widget(panel, "recent-logs").kind
|
||||||
|
);
|
||||||
|
assert.deepStrictEqual(
|
||||||
|
widget(stopped.panel, "download-artifact").kind,
|
||||||
|
widget(panel, "download-artifact").kind
|
||||||
|
);
|
||||||
|
assert(stopped.panel.control_plane_actions.includes("DebugProcess"));
|
||||||
|
assert(
|
||||||
|
stopped.panel.control_plane_actions.some(
|
||||||
|
(action) => action.DownloadArtifact === artifact
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const frozenEvent = await send(addr, {
|
||||||
|
type: "submit_panel_event",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
widget_id: "debug-process",
|
||||||
|
kind: "ButtonClicked",
|
||||||
|
max_events: 10
|
||||||
|
});
|
||||||
|
assert.strictEqual(frozenEvent.type, "error");
|
||||||
|
assert.match(frozenEvent.message, /program UI events are disabled/i);
|
||||||
|
|
||||||
|
const crossTenant = await send(addr, {
|
||||||
|
type: "render_operator_panel",
|
||||||
|
tenant: "other",
|
||||||
|
project: "project",
|
||||||
|
process,
|
||||||
|
actor_user: "user",
|
||||||
|
max_download_bytes: 1024 * 1024,
|
||||||
|
stopped: false
|
||||||
|
});
|
||||||
|
assert.strictEqual(crossTenant.type, "error");
|
||||||
|
assert.match(
|
||||||
|
crossTenant.message,
|
||||||
|
/scope|tenant|project|requires an active virtual process/i
|
||||||
|
);
|
||||||
|
assert(!crossTenant.message.includes(process));
|
||||||
|
|
||||||
|
const resumed = await send(addr, {
|
||||||
|
type: "resume_debug_epoch",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process,
|
||||||
|
epoch: breakpointHit.hit_epoch,
|
||||||
|
});
|
||||||
|
assert.strictEqual(resumed.type, "debug_epoch");
|
||||||
|
const cleanup = await send(addr, {
|
||||||
|
type: "abort_process",
|
||||||
|
tenant: "tenant",
|
||||||
|
project: "project",
|
||||||
|
actor_user: "user",
|
||||||
|
process,
|
||||||
|
});
|
||||||
|
assert.strictEqual(cleanup.type, "process_aborted");
|
||||||
|
} catch (error) {
|
||||||
|
if (coordinatorStderr) {
|
||||||
|
error.message = `${error.message}\ncoordinator stderr:\n${coordinatorStderr}`;
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
worker?.child.kill("SIGTERM");
|
||||||
|
coordinator.kill("SIGTERM");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Operator panel smoke passed");
|
||||||
|
})().catch((error) => {
|
||||||
|
console.error(error.stack || error.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
86
scripts/podman-backend-smoke.js
Executable file
86
scripts/podman-backend-smoke.js
Executable file
|
|
@ -0,0 +1,86 @@
|
||||||
|
#!/usr/bin/env node
|
||||||
|
|
||||||
|
const assert = require("assert");
|
||||||
|
const cp = require("child_process");
|
||||||
|
const path = require("path");
|
||||||
|
const { configurePodmanTestEnvironment } = require("./podman-test-env");
|
||||||
|
|
||||||
|
const repo = path.resolve(__dirname, "..");
|
||||||
|
const baseImage = "docker.io/library/alpine:3.20";
|
||||||
|
|
||||||
|
// Nix's standalone Podman package may not install the distribution-level
|
||||||
|
// containers/image policy normally found under /etc. Use an isolated test HOME
|
||||||
|
// without replacing a policy supplied by the host.
|
||||||
|
configurePodmanTestEnvironment(repo);
|
||||||
|
|
||||||
|
function run(command, args, options = {}) {
|
||||||
|
return cp.execFileSync(command, args, {
|
||||||
|
cwd: repo,
|
||||||
|
encoding: "utf8",
|
||||||
|
stdio: options.stdio || ["ignore", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function incomplete(reason) {
|
||||||
|
const error = new Error(`Linux Podman backend incomplete: ${reason}`);
|
||||||
|
error.code = "CLUSTERFLUX_PODMAN_INCOMPLETE";
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensurePodmanBaseImage() {
|
||||||
|
try {
|
||||||
|
run("podman", ["--version"]);
|
||||||
|
} catch (error) {
|
||||||
|
incomplete(`podman command is unavailable (${error.message})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let rootless;
|
||||||
|
try {
|
||||||
|
rootless = run("podman", ["info", "--format", "{{.Host.Security.Rootless}}"]).trim();
|
||||||
|
} catch (error) {
|
||||||
|
incomplete(`podman info did not report rootless status (${error.message})`);
|
||||||
|
}
|
||||||
|
if (rootless !== "true") {
|
||||||
|
incomplete(`podman is not running in rootless mode (reported ${JSON.stringify(rootless)})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
run("podman", ["image", "exists", baseImage]);
|
||||||
|
} catch (_) {
|
||||||
|
try {
|
||||||
|
run("podman", ["pull", baseImage], { stdio: "inherit" });
|
||||||
|
} catch (error) {
|
||||||
|
incomplete(`unable to make ${baseImage} available (${error.message})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
ensurePodmanBaseImage();
|
||||||
|
|
||||||
|
const stdout = run("cargo", [
|
||||||
|
"run",
|
||||||
|
"-q",
|
||||||
|
"-p",
|
||||||
|
"clusterflux-node",
|
||||||
|
"--bin",
|
||||||
|
"clusterflux-podman-smoke"
|
||||||
|
]);
|
||||||
|
const report = JSON.parse(stdout.trim().split("\n").at(-1));
|
||||||
|
|
||||||
|
assert.strictEqual(report.podman_status, "completed");
|
||||||
|
assert.strictEqual(report.status_code, 0);
|
||||||
|
assert.strictEqual(report.stdout, "podman-ok:node-local source\n");
|
||||||
|
assert.strictEqual(report.large_bytes_uploaded, false);
|
||||||
|
assert.strictEqual(report.uses_full_repo_tarball, false);
|
||||||
|
assert.strictEqual(report.coordinator_routed_file_reads, false);
|
||||||
|
assert.strictEqual(report.staged_artifact.path, "/vfs/artifacts/podman-smoke.txt");
|
||||||
|
|
||||||
|
console.log("Podman backend smoke passed");
|
||||||
|
} catch (error) {
|
||||||
|
if (error.code === "CLUSTERFLUX_PODMAN_INCOMPLETE") {
|
||||||
|
console.error(error.message);
|
||||||
|
process.exit(2);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue