michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Compare commits

base: michel:release-98d969b26488
Branches Tags
michel:main
michel:release-5edbadb22804
michel:release-501cb6d672f9
michel:release-46a4d4e84222
michel:release-7fcdc75d8eaf
michel:release-ea887c8f56cd
michel:release-e47f9c27bbeb
michel:release-6d8c5c5b2a8b
michel:release-fcace1d13a4b
michel:release-09ca780bd67a
michel:release-b8454b34d38c
michel:release-ff7f847f143d
michel:release-01875e88a3e2
michel:release-be1f654ae8de
michel:release-e570b80c3a0f
michel:release-6d1a121b0a8a
michel:release-6756e5208c8b
michel:release-5792a1a4e1cb
michel:release-f6c01dca61ee
michel:release-f70e47af061d
michel:release-55b5a563d642
michel:release-98d969b26488
...
compare: michel:main
Branches Tags
michel:main
michel:release-5edbadb22804
michel:release-501cb6d672f9
michel:release-46a4d4e84222
michel:release-7fcdc75d8eaf
michel:release-ea887c8f56cd
michel:release-e47f9c27bbeb
michel:release-6d8c5c5b2a8b
michel:release-fcace1d13a4b
michel:release-09ca780bd67a
michel:release-b8454b34d38c
michel:release-ff7f847f143d
michel:release-01875e88a3e2
michel:release-be1f654ae8de
michel:release-e570b80c3a0f
michel:release-6d1a121b0a8a
michel:release-6756e5208c8b
michel:release-5792a1a4e1cb
michel:release-f6c01dca61ee
michel:release-f70e47af061d
michel:release-55b5a563d642
michel:release-98d969b26488

No commits in common. "release-98d969b26488" and "main" have entirely different histories.
release-98 ... main

157 changed files with 19026 additions and 14393 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.gitignore vendored
View file

@ -1,8 +1,5 @@
/target/
/web/target/
/.clusterflux/
**/.clusterflux/
/vscode-extension/node_modules/
/private/*/Cargo.lock
!/private/hosted-policy/Cargo.lock
/private/*/target/
/scripts/containers-home/

22
CLUSTERFLUX_PUBLIC_TREE.json
View file

@ -1,22 +0,0 @@
{
"kind": "clusterflux-filtered-public-tree",
"source_commit": "98d969b26488b4cda8b2381fa870276a00ca98ea",
"release_name": "release-98d969b26488",
"filtered_out": [
"private/**",
"internal/**",
"experiments/**",
".git",
"target",
"git-ignored source paths",
"root/*.md except README.md",
"**/.clusterflux/**",
".forgejo/**"
],
"public_export": {
"host_neutral": true,
"include_forgejo_workflows": false
},
"forgejo_host": "git.michelpaulissen.com",
"default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
}

39
Cargo.lock generated
View file

@ -290,6 +290,20 @@ dependencies = [
"wasmparser 0.245.1",
]
[[package]]
name = "clusterflux-client"
version = "0.1.0"
dependencies = [
"base64",
"clusterflux-control",
"clusterflux-coordinator",
"clusterflux-core",
"serde",
"serde_json",
"thiserror 1.0.69",
"tokio",
]
[[package]]
name = "clusterflux-control"
version = "0.1.0"
@ -967,6 +981,13 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hello-build"
version = "0.1.0"
dependencies = [
"clusterflux-sdk",
]
[[package]]
name = "hex"
version = "0.4.3"
@ -1144,16 +1165,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "launch-build-demo"
version = "0.1.0"
dependencies = [
"clusterflux-sdk",
"futures-executor",
"serde",
"serde_json",
]
[[package]]
name = "lazy_static"
version = "1.5.0"
@ -1670,6 +1681,14 @@ dependencies = [
"yasna",
]
[[package]]
name = "recovery-build"
version = "0.1.0"
dependencies = [
"clusterflux-sdk",
"serde",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"

7
Cargo.toml
View file

@ -1,7 +1,9 @@
[workspace]
resolver = "2"
exclude = ["web"]
members = [
"crates/clusterflux-cli",
"crates/clusterflux-client",
"crates/clusterflux-control",
"crates/clusterflux-coordinator",
"crates/clusterflux-core",
@ -10,13 +12,14 @@ members = [
"crates/clusterflux-node",
"crates/clusterflux-sdk",
"crates/clusterflux-wasm-runtime",
"examples/launch-build-demo",
"examples/hello-build",
"examples/recovery-build",
]
[workspace.package]
edition = "2021"
license = "Apache-2.0 OR MIT"
repository = "https://git.michelpaulissen.com/michel/clusterflux-public"
repository = "https://clusterflux.lesstuff.com"
[workspace.dependencies]
anyhow = "1.0"

86
README.md
View file

@ -1,14 +1,82 @@
# Clusterflux
Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
coordinator hosts the async main, while attached nodes execute Wasm tasks,
rootless containers, and native commands. Tasks exchange canonical values and
portable typed handles instead of sharing host memory.
Clusterflux runs a Rust-defined workflow as one distributed virtual process. The
async main runs serverless, provisioning nodes to run tasks through rootless
containers. Tasks on nodes exchange data simply and efficiently.
The user experience is built to be as much as possible like building a regular
program. The processes are debuggable as normal through a debugger adapter.
The primary use case is to consolidate build processes into a single streamlined
developer experience. An example program can be seen below:
~~~rust
use clusterflux::prelude::*;
#[clusterflux::task(capabilities = "command")]
pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
let executable = fs::output("hello-clusterflux")?;
Command::new("cc")
.args([
"-Os",
"-static",
"-s",
"fixture/hello-clusterflux.c",
"-o",
executable.as_str(),
])
.cwd(source.mount()?)
.env("SOURCE_DATE_EPOCH", "0")
.network_disabled()
.run()
.await?;
fs::publish(&executable).await
}
#[clusterflux::main]
pub async fn build() -> Result<Artifact> {
let source = source::current_project().snapshot().await?;
let compile = clusterflux::spawn!(compile(source))
.on(clusterflux::env!("linux"))
.await?;
compile.join().await
}
~~~
After setup, this build pipeline could be deployed as easily as launching it
through your IDE. This repository includes a VS Code extension to make
development as straightforward as possible. A full collection of CLI tools is
included for advanced usage.
Clusterflux is explicitly local-first. It is trivial to provision existing
hardware as resources. Bulk data will typically not leave the local network,
allowing maximum throughput. The same capability, however, also makes it
possible to leverage cloud resources easily.
Start with [Getting started](docs/getting-started.md). It takes you through
authentication, project setup, node enrollment, a run, debugging, task restart,
and artifact download.
## Build a real executable
The primary example is intentionally small: [hello-build](examples/hello-build)
snapshots its source project, spawns one container-backed compile task, and
publishes the resulting executable as a retained artifact.
~~~bash
clusterflux bundle inspect --project examples/hello-build
clusterflux run --project examples/hello-build build
clusterflux artifact list --process <process-id>
clusterflux artifact download <artifact-id> --to ./hello-clusterflux
chmod +x ./hello-clusterflux
./hello-clusterflux
~~~
The final command prints `hello from a real Clusterflux build`. The source uses
only the public SDK path: current-project snapshot, `spawn!`, `Command::run`, and
artifact publication. See [recovery-build](examples/recovery-build) for two
same-definition task instances, a real command failure, and operator restart.
## What you get
- One virtual process with distinct task instances and restart attempts.
@ -29,6 +97,9 @@ cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinato
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
~~~
On NixOS or another system with Nix, the equivalent package is available with
`nix profile install .#clusterflux-tools`.
Rootless Podman is required on Linux nodes that build or run a declared
Containerfile environment. Install VS Code when you want the graphical debug
workflow.
@ -54,8 +125,8 @@ explicit-key option.
Choose an entrypoint and run it:
~~~bash
clusterflux bundle inspect --project examples/launch-build-demo
clusterflux run --project examples/launch-build-demo build
clusterflux bundle inspect --project examples/hello-build
clusterflux run --project examples/hello-build build
~~~
Inspect the result:
@ -65,7 +136,8 @@ clusterflux process status
clusterflux task list
clusterflux logs
clusterflux artifact list
clusterflux artifact download app.txt --to ./app.txt
# Use the `artifact` value returned by the list command, for example:
clusterflux artifact download hello-clusterflux-4f61c2... --to ./hello-clusterflux
~~~
To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md).

2
SECURITY.md
View file

@ -7,7 +7,7 @@ published Clusterflux release. Older preview releases are not maintained.
## Report a vulnerability
Email security@michelpaulissen.com with a concise description, affected version
Email security@clusterflux.lesstuff.com with a concise description, affected version
or source revision, reproduction steps, and impact. Do not open a public issue
for an unpatched vulnerability or include credentials, session tokens, private
keys, provider tokens, customer data, or operator secrets in a public report.

24
crates/clusterflux-cli/src/admin.rs
View file

@ -48,7 +48,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
.unwrap_or(json!(false)),
"response": response,
"safe_default": "read_only",
"private_website_required": false,
"external_website_required": false,
"coordinator_session_requests": session.requests(),
}));
}
@ -56,7 +56,7 @@ pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
"command": "admin status",
"mode": "self_hosted_local",
"safe_default": "read_only",
"private_website_required": false,
"external_website_required": false,
}))
}
@ -86,7 +86,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
"project": project.clone(),
"user": user,
"coordinator": scope.coordinator,
"private_website_required": false,
"external_website_required": false,
"self_hosted_cli_only": true,
"project_config_written": project_init
.get("project_config_written")
@ -107,13 +107,13 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
{
"step": "start_self_hosted_coordinator",
"command": "clusterflux-coordinator --listen 127.0.0.1:0",
"private_website_required": false,
"external_website_required": false,
},
{
"step": "create_or_link_project",
"command": "clusterflux project init --yes",
"completed": true,
"private_website_required": false,
"external_website_required": false,
},
{
"step": "create_node_enrollment_grant",
@ -121,7 +121,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
"clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project
),
"private_website_required": false,
"external_website_required": false,
},
{
"step": "attach_worker_node",
@ -129,7 +129,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
"clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
tenant, project
),
"private_website_required": false,
"external_website_required": false,
},
{
"step": "run_process",
@ -137,7 +137,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
"clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project
),
"private_website_required": false,
"external_website_required": false,
},
{
"step": "inspect_status_logs_artifacts",
@ -148,12 +148,12 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
"clusterflux artifact list",
"clusterflux quota status",
],
"private_website_required": false,
"external_website_required": false,
},
{
"step": "revoke_access",
"command": "clusterflux admin revoke-node --node <node-id> --yes",
"private_website_required": false,
"external_website_required": false,
}
],
}))
@ -209,7 +209,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
"actor_tenant": actor_tenant,
"actor_user": actor_user,
"suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
"private_website_required": false,
"external_website_required": false,
"response": response,
"coordinator_session_requests": session.requests(),
}));
@ -219,7 +219,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"tenant": tenant,
"private_website_required": false,
"external_website_required": false,
}))
}

10
crates/clusterflux-cli/src/artifact.rs
View file

@ -12,6 +12,7 @@ use crate::client::{
};
use crate::config::StoredCliSession;
use crate::errors::cli_error_summary_for_category;
use crate::process::hydrate_process_scope;
use crate::process_events::{
artifact_download_grant_disclosures, artifact_download_session_summary,
artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
@ -27,9 +28,10 @@ pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
}
pub(crate) fn artifact_list_report_with_session(
args: ArtifactListArgs,
mut args: ArtifactListArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,
@ -53,9 +55,10 @@ pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Val
}
pub(crate) fn artifact_download_report_with_session(
args: ArtifactDownloadArgs,
mut args: ArtifactDownloadArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
@ -143,9 +146,10 @@ pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value>
}
pub(crate) fn artifact_export_report_with_session(
args: ArtifactExportArgs,
mut args: ArtifactExportArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(

12
crates/clusterflux-cli/src/auth.rs
View file

@ -227,7 +227,7 @@ pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<V
"suspension_known": false,
"account_state_known": false,
"account_status": "unknown",
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
})
});
@ -263,7 +263,7 @@ fn coordinator_auth_status_summary(
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
@ -296,7 +296,7 @@ fn coordinator_auth_status_summary(
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
@ -316,7 +316,7 @@ fn coordinator_auth_status_summary(
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
@ -338,7 +338,7 @@ fn coordinator_auth_status_summary(
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(message),
"coordinator_response_type": "error",
@ -403,7 +403,7 @@ fn coordinator_auth_status_summary(
"manual_review": manual_review,
"sanitized_reason": sanitized_reason,
"next_actions": next_actions,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
"coordinator_session_requests": coordinator_session_requests,

2
crates/clusterflux-cli/src/auth_scope.rs
View file

@ -60,7 +60,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"next_actions": ["clusterflux login --browser"],
})

3
crates/clusterflux-cli/src/config.rs
View file

@ -5,8 +5,7 @@ use serde::{Deserialize, Serialize};
use crate::CliScopeArgs;
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
"https://clusterflux.michelpaulissen.com";
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://clusterflux.lesstuff.com";
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct ProjectConfig {

6
crates/clusterflux-cli/src/debug.rs
View file

@ -15,7 +15,7 @@ pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
"command": "dap",
"adapter": dap_binary_path()?.display().to_string(),
"args": args.args,
"private_website_required": false,
"external_website_required": false,
}))
}
@ -102,7 +102,7 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
.cloned()
.unwrap_or_else(|| json!(0)),
"debug_reads_quota_limited": true,
"private_website_required": false,
"external_website_required": false,
"coordinator_session_requests": session.requests(),
}));
}
@ -115,6 +115,6 @@ pub(crate) fn debug_attach_report_with_dap_and_session(
"dap": dap,
"authorized": "unknown_without_coordinator",
"debug_reads_quota_limited": "unknown_without_coordinator",
"private_website_required": false,
"external_website_required": false,
}))
}

5
crates/clusterflux-cli/src/errors.rs
View file

@ -38,7 +38,10 @@ pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &s
);
object.insert("community_tier_language".to_owned(), json!(true));
object.insert("community_tier_label".to_owned(), json!("community tier"));
object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
object.insert(
"sensitive_abuse_heuristics_exposed".to_owned(),
json!(false),
);
}
}
summary

4
crates/clusterflux-cli/src/logs.rs
View file

@ -3,6 +3,7 @@ use serde_json::{json, Value};
use crate::client::list_task_events_if_available_with_session;
use crate::config::StoredCliSession;
use crate::process::hydrate_process_scope;
use crate::process_events::log_entries;
use crate::LogsArgs;
@ -12,9 +13,10 @@ pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
}
pub(crate) fn logs_report_with_session(
args: LogsArgs,
mut args: LogsArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,

6
crates/clusterflux-cli/src/node.rs
View file

@ -152,7 +152,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
"tenant": tenant,
"project": project,
"user": user,
"private_website_required": false,
"external_website_required": false,
"enrollment_grant": enrollment_grant,
"response": response,
"coordinator_session_requests": session.requests(),
@ -161,7 +161,7 @@ pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<V
Ok(json!({
"command": "node enroll",
"status": "requires_coordinator",
"private_website_required": false,
"external_website_required": false,
"requested_ttl_seconds": args.ttl_seconds,
"enrollment_grant": null,
"reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
@ -611,6 +611,8 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
};
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &tenant,
"project": &project,
"node": &plan.node,
});
let heartbeat_signature = sign_node_request(

8
crates/clusterflux-cli/src/output.rs
View file

@ -217,10 +217,10 @@ pub(crate) fn human_report(value: &Value) -> String {
}
push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
if let Some(exposed) = account
.get("private_moderation_details_exposed")
.get("sensitive_moderation_details_exposed")
.and_then(Value::as_bool)
{
lines.push(format!("private moderation details exposed: {exposed}"));
lines.push(format!("sensitive moderation details exposed: {exposed}"));
}
}
if let Some(coordinator_selection) = value.get("coordinator") {
@ -277,10 +277,10 @@ pub(crate) fn human_report(value: &Value) -> String {
));
}
if let Some(flag) = value
.get("private_website_required")
.get("external_website_required")
.and_then(Value::as_bool)
{
lines.push(format!("private website required: {flag}"));
lines.push(format!("external website required: {flag}"));
}
if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
let actions = next_actions

5
crates/clusterflux-cli/src/process.rs
View file

@ -15,7 +15,10 @@ use crate::{
ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
};
fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
pub(crate) fn hydrate_process_scope(
scope: &mut CliScopeArgs,
stored_session: Option<&StoredCliSession>,
) {
if scope.coordinator.is_none() {
scope.coordinator = stored_session
.filter(|session| session.session_secret.is_some())

4
crates/clusterflux-cli/src/process_events.rs
View file

@ -557,7 +557,7 @@ pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
"cross_tenant_reuse_allowed": false,
"unauthorized_project_reuse_allowed": false,
"default_durable_store_assumed": false,
"private_website_required": false,
"external_website_required": false,
}])
}
@ -613,7 +613,7 @@ pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<
"current_usage": current_usage,
"limits": quota_limits_value(),
"next_blocked_action": next_blocked_action,
"private_abuse_heuristics_exposed": false,
"sensitive_abuse_heuristics_exposed": false,
})
}

12
crates/clusterflux-cli/src/project.rs
View file

@ -92,7 +92,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
Ok(json!({
"command": "project init",
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
"private_website_required": false,
"external_website_required": false,
"project_config_written": true,
"project_config_write_after_coordinator_acceptance": coordinator.is_some(),
"coordinator_create_before_local_write": coordinator.is_some(),
@ -104,7 +104,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
"config_format": "clusterflux_project_config_v1",
"links_current_directory": true,
"writes_current_directory_only": true,
"private_website_required": false,
"external_website_required": false,
},
"safe_defaults": {
"tenant": config.tenant.clone(),
@ -115,7 +115,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
"default_project_id_used": args.new_project == "project",
"default_project_name_used": args.name == "Clusterflux Project",
"browser_interaction_required": false,
"private_website_required": false,
"external_website_required": false,
},
"project_config": config,
"config_file": project_config_file(&cwd),
@ -283,7 +283,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
"user": user,
"projects": projects,
"project_count": project_count,
"private_website_required": false,
"external_website_required": false,
"response": response,
"coordinator_session_requests": session.requests(),
}));
@ -295,7 +295,7 @@ pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result
"source": "local_project_config",
"projects": projects,
"project_count": project_count,
"private_website_required": false,
"external_website_required": false,
}))
}
@ -361,7 +361,7 @@ pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Re
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
"selected_project": selected_project,
"project_config_written": true,
"private_website_required": false,
"external_website_required": false,
"project_config": config,
"coordinator_response": coordinator_response,
}))

4
crates/clusterflux-cli/src/quota.rs
View file

@ -96,7 +96,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
"user": effective_scope.user,
"coordinator": coordinator,
"project_config": config,
"policy_surface": "generic public quota categories; hosted tuning remains private policy",
"policy_surface": "generic public quota categories; hosted tuning is coordinator-defined",
"limits": limits,
"window_seconds": window_seconds,
"current_usage": current_usage,
@ -105,7 +105,7 @@ pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result
"next_blocked_action": quota_next_blocked_action(&current_usage),
"quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
"quota_tier": quota_tier,
"private_abuse_heuristics_exposed": false,
"sensitive_abuse_heuristics_exposed": false,
"quota_response": quota_status,
}))
}

107
crates/clusterflux-cli/src/run.rs
View file

@ -142,7 +142,7 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
"safe_failure": true,
"message": message,
"next_actions": next_actions,
"private_website_required": false,
"external_website_required": false,
"machine_error": machine_error,
})
}
@ -195,12 +195,14 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
);
}
let process = "vp-current".to_owned();
let launch_attempt = new_launch_attempt_id();
let mut session = JsonLineSession::connect(&coordinator)?;
let mut request = json!({
"type": "start_process",
"tenant": tenant.clone(),
"project": project.clone(),
"process": process.clone(),
"launch_attempt": launch_attempt.clone(),
"restart": false,
});
request = authenticated_human_or_local_trusted_workflow(
@ -219,6 +221,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
&tenant,
&project,
&process,
&launch_attempt,
)?;
let run_start = run_start_summary(&response);
let status = run_start
@ -286,6 +289,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
tenant: &tenant,
project: &project,
process: &process,
launch_attempt: &launch_attempt,
};
let cleanup = rollback_failed_process_launch_reconnecting(
&coordinator,
@ -325,7 +329,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
"task_launch": launch_task_response,
"coordinator_response": response,
"coordinator_session_requests": session.requests(),
"private_website_required": false,
"external_website_required": false,
}))
}
@ -340,6 +344,7 @@ fn request_process_start_with_rollback(
tenant: &str,
project: &str,
process: &str,
launch_attempt: &str,
) -> Result<Value> {
let rollback = ProcessLaunchRollback {
cli_session,
@ -348,9 +353,25 @@ fn request_process_start_with_rollback(
tenant,
project,
process,
launch_attempt,
};
match session.request_allow_error(request) {
Ok(response) => Ok(response),
Ok(response) => {
if response.get("type").and_then(Value::as_str) == Some("process_started")
&& response.get("launch_attempt").and_then(Value::as_str) != Some(launch_attempt)
{
let ownership_error = anyhow::anyhow!(
"coordinator returned a process-start acknowledgement owned by a different launch attempt"
);
rollback_failed_process_launch_reconnecting(
coordinator,
&rollback,
&json!({ "error": ownership_error.to_string(), "phase": "start_process_ownership" }),
)?;
return Err(ownership_error);
}
Ok(response)
}
Err(start_error) => {
let cleanup = rollback_failed_process_launch_reconnecting(
coordinator,
@ -370,6 +391,17 @@ fn request_process_start_with_rollback(
}
}
static NEXT_LAUNCH_ATTEMPT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
fn new_launch_attempt_id() -> String {
let sequence = NEXT_LAUNCH_ATTEMPT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
format!("launch-{}-{nanos}-{sequence}", std::process::id())
}
struct ProcessLaunchRollback<'a> {
cli_session: &'a CliSession,
fallback_user: &'a str,
@ -377,6 +409,7 @@ struct ProcessLaunchRollback<'a> {
tenant: &'a str,
project: &'a str,
process: &'a str,
launch_attempt: &'a str,
}
fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> {
@ -402,6 +435,7 @@ fn rollback_failed_process_launch(
"tenant": context.tenant,
"project": context.project,
"process": context.process,
"launch_attempt": context.launch_attempt,
}),
context.cli_session,
context.fallback_user,
@ -983,6 +1017,10 @@ mod transactional_launch_tests {
payload.get("process").and_then(Value::as_str),
Some("vp-current")
);
assert_eq!(
payload.get("launch_attempt").and_then(Value::as_str),
Some("launch-test")
);
writeln!(
stream,
"{}",
@ -1003,6 +1041,7 @@ mod transactional_launch_tests {
tenant: "tenant-a",
project: "project-a",
process: "vp-current",
launch_attempt: "launch-test",
};
rollback_failed_process_launch(
&mut session,
@ -1024,6 +1063,7 @@ mod transactional_launch_tests {
.read_line(&mut start_line)
.unwrap();
assert!(start_line.contains("\"type\":\"start_process\""));
assert!(start_line.contains("\"launch_attempt\":\"launch-test\""));
drop(stream);
let (mut cleanup_stream, _) = listener.accept().unwrap();
@ -1032,6 +1072,7 @@ mod transactional_launch_tests {
.read_line(&mut cleanup_line)
.unwrap();
assert!(cleanup_line.contains("\"type\":\"abort_process\""));
assert!(cleanup_line.contains("\"launch_attempt\":\"launch-test\""));
writeln!(
cleanup_stream,
"{}",
@ -1043,6 +1084,12 @@ mod transactional_launch_tests {
})
)
.unwrap();
listener.set_nonblocking(true).unwrap();
std::thread::sleep(std::time::Duration::from_millis(25));
assert!(matches!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
));
});
let mut session = JsonLineSession::connect(&address).unwrap();
let error = request_process_start_with_rollback(
@ -1053,6 +1100,7 @@ mod transactional_launch_tests {
"project": "project-a",
"actor_user": "user-a",
"process": "vp-current",
"launch_attempt": "launch-test",
"restart": false
}),
&address,
@ -1062,10 +1110,63 @@ mod transactional_launch_tests {
"tenant-a",
"project-a",
"vp-current",
"launch-test",
)
.unwrap_err()
.to_string();
assert!(error.contains("closed") || error.contains("response"));
server.join().unwrap();
}
#[test]
fn definitive_start_rejection_does_not_abort_existing_process() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap().to_string();
let server = std::thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut line = String::new();
std::io::BufReader::new(stream.try_clone().unwrap())
.read_line(&mut line)
.unwrap();
writeln!(
stream,
"{}",
json!({
"type": "error",
"message": "project already has active virtual process vp-existing"
})
)
.unwrap();
listener.set_nonblocking(true).unwrap();
std::thread::sleep(std::time::Duration::from_millis(25));
assert_eq!(
listener.accept().unwrap_err().kind(),
std::io::ErrorKind::WouldBlock
);
});
let mut session = JsonLineSession::connect(&address).unwrap();
let response = request_process_start_with_rollback(
&mut session,
json!({
"type": "start_process",
"tenant": "tenant-a",
"project": "project-a",
"actor_user": "user-a",
"process": "vp-current",
"launch_attempt": "launch-rejected",
"restart": false
}),
&address,
&CliSession::Anonymous,
"user-a",
None,
"tenant-a",
"project-a",
"vp-current",
"launch-rejected",
)
.unwrap();
assert_eq!(response.get("type").and_then(Value::as_str), Some("error"));
server.join().unwrap();
}
}

158
crates/clusterflux-cli/src/tests.rs
View file

@ -10,6 +10,15 @@ fn parse(args: &[&str]) -> Cli {
Cli::parse_from(args)
}
fn launch_attempt_from_wire(line: &str) -> String {
let wire: Value = serde_json::from_str(line).unwrap();
wire.pointer("/payload/request/launch_attempt")
.or_else(|| wire.pointer("/payload/launch_attempt"))
.and_then(Value::as_str)
.expect("start request must carry a launch attempt")
.to_owned()
}
fn write_runnable_wasm_project(project: &Path) {
let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let sdk = cli_manifest_dir.parent().unwrap().join("clusterflux-sdk");
@ -115,7 +124,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
assert_eq!(summary["resource_category"], "api_calls");
assert_eq!(summary["community_tier_language"], true);
assert_eq!(summary["community_tier_label"], "community tier");
assert_eq!(summary["private_abuse_heuristics_exposed"], false);
assert_eq!(summary["sensitive_abuse_heuristics_exposed"], false);
let rendered = human_report(&json!({
"command": "run",
"machine_error": summary,
@ -198,7 +207,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
assert_eq!(report["status"], "authentication_required");
assert_eq!(report["non_interactive"], true);
assert_eq!(report["browser_opened"], false);
assert_eq!(report["private_website_required"], false);
assert_eq!(report["external_website_required"], false);
assert_eq!(report["machine_error"]["category"], "authentication");
assert_eq!(report["machine_error"]["stable_exit_code"], 20);
assert_eq!(report["machine_error"]["browser_opened"], false);
@ -438,10 +447,11 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
assert!(line.contains(r#""nonce":"agent-signature-"#));
assert!(line.contains(r#""signature":"ed25519:"#));
assert!(!line.contains(r#""actor_user""#));
let launch_attempt = launch_attempt_from_wire(&line);
stream
.write_all(
format!(
r#"{{"type":"process_started","process":"vp-current","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
r#"{{"type":"process_started","process":"vp-current","launch_attempt":"{launch_attempt}","epoch":7,"actor":{{"kind":"agent","user":"user-live","agent":"agent-ci","credential_kind":"public_key","public_key_fingerprint":"{expected_fingerprint}","authenticated_without_browser":true,"scopes":["project:read","project:run"]}}}}"#
)
.as_bytes(),
)
@ -588,7 +598,32 @@ fn run_with_human_session_derives_workflow_scope_from_authenticated_envelope() {
assert!(wire["payload"]["request"].get("tenant").is_none());
assert!(wire["payload"]["request"].get("project").is_none());
assert!(wire["payload"]["request"].get("actor_user").is_none());
stream.write_all(response).unwrap();
if expected_operation == "start_process" {
let launch_attempt = launch_attempt_from_wire(&line);
stream
.write_all(
serde_json::to_string(&json!({
"type": "process_started",
"process": "vp-current",
"launch_attempt": launch_attempt,
"epoch": 7,
"actor": {
"kind": "user",
"user": "user-session",
"agent": null,
"credential_kind": "cli_device_session",
"public_key_fingerprint": null,
"authenticated_without_browser": false,
"scopes": ["project:read", "project:run"]
}
}))
.unwrap()
.as_bytes(),
)
.unwrap();
} else {
stream.write_all(response).unwrap();
}
stream.write_all(b"\n").unwrap();
}
});
@ -661,6 +696,18 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
assert!(line.contains(r#""project":"project-live""#));
assert!(line.contains(r#""process":"vp-current""#));
assert!(line.contains(r#""restart":false"#));
let response = if index == 0 {
let launch_attempt = launch_attempt_from_wire(&line);
serde_json::to_string(&json!({
"type": "process_started",
"process": "vp-current",
"launch_attempt": launch_attempt,
"epoch": 7,
}))
.unwrap()
} else {
response.to_owned()
};
stream.write_all(response.as_bytes()).unwrap();
stream.write_all(b"\n").unwrap();
if index == 0 {
@ -780,7 +827,7 @@ fn run_rejection_reports_machine_readable_error_category() {
assert_eq!(rejected["machine_error"]["resource_category"], "api_calls");
assert_eq!(rejected["machine_error"]["community_tier_language"], true);
assert_eq!(
rejected["machine_error"]["private_abuse_heuristics_exposed"],
rejected["machine_error"]["sensitive_abuse_heuristics_exposed"],
false
);
assert!(rejected["machine_error"]["next_actions"]
@ -1618,12 +1665,11 @@ fn node_attach_refuses_a_symlink_credential_target() {
fn hosted_coordinator_remains_a_real_https_control_endpoint() {
assert_eq!(
control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
"https://clusterflux.lesstuff.com/api/v1/control"
);
assert_eq!(
control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
.unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
"https://clusterflux.lesstuff.com/api/v1/control"
);
assert!(control_endpoint_identity("http://operator.example.test").is_err());
assert_eq!(
@ -1741,7 +1787,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
assert!(!line.contains(r#""actor_user":"user-session""#));
stream
.write_all(
br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
br#"{"type":"auth_status","tenant":"tenant-session","project":"project-session","actor":"user-session","authenticated":true,"account_status":"active","suspended":false,"disabled":false,"sanitized_reason":null,"next_actions":[],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false}"#,
)
.unwrap();
stream.write_all(b"\n").unwrap();
@ -1804,7 +1850,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
"active"
);
assert_eq!(
report["coordinator_account_status"]["private_moderation_details_exposed"],
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
false
);
}
@ -1886,7 +1932,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
}
#[test]
fn auth_status_queries_coordinator_account_state_without_private_moderation_details() {
fn auth_status_queries_coordinator_account_state_without_sensitive_moderation_details() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap().to_string();
let server = std::thread::spawn(move || {
@ -1900,7 +1946,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
assert!(line.contains(r#""actor_user":"user-live""#));
stream
.write_all(
br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"private_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"private moderation note"}"#,
br#"{"type":"auth_status","tenant":"tenant-live","project":"project-live","actor":"user-live","authenticated":true,"account_status":"suspended","suspended":true,"disabled":false,"sanitized_reason":"account or tenant is suspended by hosted policy","next_actions":["contact the hosted operator"],"sensitive_moderation_details_exposed":false,"signup_failure_details_exposed":false,"abuse_score":99,"moderation_notes":"sensitive moderation note"}"#,
)
.unwrap();
stream.write_all(b"\n").unwrap();
@ -1940,7 +1986,7 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
"account or tenant is suspended by hosted policy"
);
assert_eq!(
report["coordinator_account_status"]["private_moderation_details_exposed"],
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
false
);
assert_eq!(
@ -1958,13 +2004,13 @@ fn auth_status_queries_coordinator_account_state_without_private_moderation_deta
let serialized = serde_json::to_string(&report).unwrap();
assert!(!serialized.contains("abuse_score"));
assert!(!serialized.contains("moderation_notes"));
assert!(!serialized.contains("private moderation note"));
assert!(!serialized.contains("sensitive moderation note"));
let rendered = human_report(&report);
assert!(rendered.contains("account status: suspended"));
assert!(rendered.contains("account suspended: true"));
assert!(rendered.contains("private moderation details exposed: false"));
assert!(!rendered.contains("private moderation note"));
assert!(rendered.contains("sensitive moderation details exposed: false"));
assert!(!rendered.contains("sensitive moderation note"));
}
#[test]
@ -2015,11 +2061,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
"manual_review": manual_review,
"sanitized_reason": reason,
"next_actions": ["contact the hosted operator"],
"private_moderation_details_exposed": false,
"sensitive_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"abuse_score": 99,
"moderation_notes": "private moderation note",
"signup_policy_trace": "private signup trace",
"moderation_notes": "sensitive moderation note",
"signup_policy_trace": "sensitive signup trace",
})
)
.unwrap();
@ -2058,7 +2104,7 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
true
);
assert_eq!(
report["coordinator_account_status"]["private_moderation_details_exposed"],
report["coordinator_account_status"]["sensitive_moderation_details_exposed"],
false
);
assert_eq!(
@ -2069,11 +2115,11 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
assert!(!serialized.contains("abuse_score"));
assert!(!serialized.contains("moderation_notes"));
assert!(!serialized.contains("signup_policy_trace"));
assert!(!serialized.contains("private moderation note"));
assert!(!serialized.contains("sensitive moderation note"));
let rendered = human_report(&report);
assert!(rendered.contains(&format!("account status: {status}")));
assert!(rendered.contains(rendered_marker));
assert!(!rendered.contains("private moderation note"));
assert!(!rendered.contains("sensitive moderation note"));
}
server.join().unwrap();
}
@ -2188,11 +2234,11 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
assert_eq!(report["command"], "admin bootstrap");
assert_eq!(report["mode"], "self_hosted_local");
assert_eq!(report["private_website_required"], false);
assert_eq!(report["external_website_required"], false);
assert_eq!(report["self_hosted_cli_only"], true);
assert_eq!(report["project_config_written"], true);
assert_eq!(report["project_init"]["command"], "project init");
assert_eq!(report["project_init"]["private_website_required"], false);
assert_eq!(report["project_init"]["external_website_required"], false);
assert_eq!(
report["project_init"]["project_config"]["project"],
"self-hosted"
@ -2218,7 +2264,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
}
assert!(steps.iter().all(|step| {
!step
.get("private_website_required")
.get("external_website_required")
.and_then(Value::as_bool)
.unwrap_or(false)
}));
@ -2846,13 +2892,13 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
assert_eq!(status["command"], "admin status");
assert_eq!(status["safe_default"], "read_only");
assert_eq!(status["private_website_required"], false);
assert_eq!(status["external_website_required"], false);
assert_eq!(status["suspended"], false);
assert_eq!(suspended["command"], "admin suspend-tenant");
assert_eq!(suspended["tenant"], "tenant");
assert_eq!(suspended["actor_tenant"], "admin-tenant");
assert_eq!(suspended["suspended"], true);
assert_eq!(suspended["private_website_required"], false);
assert_eq!(suspended["external_website_required"], false);
}
#[test]
@ -2921,7 +2967,7 @@ fn debug_attach_reports_public_authorization() {
assert_eq!(report["charged_debug_read_bytes"], 1024);
assert_eq!(report["used_debug_read_bytes"], 1024);
assert_eq!(report["debug_reads_quota_limited"], true);
assert_eq!(report["private_website_required"], false);
assert_eq!(report["external_website_required"], false);
}
#[test]
@ -2958,6 +3004,14 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
"restart_task",
br#"{"type":"task_restart","process":"vp","task":"task-a","actor":"user-session","accepted":false,"clean_boundary_available":false,"active_task":false,"completed_event_observed":false,"requires_whole_process_restart":true,"message":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024,"audit_event":{"tenant":"tenant-session","project":"project-session","process":"vp","task":"task-a","actor":"user-session","operation":"restart_task","allowed":true,"reason":"restart requires checkpoint","charged_debug_read_bytes":1024,"used_debug_read_bytes":1024}}"#.as_slice(),
),
(
"list_task_events",
br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","stdout_tail":"compiled\n","stderr_tail":""}]}"#.as_slice(),
),
(
"list_task_events",
br#"{"type":"task_events","events":[{"process":"vp","task":"task-a","terminal_state":"completed","artifact_path":"/vfs/artifacts/app.txt","artifact_digest":"sha256:app","artifact_size_bytes":3}]}"#.as_slice(),
),
(
"create_artifact_download_link",
br#"{"type":"artifact_download_link","link":{"artifact":"app.txt","source":{"RetainedNode":"node-a"},"url_path":"/artifacts/tenant-session/project-session/vp/app.txt","scoped_token_digest":"sha256:token","expires_at_epoch_seconds":60,"tenant":"tenant-session","project":"project-session","process":"vp","actor":{"User":"user-session"},"max_bytes":2048,"policy_context_digest":"sha256:policy"}}"#.as_slice(),
@ -3062,9 +3116,26 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
Some(&session),
)
.unwrap();
artifact_download_report_with_session(
let logs = logs_report_with_session(
LogsArgs {
scope: scope.clone(),
process: Some("vp".to_owned()),
task: Some("task-a".to_owned()),
},
Some(&session),
)
.unwrap();
let artifacts = artifact_list_report_with_session(
ArtifactListArgs {
scope: scope.clone(),
process: Some("vp".to_owned()),
},
Some(&session),
)
.unwrap();
let download = artifact_download_report_with_session(
ArtifactDownloadArgs {
scope: coordinator_scope.clone(),
scope,
artifact: "app.txt".to_owned(),
to: None,
max_bytes: 2048,
@ -3072,6 +3143,9 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
Some(&session),
)
.unwrap();
assert_eq!(logs["log_entries"][0]["stdout_tail"], "compiled\n");
assert_eq!(artifacts["artifacts"][0]["artifact"], "app.txt");
assert_eq!(download["coordinator"], session.coordinator);
debug_attach_report_with_dap_and_session(
DebugAttachArgs {
scope: coordinator_scope,
@ -3264,7 +3338,7 @@ fn project_init_uses_public_create_before_writing_local_config() {
created["safe_defaults"]["browser_interaction_required"],
false
);
assert_eq!(created["private_website_required"], false);
assert_eq!(created["external_website_required"], false);
assert_eq!(
read_project_config(temp_success.path())
.unwrap()
@ -3364,7 +3438,7 @@ fn project_list_and_select_use_public_api_without_website() {
assert_eq!(list["source"], "public_coordinator_api");
assert_eq!(list["project_count"], 1);
assert_eq!(list["projects"][0]["id"], "project-a");
assert_eq!(list["private_website_required"], false);
assert_eq!(list["external_website_required"], false);
assert_eq!(list["coordinator_session_requests"], 1);
let selected = project_select_report(
@ -3379,7 +3453,7 @@ fn project_list_and_select_use_public_api_without_website() {
assert_eq!(selected["source"], "public_coordinator_api");
assert_eq!(selected["selected_project"]["id"], "project-a");
assert_eq!(selected["project_config_written"], true);
assert_eq!(selected["private_website_required"], false);
assert_eq!(selected["external_website_required"], false);
assert_eq!(
read_project_config(temp.path()).unwrap().unwrap().project,
"project-a"
@ -4408,7 +4482,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
let temp = tempfile::tempdir().unwrap();
let project = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../..")
.join("examples/launch-build-demo");
.join("examples/hello-build");
let output = temp.path().join("bundle");
let report = build_report(
@ -4426,8 +4500,8 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
assert_eq!(report["command"], "build");
assert_eq!(report["content_addressed"], true);
assert_eq!(report["contains_full_repository_upload"], false);
assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 10);
assert_eq!(report["bundle_artifact"]["entrypoint_count"], 5);
assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 2);
assert_eq!(report["bundle_artifact"]["entrypoint_count"], 1);
assert!(output.join("module.wasm").is_file());
assert!(output.join("manifest.json").is_file());
assert!(output.join("task-descriptors.json").is_file());
@ -4438,9 +4512,9 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
.unwrap()
.iter()
.any(|descriptor| {
descriptor["name"] == "task_add_one"
&& descriptor["argument_schema"] == "input : i32"
&& descriptor["result_schema"] == "i32"
descriptor["name"] == "compile"
&& descriptor["argument_schema"] == "source : SourceSnapshot"
&& descriptor["result_schema"] == "Result < Artifact >"
&& descriptor["restart_compatibility_hash"]
.as_str()
.unwrap()
@ -4601,7 +4675,7 @@ fn node_enroll_reports_short_lived_public_api_grant() {
assert_eq!(report["command"], "node enroll");
assert_eq!(report["status"], "created");
assert_eq!(report["private_website_required"], false);
assert_eq!(report["external_website_required"], false);
assert_eq!(report["tenant"], "tenant");
assert_eq!(report["project"], "project");
assert_eq!(report["user"], "user");
@ -4744,7 +4818,7 @@ fn node_enroll_and_process_commands_have_safe_plan_without_coordinator() {
)
.unwrap();
assert_eq!(enroll["status"], "requires_coordinator");
assert_eq!(enroll["private_website_required"], false);
assert_eq!(enroll["external_website_required"], false);
assert_eq!(enroll["enrollment_grant"], serde_json::Value::Null);
assert_eq!(enroll["requested_ttl_seconds"], 60);

18
crates/clusterflux-client/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "clusterflux-client"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
clusterflux-control = { path = "../clusterflux-control" }
clusterflux-core = { path = "../clusterflux-core" }
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["sync", "time"] }
[dev-dependencies]
clusterflux-coordinator = { path = "../clusterflux-coordinator" }

68
crates/clusterflux-client/README.md Normal file
View file

@ -0,0 +1,68 @@
# clusterflux-client
`clusterflux-client` is the public, typed Rust boundary for a Clusterflux web
backend. It talks to the same versioned control and login endpoints as the CLI.
Callers do not construct JSON envelopes or place session secrets into request
objects.
The crate intentionally contains no coordinator implementation, persistence
types, scheduler policy, or website-specific business rules. A web backend only
needs this crate to authenticate and work with account status, projects, Agent
keys, node enrollment and liveness, current/recent processes, task attempts,
recent logs, artifacts, quota state, process control, task recovery, and Debug
Epoch state. It does not expose workflow launch or whole-process replay.
```rust,no_run
use clusterflux_client::{ClusterfluxClient, SessionCredential};
# async fn example() -> Result<(), clusterflux_client::ClientError> {
let credential = SessionCredential::from_secret(
std::env::var("CLUSTERFLUX_SESSION_SECRET").expect("server-side session secret"),
);
let client = ClusterfluxClient::connect("https://clusterflux.lesstuff.com")?
.with_session_credential(&credential);
let account = client.account_status().await?;
let nodes = client.list_nodes().await?;
let processes = client.list_processes(None, 20).await?;
# let _ = (account, nodes, processes);
# Ok(())
# }
```
## Login boundary
`begin_browser_login` starts the hosted Authentik flow. The identity callback
returns only to the fixed configured website URL with a short-lived handoff.
The website backend calls `exchange_browser_login_handoff` once and stores the
returned `SessionCredential` only in encrypted server-side session storage.
The credential has redacted `Debug` output and deliberately does not implement
Serde serialization. Logout calls the existing session-revocation operation.
The browser must not receive or persist the Clusterflux session credential,
provider tokens, authorization code, PKCE verifier, or CLI polling secret.
## Errors, bounds, and transport
API failures are returned as `ClientError::Api(ApiError)`. Decisions should use
the stable `code`, `category`, `retryable`, and `request_id` fields, while
`message` is for people. The client rejects an error whose request ID does not
match the originating request.
Paginated list methods require an explicit bounded page size. Process pages are
limited to 100 entries; node, artifact, and recent-log pages to 200.
`list_nodes()` is a bounded first-page convenience; `list_nodes_page()` exposes
its cursor. Recent logs are ephemeral and can report truncation or sequence
loss. Artifact bytes use `ArtifactDownload::next_chunk`, which validates
offsets and decodes bounded chunks without materializing the complete artifact.
`ControlTransport` has bounded connect and I/O timeouts and performs blocking
network work outside the async executor. Dropping a request future cancels the
caller’s wait; any already-started blocking I/O remains bounded by its timeout.
`MockTransport` supplies deterministic responses and records exact envelopes
for application tests.
`CLIENT_API_VERSION` is the one supported control protocol version. The
checked-in `web_operations.json` contract fixture records every website
operation and its stable error shape, and the crate’s contract suite checks the
fixture.

924
crates/clusterflux-client/src/lib.rs Normal file
View file

@ -0,0 +1,924 @@
mod protocol;
mod transport;
mod types;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
use clusterflux_core::{coordinator_wire_request, ApiError, COORDINATOR_PROTOCOL_VERSION};
use protocol::{AuthenticatedRequest, LoginRequest, WireResponse};
use serde_json::json;
use thiserror::Error;
use transport::{ClientTransport, TransportRequest};
pub use clusterflux_core::{
AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Capability, CredentialKind,
Digest, DownloadLink, EnvironmentBackend, LimitKind, NodeCapabilities, NodeId, Os, ProcessId,
ProjectId, ResourceLimits, TaskDefinitionId, TaskFailurePolicy, TaskInstanceId, TenantId,
UserId, VfsPath,
};
pub use transport::{
ClientTransportError, ControlTransport, MockTransport, TransportFuture, TransportResponse,
};
pub use types::*;
pub const CLIENT_API_VERSION: u64 = COORDINATOR_PROTOCOL_VERSION;
#[derive(Debug, Error)]
pub enum ClientError {
#[error("Clusterflux API error: {0}")]
Api(ApiError),
#[error(transparent)]
Transport(#[from] ClientTransportError),
#[error("Clusterflux client protocol error: {0}")]
Protocol(String),
}
#[derive(Clone)]
pub struct ClusterfluxClient {
transport: Arc<dyn ClientTransport>,
session_secret: Arc<Mutex<Option<String>>>,
next_request: Arc<AtomicU64>,
}
impl ClusterfluxClient {
pub fn connect(endpoint: impl Into<String>) -> Result<Self, ClientError> {
Ok(Self::with_transport(ControlTransport::new(endpoint)?))
}
pub fn with_transport(transport: impl ClientTransport) -> Self {
Self {
transport: Arc::new(transport),
session_secret: Arc::new(Mutex::new(None)),
next_request: Arc::new(AtomicU64::new(1)),
}
}
pub fn with_session_credential(mut self, credential: &SessionCredential) -> Self {
self.session_secret = Arc::new(Mutex::new(Some(credential.0.clone())));
self
}
pub fn is_session_configured(&self) -> bool {
self.session_secret
.lock()
.map(|secret| secret.is_some())
.unwrap_or(false)
}
pub async fn begin_browser_login(&self) -> Result<BrowserLoginStart, ClientError> {
match self
.send_login(LoginRequest::BeginWebBrowserLogin {})
.await?
{
WireResponse::WebBrowserLoginStarted {
transaction_id,
authorization_url,
expires_at_epoch_seconds,
} => Ok(BrowserLoginStart {
transaction_id,
authorization_url,
expires_at_epoch_seconds,
}),
_ => Err(unexpected_response("web_browser_login_started")),
}
}
pub async fn exchange_browser_login_handoff(
&self,
transaction_id: impl Into<String>,
handoff_code: impl Into<String>,
) -> Result<BrowserSession, ClientError> {
match self
.send_login(LoginRequest::ExchangeWebLoginHandoff {
transaction_id: transaction_id.into(),
handoff_code: handoff_code.into(),
})
.await?
{
WireResponse::WebBrowserSession { session } => Ok(BrowserSession {
tenant: session.tenant,
project: session.project,
user: session.user,
credential: SessionCredential(session.session_secret),
expires_at_epoch_seconds: session.expires_at_epoch_seconds,
}),
_ => Err(unexpected_response("web_browser_session")),
}
}
pub async fn cancel_browser_login(
&self,
transaction_id: impl Into<String>,
) -> Result<(), ClientError> {
match self
.send_login(LoginRequest::CancelWebBrowserLogin {
transaction_id: transaction_id.into(),
})
.await?
{
WireResponse::WebBrowserLoginCancelled {} => Ok(()),
_ => Err(unexpected_response("web_browser_login_cancelled")),
}
}
pub async fn account_status(&self) -> Result<AccountStatus, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::AuthStatus)
.await?
{
WireResponse::AuthStatus {
tenant,
project,
actor,
authenticated,
account_status,
suspended,
disabled,
deleted,
manual_review,
sanitized_reason,
next_actions,
} => Ok(AccountStatus {
tenant,
project,
actor,
authenticated,
account_status,
suspended,
disabled,
deleted,
manual_review,
sanitized_reason,
next_actions,
}),
_ => Err(unexpected_response("auth_status")),
}
}
pub async fn logout(&self) -> Result<(), ClientError> {
match self
.send_authenticated(AuthenticatedRequest::RevokeCliSession)
.await?
{
WireResponse::CliSessionRevoked {} => {
*self.session_secret.lock().map_err(|_| {
ClientError::Protocol("session credential lock was poisoned".to_owned())
})? = None;
Ok(())
}
_ => Err(unexpected_response("cli_session_revoked")),
}
}
pub async fn list_projects(&self) -> Result<Vec<Project>, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListProjects)
.await?
{
WireResponse::Projects { projects } => Ok(projects),
_ => Err(unexpected_response("projects")),
}
}
pub async fn create_project(
&self,
project: ProjectId,
name: impl Into<String>,
) -> Result<Project, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::CreateProject {
project,
name: name.into(),
})
.await?
{
WireResponse::ProjectCreated { project } => Ok(project),
_ => Err(unexpected_response("project_created")),
}
}
pub async fn select_project(&self, project: ProjectId) -> Result<Project, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::SelectProject { project })
.await?
{
WireResponse::ProjectSelected { project } => Ok(project),
_ => Err(unexpected_response("project_selected")),
}
}
pub async fn register_agent_public_key(
&self,
agent: AgentId,
public_key: impl Into<String>,
) -> Result<AgentPublicKey, ClientError> {
self.agent_key_mutation(AuthenticatedRequest::RegisterAgentPublicKey {
agent,
public_key: public_key.into(),
})
.await
}
pub async fn list_agent_public_keys(&self) -> Result<Vec<AgentPublicKey>, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListAgentPublicKeys)
.await?
{
WireResponse::AgentPublicKeys { records } => Ok(records),
_ => Err(unexpected_response("agent_public_keys")),
}
}
pub async fn rotate_agent_public_key(
&self,
agent: AgentId,
public_key: impl Into<String>,
) -> Result<AgentPublicKey, ClientError> {
self.agent_key_mutation(AuthenticatedRequest::RotateAgentPublicKey {
agent,
public_key: public_key.into(),
})
.await
}
pub async fn revoke_agent_public_key(
&self,
agent: AgentId,
) -> Result<AgentPublicKey, ClientError> {
self.agent_key_mutation(AuthenticatedRequest::RevokeAgentPublicKey { agent })
.await
}
async fn agent_key_mutation(
&self,
request: AuthenticatedRequest,
) -> Result<AgentPublicKey, ClientError> {
match self.send_authenticated(request).await? {
WireResponse::AgentPublicKey { record } => Ok(record),
_ => Err(unexpected_response("agent_public_key")),
}
}
pub async fn create_node_enrollment_grant(
&self,
ttl_seconds: u64,
) -> Result<NodeEnrollmentGrant, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::CreateNodeEnrollmentGrant { ttl_seconds })
.await?
{
WireResponse::NodeEnrollmentGrantCreated {
tenant,
project,
grant,
scope,
expires_at_epoch_seconds,
} => Ok(NodeEnrollmentGrant {
tenant,
project,
grant,
scope,
expires_at_epoch_seconds,
}),
_ => Err(unexpected_response("node_enrollment_grant_created")),
}
}
pub async fn list_nodes(&self) -> Result<Vec<NodeSummary>, ClientError> {
Ok(self.list_nodes_page(None, 200).await?.nodes)
}
pub async fn list_nodes_page(
&self,
cursor: Option<String>,
limit: u32,
) -> Result<NodePage, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListNodeSummaries { cursor, limit })
.await?
{
WireResponse::NodeSummaries { nodes, next_cursor } => {
Ok(NodePage { nodes, next_cursor })
}
_ => Err(unexpected_response("node_summaries")),
}
}
pub async fn revoke_node(&self, node: NodeId) -> Result<NodeRevocation, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::RevokeNodeCredential { node })
.await?
{
WireResponse::NodeCredentialRevoked {
node,
tenant,
project,
actor,
descriptor_removed,
queued_assignments_removed,
} => Ok(NodeRevocation {
node,
tenant,
project,
actor,
descriptor_removed,
queued_assignments_removed,
}),
_ => Err(unexpected_response("node_credential_revoked")),
}
}
pub async fn list_processes(
&self,
cursor: Option<String>,
limit: u32,
) -> Result<ProcessPage, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListProcessSummaries { cursor, limit })
.await?
{
WireResponse::ProcessSummaries {
processes,
next_cursor,
} => Ok(ProcessPage {
processes,
next_cursor,
}),
_ => Err(unexpected_response("process_summaries")),
}
}
pub async fn cancel_process(
&self,
process: ProcessId,
) -> Result<ProcessCancellation, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::CancelProcess { process })
.await?
{
WireResponse::ProcessCancellationRequested {
process,
cancelled_tasks,
affected_nodes,
} => Ok(ProcessCancellation {
process,
affected_tasks: cancelled_tasks,
affected_nodes,
aborted: false,
}),
_ => Err(unexpected_response("process_cancellation_requested")),
}
}
pub async fn abort_process(
&self,
process: ProcessId,
) -> Result<ProcessCancellation, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::AbortProcess {
process,
launch_attempt: None,
})
.await?
{
WireResponse::ProcessAborted {
process,
aborted_tasks,
affected_nodes,
} => Ok(ProcessCancellation {
process,
affected_tasks: aborted_tasks,
affected_nodes,
aborted: true,
}),
_ => Err(unexpected_response("process_aborted")),
}
}
pub async fn quota_status(&self) -> Result<QuotaStatus, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::QuotaStatus)
.await?
{
WireResponse::QuotaStatus {
tenant,
project,
actor,
policy_label,
limits,
window_seconds,
usage,
window_started_epoch_seconds,
} => Ok(QuotaStatus {
tenant,
project,
actor,
policy_label,
limits,
window_seconds,
usage,
window_started_epoch_seconds,
}),
_ => Err(unexpected_response("quota_status")),
}
}
pub async fn list_task_events(
&self,
process: Option<ProcessId>,
) -> Result<Vec<TaskCompletionEvent>, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListTaskEvents { process })
.await?
{
WireResponse::TaskEvents { events } => Ok(events),
_ => Err(unexpected_response("task_events")),
}
}
pub async fn list_task_snapshots(
&self,
process: ProcessId,
) -> Result<Vec<TaskAttemptSnapshot>, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListTaskSnapshots { process })
.await?
{
WireResponse::TaskSnapshots { snapshots } => Ok(snapshots),
_ => Err(unexpected_response("task_snapshots")),
}
}
pub async fn list_recent_logs(
&self,
process: ProcessId,
task: Option<TaskInstanceId>,
after_sequence: Option<u64>,
limit: u32,
) -> Result<RecentLogPage, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListRecentLogs {
process,
task,
after_sequence,
limit,
})
.await?
{
WireResponse::RecentLogs {
entries,
next_sequence,
history_truncated,
} => Ok(RecentLogPage {
entries,
next_sequence,
history_truncated,
}),
_ => Err(unexpected_response("recent_logs")),
}
}
pub async fn restart_task(
&self,
process: ProcessId,
task: TaskInstanceId,
replacement_bundle: Option<TaskReplacementBundle>,
) -> Result<TaskRestart, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::RestartTask {
process,
task,
replacement_bundle,
})
.await?
{
WireResponse::TaskRestart {
process,
task,
restarted_task_instance,
restarted_attempt_id,
actor,
accepted,
clean_boundary_available,
active_task,
completed_event_observed,
requires_whole_process_restart,
message,
audit_event,
} => Ok(TaskRestart {
process,
task,
restarted_task_instance,
restarted_attempt_id,
actor,
accepted,
clean_boundary_available,
active_task,
completed_event_observed,
requires_whole_process_restart,
message,
audit_event,
}),
_ => Err(unexpected_response("task_restart")),
}
}
pub async fn resolve_task_failure(
&self,
process: ProcessId,
task: TaskInstanceId,
resolution: TaskFailureResolution,
) -> Result<TaskFailureResolutionResult, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ResolveTaskFailure {
process,
task,
resolution,
})
.await?
{
WireResponse::TaskFailureResolved {
process,
task,
attempt_id,
resolution,
} => Ok(TaskFailureResolutionResult {
process,
task,
attempt_id,
resolution,
}),
_ => Err(unexpected_response("task_failure_resolved")),
}
}
pub async fn debug_attach(&self, process: ProcessId) -> Result<DebugAttach, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::DebugAttach { process })
.await?
{
WireResponse::DebugAttach {
process,
actor,
authorization,
audit_event,
} => Ok(DebugAttach {
process,
actor,
authorization,
audit_event,
}),
_ => Err(unexpected_response("debug_attach")),
}
}
pub async fn create_debug_epoch(
&self,
process: ProcessId,
stopped_task: TaskInstanceId,
reason: impl Into<String>,
) -> Result<DebugEpochControl, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::CreateDebugEpoch {
process,
stopped_task,
reason: reason.into(),
})
.await?
{
WireResponse::DebugEpoch {
process,
actor,
epoch,
command,
affected_tasks,
all_stop_requested,
audit_event,
} => Ok(DebugEpochControl {
process,
actor,
epoch,
command,
affected_tasks,
all_stop_requested,
audit_event,
}),
_ => Err(unexpected_response("debug_epoch")),
}
}
pub async fn resume_debug_epoch(
&self,
process: ProcessId,
epoch: u64,
) -> Result<DebugEpochControl, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ResumeDebugEpoch { process, epoch })
.await?
{
WireResponse::DebugEpoch {
process,
actor,
epoch,
command,
affected_tasks,
all_stop_requested,
audit_event,
} => Ok(DebugEpochControl {
process,
actor,
epoch,
command,
affected_tasks,
all_stop_requested,
audit_event,
}),
_ => Err(unexpected_response("debug_epoch")),
}
}
pub async fn inspect_debug_epoch(
&self,
process: ProcessId,
epoch: u64,
) -> Result<DebugEpochStatus, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::InspectDebugEpoch { process, epoch })
.await?
{
WireResponse::DebugEpochStatus {
process,
actor,
epoch,
command,
expected_tasks,
acknowledgements,
fully_frozen,
partially_frozen,
fully_resumed,
failed,
failure_messages,
audit_event,
} => Ok(DebugEpochStatus {
process,
actor,
epoch,
command,
expected_tasks,
acknowledgements,
fully_frozen,
partially_frozen,
fully_resumed,
failed,
failure_messages,
audit_event,
}),
_ => Err(unexpected_response("debug_epoch_status")),
}
}
pub async fn list_artifacts(
&self,
process: Option<ProcessId>,
cursor: Option<String>,
limit: u32,
) -> Result<ArtifactPage, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::ListArtifacts {
process,
cursor,
limit,
})
.await?
{
WireResponse::Artifacts {
artifacts,
next_cursor,
} => Ok(ArtifactPage {
artifacts,
next_cursor,
}),
_ => Err(unexpected_response("artifacts")),
}
}
pub async fn get_artifact(&self, artifact: ArtifactId) -> Result<ArtifactSummary, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::GetArtifact { artifact })
.await?
{
WireResponse::Artifact { artifact } => Ok(artifact),
_ => Err(unexpected_response("artifact")),
}
}
pub async fn begin_artifact_download(
&self,
artifact: ArtifactId,
max_bytes: u64,
ttl_seconds: u64,
chunk_bytes: u64,
) -> Result<ArtifactDownload, ClientError> {
match self
.send_authenticated(AuthenticatedRequest::CreateArtifactDownloadLink {
artifact: artifact.clone(),
max_bytes,
ttl_seconds,
})
.await?
{
WireResponse::ArtifactDownloadLink { link } => Ok(ArtifactDownload {
client: self.clone(),
artifact,
max_bytes,
chunk_bytes,
token_digest: link.scoped_token_digest.clone(),
link,
expected_offset: 0,
finished: false,
}),
_ => Err(unexpected_response("artifact_download_link")),
}
}
async fn send_authenticated(
&self,
request: AuthenticatedRequest,
) -> Result<WireResponse, ClientError> {
let session_secret = self
.session_secret
.lock()
.map_err(|_| ClientError::Protocol("session credential lock was poisoned".to_owned()))?
.clone()
.ok_or_else(|| {
ClientError::Api(ApiError::from_message(
"client",
"no authenticated Clusterflux session is configured",
))
})?;
self.send(
CONTROL_API_PATH,
json!({
"type": "authenticated",
"session_secret": session_secret,
"request": request,
}),
)
.await
}
async fn send_login(&self, request: LoginRequest) -> Result<WireResponse, ClientError> {
let payload = serde_json::to_value(request)
.map_err(|error| ClientError::Protocol(error.to_string()))?;
self.send(LOGIN_API_PATH, payload).await
}
async fn send(
&self,
api_path: &str,
payload: serde_json::Value,
) -> Result<WireResponse, ClientError> {
let request_number = self.next_request.fetch_add(1, Ordering::Relaxed);
let request_id = format!("client-{request_number}");
let envelope = coordinator_wire_request(&request_id, payload);
let body = serde_json::to_vec(&envelope)
.map_err(|error| ClientError::Protocol(error.to_string()))?;
let response = self
.transport
.send(TransportRequest {
api_path: api_path.to_owned(),
body,
})
.await?;
let response: WireResponse = serde_json::from_slice(&response.body).map_err(|error| {
ClientError::Protocol(format!("decode typed API response: {error}"))
})?;
match response {
WireResponse::Error {
code,
category,
message,
retryable,
request_id: response_request_id,
} => {
if response_request_id != request_id {
return Err(ClientError::Protocol(format!(
"error response request_id {response_request_id} does not match {request_id}"
)));
}
Err(ClientError::Api(ApiError::new(
code,
category,
message,
retryable,
response_request_id,
)))
}
response => Ok(response),
}
}
}
pub struct ArtifactDownload {
client: ClusterfluxClient,
artifact: ArtifactId,
max_bytes: u64,
chunk_bytes: u64,
token_digest: Digest,
link: clusterflux_core::DownloadLink,
expected_offset: u64,
finished: bool,
}
impl ArtifactDownload {
pub fn link(&self) -> &clusterflux_core::DownloadLink {
&self.link
}
pub async fn next_chunk(&mut self) -> Result<ArtifactDownloadPoll, ClientError> {
if self.finished {
return Ok(ArtifactDownloadPoll::Chunk {
offset: self.expected_offset,
bytes: Vec::new(),
eof: true,
});
}
match self
.client
.send_authenticated(AuthenticatedRequest::OpenArtifactDownloadStream {
artifact: self.artifact.clone(),
max_bytes: self.max_bytes,
token_digest: self.token_digest.clone(),
chunk_bytes: self.chunk_bytes,
})
.await?
{
WireResponse::ArtifactDownloadStream {
content_bytes_available,
content_offset,
content_eof,
content_base64,
..
} => {
if !content_bytes_available {
return Ok(ArtifactDownloadPoll::Pending);
}
let offset = content_offset.ok_or_else(|| {
ClientError::Protocol(
"artifact response contained bytes without an offset".to_owned(),
)
})?;
if offset != self.expected_offset {
return Err(ClientError::Protocol(format!(
"artifact chunk offset {offset} does not match expected {}",
self.expected_offset
)));
}
let bytes = BASE64_STANDARD
.decode(content_base64.unwrap_or_default())
.map_err(|error| {
ClientError::Protocol(format!(
"artifact response contains invalid base64: {error}"
))
})?;
self.expected_offset = self.expected_offset.saturating_add(bytes.len() as u64);
self.finished = content_eof;
Ok(ArtifactDownloadPoll::Chunk {
offset,
bytes,
eof: content_eof,
})
}
_ => Err(unexpected_response("artifact_download_stream")),
}
}
pub async fn cancel(mut self) -> Result<(), ClientError> {
if self.finished {
return Ok(());
}
match self
.client
.send_authenticated(AuthenticatedRequest::RevokeArtifactDownloadLink {
artifact: self.artifact.clone(),
token_digest: self.token_digest.clone(),
})
.await?
{
WireResponse::ArtifactDownloadLinkRevoked {} => {
self.finished = true;
Ok(())
}
_ => Err(unexpected_response("artifact_download_link_revoked")),
}
}
}
fn unexpected_response(expected: &str) -> ClientError {
ClientError::Protocol(format!(
"expected typed response {expected}, received another response variant"
))
}

403
crates/clusterflux-client/src/protocol.rs Normal file
View file

@ -0,0 +1,403 @@
use std::collections::BTreeMap;
use clusterflux_core::{
AgentId, ApiErrorCategory, ApiErrorCode, ArtifactId, Authorization, Digest, DownloadLink,
LimitKind, NodeId, ProcessId, ProjectId, ResourceLimits, TaskInstanceId, TenantId, UserId,
};
use serde::{Deserialize, Serialize};
use crate::types::*;
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum AuthenticatedRequest {
AuthStatus,
RevokeCliSession,
CreateProject {
project: ProjectId,
name: String,
},
SelectProject {
project: ProjectId,
},
ListProjects,
RegisterAgentPublicKey {
agent: AgentId,
public_key: String,
},
ListAgentPublicKeys,
RotateAgentPublicKey {
agent: AgentId,
public_key: String,
},
RevokeAgentPublicKey {
agent: AgentId,
},
CreateNodeEnrollmentGrant {
ttl_seconds: u64,
},
ListNodeSummaries {
cursor: Option<String>,
limit: u32,
},
RevokeNodeCredential {
node: NodeId,
},
ListProcessSummaries {
cursor: Option<String>,
limit: u32,
},
CancelProcess {
process: ProcessId,
},
AbortProcess {
process: ProcessId,
launch_attempt: Option<String>,
},
QuotaStatus,
ListTaskEvents {
process: Option<ProcessId>,
},
ListTaskSnapshots {
process: ProcessId,
},
ListRecentLogs {
process: ProcessId,
task: Option<TaskInstanceId>,
after_sequence: Option<u64>,
limit: u32,
},
RestartTask {
process: ProcessId,
task: TaskInstanceId,
replacement_bundle: Option<TaskReplacementBundle>,
},
ResolveTaskFailure {
process: ProcessId,
task: TaskInstanceId,
resolution: TaskFailureResolution,
},
DebugAttach {
process: ProcessId,
},
CreateDebugEpoch {
process: ProcessId,
stopped_task: TaskInstanceId,
reason: String,
},
ResumeDebugEpoch {
process: ProcessId,
epoch: u64,
},
InspectDebugEpoch {
process: ProcessId,
epoch: u64,
},
ListArtifacts {
process: Option<ProcessId>,
cursor: Option<String>,
limit: u32,
},
GetArtifact {
artifact: ArtifactId,
},
CreateArtifactDownloadLink {
artifact: ArtifactId,
max_bytes: u64,
ttl_seconds: u64,
},
OpenArtifactDownloadStream {
artifact: ArtifactId,
max_bytes: u64,
token_digest: Digest,
chunk_bytes: u64,
},
RevokeArtifactDownloadLink {
artifact: ArtifactId,
token_digest: Digest,
},
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum LoginRequest {
BeginWebBrowserLogin {},
CancelWebBrowserLogin {
transaction_id: String,
},
ExchangeWebLoginHandoff {
transaction_id: String,
handoff_code: String,
},
}
#[derive(Clone, Deserialize)]
pub(crate) struct WireBrowserSession {
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub session_secret: String,
pub expires_at_epoch_seconds: u64,
}
#[allow(clippy::large_enum_variant)]
#[derive(Clone, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub(crate) enum WireResponse {
AuthStatus {
tenant: TenantId,
project: ProjectId,
actor: UserId,
authenticated: bool,
account_status: String,
suspended: bool,
disabled: bool,
deleted: bool,
manual_review: bool,
sanitized_reason: Option<String>,
next_actions: Vec<String>,
},
CliSessionRevoked {},
ProjectCreated {
project: Project,
},
ProjectSelected {
project: Project,
},
Projects {
projects: Vec<Project>,
},
AgentPublicKey {
record: AgentPublicKey,
},
AgentPublicKeys {
records: Vec<AgentPublicKey>,
},
NodeEnrollmentGrantCreated {
tenant: TenantId,
project: ProjectId,
grant: String,
scope: String,
expires_at_epoch_seconds: u64,
},
NodeSummaries {
nodes: Vec<NodeSummary>,
next_cursor: Option<String>,
},
NodeCredentialRevoked {
node: NodeId,
tenant: TenantId,
project: ProjectId,
actor: UserId,
descriptor_removed: bool,
queued_assignments_removed: usize,
},
ProcessSummaries {
processes: Vec<ProcessSummary>,
next_cursor: Option<String>,
},
ProcessCancellationRequested {
process: ProcessId,
cancelled_tasks: Vec<TaskCancellationTarget>,
affected_nodes: Vec<NodeId>,
},
ProcessAborted {
process: ProcessId,
aborted_tasks: Vec<TaskCancellationTarget>,
affected_nodes: Vec<NodeId>,
},
QuotaStatus {
tenant: TenantId,
project: ProjectId,
actor: UserId,
policy_label: Option<String>,
limits: ResourceLimits,
window_seconds: BTreeMap<LimitKind, u64>,
usage: BTreeMap<LimitKind, u64>,
window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
},
TaskEvents {
events: Vec<TaskCompletionEvent>,
},
TaskSnapshots {
snapshots: Vec<TaskAttemptSnapshot>,
},
RecentLogs {
entries: Vec<RecentLogEntry>,
next_sequence: Option<u64>,
history_truncated: bool,
},
TaskRestart {
process: ProcessId,
task: TaskInstanceId,
restarted_task_instance: Option<TaskInstanceId>,
restarted_attempt_id: Option<String>,
actor: UserId,
accepted: bool,
clean_boundary_available: bool,
active_task: bool,
completed_event_observed: bool,
requires_whole_process_restart: bool,
message: String,
audit_event: DebugAuditEvent,
},
TaskFailureResolved {
process: ProcessId,
task: TaskInstanceId,
attempt_id: String,
resolution: TaskFailureResolution,
},
DebugAttach {
process: ProcessId,
actor: UserId,
authorization: Authorization,
audit_event: DebugAuditEvent,
},
DebugEpoch {
process: ProcessId,
actor: UserId,
epoch: u64,
command: String,
affected_tasks: Vec<TaskCancellationTarget>,
all_stop_requested: bool,
audit_event: DebugAuditEvent,
},
DebugEpochStatus {
process: ProcessId,
actor: UserId,
epoch: u64,
command: String,
expected_tasks: Vec<TaskCancellationTarget>,
acknowledgements: Vec<DebugParticipantAcknowledgement>,
fully_frozen: bool,
partially_frozen: bool,
fully_resumed: bool,
failed: bool,
failure_messages: Vec<String>,
audit_event: DebugAuditEvent,
},
Artifacts {
artifacts: Vec<ArtifactSummary>,
next_cursor: Option<String>,
},
Artifact {
artifact: ArtifactSummary,
},
ArtifactDownloadLink {
link: DownloadLink,
},
ArtifactDownloadLinkRevoked {},
ArtifactDownloadStream {
#[serde(rename = "link")]
_link: DownloadLink,
content_bytes_available: bool,
content_offset: Option<u64>,
content_eof: bool,
content_base64: Option<String>,
},
WebBrowserLoginStarted {
transaction_id: String,
authorization_url: String,
expires_at_epoch_seconds: u64,
},
WebBrowserLoginCancelled {},
WebBrowserSession {
session: WireBrowserSession,
},
Error {
code: ApiErrorCode,
category: ApiErrorCategory,
message: String,
retryable: bool,
request_id: String,
},
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use serde::Deserialize;
use serde_json::Value;
use super::*;
#[derive(Deserialize)]
struct OperationFixture {
operation: String,
boundary: String,
request: Value,
response: Value,
}
#[test]
fn website_operation_contract_fixtures_cover_every_typed_request() {
let fixtures: Vec<OperationFixture> =
serde_json::from_str(include_str!("../tests/fixtures/web_operations.json")).unwrap();
let expected = BTreeSet::from([
"abort_process",
"auth_status",
"begin_web_browser_login",
"cancel_web_browser_login",
"cancel_process",
"create_artifact_download_link",
"create_debug_epoch",
"create_node_enrollment_grant",
"create_project",
"debug_attach",
"exchange_web_login_handoff",
"get_artifact",
"inspect_debug_epoch",
"list_agent_public_keys",
"list_artifacts",
"list_node_summaries",
"list_process_summaries",
"list_projects",
"list_recent_logs",
"list_task_events",
"list_task_snapshots",
"open_artifact_download_stream",
"quota_status",
"register_agent_public_key",
"resolve_task_failure",
"restart_task",
"resume_debug_epoch",
"revoke_agent_public_key",
"revoke_artifact_download_link",
"revoke_cli_session",
"revoke_node_credential",
"rotate_agent_public_key",
"select_project",
])
.into_iter()
.map(str::to_owned)
.collect();
let mut observed = BTreeSet::new();
for fixture in fixtures {
assert_eq!(fixture.request["type"], fixture.operation);
let round_trip = match fixture.boundary.as_str() {
"control" => {
let typed: AuthenticatedRequest =
serde_json::from_value(fixture.request.clone()).unwrap();
serde_json::to_value(typed).unwrap()
}
"login" => {
let typed: LoginRequest =
serde_json::from_value(fixture.request.clone()).unwrap();
serde_json::to_value(typed).unwrap()
}
boundary => panic!("unknown fixture boundary {boundary}"),
};
assert_eq!(round_trip, fixture.request);
let response: WireResponse = serde_json::from_value(fixture.response).unwrap();
assert!(
!matches!(response, WireResponse::Error { .. }),
"{} must carry its operation-specific success response fixture",
fixture.operation
);
assert!(observed.insert(fixture.operation));
}
assert_eq!(observed, expected);
}
}

250
crates/clusterflux-client/src/transport.rs Normal file
View file

@ -0,0 +1,250 @@
use std::collections::{BTreeMap, VecDeque};
use std::future::Future;
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use clusterflux_control::{endpoint_identity, ControlSession};
use thiserror::Error;
pub type TransportFuture =
Pin<Box<dyn Future<Output = Result<TransportResponse, ClientTransportError>> + Send + 'static>>;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TransportRequest {
pub api_path: String,
pub body: Vec<u8>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TransportResponse {
pub body: Vec<u8>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ClientTransportError {
#[error("client transport failed: {0}")]
Failed(String),
#[error("client transport task failed: {0}")]
Task(String),
}
pub trait ClientTransport: Send + Sync + 'static {
fn send(&self, request: TransportRequest) -> TransportFuture;
}
type ControlSessionPool = Arc<Vec<Mutex<Option<ControlSession>>>>;
pub struct ControlTransport {
endpoint: String,
connect_timeout: Duration,
io_timeout: Duration,
sessions: Arc<Mutex<BTreeMap<String, ControlSessionPool>>>,
next_session: Arc<AtomicU64>,
}
// A small pool allows independent HTMX reads to progress concurrently while
// keeping connection and blocking-worker use strictly bounded.
const SESSIONS_PER_API_PATH: usize = 4;
impl ControlTransport {
pub fn new(endpoint: impl Into<String>) -> Result<Self, ClientTransportError> {
Self::with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
}
pub fn with_timeouts(
endpoint: impl Into<String>,
connect_timeout: Duration,
io_timeout: Duration,
) -> Result<Self, ClientTransportError> {
let endpoint = endpoint.into();
endpoint_identity(&endpoint)
.map_err(|error| ClientTransportError::Failed(error.to_string()))?;
Ok(Self {
endpoint,
connect_timeout,
io_timeout,
sessions: Arc::new(Mutex::new(BTreeMap::new())),
next_session: Arc::new(AtomicU64::new(0)),
})
}
}
impl ClientTransport for ControlTransport {
fn send(&self, request: TransportRequest) -> TransportFuture {
let endpoint = self.endpoint.clone();
let connect_timeout = self.connect_timeout;
let io_timeout = self.io_timeout;
let sessions = Arc::clone(&self.sessions);
let next_session = Arc::clone(&self.next_session);
Box::pin(async move {
let pool = {
let mut sessions = sessions.lock().map_err(|_| {
ClientTransportError::Failed(
"client transport pool lock was poisoned".to_owned(),
)
})?;
Arc::clone(sessions.entry(request.api_path.clone()).or_insert_with(|| {
Arc::new(
(0..SESSIONS_PER_API_PATH)
.map(|_| Mutex::new(None))
.collect(),
)
}))
};
let slot_index =
next_session.fetch_add(1, Ordering::Relaxed) as usize % SESSIONS_PER_API_PATH;
tokio::task::spawn_blocking(move || {
let value = serde_json::from_slice(&request.body)
.map_err(|error| ClientTransportError::Failed(error.to_string()))?;
let mut selected = None;
for offset in 0..SESSIONS_PER_API_PATH {
let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
match pool[index].try_lock() {
Ok(session) if session.is_some() => {
selected = Some(session);
break;
}
Ok(_) | Err(std::sync::TryLockError::WouldBlock) => {}
Err(std::sync::TryLockError::Poisoned(_)) => {
return Err(ClientTransportError::Failed(
"client transport session lock was poisoned".to_owned(),
));
}
}
}
if selected.is_none() {
for offset in 0..SESSIONS_PER_API_PATH {
let index = (slot_index + offset) % SESSIONS_PER_API_PATH;
match pool[index].try_lock() {
Ok(session) => {
selected = Some(session);
break;
}
Err(std::sync::TryLockError::WouldBlock) => {}
Err(std::sync::TryLockError::Poisoned(_)) => {
return Err(ClientTransportError::Failed(
"client transport session lock was poisoned".to_owned(),
));
}
}
}
}
let mut session = match selected {
Some(session) => session,
None => pool[slot_index].lock().map_err(|_| {
ClientTransportError::Failed(
"client transport session lock was poisoned".to_owned(),
)
})?,
};
if session.is_none() {
*session = Some(
ControlSession::connect_to_api_path_with_timeouts(
&endpoint,
&request.api_path,
connect_timeout,
io_timeout,
)
.map_err(|error| ClientTransportError::Failed(error.to_string()))?,
);
}
let response = session
.as_mut()
.expect("session was initialized for the requested API path")
.request(&value);
match response {
Ok(response) => serde_json::to_vec(&response)
.map(|body| TransportResponse { body })
.map_err(|error| ClientTransportError::Failed(error.to_string())),
Err(error) => {
*session = None;
Err(ClientTransportError::Failed(error.to_string()))
}
}
})
.await
.map_err(|error| ClientTransportError::Task(error.to_string()))?
})
}
}
#[derive(Clone, Default)]
pub struct MockTransport {
state: Arc<Mutex<MockTransportState>>,
}
#[derive(Default)]
struct MockTransportState {
responses: VecDeque<Result<Vec<u8>, ClientTransportError>>,
requests: Vec<TransportRequest>,
}
impl MockTransport {
pub fn from_json_responses(responses: impl IntoIterator<Item = impl Into<String>>) -> Self {
let responses = responses
.into_iter()
.map(|response| Ok(response.into().into_bytes()))
.collect();
Self {
state: Arc::new(Mutex::new(MockTransportState {
responses,
requests: Vec::new(),
})),
}
}
pub fn push_json_response(&self, response: impl Into<String>) {
self.state
.lock()
.expect("mock transport lock is not poisoned")
.responses
.push_back(Ok(response.into().into_bytes()));
}
pub fn push_error(&self, message: impl Into<String>) {
self.state
.lock()
.expect("mock transport lock is not poisoned")
.responses
.push_back(Err(ClientTransportError::Failed(message.into())));
}
pub fn request_bodies(&self) -> Vec<String> {
self.state
.lock()
.expect("mock transport lock is not poisoned")
.requests
.iter()
.map(|request| String::from_utf8_lossy(&request.body).into_owned())
.collect()
}
pub fn requests(&self) -> Vec<TransportRequest> {
self.state
.lock()
.expect("mock transport lock is not poisoned")
.requests
.clone()
}
}
impl ClientTransport for MockTransport {
fn send(&self, request: TransportRequest) -> TransportFuture {
let state = Arc::clone(&self.state);
Box::pin(async move {
let mut state = state.lock().map_err(|_| {
ClientTransportError::Failed("mock transport lock was poisoned".to_owned())
})?;
state.requests.push(request);
state
.responses
.pop_front()
.ok_or_else(|| {
ClientTransportError::Failed("mock transport has no queued response".to_owned())
})?
.map(|body| TransportResponse { body })
})
}
}

468
crates/clusterflux-client/src/types.rs Normal file
View file

@ -0,0 +1,468 @@
use std::collections::BTreeMap;
use clusterflux_core::{
AgentId, ArtifactId, Authorization, Digest, LimitKind, NodeCapabilities, NodeId, Placement,
ProcessId, ProjectId, ResourceLimits, TaskBoundaryValue, TaskDefinitionId, TaskFailurePolicy,
TaskInstanceId, TenantId, UserId, VfsPath,
};
use serde::{Deserialize, Serialize};
use std::fmt;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AccountStatus {
pub tenant: TenantId,
pub project: ProjectId,
pub actor: UserId,
pub authenticated: bool,
pub account_status: String,
pub suspended: bool,
pub disabled: bool,
pub deleted: bool,
pub manual_review: bool,
pub sanitized_reason: Option<String>,
pub next_actions: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Project {
pub id: ProjectId,
pub tenant: TenantId,
pub name: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgentPublicKey {
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub agent: AgentId,
pub public_key: String,
pub public_key_fingerprint: Digest,
pub version: u64,
pub revoked: bool,
pub scopes: Vec<String>,
pub human_account_creation_privilege: bool,
pub browser_interaction_required_each_run: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeEnrollmentGrant {
pub tenant: TenantId,
pub project: ProjectId,
pub grant: String,
pub scope: String,
pub expires_at_epoch_seconds: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeSummary {
pub id: NodeId,
pub display_name: String,
pub online: bool,
pub stale: bool,
pub last_seen_epoch_seconds: Option<u64>,
pub capabilities: NodeCapabilities,
pub direct_connectivity: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodePage {
pub nodes: Vec<NodeSummary>,
pub next_cursor: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeRevocation {
pub node: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub actor: UserId,
pub descriptor_removed: bool,
pub queued_assignments_removed: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessLifecycleState {
Active,
RecentTerminal,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessActivityState {
Running,
WaitingForNode,
WaitingForTask,
AwaitingAction,
DebugEpochPartial,
Cancelling,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessFinalResult {
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpochSummary {
pub epoch: u64,
pub command: String,
pub fully_frozen: bool,
pub partially_frozen: bool,
pub fully_resumed: bool,
pub failed: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProcessSummary {
pub process: ProcessId,
pub lifecycle: ProcessLifecycleState,
pub activity: ProcessActivityState,
pub main_wait_state: Option<String>,
pub started_at_epoch_seconds: u64,
pub ended_at_epoch_seconds: Option<u64>,
pub final_result: Option<ProcessFinalResult>,
pub connected_nodes: Vec<NodeId>,
pub current_debug_epoch: Option<DebugEpochSummary>,
pub order_cursor: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProcessPage {
pub processes: Vec<ProcessSummary>,
pub next_cursor: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskTerminalState {
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskExecutor {
CoordinatorMain,
Node,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCompletionEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub node: NodeId,
pub executor: TaskExecutor,
pub task_definition: TaskDefinitionId,
pub task: TaskInstanceId,
pub attempt_id: Option<String>,
pub placement: Option<Placement>,
pub terminal_state: TaskTerminalState,
pub status_code: Option<i32>,
pub stdout_bytes: u64,
pub stderr_bytes: u64,
pub stdout_tail: String,
pub stderr_tail: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub artifact_path: Option<VfsPath>,
pub artifact_digest: Option<Digest>,
pub artifact_size_bytes: Option<u64>,
pub result: Option<TaskBoundaryValue>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskAttemptState {
Queued,
Running,
FailedAwaitingAction,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskAttemptSnapshot {
pub process: ProcessId,
pub task: TaskInstanceId,
pub attempt_id: String,
pub attempt_number: u32,
pub task_definition: TaskDefinitionId,
pub display_name: String,
pub state: TaskAttemptState,
pub current: bool,
pub node: Option<NodeId>,
pub environment_id: Option<String>,
pub environment_digest: Option<Digest>,
pub argument_summary: Vec<String>,
pub handle_summary: Vec<String>,
pub command_state: Option<String>,
pub vfs_checkpoint: String,
pub probe_symbol: Option<String>,
pub source_path: Option<String>,
pub source_line: Option<u32>,
pub restart_compatible: bool,
pub failure_policy: TaskFailurePolicy,
pub artifact_path: Option<VfsPath>,
pub artifact_digest: Option<Digest>,
pub artifact_size_bytes: Option<u64>,
pub status_code: Option<i32>,
pub error: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskLogStream {
Stdout,
Stderr,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RecentLogEntry {
pub sequence: u64,
pub process: ProcessId,
pub task: TaskInstanceId,
pub stream: TaskLogStream,
pub text: String,
pub server_timestamp_epoch_seconds: u64,
pub truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RecentLogPage {
pub entries: Vec<RecentLogEntry>,
pub next_sequence: Option<u64>,
pub history_truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactAvailability {
Available,
NodeOffline,
Unavailable,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactRetentionState {
NodeRetained,
ExplicitStorage,
Lost,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactSummary {
pub id: ArtifactId,
pub display_path: String,
pub display_name: String,
pub process: ProcessId,
pub producer_task: TaskInstanceId,
pub safe_node: Option<NodeId>,
pub digest: Digest,
pub size_bytes: u64,
pub availability: ArtifactAvailability,
pub downloadable_now: bool,
pub retention_state: ArtifactRetentionState,
pub explicit_storage: bool,
pub order_cursor: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactPage {
pub artifacts: Vec<ArtifactSummary>,
pub next_cursor: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct QuotaStatus {
pub tenant: TenantId,
pub project: ProjectId,
pub actor: UserId,
pub policy_label: Option<String>,
pub limits: ResourceLimits,
pub window_seconds: BTreeMap<LimitKind, u64>,
pub usage: BTreeMap<LimitKind, u64>,
pub window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCancellationTarget {
pub process: ProcessId,
pub task: TaskInstanceId,
pub node: NodeId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProcessCancellation {
pub process: ProcessId,
pub affected_tasks: Vec<TaskCancellationTarget>,
pub affected_nodes: Vec<NodeId>,
pub aborted: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugAuditEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub task: Option<TaskInstanceId>,
pub actor: UserId,
pub operation: String,
pub allowed: bool,
pub reason: String,
pub charged_debug_read_bytes: u64,
pub used_debug_read_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugAttach {
pub process: ProcessId,
pub actor: UserId,
pub authorization: Authorization,
pub audit_event: DebugAuditEvent,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpochControl {
pub process: ProcessId,
pub actor: UserId,
pub epoch: u64,
pub command: String,
pub affected_tasks: Vec<TaskCancellationTarget>,
pub all_stop_requested: bool,
pub audit_event: DebugAuditEvent,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DebugAcknowledgementState {
Frozen,
Running,
Failed,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugParticipantAcknowledgement {
pub node: NodeId,
pub task_definition: TaskDefinitionId,
pub task: TaskInstanceId,
pub epoch: u64,
pub state: DebugAcknowledgementState,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
pub message: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpochStatus {
pub process: ProcessId,
pub actor: UserId,
pub epoch: u64,
pub command: String,
pub expected_tasks: Vec<TaskCancellationTarget>,
pub acknowledgements: Vec<DebugParticipantAcknowledgement>,
pub fully_frozen: bool,
pub partially_frozen: bool,
pub fully_resumed: bool,
pub failed: bool,
pub failure_messages: Vec<String>,
pub audit_event: DebugAuditEvent,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskRestart {
pub process: ProcessId,
pub task: TaskInstanceId,
pub restarted_task_instance: Option<TaskInstanceId>,
pub restarted_attempt_id: Option<String>,
pub actor: UserId,
pub accepted: bool,
pub clean_boundary_available: bool,
pub active_task: bool,
pub completed_event_observed: bool,
pub requires_whole_process_restart: bool,
pub message: String,
pub audit_event: DebugAuditEvent,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskFailureResolutionResult {
pub process: ProcessId,
pub task: TaskInstanceId,
pub attempt_id: String,
pub resolution: TaskFailureResolution,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskFailureResolution {
AcceptFailure,
Cancel,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskReplacementBundle {
pub bundle_digest: Digest,
pub wasm_module_base64: String,
pub source_snapshot: Option<Digest>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowserLoginStart {
pub transaction_id: String,
pub authorization_url: String,
pub expires_at_epoch_seconds: u64,
}
#[derive(Clone, PartialEq, Eq)]
pub struct SessionCredential(pub(crate) String);
impl SessionCredential {
pub fn from_secret(secret: impl Into<String>) -> Self {
Self(secret.into())
}
pub fn expose_secret(&self) -> &str {
&self.0
}
}
impl fmt::Debug for SessionCredential {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("SessionCredential([REDACTED])")
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BrowserSession {
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub credential: SessionCredential,
pub expires_at_epoch_seconds: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ArtifactDownloadPoll {
Pending,
Chunk {
offset: u64,
bytes: Vec<u8>,
eof: bool,
},
}

254
crates/clusterflux-client/tests/client_contract.rs Normal file
View file

@ -0,0 +1,254 @@
use std::net::TcpListener;
use std::time::Duration;
use clusterflux_client::{
ApiErrorCategory, ApiErrorCode, ArtifactDownloadPoll, ArtifactId, ClientError,
ClusterfluxClient, ControlTransport, MockTransport, ProjectId, SessionCredential, TenantId,
UserId, CLIENT_API_VERSION,
};
use clusterflux_control::{CONTROL_API_PATH, LOGIN_API_PATH};
use clusterflux_coordinator::service::CoordinatorService;
use serde_json::{json, Value};
#[tokio::test]
async fn mock_transport_exercises_typed_envelope_and_session_plumbing() {
let transport = MockTransport::from_json_responses([json!({
"type": "projects",
"projects": [{
"id": "project-one",
"tenant": "tenant-one",
"name": "Project one"
}],
"actor": "user-one"
})
.to_string()]);
let client = ClusterfluxClient::with_transport(transport.clone())
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
let projects = client.list_projects().await.unwrap();
assert_eq!(projects.len(), 1);
assert_eq!(projects[0].id, ProjectId::from("project-one"));
let requests = transport.requests();
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].api_path, CONTROL_API_PATH);
let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
assert_eq!(envelope["protocol_version"], CLIENT_API_VERSION);
assert_eq!(envelope["request_id"], "client-1");
assert_eq!(envelope["operation"], "authenticated");
assert_eq!(envelope["payload"]["type"], "authenticated");
assert_eq!(envelope["payload"]["request"]["type"], "list_projects");
assert_eq!(envelope["payload"]["session_secret"], "test-session-secret");
}
#[tokio::test]
async fn structured_errors_retain_machine_fields_and_originating_request_id() {
let transport = MockTransport::from_json_responses([json!({
"type": "error",
"code": "account_suspended",
"category": "authorization",
"message": "account access is suspended",
"retryable": false,
"request_id": "client-1"
})
.to_string()]);
let client = ClusterfluxClient::with_transport(transport)
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
let ClientError::Api(error) = client.account_status().await.unwrap_err() else {
panic!("expected typed API error");
};
assert_eq!(error.code, ApiErrorCode::AccountSuspended);
assert_eq!(error.category, ApiErrorCategory::Authorization);
assert_eq!(error.request_id, "client-1");
assert!(!error.retryable);
}
#[tokio::test]
async fn browser_login_cancellation_uses_the_login_boundary_and_exact_transaction() {
let transport = MockTransport::from_json_responses([
json!({ "type": "web_browser_login_cancelled" }).to_string(),
]);
let client = ClusterfluxClient::with_transport(transport.clone());
client
.cancel_browser_login("login-transaction")
.await
.unwrap();
let requests = transport.requests();
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].api_path, LOGIN_API_PATH);
let envelope: Value = serde_json::from_slice(&requests[0].body).unwrap();
assert_eq!(envelope["operation"], "cancel_web_browser_login");
assert_eq!(
envelope["payload"],
json!({
"type": "cancel_web_browser_login",
"transaction_id": "login-transaction"
})
);
}
#[tokio::test]
async fn a_mismatched_error_request_id_is_rejected_as_a_protocol_error() {
let transport = MockTransport::from_json_responses([json!({
"type": "error",
"code": "validation_error",
"category": "validation",
"message": "bad request",
"retryable": false,
"request_id": "another-request"
})
.to_string()]);
let client = ClusterfluxClient::with_transport(transport)
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
let ClientError::Protocol(message) = client.list_projects().await.unwrap_err() else {
panic!("expected protocol error");
};
assert!(message.contains("does not match client-1"));
}
#[test]
fn session_credentials_are_redacted_from_debug_output() {
let credential = SessionCredential::from_secret("must-not-appear");
assert_eq!(format!("{credential:?}"), "SessionCredential([REDACTED])");
}
#[tokio::test]
async fn artifact_download_is_a_typed_bounded_chunk_stream() {
let link = json!({
"artifact": "artifact-one",
"artifact_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"artifact_size_bytes": 5,
"source": { "RetainedNode": "node-one" },
"url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one",
"scoped_token_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
"expires_at_epoch_seconds": 1000,
"tenant": "tenant-one",
"project": "project-one",
"process": "process-one",
"actor": { "User": "user-one" },
"max_bytes": 1024,
"policy_context_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
});
let transport = MockTransport::from_json_responses([
json!({ "type": "artifact_download_link", "link": link.clone() }).to_string(),
json!({
"type": "artifact_download_stream",
"link": link.clone(),
"streamed_bytes": 0,
"charged_download_bytes": 0,
"content_bytes_available": false,
"content_eof": false
})
.to_string(),
json!({
"type": "artifact_download_stream",
"link": link,
"streamed_bytes": 5,
"charged_download_bytes": 5,
"content_bytes_available": true,
"content_offset": 0,
"content_eof": true,
"content_base64": "aGVsbG8="
})
.to_string(),
]);
let client = ClusterfluxClient::with_transport(transport)
.with_session_credential(&SessionCredential::from_secret("test-session-secret"));
let mut download = client
.begin_artifact_download(ArtifactId::from("artifact-one"), 1024, 60, 64)
.await
.unwrap();
assert_eq!(
download.next_chunk().await.unwrap(),
ArtifactDownloadPoll::Pending
);
assert_eq!(
download.next_chunk().await.unwrap(),
ArtifactDownloadPoll::Chunk {
offset: 0,
bytes: b"hello".to_vec(),
eof: true,
}
);
}
#[tokio::test]
async fn typed_client_runs_against_the_real_strict_control_endpoint() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let mut service = CoordinatorService::new(7);
service
.issue_cli_session(
TenantId::from("tenant-one"),
ProjectId::from("project-one"),
UserId::from("user-one"),
"real-endpoint-session",
None,
)
.unwrap();
let (stream, _) = listener.accept().unwrap();
service.handle_stream(stream).unwrap();
});
let client = ClusterfluxClient::connect(format!("clusterflux+tcp://{address}"))
.unwrap()
.with_session_credential(&SessionCredential::from_secret("real-endpoint-session"));
let projects = client.list_projects().await.unwrap();
assert_eq!(projects.len(), 1);
assert_eq!(projects[0].id, ProjectId::from("project-one"));
let status = client.account_status().await.unwrap();
assert!(status.authenticated);
assert_eq!(status.actor, UserId::from("user-one"));
drop(client);
server.join().unwrap();
}
#[tokio::test]
async fn concurrent_requests_expand_the_bounded_pool_without_serializing() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
let server = std::thread::spawn(move || {
let handlers = (0..2)
.map(|_| {
let (stream, _) = listener.accept().unwrap();
std::thread::spawn(move || {
let mut service = CoordinatorService::new(7);
service
.issue_cli_session(
TenantId::from("tenant-one"),
ProjectId::from("project-one"),
UserId::from("user-one"),
"concurrent-session",
None,
)
.unwrap();
service.handle_stream(stream).unwrap();
})
})
.collect::<Vec<_>>();
for handler in handlers {
handler.join().unwrap();
}
});
let transport = ControlTransport::with_timeouts(
format!("clusterflux+tcp://{address}"),
Duration::from_secs(2),
Duration::from_secs(5),
)
.unwrap();
let client = ClusterfluxClient::with_transport(transport)
.with_session_credential(&SessionCredential::from_secret("concurrent-session"));
let (first, second) = tokio::join!(client.account_status(), client.account_status());
assert!(first.unwrap().authenticated);
assert!(second.unwrap().authenticated);
drop(client);
server.join().unwrap();
}

200
crates/clusterflux-client/tests/fixtures/web_operations.json vendored Normal file
View file

@ -0,0 +1,200 @@
[
{
"operation": "begin_web_browser_login",
"boundary": "login",
"request": { "type": "begin_web_browser_login" },
"response": { "type": "web_browser_login_started", "transaction_id": "login-transaction", "authorization_url": "https://auth.clusterflux.lesstuff.com/authorize?state=opaque", "expires_at_epoch_seconds": 1000 }
},
{
"operation": "cancel_web_browser_login",
"boundary": "login",
"request": { "type": "cancel_web_browser_login", "transaction_id": "login-transaction" },
"response": { "type": "web_browser_login_cancelled" }
},
{
"operation": "exchange_web_login_handoff",
"boundary": "login",
"request": { "type": "exchange_web_login_handoff", "transaction_id": "login-transaction", "handoff_code": "one-time-handoff" },
"response": { "type": "web_browser_session", "session": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "session_secret": "server-side-session", "expires_at_epoch_seconds": 2000 } }
},
{
"operation": "auth_status",
"boundary": "control",
"request": { "type": "auth_status" },
"response": { "type": "auth_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "authenticated": true, "account_status": "active", "suspended": false, "disabled": false, "deleted": false, "manual_review": false, "sanitized_reason": null, "next_actions": [] }
},
{
"operation": "revoke_cli_session",
"boundary": "control",
"request": { "type": "revoke_cli_session" },
"response": { "type": "cli_session_revoked" }
},
{
"operation": "create_project",
"boundary": "control",
"request": { "type": "create_project", "project": "project-new", "name": "New project" },
"response": { "type": "project_created", "project": { "id": "project-new", "tenant": "tenant-one", "name": "New project" } }
},
{
"operation": "select_project",
"boundary": "control",
"request": { "type": "select_project", "project": "project-selected" },
"response": { "type": "project_selected", "project": { "id": "project-selected", "tenant": "tenant-one", "name": "Selected project" } }
},
{
"operation": "list_projects",
"boundary": "control",
"request": { "type": "list_projects" },
"response": { "type": "projects", "projects": [{ "id": "project-one", "tenant": "tenant-one", "name": "Project one" }] }
},
{
"operation": "register_agent_public_key",
"boundary": "control",
"request": { "type": "register_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:test-public-key" },
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:test-public-key", "public_key_fingerprint": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "version": 1, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
},
{
"operation": "list_agent_public_keys",
"boundary": "control",
"request": { "type": "list_agent_public_keys" },
"response": { "type": "agent_public_keys", "records": [] }
},
{
"operation": "rotate_agent_public_key",
"boundary": "control",
"request": { "type": "rotate_agent_public_key", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key" },
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 2, "revoked": false, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
},
{
"operation": "revoke_agent_public_key",
"boundary": "control",
"request": { "type": "revoke_agent_public_key", "agent": "agent-browser" },
"response": { "type": "agent_public_key", "record": { "tenant": "tenant-one", "project": "project-one", "user": "user-one", "agent": "agent-browser", "public_key": "ed25519:rotated-public-key", "public_key_fingerprint": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "version": 3, "revoked": true, "scopes": ["project"], "human_account_creation_privilege": false, "browser_interaction_required_each_run": false } }
},
{
"operation": "create_node_enrollment_grant",
"boundary": "control",
"request": { "type": "create_node_enrollment_grant", "ttl_seconds": 300 },
"response": { "type": "node_enrollment_grant_created", "tenant": "tenant-one", "project": "project-one", "grant": "one-time-grant", "scope": "tenant-one/project-one", "expires_at_epoch_seconds": 1000 }
},
{
"operation": "list_node_summaries",
"boundary": "control",
"request": { "type": "list_node_summaries", "cursor": null, "limit": 200 },
"response": { "type": "node_summaries", "nodes": [], "next_cursor": null }
},
{
"operation": "revoke_node_credential",
"boundary": "control",
"request": { "type": "revoke_node_credential", "node": "node-one" },
"response": { "type": "node_credential_revoked", "node": "node-one", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "descriptor_removed": true, "queued_assignments_removed": 0 }
},
{
"operation": "list_process_summaries",
"boundary": "control",
"request": { "type": "list_process_summaries", "cursor": "process:1", "limit": 20 },
"response": { "type": "process_summaries", "processes": [], "next_cursor": null }
},
{
"operation": "cancel_process",
"boundary": "control",
"request": { "type": "cancel_process", "process": "process-one" },
"response": { "type": "process_cancellation_requested", "process": "process-one", "cancelled_tasks": [], "affected_nodes": [] }
},
{
"operation": "abort_process",
"boundary": "control",
"request": { "type": "abort_process", "process": "process-one", "launch_attempt": null },
"response": { "type": "process_aborted", "process": "process-one", "aborted_tasks": [], "affected_nodes": [] }
},
{
"operation": "quota_status",
"boundary": "control",
"request": { "type": "quota_status" },
"response": { "type": "quota_status", "tenant": "tenant-one", "project": "project-one", "actor": "user-one", "policy_label": "community tier", "limits": { "limits": { "ApiCall": 10000 } }, "window_seconds": { "ApiCall": 60 }, "usage": { "ApiCall": 12 }, "window_started_epoch_seconds": { "ApiCall": 900 } }
},
{
"operation": "list_task_events",
"boundary": "control",
"request": { "type": "list_task_events", "process": "process-one" },
"response": { "type": "task_events", "events": [] }
},
{
"operation": "list_task_snapshots",
"boundary": "control",
"request": { "type": "list_task_snapshots", "process": "process-one" },
"response": { "type": "task_snapshots", "snapshots": [] }
},
{
"operation": "list_recent_logs",
"boundary": "control",
"request": { "type": "list_recent_logs", "process": "process-one", "task": "task-one", "after_sequence": 7, "limit": 100 },
"response": { "type": "recent_logs", "entries": [{ "sequence": 1, "process": "process-one", "task": "task-one", "stream": "stdout", "text": "building", "server_timestamp_epoch_seconds": 1000, "truncated": false }], "next_sequence": 1, "history_truncated": false }
},
{
"operation": "restart_task",
"boundary": "control",
"request": { "type": "restart_task", "process": "process-one", "task": "task-one", "replacement_bundle": null },
"response": { "type": "task_restart", "process": "process-one", "task": "task-one", "restarted_task_instance": "task-one-retry", "restarted_attempt_id": "attempt-2", "actor": "user-one", "accepted": true, "clean_boundary_available": true, "active_task": false, "completed_event_observed": true, "requires_whole_process_restart": false, "message": "task restart accepted", "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "restart_task", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
},
{
"operation": "resolve_task_failure",
"boundary": "control",
"request": { "type": "resolve_task_failure", "process": "process-one", "task": "task-one", "resolution": "accept_failure" },
"response": { "type": "task_failure_resolved", "process": "process-one", "task": "task-one", "attempt_id": "attempt-1", "resolution": "accept_failure" }
},
{
"operation": "debug_attach",
"boundary": "control",
"request": { "type": "debug_attach", "process": "process-one" },
"response": { "type": "debug_attach", "process": "process-one", "actor": "user-one", "authorization": { "allowed": true, "reason": "authorized" }, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "debug_attach", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
},
{
"operation": "create_debug_epoch",
"boundary": "control",
"request": { "type": "create_debug_epoch", "process": "process-one", "stopped_task": "task-one", "reason": "breakpoint" },
"response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "affected_tasks": [], "all_stop_requested": true, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": "task-one", "actor": "user-one", "operation": "create_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
},
{
"operation": "resume_debug_epoch",
"boundary": "control",
"request": { "type": "resume_debug_epoch", "process": "process-one", "epoch": 3 },
"response": { "type": "debug_epoch", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "resume", "affected_tasks": [], "all_stop_requested": false, "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "resume_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
},
{
"operation": "inspect_debug_epoch",
"boundary": "control",
"request": { "type": "inspect_debug_epoch", "process": "process-one", "epoch": 3 },
"response": { "type": "debug_epoch_status", "process": "process-one", "actor": "user-one", "epoch": 3, "command": "freeze", "expected_tasks": [], "acknowledgements": [], "fully_frozen": false, "partially_frozen": true, "fully_resumed": false, "failed": false, "failure_messages": [], "audit_event": { "tenant": "tenant-one", "project": "project-one", "process": "process-one", "task": null, "actor": "user-one", "operation": "inspect_debug_epoch", "allowed": true, "reason": "authorized", "charged_debug_read_bytes": 0, "used_debug_read_bytes": 0 } }
},
{
"operation": "list_artifacts",
"boundary": "control",
"request": { "type": "list_artifacts", "process": "process-one", "cursor": "artifact:1", "limit": 50 },
"response": { "type": "artifacts", "artifacts": [], "next_cursor": null }
},
{
"operation": "get_artifact",
"boundary": "control",
"request": { "type": "get_artifact", "artifact": "artifact-one" },
"response": { "type": "artifact", "artifact": { "id": "artifact-one", "display_path": "/out/artifact-one", "display_name": "artifact-one", "process": "process-one", "producer_task": "task-one", "safe_node": "node-one", "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "size_bytes": 5, "availability": "available", "downloadable_now": true, "retention_state": "node_retained", "explicit_storage": false, "order_cursor": "artifact:1" } }
},
{
"operation": "create_artifact_download_link",
"boundary": "control",
"request": { "type": "create_artifact_download_link", "artifact": "artifact-one", "max_bytes": 1048576, "ttl_seconds": 120 },
"response": { "type": "artifact_download_link", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" } }
},
{
"operation": "open_artifact_download_stream",
"boundary": "control",
"request": { "type": "open_artifact_download_stream", "artifact": "artifact-one", "max_bytes": 1048576, "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "chunk_bytes": 65536 },
"response": { "type": "artifact_download_stream", "link": { "artifact": "artifact-one", "artifact_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "artifact_size_bytes": 5, "source": { "RetainedNode": "node-one" }, "url_path": "/artifacts/tenant-one/project-one/process-one/artifact-one", "scoped_token_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "expires_at_epoch_seconds": 1100, "tenant": "tenant-one", "project": "project-one", "process": "process-one", "actor": { "User": "user-one" }, "max_bytes": 1048576, "policy_context_digest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" }, "streamed_bytes": 5, "charged_download_bytes": 5, "content_bytes_available": true, "content_offset": 0, "content_eof": true, "content_base64": "aGVsbG8=" }
},
{
"operation": "revoke_artifact_download_link",
"boundary": "control",
"request": { "type": "revoke_artifact_download_link", "artifact": "artifact-one", "token_digest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },
"response": { "type": "artifact_download_link_revoked" }
}
]

57
crates/clusterflux-control/src/lib.rs
View file

@ -3,6 +3,8 @@ use std::io::{BufRead, BufReader, Read, Write};
#[cfg(not(target_arch = "wasm32"))]
use std::net::TcpStream;
use std::net::{IpAddr, ToSocketAddrs};
#[cfg(not(target_arch = "wasm32"))]
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use serde_json::Value;
@ -59,12 +61,34 @@ impl ControlSession {
endpoint: &str,
api_path: &str,
) -> Result<Self, ControlTransportError> {
let mut session = Self::connect(endpoint)?;
Self::connect_to_api_path_with_timeouts(
endpoint,
api_path,
Duration::from_secs(10),
Duration::from_secs(30),
)
}
pub fn connect_to_api_path_with_timeouts(
endpoint: &str,
api_path: &str,
connect_timeout: Duration,
io_timeout: Duration,
) -> Result<Self, ControlTransportError> {
let session = Self::connect_with_timeouts(endpoint, connect_timeout, io_timeout)?;
#[cfg(not(target_arch = "wasm32"))]
if let ControlTransport::Https { url, .. } = &mut session.transport {
*url = endpoint_api_url(endpoint, api_path)?;
{
let mut session = session;
if let ControlTransport::Https { url, .. } = &mut session.transport {
*url = endpoint_api_url(endpoint, api_path)?;
}
Ok(session)
}
#[cfg(target_arch = "wasm32")]
{
let _ = api_path;
Ok(session)
}
Ok(session)
}
pub fn connect_with_timeouts(
@ -127,6 +151,7 @@ impl ControlSession {
if encoded.len() > MAX_CONTROL_FRAME_BYTES {
return Err(ControlTransportError::FrameTooLarge);
}
let inject_response_loss = should_inject_response_loss(value);
let response = match &mut self.transport {
#[cfg(not(target_arch = "wasm32"))]
ControlTransport::Https { agent, url } => {
@ -136,6 +161,9 @@ impl ControlSession {
.set("Accept", "application/json")
.send_bytes(&encoded)
.map_err(|error| ControlTransportError::Http(error.to_string()))?;
if inject_response_loss {
return Err(ControlTransportError::Closed);
}
if response.status() != 200 {
return Err(ControlTransportError::Http(format!(
"coordinator returned HTTP {} {}",
@ -157,6 +185,9 @@ impl ControlSession {
writer.write_all(&encoded)?;
writer.write_all(b"\n")?;
writer.flush()?;
if inject_response_loss {
return Err(ControlTransportError::Closed);
}
let mut bytes = Vec::new();
reader
.take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
@ -180,6 +211,24 @@ impl ControlSession {
}
}
#[cfg(not(target_arch = "wasm32"))]
fn should_inject_response_loss(value: &Value) -> bool {
static INJECTED: AtomicBool = AtomicBool::new(false);
let Ok(expected_operation) = std::env::var("CLUSTERFLUX_TEST_DROP_RESPONSE_AFTER_OPERATION")
else {
return false;
};
let operation = value
.pointer("/payload/request/type")
.or_else(|| value.pointer("/payload/type"))
.or_else(|| value.get("type"))
.and_then(Value::as_str);
operation == Some(expected_operation.as_str())
&& INJECTED
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_ok()
}
pub fn control_api_url(endpoint: &str) -> Result<String, ControlTransportError> {
endpoint_api_url(endpoint, CONTROL_API_PATH)
}

35
crates/clusterflux-coordinator/src/durable.rs
View file

@ -33,6 +33,39 @@ pub struct NodeIdentityRecord {
pub enrollment_scope: String,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct NodeScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub node: NodeId,
}
impl NodeScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, node: NodeId) -> Self {
Self {
tenant,
project,
node,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, node: &NodeId) -> Self {
Self::new(tenant.clone(), project.clone(), node.clone())
}
pub fn credential_subject(&self) -> String {
format!(
"node:{}:{}:{}:{}:{}:{}",
self.tenant.as_str().len(),
self.tenant,
self.project.as_str().len(),
self.project,
self.node.as_str().len(),
self.node
)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CredentialRecord {
pub subject: String,
@ -107,7 +140,7 @@ pub struct DurableState {
pub tenants: BTreeMap<TenantId, TenantRecord>,
pub users: BTreeMap<UserId, UserRecord>,
pub projects: BTreeMap<ProjectId, ProjectRecord>,
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
pub node_identities: BTreeMap<NodeScopeKey, NodeIdentityRecord>,
pub credentials: BTreeMap<String, CredentialRecord>,
pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
pub source_provider_configs:

258
crates/clusterflux-coordinator/src/lib.rs
View file

@ -13,7 +13,7 @@ pub mod service;
mod sessions;
pub use durable::{
AccountPolicyState, AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord,
DurableStore, FallibleDurableStore, InMemoryDurableStore, NodeIdentityRecord, NodeScopeKey,
ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord,
TenantRecord, UserRecord,
};
@ -33,6 +33,7 @@ pub use service::{
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ActiveProcess {
pub id: ProcessId,
pub launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
pub tenant: TenantId,
pub project: ProjectId,
pub connected_nodes: BTreeSet<NodeId>,
@ -127,8 +128,9 @@ impl Coordinator {
public_key: impl Into<String>,
enrollment_scope: impl Into<String>,
) {
let key = NodeScopeKey::new(tenant.clone(), project.clone(), node.clone());
self.durable.node_identities.insert(
node.clone(),
key,
NodeIdentityRecord {
id: node,
tenant,
@ -180,10 +182,13 @@ impl Coordinator {
public_key,
credential.scope.clone(),
);
let subject =
NodeScopeKey::new(credential.tenant.clone(), credential.project.clone(), node)
.credential_subject();
self.durable.credentials.insert(
format!("node:{node}"),
subject.clone(),
CredentialRecord {
subject: format!("node:{node}"),
subject,
tenant: credential.tenant.clone(),
project: Some(credential.project.clone()),
kind: credential.credential_kind.clone(),
@ -322,9 +327,20 @@ impl Coordinator {
tenant: TenantId,
project: ProjectId,
id: ProcessId,
) -> ActiveProcess {
self.start_process_for_launch_attempt(tenant, project, id, None)
}
pub fn start_process_for_launch_attempt(
&mut self,
tenant: TenantId,
project: ProjectId,
id: ProcessId,
launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
) -> ActiveProcess {
let process = ActiveProcess {
id: id.clone(),
launch_attempt,
tenant,
project,
connected_nodes: BTreeSet::new(),
@ -344,15 +360,12 @@ impl Coordinator {
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node identity is outside the requested tenant/project scope".to_owned(),
));
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if !self
.active_processes
@ -367,14 +380,18 @@ impl Coordinator {
pub fn reconnect_node(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
process: Option<(&ProcessId, u64)>,
) -> Result<(), CoordinatorError> {
let identity = self
if !self
.durable
.node_identities
.get(node)
.ok_or(CoordinatorError::UnknownNode)?;
.contains_key(&NodeScopeKey::from_refs(tenant, project, node))
{
return Err(CoordinatorError::UnknownNode);
}
if let Some((process_id, stale_epoch)) = process {
if stale_epoch != self.coordinator_epoch {
@ -383,11 +400,7 @@ impl Coordinator {
current_epoch: self.coordinator_epoch,
});
}
let key = (
identity.tenant.clone(),
identity.project.clone(),
process_id.clone(),
);
let key = (tenant.clone(), project.clone(), process_id.clone());
if let Some(active) = self.active_processes.get_mut(&key) {
active.connected_nodes.insert(node.clone());
}
@ -404,23 +417,27 @@ impl Coordinator {
let identity = self
.durable
.node_identities
.get(node)
.get(&NodeScopeKey::from_refs(
&context.tenant,
&context.project,
node,
))
.ok_or(CoordinatorError::UnknownNode)?
.clone();
if identity.tenant != context.tenant || identity.project != context.project {
return Err(CoordinatorError::Unauthorized(
"node credential is outside the signed-in tenant/project scope".to_owned(),
));
}
if !matches!(context.actor, Actor::User(_)) {
return Err(CoordinatorError::Unauthorized(
"node credential revocation requires a user identity".to_owned(),
));
}
self.durable.node_identities.remove(node);
self.durable.credentials.remove(&format!("node:{node}"));
for active in self.active_processes.values_mut() {
active.connected_nodes.remove(node);
let key = NodeScopeKey::from_refs(&context.tenant, &context.project, node);
self.durable.node_identities.remove(&key);
self.durable.credentials.remove(&key.credential_subject());
for active in self
.active_processes
.values_mut()
.filter(|active| active.tenant == context.tenant && active.project == context.project)
{
active.connected_nodes.remove(&key.node);
}
Ok(identity)
}
@ -531,6 +548,32 @@ impl Coordinator {
.expect("active process was checked immediately before removal"))
}
pub fn abort_process_for_launch_attempt(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
launch_attempt: &clusterflux_core::LaunchAttemptId,
) -> Result<ActiveProcess, CoordinatorError> {
let key = (tenant.clone(), project.clone(), process.clone());
let active = self.active_processes.get(&key).ok_or_else(|| {
CoordinatorError::Unauthorized(
"launch rollback requires an active virtual process".to_owned(),
)
})?;
if active.launch_attempt.as_ref() != Some(launch_attempt) {
return Err(CoordinatorError::Unauthorized(format!(
"launch rollback denied: attempt {} does not own process {}",
launch_attempt.as_str(),
process.as_str()
)));
}
Ok(self
.active_processes
.remove(&key)
.expect("active process was checked immediately before removal"))
}
pub fn active_process_count(&self) -> usize {
self.active_processes.len()
}
@ -542,8 +585,31 @@ impl Coordinator {
.count()
}
pub fn node_identity(&self, id: &NodeId) -> Option<&NodeIdentityRecord> {
self.durable.node_identities.get(id)
pub fn tenant_count(&self) -> usize {
self.durable.tenants.len()
}
pub fn user_count(&self) -> usize {
self.durable.users.len()
}
pub fn project_count(&self) -> usize {
self.durable.projects.len()
}
pub fn node_identity_count(&self) -> usize {
self.durable.node_identities.len()
}
pub fn node_identity(
&self,
tenant: &TenantId,
project: &ProjectId,
id: &NodeId,
) -> Option<&NodeIdentityRecord> {
self.durable
.node_identities
.get(&NodeScopeKey::from_refs(tenant, project, id))
}
pub fn node_identity_count_for_tenant(&self, tenant: &TenantId) -> usize {
@ -645,12 +711,24 @@ mod tests {
.contains_key(&TenantId::from("tenant")));
assert!(restarted.durable.users.contains_key(&UserId::from("user")));
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
assert_eq!(
restarted
.durable
.credentials
.get("node:node")
.get(&node_subject)
.map(|credential| &credential.kind),
Some(&CredentialKind::NodeCredential)
);
@ -674,7 +752,12 @@ mod tests {
);
assert_eq!(rerun.coordinator_epoch, 2);
restarted
.reconnect_node(&NodeId::from("node"), Some((&process, 2)))
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&process, 2)),
)
.unwrap();
assert!(restarted
.active_process(
@ -687,6 +770,68 @@ mod tests {
.contains(&NodeId::from("node")));
}
#[test]
fn duplicate_node_ids_use_distinct_durable_identities_and_credential_subjects() {
let store = InMemoryDurableStore::default();
let mut coordinator = Coordinator::boot(&store, 1);
let node = NodeId::from("shared-node");
let scopes = [
(TenantId::from("tenant-a"), ProjectId::from("project-a")),
(TenantId::from("tenant-b"), ProjectId::from("project-b")),
(TenantId::from("tenant-a"), ProjectId::from("project-c")),
];
for (index, (tenant, project)) in scopes.iter().enumerate() {
let mut grant = coordinator.create_node_enrollment_grant(
tenant.clone(),
project.clone(),
format!("grant-{index}"),
"node:attach",
100,
);
coordinator
.exchange_node_enrollment_grant(
&mut grant,
node.clone(),
&format!("public-key-{index}"),
"node:attach",
99,
)
.unwrap();
}
let scope_a = NodeScopeKey::from_refs(&scopes[0].0, &scopes[0].1, &node);
let scope_b = NodeScopeKey::from_refs(&scopes[1].0, &scopes[1].1, &node);
let scope_c = NodeScopeKey::from_refs(&scopes[2].0, &scopes[2].1, &node);
assert_ne!(scope_a.credential_subject(), scope_b.credential_subject());
assert_ne!(scope_a.credential_subject(), scope_c.credential_subject());
assert_eq!(
coordinator.node_identity(&scope_a.tenant, &scope_a.project, &scope_a.node),
coordinator.durable.node_identities.get(&scope_a)
);
assert_eq!(
coordinator.node_identity(&scope_b.tenant, &scope_b.project, &scope_b.node),
coordinator.durable.node_identities.get(&scope_b)
);
assert_eq!(
coordinator.node_identity(&scope_c.tenant, &scope_c.project, &scope_c.node),
coordinator.durable.node_identities.get(&scope_c)
);
assert!(coordinator
.durable
.credentials
.contains_key(&scope_a.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_b.credential_subject()));
assert!(coordinator
.durable
.credentials
.contains_key(&scope_c.credential_subject()));
assert_eq!(coordinator.durable.node_identities.len(), 3);
assert_eq!(coordinator.durable.credentials.len(), 3);
}
#[test]
fn identical_process_ids_are_isolated_by_tenant_and_project() {
let store = InMemoryDurableStore::default();
@ -735,11 +880,18 @@ mod tests {
let mut restarted = Coordinator::boot(&store, 2);
restarted
.reconnect_node(&NodeId::from("node"), None)
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
None,
)
.unwrap();
let error = restarted
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -771,7 +923,13 @@ mod tests {
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert!(coordinator.node_identity(&NodeId::from("node")).is_some());
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
}
#[test]
@ -785,10 +943,16 @@ mod tests {
"public-key",
"node:attach",
);
let node_subject = NodeScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
)
.credential_subject();
coordinator.durable.credentials.insert(
"node:node".to_owned(),
node_subject.clone(),
CredentialRecord {
subject: "node:node".to_owned(),
subject: node_subject.clone(),
tenant: TenantId::from("tenant"),
project: Some(ProjectId::from("project")),
kind: CredentialKind::NodeCredential,
@ -802,6 +966,8 @@ mod tests {
);
coordinator
.reconnect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
Some((&ProcessId::from("process"), 1)),
)
@ -817,7 +983,7 @@ mod tests {
&NodeId::from("node"),
)
.unwrap_err();
assert!(matches!(foreign, CoordinatorError::Unauthorized(_)));
assert!(matches!(foreign, CoordinatorError::UnknownNode));
let revoked = coordinator
.revoke_node_credential(
@ -830,8 +996,14 @@ mod tests {
)
.unwrap();
assert_eq!(revoked.id, NodeId::from("node"));
assert!(coordinator.node_identity(&NodeId::from("node")).is_none());
assert!(!coordinator.durable.credentials.contains_key("node:node"));
assert!(coordinator
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_none());
assert!(!coordinator.durable.credentials.contains_key(&node_subject));
assert!(!coordinator
.active_process(
&TenantId::from("tenant"),
@ -890,7 +1062,7 @@ mod tests {
}
#[test]
fn account_policy_state_summarizes_private_admin_records_safely() {
fn account_policy_state_summarizes_sensitive_admin_records_safely() {
let tenant = TenantId::from("tenant");
let mut coordinator = Coordinator::boot(&InMemoryDurableStore::default(), 1);
@ -1046,7 +1218,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, CoordinatorError::Unauthorized(_)));
assert!(matches!(error, CoordinatorError::UnknownNode));
}
#[test]

25
crates/clusterflux-coordinator/src/main.rs
View file

@ -5,17 +5,40 @@ use clusterflux_core::{ProjectId, TenantId, UserId};
use serde_json::json;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
match raw_args.as_slice() {
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
println!("clusterflux-coordinator {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
println!(
"Clusterflux coordinator.\n\n\
Usage: clusterflux-coordinator [OPTIONS]\n\n\
Options:\n \
--listen <ADDRESS> [default: 127.0.0.1:0]\n \
--allow-local-trusted-loopback\n \
-h, --help\n \
-V, --version"
);
return Ok(());
}
_ => {}
}
let mut listen = "127.0.0.1:0".to_owned();
let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
.ok()
.as_deref()
== Some("1");
let mut args = std::env::args().skip(1);
let mut args = raw_args.into_iter();
while let Some(arg) = args.next() {
if arg == "--listen" {
listen = args.next().ok_or("--listen requires an address")?;
} else if arg == "--allow-local-trusted-loopback" {
allow_local_trusted = true;
} else {
return Err(format!("unknown argument: {arg}").into());
}
}

142
crates/clusterflux-coordinator/src/postgres_store.rs
View file

@ -4,8 +4,8 @@ use thiserror::Error;
use crate::{
AgentPublicKeyRecord, ArtifactRelayDurableState, CliSessionRecord, CredentialRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord,
ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
DurableState, FallibleDurableStore, NodeIdentityRecord, NodeScopeKey, ProjectPermissionRecord,
ProjectRecord, ServicePolicyRecord, SourceProviderConfigRecord, TenantRecord, UserRecord,
};
#[derive(Clone, Debug, PartialEq, Eq)]
@ -154,9 +154,16 @@ impl FallibleDurableStore for PostgresDurableStore {
state.projects.insert(record.id.clone(), record);
}
for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM clusterflux_node_identities ORDER BY node_id",
"SELECT record FROM clusterflux_node_identities ORDER BY tenant_id, project_id, node_id",
)? {
state.node_identities.insert(record.id.clone(), record);
state.node_identities.insert(
NodeScopeKey::new(
record.tenant.clone(),
record.project.clone(),
record.id.clone(),
),
record,
);
}
for record in self.query_records::<CredentialRecord>(
"SELECT record FROM clusterflux_credentials ORDER BY subject",
@ -376,12 +383,59 @@ CREATE TABLE IF NOT EXISTS clusterflux_projects (
);
CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
node_id TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, node_id)
);
DO $clusterflux_node_scope_migration$
DECLARE
current_primary_key TEXT;
current_primary_key_columns TEXT[];
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_node_identities
WHERE record->>'id' IS DISTINCT FROM node_id
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node identity migration refused an internally inconsistent legacy row';
END IF;
SELECT
constraint_record.conname,
array_agg(attribute.attname ORDER BY key_column.ordinality)
INTO current_primary_key, current_primary_key_columns
FROM pg_constraint AS constraint_record
CROSS JOIN LATERAL unnest(constraint_record.conkey)
WITH ORDINALITY AS key_column(attribute_number, ordinality)
JOIN pg_attribute AS attribute
ON attribute.attrelid = constraint_record.conrelid
AND attribute.attnum = key_column.attribute_number
WHERE constraint_record.conrelid = 'clusterflux_node_identities'::regclass
AND constraint_record.contype = 'p'
GROUP BY constraint_record.conname;
IF current_primary_key_columns = ARRAY['node_id']::TEXT[] THEN
EXECUTE format(
'ALTER TABLE clusterflux_node_identities DROP CONSTRAINT %I',
current_primary_key
);
ALTER TABLE clusterflux_node_identities
ADD PRIMARY KEY (tenant_id, project_id, node_id);
ELSIF current_primary_key_columns
IS DISTINCT FROM ARRAY['tenant_id', 'project_id', 'node_id']::TEXT[]
THEN
RAISE EXCEPTION
'clusterflux node identity migration found an unexpected primary key shape';
END IF;
END
$clusterflux_node_scope_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -389,6 +443,74 @@ CREATE TABLE IF NOT EXISTS clusterflux_credentials (
record JSONB NOT NULL
);
DO $clusterflux_node_credential_migration$
BEGIN
IF EXISTS (
SELECT 1
FROM clusterflux_credentials
WHERE record->>'subject' IS DISTINCT FROM subject
OR record->>'tenant' IS DISTINCT FROM tenant_id
OR record->>'project' IS DISTINCT FROM project_id
) THEN
RAISE EXCEPTION
'clusterflux node credential migration refused an internally inconsistent legacy row';
END IF;
UPDATE clusterflux_credentials AS credential
SET subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
),
record = jsonb_set(
credential.record,
'{subject}',
to_jsonb(format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)),
false
)
FROM clusterflux_node_identities AS identity
WHERE credential.subject = 'node:' || identity.node_id
AND credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id;
IF EXISTS (
SELECT 1
FROM clusterflux_credentials AS credential
WHERE credential.subject LIKE 'node:%'
AND NOT EXISTS (
SELECT 1
FROM clusterflux_node_identities AS identity
WHERE credential.tenant_id = identity.tenant_id
AND credential.project_id = identity.project_id
AND credential.subject = format(
'node:%s:%s:%s:%s:%s:%s',
octet_length(identity.tenant_id),
identity.tenant_id,
octet_length(identity.project_id),
identity.project_id,
octet_length(identity.node_id),
identity.node_id
)
)
) THEN
RAISE EXCEPTION
'clusterflux node credential migration found an unscoped or orphaned node subject';
END IF;
END
$clusterflux_node_credential_migration$;
CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
@ -529,7 +651,13 @@ mod tests {
let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert!(restarted
.node_identity(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
)
.is_some());
assert_eq!(restarted.active_process_count(), 0);
}

223
crates/clusterflux-coordinator/src/service.rs
View file

@ -7,13 +7,14 @@ use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
RateLimit, TenantId, TransportError, UserId,
Actor, AgentId, ApiError, ApiErrorCategory, ApiErrorCode, ArtifactRegistry,
CapabilityReportError, CredentialKind, Digest, DownloadError, LimitError, NativeQuicTransport,
NodeDescriptor, NodeId, PanelError, PanelState, Placement, ProcessId, ProjectId, RateLimit,
TenantId, TransportError, UserId,
};
use thiserror::Error;
use crate::{Coordinator, CoordinatorError};
use crate::{Coordinator, CoordinatorError, NodeScopeKey};
mod admin;
mod artifacts;
@ -34,6 +35,7 @@ mod quota;
mod relay;
mod routing;
mod signed_nodes;
mod summaries;
mod tcp;
mod wire_protocol;
use authorization::authorize_authenticated_user_operation;
@ -43,12 +45,14 @@ use keys::{
ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
};
pub use protocol::{
ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
TaskTerminalState, VirtualProcessStatus, WorkflowActor,
ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, ArtifactTransferAssignment,
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse,
DebugAcknowledgementState, DebugAuditEvent, DebugEpochSummary, DebugParticipantAcknowledgement,
NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
RecentLogEntry, SourcePreparationDisposition, SourcePreparationStatus, TaskAssignment,
TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget, TaskCompletionEvent,
TaskExecutor, TaskFailureResolution, TaskLogStream, TaskReplacementBundle, TaskTerminalState,
VirtualProcessStatus, WorkflowActor,
};
pub use quota::CoordinatorQuotaConfiguration;
pub use relay::{
@ -69,9 +73,15 @@ const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 1_000_000;
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
const MAX_RECENT_LOG_ENTRIES_PER_PROCESS: usize = 256;
const MAX_RECENT_LOG_ENTRIES_PER_PROJECT: usize = 1_024;
const MAX_RECENT_LOG_BYTES_PER_PROJECT: usize = 512 * 1024;
const MAX_RECENT_LOG_CHUNK_BYTES: usize = 16 * 1024;
const MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT: usize = 32;
const MAX_RECENT_PROCESS_SUMMARIES_TOTAL: usize = 8_192;
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
requested.clamp(1, maximum)
@ -111,6 +121,24 @@ pub struct CoordinatorAdmission {
pub max_artifact_download_ttl_seconds: u64,
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct CoordinatorOperationalMetrics {
pub tenants: usize,
pub users: usize,
pub projects: usize,
pub enrolled_nodes: usize,
pub reported_nodes: usize,
pub live_nodes: usize,
pub active_processes: usize,
pub active_coordinator_mains: usize,
pub max_active_coordinator_mains: usize,
pub active_tasks: usize,
pub queued_tasks: usize,
pub artifacts: usize,
pub retained_download_links: usize,
pub relay: ArtifactRelayUsage,
}
impl Default for CoordinatorAdmission {
fn default() -> Self {
Self {
@ -151,16 +179,139 @@ pub enum CoordinatorServiceError {
Durable(String),
}
impl CoordinatorServiceError {
pub fn api_error(&self, request_id: impl Into<String>) -> ApiError {
let request_id = request_id.into();
let message = self.to_string();
let (code, category, retryable) = match self {
Self::Io(_) => (
ApiErrorCode::TemporaryCapacity,
ApiErrorCategory::Availability,
true,
),
Self::Json(_)
| Self::CapabilityReport(_)
| Self::InvalidArtifactPath(_)
| Self::InvalidTaskLogTail(_) => (
ApiErrorCode::ValidationError,
ApiErrorCategory::Validation,
false,
),
Self::Protocol(_) | Self::Coordinator(CoordinatorError::Unauthorized(_)) => {
return ApiError::from_message(request_id, message);
}
Self::Coordinator(CoordinatorError::UnknownNode) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Coordinator(CoordinatorError::Enrollment(_)) => (
ApiErrorCode::Unauthenticated,
ApiErrorCategory::Authentication,
false,
),
Self::Coordinator(CoordinatorError::StaleProcessEpoch { .. }) => {
(ApiErrorCode::Conflict, ApiErrorCategory::State, true)
}
Self::Download(DownloadError::NotFound) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Download(DownloadError::Unavailable)
| Self::Download(DownloadError::DirectConnectivityUnavailable(_)) => (
ApiErrorCode::ArtifactUnavailable,
ApiErrorCategory::Availability,
true,
),
Self::Download(DownloadError::LimitExceeded { .. }) => (
ApiErrorCode::ArtifactLimitExceeded,
ApiErrorCategory::Resource,
false,
),
Self::Download(DownloadError::Unauthorized(_))
| Self::Download(DownloadError::InvalidToken)
| Self::Download(DownloadError::Expired)
| Self::Download(DownloadError::Revoked) => (
ApiErrorCode::Forbidden,
ApiErrorCategory::Authorization,
false,
),
Self::Download(DownloadError::Usage(_)) | Self::Resource(_) => (
ApiErrorCode::QuotaExceeded,
ApiErrorCategory::Resource,
true,
),
Self::Scheduler(_) => (
ApiErrorCode::NoCapableNode,
ApiErrorCategory::Availability,
true,
),
Self::Transport(_) => (
ApiErrorCode::NodeOffline,
ApiErrorCategory::Availability,
true,
),
Self::Panel(PanelError::RateLimited) => (
ApiErrorCode::QuotaExceeded,
ApiErrorCategory::Resource,
true,
),
Self::Panel(PanelError::UnknownWidget(_)) => {
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
}
Self::Panel(_) => (
ApiErrorCode::Forbidden,
ApiErrorCategory::Authorization,
false,
),
Self::Durable(_) => (
ApiErrorCode::InternalError,
ApiErrorCategory::Internal,
true,
),
};
ApiError::new(code, category, message, retryable, request_id)
}
}
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_descriptors: BTreeMap<NodeScopeKey, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeScopeKey, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
task_events: VecDeque<TaskCompletionEvent>,
process_scope_history: VecDeque<ProcessControlKey>,
process_summaries: BTreeMap<ProcessControlKey, summaries::StoredProcessSummary>,
process_summary_order: VecDeque<ProcessControlKey>,
next_process_summary_order: u64,
task_terminal_states: BTreeMap<TaskRestartKey, TaskTerminalState>,
recent_logs: BTreeMap<(TenantId, ProjectId), VecDeque<RecentLogEntry>>,
recent_log_dropped_through: BTreeMap<ProcessControlKey, u64>,
recent_log_accounted_bytes: BTreeMap<
(
TenantId,
ProjectId,
ProcessId,
clusterflux_core::TaskInstanceId,
String,
),
u64,
>,
recent_log_truncated_streams: BTreeSet<(
TenantId,
ProjectId,
ProcessId,
clusterflux_core::TaskInstanceId,
String,
)>,
recent_log_quota_truncated_streams: BTreeSet<(
TenantId,
ProjectId,
ProcessId,
clusterflux_core::TaskInstanceId,
String,
)>,
next_recent_log_sequence: u64,
debug_audit_events: VecDeque<DebugAuditEvent>,
debug_epochs: BTreeMap<ProcessControlKey, u64>,
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@ -180,7 +331,7 @@ pub struct CoordinatorService {
process_cancellations: BTreeSet<ProcessControlKey>,
process_aborts: BTreeSet<ProcessControlKey>,
agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
node_replay_nonces: BTreeMap<(NodeId, String), u64>,
node_replay_nonces: BTreeMap<(NodeScopeKey, String), u64>,
panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
stopped_panels: BTreeSet<PanelStopKey>,
panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
@ -215,6 +366,29 @@ impl CoordinatorService {
self.artifact_relay.usage()
}
pub fn operational_metrics(&self) -> CoordinatorOperationalMetrics {
CoordinatorOperationalMetrics {
tenants: self.coordinator.tenant_count(),
users: self.coordinator.user_count(),
projects: self.coordinator.project_count(),
enrolled_nodes: self.coordinator.node_identity_count(),
reported_nodes: self.node_descriptors.len(),
live_nodes: self
.node_descriptors
.keys()
.filter(|scope| self.node_is_live(scope))
.count(),
active_processes: self.coordinator.active_process_count(),
active_coordinator_mains: self.main_runtime.active_main_count(),
max_active_coordinator_mains: self.main_runtime.max_active_mains(),
active_tasks: self.active_tasks.len(),
queued_tasks: self.pending_task_launches.len(),
artifacts: self.artifact_registry.artifact_count(),
retained_download_links: self.artifact_registry.retained_download_link_count(),
relay: self.artifact_relay.usage(),
}
}
fn commit_artifact_relay(
&mut self,
candidate: relay::ArtifactRelayLedger,
@ -284,15 +458,10 @@ impl CoordinatorService {
{
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node process-control request is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
return Ok(());
}
self.coordinator
@ -409,6 +578,16 @@ impl CoordinatorService {
enrollment_grants: BTreeMap::new(),
task_events: VecDeque::new(),
process_scope_history: VecDeque::new(),
process_summaries: BTreeMap::new(),
process_summary_order: VecDeque::new(),
next_process_summary_order: 1,
task_terminal_states: BTreeMap::new(),
recent_logs: BTreeMap::new(),
recent_log_dropped_through: BTreeMap::new(),
recent_log_accounted_bytes: BTreeMap::new(),
recent_log_truncated_streams: BTreeSet::new(),
recent_log_quota_truncated_streams: BTreeSet::new(),
next_recent_log_sequence: 1,
debug_audit_events: VecDeque::new(),
debug_epochs: BTreeMap::new(),
debug_epoch_runtime: BTreeMap::new(),

51
crates/clusterflux-coordinator/src/service/artifacts.rs
View file

@ -8,7 +8,7 @@ use clusterflux_core::{
};
use sha2::{Digest as _, Sha256};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::relay::RelayFinishReason;
use super::{
@ -52,7 +52,11 @@ impl CoordinatorService {
let action = self
.artifact_registry
.download_action(&context, &artifact, &policy)?;
self.ensure_download_source_connectivity(&action.source)?;
self.ensure_download_source_connectivity(
&context.tenant,
&context.project,
&action.source,
)?;
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
@ -163,7 +167,11 @@ impl CoordinatorService {
},
&mut validation_meter,
)?;
self.ensure_download_source_connectivity(&stream.link.source)?;
self.ensure_download_source_connectivity(
&stream.link.tenant,
&stream.link.project,
&stream.link.source,
)?;
self.expire_artifact_reverse_transfers(now_epoch_seconds)?;
if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
@ -302,7 +310,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?;
@ -410,7 +418,7 @@ impl CoordinatorService {
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.metadata(&context.tenant, &context.project, &artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination =
@ -682,15 +690,10 @@ impl CoordinatorService {
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer node is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
Ok(())
}
@ -725,15 +728,12 @@ impl CoordinatorService {
) -> Result<NodeEndpoint, CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(tenant, project, node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact export node is outside the tenant/project scope".to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
debug_assert_eq!(&identity.tenant, tenant);
debug_assert_eq!(&identity.project, project);
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity"
))
@ -744,7 +744,7 @@ impl CoordinatorService {
)
.into());
}
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export"
@ -769,17 +769,20 @@ impl CoordinatorService {
fn ensure_download_source_connectivity(
&self,
tenant: &TenantId,
project: &ProjectId,
source: &StorageLocation,
) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else {
return Ok(());
};
let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
let node_scope = NodeScopeKey::from_refs(tenant, project, node);
let _descriptor = self.node_descriptors.get(&node_scope).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download"
))
})?;
if !self.node_is_live(node) {
if !self.node_is_live(&node_scope) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} is offline for artifact download"

17
crates/clusterflux-coordinator/src/service/authorization.rs
View file

@ -17,6 +17,7 @@ pub(super) enum PublicUserOperation {
RevokeAgentPublicKey,
CreateNodeEnrollmentGrant,
ListNodeDescriptors,
ListNodeSummaries,
RevokeNodeCredential,
StartProcess,
ScheduleTask,
@ -24,6 +25,7 @@ pub(super) enum PublicUserOperation {
CancelProcess,
AbortProcess,
ListProcesses,
ListProcessSummaries,
QuotaStatus,
RestartTask,
ResolveTaskFailure,
@ -35,7 +37,10 @@ pub(super) enum PublicUserOperation {
InspectDebugEpoch,
ListTaskEvents,
ListTaskSnapshots,
ListRecentLogs,
JoinTask,
ListArtifacts,
GetArtifact,
CreateArtifactDownloadLink,
OpenArtifactDownloadStream,
RevokeArtifactDownloadLink,
@ -56,6 +61,7 @@ impl PublicUserOperation {
Self::RevokeAgentPublicKey => "revoke_agent_public_key",
Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
Self::ListNodeDescriptors => "list_node_descriptors",
Self::ListNodeSummaries => "list_node_summaries",
Self::RevokeNodeCredential => "revoke_node_credential",
Self::StartProcess => "start_process",
Self::ScheduleTask => "schedule_task",
@ -63,6 +69,7 @@ impl PublicUserOperation {
Self::CancelProcess => "cancel_process",
Self::AbortProcess => "abort_process",
Self::ListProcesses => "list_processes",
Self::ListProcessSummaries => "list_process_summaries",
Self::QuotaStatus => "quota_status",
Self::RestartTask => "restart_task",
Self::ResolveTaskFailure => "resolve_task_failure",
@ -74,7 +81,10 @@ impl PublicUserOperation {
Self::InspectDebugEpoch => "inspect_debug_epoch",
Self::ListTaskEvents => "list_task_events",
Self::ListTaskSnapshots => "list_task_snapshots",
Self::ListRecentLogs => "list_recent_logs",
Self::JoinTask => "join_task",
Self::ListArtifacts => "list_artifacts",
Self::GetArtifact => "get_artifact",
Self::CreateArtifactDownloadLink => "create_artifact_download_link",
Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
@ -105,6 +115,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
Self::CreateNodeEnrollmentGrant
}
AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
AuthenticatedCoordinatorRequest::ListNodeSummaries { .. } => Self::ListNodeSummaries,
AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
Self::RevokeNodeCredential
}
@ -114,6 +125,9 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
AuthenticatedCoordinatorRequest::ListProcessSummaries { .. } => {
Self::ListProcessSummaries
}
AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
@ -129,7 +143,10 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
AuthenticatedCoordinatorRequest::ListRecentLogs { .. } => Self::ListRecentLogs,
AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
AuthenticatedCoordinatorRequest::ListArtifacts { .. } => Self::ListArtifacts,
AuthenticatedCoordinatorRequest::GetArtifact { .. } => Self::GetArtifact,
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
Self::CreateArtifactDownloadLink
}

24
crates/clusterflux-coordinator/src/service/debug.rs
View file

@ -35,6 +35,7 @@ pub(super) struct DebugEpochRuntime {
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct DebugBreakpointPlan {
pub(super) actor: UserId,
pub(super) revision: u64,
pub(super) probe_symbols: BTreeSet<String>,
pub(super) hit_epoch: Option<u64>,
pub(super) hit_task: Option<TaskInstanceId>,
@ -85,6 +86,7 @@ impl CoordinatorService {
project: String,
actor_user: String,
process: String,
revision: u64,
probe_symbols: Vec<String>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let probe_symbols = validate_probe_symbols(probe_symbols)?;
@ -115,10 +117,28 @@ impl CoordinatorService {
))
.into());
}
let key = process_control_key(&tenant, &project, &process);
if let Some(current) = self.debug_breakpoints.get(&key) {
if current.actor == actor && revision < current.revision {
return Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision: current.revision,
probe_symbols: current.probe_symbols.iter().cloned().collect(),
hit_epoch: current.hit_epoch,
hit_task: current.hit_task.clone(),
hit_probe_symbol: current.hit_probe_symbol.clone(),
charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
used_debug_read_bytes: audit_event.used_debug_read_bytes,
audit_event,
});
}
}
self.debug_breakpoints.insert(
process_control_key(&tenant, &project, &process),
key,
DebugBreakpointPlan {
actor: actor.clone(),
revision,
probe_symbols: probe_symbols.iter().cloned().collect(),
hit_epoch: None,
hit_task: None,
@ -128,6 +148,7 @@ impl CoordinatorService {
Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision,
probe_symbols,
hit_epoch: None,
hit_task: None,
@ -190,6 +211,7 @@ impl CoordinatorService {
Ok(CoordinatorResponse::DebugBreakpoints {
process,
actor,
revision: plan.revision,
probe_symbols: plan.probe_symbols.into_iter().collect(),
hit_epoch: plan.hit_epoch,
hit_task: plan.hit_task,

61
crates/clusterflux-coordinator/src/service/debug_requests.rs
View file

@ -33,12 +33,14 @@ impl CoordinatorService {
project,
actor_user,
process,
revision,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant,
project,
actor_user,
process,
revision,
probe_symbols,
),
CoordinatorRequest::InspectDebugBreakpoints {
@ -96,12 +98,14 @@ impl CoordinatorService {
),
AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
process,
revision,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
revision,
probe_symbols,
),
AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
@ -222,24 +226,25 @@ impl CoordinatorService {
} else if !completed_event_observed {
"selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
} else if let Some(checkpoint) = checkpoint {
let vfs_available =
checkpoint
.checkpoint
.vfs_manifest
.objects
.iter()
.all(|(path, object)| {
let artifact = super::keys::artifact_id_from_path(path);
self.artifact_registry
.metadata(&artifact)
.is_some_and(|metadata| {
metadata.tenant == tenant
&& metadata.project == project
&& metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
})
});
let vfs_available = checkpoint.checkpoint.vfs_manifest.objects.iter().try_fold(
true,
|available, (path, object)| {
let artifact = super::keys::artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?;
Ok::<_, CoordinatorServiceError>(
available
&& self
.artifact_registry
.metadata(&tenant, &project, &artifact)
.is_some_and(|metadata| {
metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
}),
)
},
)?;
if !vfs_available {
"selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
} else {
@ -285,6 +290,25 @@ impl CoordinatorService {
task.as_str()
);
if let Some(replacement) = replacement {
if assignment.task_spec.source_snapshot.is_some() {
let replacement_source = replacement_bundle
.as_ref()
.and_then(|bundle| bundle.source_snapshot.clone())
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"replacement task omitted the current SourceSnapshot for source-bound arguments"
.to_owned(),
)
})?;
assignment
.task_spec
.rebind_source_snapshot(replacement_source)
.map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"replacement task SourceSnapshot is invalid: {error}"
))
})?;
}
assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
export: Some(replacement.export),
abi: WasmExportAbi::TaskV1,
@ -456,6 +480,7 @@ impl CoordinatorService {
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
self.maybe_retire_terminal_process(&tenant, &project, &process)?;
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,

45
crates/clusterflux-coordinator/src/service/keys.rs
View file

@ -1,6 +1,7 @@
use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
use thiserror::Error;
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
@ -63,11 +64,47 @@ pub(super) fn enrollment_grant_key(
(tenant.clone(), project.clone(), grant.to_owned())
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub(super) enum ArtifactPathError {
#[error("path must start with the exact /vfs/artifacts/ prefix")]
WrongPrefix,
#[error("path must name an artifact after /vfs/artifacts/")]
EmptyArtifact,
#[error("mapped artifact identifier is invalid: {0}")]
InvalidArtifactId(#[from] clusterflux_core::IdParseError),
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> Result<ArtifactId, ArtifactPathError> {
let value = path
.as_str()
.strip_prefix("/vfs/artifacts/")
.unwrap_or(path.as_str())
.replace('/', ":");
ArtifactId::new(value)
.ok_or(ArtifactPathError::WrongPrefix)?;
if value.is_empty() {
return Err(ArtifactPathError::EmptyArtifact);
}
ArtifactId::try_new(value.replace('/', ":")).map_err(ArtifactPathError::from)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn artifact_id_conversion_is_fallible_and_structural() {
assert_eq!(
artifact_id_from_path(&VfsPath::new("/vfs/artifacts/build/output").unwrap()).unwrap(),
ArtifactId::from("build:output")
);
for path in [
"/vfs/other/output",
"/vfs/artifacts",
"/vfs/artifacts/bad artifact!",
] {
let path = VfsPath::new(path).unwrap();
assert!(artifact_id_from_path(&path).is_err(), "{path:?}");
}
let mapped = format!("/vfs/artifacts/{}", "x".repeat(256));
assert!(artifact_id_from_path(&VfsPath::new(mapped).unwrap()).is_err());
}
}

858
crates/clusterflux-coordinator/src/service/logs.rs
View file

@ -1,5 +1,5 @@
use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
ArtifactFlush, ArtifactScopeKey, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
};
@ -9,7 +9,10 @@ use super::keys::{process_control_key, task_control_key, task_restart_key};
use super::protocol::TaskAttemptState;
use super::{
artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
RecentLogEntry, TaskCompletionEvent, TaskLogStream, TaskTerminalState,
MAX_RECENT_LOG_BYTES_PER_PROJECT, MAX_RECENT_LOG_CHUNK_BYTES,
MAX_RECENT_LOG_ENTRIES_PER_PROCESS, MAX_RECENT_LOG_ENTRIES_PER_PROJECT,
MAX_TASK_LOG_TAIL_BYTES,
};
impl CoordinatorService {
@ -36,23 +39,44 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
validate_task_log_tail("stdout_tail", &stdout_tail)?;
validate_task_log_tail("stderr_tail", &stderr_tail)?;
let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota
.can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
self.quota
.charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
let stdout_retained = self.accept_final_log_stream(
&tenant,
&project,
&process,
&task,
TaskLogStream::Stdout,
stdout_bytes,
&stdout_tail,
stdout_truncated,
now_epoch_seconds,
)?;
let stderr_retained = self.accept_final_log_stream(
&tenant,
&project,
&process,
&task,
TaskLogStream::Stderr,
stderr_bytes,
&stderr_tail,
stderr_truncated,
now_epoch_seconds,
)?;
Ok(CoordinatorResponse::TaskLogRecorded {
process,
task,
stdout_bytes,
stderr_bytes,
stdout_tail: if stdout_truncated {
stdout_tail: if !stdout_retained {
"[log output truncated at project log quota]".to_owned()
} else if stdout_truncated {
format!("{stdout_tail}\n... truncated")
} else {
stdout_tail
},
stderr_tail: if stderr_truncated {
stderr_tail: if !stderr_retained {
"[log output truncated at project log quota]".to_owned()
} else if stderr_truncated {
format!("{stderr_tail}\n... truncated")
} else {
stderr_tail
@ -61,6 +85,158 @@ impl CoordinatorService {
})
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_report_task_log_chunk(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
stream: TaskLogStream,
offset: u64,
source_bytes: u64,
text: String,
truncated: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
"live log chunk is {} bytes; max is {MAX_RECENT_LOG_CHUNK_BYTES}",
text.len()
)));
}
if source_bytes == 0 && !text.is_empty() && !truncated {
return Err(CoordinatorServiceError::Protocol(
"live log chunk source_bytes must describe non-empty text".to_owned(),
));
}
if source_bytes > (MAX_RECENT_LOG_CHUNK_BYTES as u64).saturating_mul(4) {
return Err(CoordinatorServiceError::Protocol(
"live log chunk source_bytes exceeds the bounded chunk allowance".to_owned(),
));
}
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
let key = recent_log_offset_key(&tenant, &project, &process, &task, &stream);
let expected = self
.recent_log_accounted_bytes
.get(&key)
.copied()
.unwrap_or(0);
let end = offset.checked_add(source_bytes).ok_or_else(|| {
CoordinatorServiceError::Protocol(
"live log chunk offset exceeds the supported range".to_owned(),
)
})?;
let state_marker = source_bytes == 0 && truncated;
if end < expected || (end == expected && !state_marker) {
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
sequence: None,
next_offset: expected,
});
}
let now_epoch_seconds = self.current_epoch_seconds()?;
if self.recent_log_quota_truncated_streams.contains(&key) {
if end > expected {
self.recent_log_accounted_bytes.insert(key, end);
}
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
sequence: None,
next_offset: end.max(expected),
});
}
let newly_accounted = end.saturating_sub(expected);
if self
.quota
.charge_log_bytes(&tenant, &project, newly_accounted, now_epoch_seconds)
.is_err()
{
if end > expected {
self.recent_log_accounted_bytes.insert(key.clone(), end);
}
let sequence = self.mark_log_quota_truncated(
&tenant,
&project,
&process,
&task,
&stream,
now_epoch_seconds,
);
return Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
sequence,
next_offset: end.max(expected),
});
}
if offset > expected {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!("[log output lost: {} bytes]", offset - expected),
true,
now_epoch_seconds,
);
} else if offset < expected {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!(
"[log output overlap omitted: {} new source bytes]",
end - expected
),
true,
now_epoch_seconds,
);
}
let marker_is_new = if truncated {
self.recent_log_truncated_streams.insert(key.clone())
} else {
false
};
let text = if state_marker && text.is_empty() {
"[log output truncated at source]".to_owned()
} else {
text
};
let sequence = (!text.is_empty() && offset >= expected && (!state_marker || marker_is_new))
.then(|| {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream,
text,
truncated,
now_epoch_seconds,
)
});
if end > expected {
self.recent_log_accounted_bytes.insert(key, end);
}
Ok(CoordinatorResponse::TaskLogChunkRecorded {
process,
task,
sequence,
next_offset: end,
})
}
pub(super) fn handle_report_vfs_metadata(
&mut self,
tenant: String,
@ -85,7 +261,9 @@ impl CoordinatorService {
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant,
project,
process: process.clone(),
@ -177,20 +355,37 @@ impl CoordinatorService {
artifact_size_bytes,
result,
};
let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota.can_charge_log_bytes(
let stdout_retained = self.accept_final_log_stream(
&event.tenant,
&event.project,
reported_bytes,
&event.process,
&event.task,
TaskLogStream::Stdout,
event.stdout_bytes,
&event.stdout_tail,
event.stdout_truncated,
now_epoch_seconds,
)?;
self.quota.charge_log_bytes(
let stderr_retained = self.accept_final_log_stream(
&event.tenant,
&event.project,
reported_bytes,
&event.process,
&event.task,
TaskLogStream::Stderr,
event.stderr_bytes,
&event.stderr_tail,
event.stderr_truncated,
now_epoch_seconds,
)?;
if !stdout_retained {
event.stdout_tail = "[log output truncated at project log quota]".to_owned();
event.stdout_truncated = true;
}
if !stderr_retained {
event.stderr_tail = "[log output truncated at project log quota]".to_owned();
event.stderr_truncated = true;
}
let task_key = task_control_key(
&event.tenant,
&event.project,
@ -203,7 +398,9 @@ impl CoordinatorService {
event.placement = self.task_placements.remove(&task_key);
if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
id: artifact_id_from_path(path).map_err(|error| {
CoordinatorServiceError::InvalidArtifactPath(error.to_string())
})?,
tenant: event.tenant.clone(),
project: event.project.clone(),
process: event.process.clone(),
@ -217,26 +414,22 @@ impl CoordinatorService {
self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key);
let no_active_tasks =
!self
.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &event.tenant
&& task_project == &event.project
&& task_process == &event.process
});
if no_active_tasks {
self.process_aborts.remove(&process_key);
if self.process_cancellations.remove(&process_key)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
}
self.clear_recent_log_offsets_for_task(
&event.tenant,
&event.project,
&event.process,
&event.task,
);
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != event.tenant
|| assignment.project != event.project
|| assignment.process != event.process
|| assignment.node != event.node
|| assignment.task != event.task
});
!assignments.is_empty()
});
if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key(
&event.tenant,
@ -253,10 +446,20 @@ impl CoordinatorService {
if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
let events_recorded = self
.task_events
.iter()
.filter(|recorded| {
recorded.tenant == event.tenant
&& recorded.project == event.project
&& recorded.process == event.process
})
.count();
Ok(CoordinatorResponse::TaskRecorded {
process: event.process,
task: event.task,
events_recorded: self.task_events.len(),
events_recorded,
})
}
@ -316,7 +519,7 @@ impl CoordinatorService {
Ok(CoordinatorResponse::TaskSnapshots { snapshots })
}
fn authorize_task_event_process_scope(
pub(super) fn authorize_task_event_process_scope(
&self,
tenant: &TenantId,
project: &ProjectId,
@ -356,6 +559,47 @@ impl CoordinatorService {
Ok(())
}
pub(super) fn handle_list_recent_logs(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
task: Option<String>,
after_sequence: Option<u64>,
limit: u32,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = ProcessId::new(process);
let task = task.map(TaskInstanceId::new);
self.authorize_task_event_process_scope(&tenant, &project, &process)?;
let after_sequence = after_sequence.unwrap_or(0);
let retained = self.recent_logs.get(&(tenant.clone(), project.clone()));
let history_truncated = self
.recent_log_dropped_through
.get(&process_control_key(&tenant, &project, &process))
.is_some_and(|dropped_through| *dropped_through > after_sequence);
let entries = retained
.into_iter()
.flatten()
.filter(|entry| {
entry.process == process
&& entry.sequence > after_sequence
&& task.as_ref().is_none_or(|task| &entry.task == task)
})
.take(limit as usize)
.cloned()
.collect::<Vec<_>>();
let next_sequence = entries.last().map(|entry| entry.sequence);
Ok(CoordinatorResponse::RecentLogs {
entries,
next_sequence,
history_truncated,
})
}
pub(super) fn handle_join_task(
&mut self,
tenant: String,
@ -474,6 +718,22 @@ impl CoordinatorService {
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
match event.executor {
super::TaskExecutor::CoordinatorMain => self.record_main_terminal_state(
&event.tenant,
&event.project,
&event.process,
event.task_definition.clone(),
event.task.clone(),
event.terminal_state.clone(),
),
super::TaskExecutor::Node => {
self.task_terminal_states.insert(
task_restart_key(&event.tenant, &event.project, &event.process, &event.task),
event.terminal_state.clone(),
);
}
}
let process_scope = (
event.tenant.clone(),
event.project.clone(),
@ -511,6 +771,321 @@ impl CoordinatorService {
self.task_events.push_back(event);
}
#[allow(clippy::too_many_arguments)]
fn record_recent_log(
&mut self,
tenant: TenantId,
project: ProjectId,
process: ProcessId,
task: TaskInstanceId,
stream: TaskLogStream,
mut text: String,
mut truncated: bool,
server_timestamp_epoch_seconds: u64,
) -> u64 {
if text.len() > MAX_RECENT_LOG_CHUNK_BYTES {
let mut boundary = MAX_RECENT_LOG_CHUNK_BYTES;
while !text.is_char_boundary(boundary) {
boundary -= 1;
}
text.truncate(boundary);
truncated = true;
}
let sequence = self.next_recent_log_sequence;
self.next_recent_log_sequence = self.next_recent_log_sequence.saturating_add(1);
let logs = self
.recent_logs
.entry((tenant.clone(), project.clone()))
.or_default();
let mut dropped = Vec::new();
while logs.iter().filter(|entry| entry.process == process).count()
>= MAX_RECENT_LOG_ENTRIES_PER_PROCESS
{
let Some(index) = logs.iter().position(|entry| entry.process == process) else {
break;
};
if let Some(entry) = logs.remove(index) {
dropped.push(entry);
}
}
while logs.len() >= MAX_RECENT_LOG_ENTRIES_PER_PROJECT
|| logs
.iter()
.map(|entry| entry.text.len())
.sum::<usize>()
.saturating_add(text.len())
> MAX_RECENT_LOG_BYTES_PER_PROJECT
{
match logs.pop_front() {
Some(entry) => dropped.push(entry),
None => break,
}
}
logs.push_back(RecentLogEntry {
sequence,
process,
task,
stream,
text,
server_timestamp_epoch_seconds,
truncated,
});
for entry in dropped {
let key = process_control_key(&tenant, &project, &entry.process);
self.recent_log_dropped_through
.entry(key)
.and_modify(|dropped_through| {
*dropped_through = (*dropped_through).max(entry.sequence);
})
.or_insert(entry.sequence);
}
sequence
}
#[allow(clippy::too_many_arguments)]
fn accept_final_log_stream(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stream: TaskLogStream,
total_source_bytes: u64,
final_tail: &str,
source_truncated: bool,
now_epoch_seconds: u64,
) -> Result<bool, CoordinatorServiceError> {
let key = recent_log_offset_key(tenant, project, process, task, &stream);
let accounted = self
.recent_log_accounted_bytes
.get(&key)
.copied()
.unwrap_or(0);
let remaining = total_source_bytes.checked_sub(accounted).ok_or_else(|| {
let stream_name = match stream {
TaskLogStream::Stdout => "stdout",
TaskLogStream::Stderr => "stderr",
};
CoordinatorServiceError::Protocol(format!(
"final {stream_name} byte count {total_source_bytes} is below the {accounted} live bytes already accounted"
))
})?;
if self.recent_log_quota_truncated_streams.contains(&key) {
self.recent_log_accounted_bytes
.insert(key, total_source_bytes);
return Ok(false);
}
if self
.quota
.charge_log_bytes(tenant, project, remaining, now_epoch_seconds)
.is_err()
{
self.recent_log_accounted_bytes
.insert(key, total_source_bytes);
self.mark_log_quota_truncated(
tenant,
project,
process,
task,
&stream,
now_epoch_seconds,
);
return Ok(false);
}
self.reconcile_final_log_stream(
tenant,
project,
process,
task,
stream,
total_source_bytes,
final_tail,
source_truncated,
now_epoch_seconds,
);
Ok(true)
}
#[allow(clippy::too_many_arguments)]
fn mark_log_quota_truncated(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stream: &TaskLogStream,
now_epoch_seconds: u64,
) -> Option<u64> {
let key = recent_log_offset_key(tenant, project, process, task, stream);
if !self.recent_log_quota_truncated_streams.insert(key.clone()) {
return None;
}
self.recent_log_truncated_streams.insert(key);
Some(self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
"[log output truncated at project log quota]".to_owned(),
true,
now_epoch_seconds,
))
}
#[allow(clippy::too_many_arguments)]
fn reconcile_final_log_stream(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stream: TaskLogStream,
total_source_bytes: u64,
final_tail: &str,
source_truncated: bool,
now_epoch_seconds: u64,
) {
let key = recent_log_offset_key(tenant, project, process, task, &stream);
let accounted = self
.recent_log_accounted_bytes
.get(&key)
.copied()
.unwrap_or(0);
let mut visible_truncation = false;
if total_source_bytes > accounted {
let missing = total_source_bytes - accounted;
if final_tail.is_empty() {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!("[log output unavailable: {missing} source bytes]"),
true,
now_epoch_seconds,
);
visible_truncation = true;
} else if (final_tail.len() as u64) <= total_source_bytes {
let tail_source_start = total_source_bytes - final_tail.len() as u64;
if accounted < tail_source_start {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!(
"[log output lost before final tail: {} source bytes]",
tail_source_start - accounted
),
true,
now_epoch_seconds,
);
visible_truncation = true;
}
let source_start = accounted.max(tail_source_start) - tail_source_start;
let mut byte_start = usize::try_from(source_start)
.unwrap_or(final_tail.len())
.min(final_tail.len());
while byte_start < final_tail.len() && !final_tail.is_char_boundary(byte_start) {
byte_start += 1;
}
let suffix = &final_tail[byte_start..];
if !suffix.is_empty() {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
suffix.to_owned(),
source_truncated || visible_truncation,
now_epoch_seconds,
);
visible_truncation |= source_truncated;
}
} else if accounted == 0 {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
final_tail.to_owned(),
source_truncated,
now_epoch_seconds,
);
visible_truncation |= source_truncated;
} else {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream.clone(),
format!(
"[{missing} additional source bytes could not be merged without duplicating redacted output]"
),
true,
now_epoch_seconds,
);
visible_truncation = true;
}
self.recent_log_accounted_bytes
.insert(key.clone(), total_source_bytes);
}
let marker_is_new = (source_truncated || visible_truncation)
&& self.recent_log_truncated_streams.insert(key);
if marker_is_new && !visible_truncation {
self.record_recent_log(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
stream,
"[log output truncated at source]".to_owned(),
true,
now_epoch_seconds,
);
}
}
pub(super) fn clear_recent_log_offsets_for_task(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
) {
self.recent_log_accounted_bytes.retain(
|(entry_tenant, entry_project, entry_process, entry_task, _), _| {
entry_tenant != tenant
|| entry_project != project
|| entry_process != process
|| entry_task != task
},
);
self.recent_log_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, entry_task, _)| {
entry_tenant != tenant
|| entry_project != project
|| entry_process != process
|| entry_task != task
},
);
self.recent_log_quota_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, entry_task, _)| {
entry_tenant != tenant
|| entry_project != project
|| entry_process != process
|| entry_task != task
},
);
}
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
let Some(attempt) = self
@ -545,6 +1120,118 @@ impl CoordinatorService {
awaiting_operator
}
pub(super) fn maybe_retire_terminal_process(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<bool, CoordinatorServiceError> {
let process_key = process_control_key(tenant, project, process);
if self.main_runtime.controls.contains_key(&process_key) {
return Ok(false);
}
let has_runnable_remote_work =
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == tenant && task_project == project && task_process == process
})
|| self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
})
|| self.task_assignments.values().any(|assignments| {
assignments.iter().any(|assignment| {
&assignment.tenant == tenant
&& &assignment.project == project
&& &assignment.process == process
})
})
|| self.task_attempts.iter().any(
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
attempt_tenant == tenant
&& attempt_project == project
&& attempt_process == process
&& attempts.iter().rev().any(|attempt| {
attempt.current
&& matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
},
);
if has_runnable_remote_work {
return Ok(false);
}
let main_completed = self
.process_summaries
.get(&process_key)
.and_then(|summary| summary.main_terminal_state.as_ref())
.is_some_and(|state| matches!(state, TaskTerminalState::Completed));
let cancellation_completed = self.process_cancellations.contains(&process_key);
if !main_completed && !cancellation_completed {
return Ok(false);
}
self.process_aborts.remove(&process_key);
self.process_cancellations.remove(&process_key);
if self
.coordinator
.active_process(tenant, project, process)
.is_none()
{
return Ok(false);
}
let final_result = if cancellation_completed {
super::ProcessFinalResult::Cancelled
} else if self.task_terminal_states.iter().any(
|((task_tenant, task_project, task_process, _), terminal_state)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& terminal_state == &TaskTerminalState::Failed
},
) {
super::ProcessFinalResult::Failed
} else if self.task_terminal_states.iter().any(
|((task_tenant, task_project, task_process, _), terminal_state)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& terminal_state == &TaskTerminalState::Cancelled
},
) {
super::ProcessFinalResult::Cancelled
} else {
super::ProcessFinalResult::Completed
};
self.record_process_terminal(
tenant,
project,
process,
final_result,
self.current_epoch_seconds()?,
);
self.coordinator.abort_process(tenant, project, process)?;
self.clear_debug_state_for_process(tenant, project, process);
self.clear_operator_panel_state(tenant, project, process);
let (pinned, protected_processes) =
self.artifact_retention_guards_for_project(tenant, project);
self.artifact_registry.enforce_project_metadata_limit(
tenant,
project,
&pinned,
&protected_processes,
);
Ok(true)
}
fn flush_artifact_metadata(
&mut self,
flush: ArtifactFlush,
@ -552,23 +1239,59 @@ impl CoordinatorService {
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let pinned = self
.task_restart_checkpoints
.values()
.flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
.chain(
self.pending_task_launches
.iter()
.flat_map(|pending| pending.task_spec.required_artifacts.iter()),
)
.cloned()
.collect::<std::collections::BTreeSet<ArtifactId>>();
let tenant = flush.tenant.clone();
let project = flush.project.clone();
let (pinned, protected_processes) =
self.artifact_retention_guards_for_project(&tenant, &project);
self.artifact_registry
.flush_metadata_bounded(flush, &pinned)
.flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
.map(|_| ())
.map_err(CoordinatorServiceError::Protocol)
}
fn artifact_retention_guards_for_project(
&self,
tenant: &TenantId,
project: &ProjectId,
) -> (
std::collections::BTreeSet<ArtifactScopeKey>,
std::collections::BTreeSet<ProcessId>,
) {
let mut pinned = std::collections::BTreeSet::new();
for checkpoint in self.task_restart_checkpoints.values() {
if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
{
continue;
}
for artifact in &checkpoint.assignment.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&checkpoint.assignment.tenant,
&checkpoint.assignment.project,
artifact,
));
}
}
for pending in &self.pending_task_launches {
if &pending.tenant != tenant || &pending.project != project {
continue;
}
for artifact in &pending.task_spec.required_artifacts {
pinned.insert(ArtifactScopeKey::from_refs(
&pending.tenant,
&pending.project,
artifact,
));
}
}
let protected_processes = self
.coordinator
.active_processes_for_project(tenant, project)
.into_iter()
.map(|process| process.id)
.collect();
(pinned, protected_processes)
}
fn task_is_known_or_active(
&self,
tenant: &TenantId,
@ -601,15 +1324,24 @@ impl CoordinatorService {
}
}
fn checked_reported_log_bytes(
stdout_bytes: u64,
stderr_bytes: u64,
) -> Result<u64, CoordinatorServiceError> {
stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
CoordinatorServiceError::Protocol(
"reported task log byte counts exceed the supported range".to_owned(),
)
})
fn recent_log_offset_key(
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
stream: &TaskLogStream,
) -> (TenantId, ProjectId, ProcessId, TaskInstanceId, String) {
(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
match stream {
TaskLogStream::Stdout => "stdout",
TaskLogStream::Stderr => "stderr",
}
.to_owned(),
)
}
fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
@ -626,11 +1358,11 @@ fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
return value;
}
let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
while !value.is_char_boundary(boundary) {
boundary -= 1;
let mut boundary = value.len() - MAX_TASK_LOG_TAIL_BYTES;
while boundary < value.len() && !value.is_char_boundary(boundary) {
boundary += 1;
}
value.truncate(boundary);
value.drain(..boundary);
*truncated = true;
value
}

162
crates/clusterflux-coordinator/src/service/main_runtime.rs
View file

@ -30,14 +30,14 @@ use super::{
};
#[derive(Clone)]
struct MainScope {
tenant: TenantId,
project: ProjectId,
process: ProcessId,
task_definition: TaskDefinitionId,
task_instance: TaskInstanceId,
epoch: u64,
launch_id: u64,
pub(super) struct MainScope {
pub(super) tenant: TenantId,
pub(super) project: ProjectId,
pub(super) process: ProcessId,
pub(super) task_definition: TaskDefinitionId,
pub(super) task_instance: TaskInstanceId,
pub(super) epoch: u64,
pub(super) launch_id: u64,
}
enum MainCommand {
@ -109,6 +109,14 @@ impl Default for CoordinatorMainRuntime {
}
impl CoordinatorMainRuntime {
pub(super) fn active_main_count(&self) -> usize {
self.controls.len()
}
pub(super) fn max_active_mains(&self) -> usize {
self.max_active_mains
}
pub(super) fn configure(
&mut self,
configuration: super::CoordinatorMainRuntimeConfiguration,
@ -729,6 +737,13 @@ impl CoordinatorService {
&process,
"coordinator main launch failed admission or validation",
);
self.record_process_terminal(
&tenant,
&project,
&process,
super::ProcessFinalResult::Failed,
self.liveness_now_epoch_seconds(),
);
let _ = self.coordinator.abort_process(&tenant, &project, &process);
}
result
@ -889,7 +904,7 @@ impl CoordinatorService {
}
}
fn record_coordinator_main_completion(
pub(super) fn record_coordinator_main_completion(
&mut self,
scope: MainScope,
result: Result<WasmTaskResult, String>,
@ -929,6 +944,7 @@ impl CoordinatorService {
),
Err(error) => (TaskTerminalState::Failed, None, error),
};
let main_completed = matches!(terminal_state, TaskTerminalState::Completed);
let main_state = match terminal_state {
TaskTerminalState::Completed => "completed",
TaskTerminalState::Failed => "failed",
@ -970,6 +986,12 @@ impl CoordinatorService {
}
}
let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
self.main_runtime.controls.remove(&process_key);
if main_completed {
let _ =
self.maybe_retire_terminal_process(&scope.tenant, &scope.project, &scope.process);
return;
}
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
if task_tenant == scope.tenant
&& task_project == scope.project
@ -985,10 +1007,16 @@ impl CoordinatorService {
}
}
self.process_aborts.insert(process_key.clone());
self.record_process_terminal(
&scope.tenant,
&scope.project,
&scope.process,
super::ProcessFinalResult::Failed,
self.liveness_now_epoch_seconds(),
);
let _ = self
.coordinator
.abort_process(&scope.tenant, &scope.project, &scope.process);
self.main_runtime.controls.remove(&process_key);
self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process);
self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process);
}
@ -1157,6 +1185,120 @@ mod tests {
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
assert!(!service.process_aborts.contains(&process_key));
}
#[test]
fn completed_main_keeps_process_and_debug_state_for_active_children() {
let mut service = CoordinatorService::new(7);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("vp-current");
let main_task = TaskInstanceId::from("ti:vp-current:main");
let child_task = TaskInstanceId::from("ti:vp-current:child:1");
let child_node = NodeId::from("worker");
let scope = MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main_task.clone(),
epoch: 7,
launch_id: 1,
};
service
.coordinator
.start_process(tenant.clone(), project.clone(), process.clone());
let process_key = process_control_key(&tenant, &project, &process);
service.main_runtime.controls.insert(
process_key.clone(),
CoordinatorMainControl {
task_definition: scope.task_definition.clone(),
task_instance: main_task.clone(),
abort: Arc::new(AtomicBool::new(false)),
debug: Arc::new(WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: Arc::new(Mutex::new(HashMap::new())),
launch_id: 1,
},
);
let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
service.active_tasks.insert(child_key.clone());
service.debug_epochs.insert(process_key.clone(), 2);
service.record_coordinator_main_completion(
scope,
Ok(WasmTaskResult::completed(
main_task,
TaskBoundaryValue::SmallJson(serde_json::Value::Null),
)),
);
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_some());
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(service.debug_epochs.contains_key(&process_key));
assert!(!service.process_aborts.contains(&process_key));
assert!(service.active_tasks.contains(&child_key));
assert!(!service.task_aborts.contains(&child_key));
}
#[test]
fn failed_main_aborts_unfinished_children_and_clears_process_debug_state() {
let mut service = CoordinatorService::new(7);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("vp-failed-main");
let main_task = TaskInstanceId::from("ti:vp-failed-main:main");
let child_task = TaskInstanceId::from("ti:vp-failed-main:child:1");
let child_node = NodeId::from("worker");
let scope = MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main_task.clone(),
epoch: 7,
launch_id: 1,
};
service
.coordinator
.start_process(tenant.clone(), project.clone(), process.clone());
let process_key = process_control_key(&tenant, &project, &process);
service.main_runtime.controls.insert(
process_key.clone(),
CoordinatorMainControl {
task_definition: scope.task_definition.clone(),
task_instance: main_task,
abort: Arc::new(AtomicBool::new(false)),
debug: Arc::new(WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: Arc::new(Mutex::new(HashMap::new())),
launch_id: 1,
},
);
let child_key = task_control_key(&tenant, &project, &process, &child_node, &child_task);
service.active_tasks.insert(child_key.clone());
service.debug_epochs.insert(process_key.clone(), 2);
service.record_coordinator_main_completion(scope, Err("main crashed".to_owned()));
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
assert!(service.active_tasks.contains(&child_key));
assert!(service.task_aborts.contains(&child_key));
assert!(service.process_aborts.contains(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(service.task_events.iter().any(|event| {
event.process == process
&& event.executor == TaskExecutor::CoordinatorMain
&& event.terminal_state == TaskTerminalState::Failed
}));
}
}

87
crates/clusterflux-coordinator/src/service/nodes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
SourceProviderKind, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{
bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
@ -27,9 +27,9 @@ impl CoordinatorService {
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
pub(super) fn node_is_live(&self, scope: &NodeScopeKey) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.get(scope)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
@ -41,7 +41,11 @@ impl CoordinatorService {
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor.online = self.node_is_live(&NodeScopeKey::from_refs(
&descriptor.tenant,
&descriptor.project,
&descriptor.id,
));
descriptor
})
.collect()
@ -165,7 +169,11 @@ impl CoordinatorService {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
self.coordinator.ensure_tenant_active(&tenant)?;
if self.coordinator.node_identity(&node).is_none() {
if self
.coordinator
.node_identity(&tenant, &project, &node)
.is_none()
{
self.quota.ensure_node_admission(
&tenant,
self.coordinator.node_identity_count_for_tenant(&tenant),
@ -197,12 +205,21 @@ impl CoordinatorService {
pub(super) fn handle_node_heartbeat(
&mut self,
tenant: String,
project: String,
node: String,
node_signature: Option<NodeSignedRequest>,
payload_digest: &Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
self.authenticate_node_request(
&NodeScopeKey::new(tenant, project, node.clone()),
node_signature,
"node_heartbeat",
payload_digest,
)?;
Ok(CoordinatorResponse::NodeHeartbeat {
node,
epoch: self.coordinator.coordinator_epoch(),
@ -225,16 +242,13 @@ impl CoordinatorService {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node capability report is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
capabilities.validate_public_report()?;
for (kind, count) in [
("cached environments", cached_environment_digests.len()),
@ -274,16 +288,20 @@ impl CoordinatorService {
.into_iter()
.map(ArtifactId::new)
.collect::<BTreeSet<_>>();
self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations);
self.artifact_registry.reconcile_node_retention(
&tenant,
&project,
&node,
&artifact_locations,
);
let online = self.node_is_live(&node);
let online = self.node_is_live(&node_scope);
self.node_descriptors.insert(
node.clone(),
node_scope,
NodeDescriptor {
id: node.clone(),
tenant,
project,
tenant: tenant.clone(),
project: project.clone(),
capabilities,
cached_environments: cached_environment_digests.into_iter().collect(),
dependency_caches: dependency_cache_digests.into_iter().collect(),
@ -293,9 +311,14 @@ impl CoordinatorService {
online,
},
);
let node_descriptors = self
.node_descriptors
.values()
.filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
.count();
Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
node,
node_descriptors: self.node_descriptors.len(),
node_descriptors,
})
}
@ -333,9 +356,13 @@ impl CoordinatorService {
actor: Actor::User(actor.clone()),
};
self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node);
let node_scope = NodeScopeKey::from_refs(&tenant, &project, &node);
let descriptor_removed = self.node_descriptors.remove(&node_scope).is_some();
self.node_last_seen_epoch_seconds.remove(&node_scope);
self.node_replay_nonces
.retain(|(retained_scope, _), _| retained_scope != &node_scope);
self.artifact_registry
.garbage_collect_node(&tenant, &project, &node);
let queued_assignments_removed = self
.task_assignments
.remove(&(tenant.clone(), project.clone(), node.clone()))
@ -369,14 +396,14 @@ impl CoordinatorService {
pub(super) fn authenticate_node_request(
&mut self,
node: &NodeId,
scope: &NodeScopeKey,
node_signature: Option<NodeSignedRequest>,
request_kind: &str,
payload_digest: &Digest,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.node_identity(&scope.tenant, &scope.project, &scope.node)
.ok_or(CoordinatorError::UnknownNode)?;
let signature = node_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
@ -401,7 +428,7 @@ impl CoordinatorService {
)
.into());
}
let replay_key = (node.clone(), signature.nonce.clone());
let replay_key = (scope.clone(), signature.nonce.clone());
self.node_replay_nonces.retain(|_, accepted_at| {
now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
});
@ -413,7 +440,7 @@ impl CoordinatorService {
}
verify_node_request_signature(
&identity.public_key,
node,
&scope.node,
request_kind,
payload_digest,
&signature,
@ -422,7 +449,7 @@ impl CoordinatorService {
if self
.node_replay_nonces
.keys()
.filter(|(retained_node, _)| retained_node == node)
.filter(|(retained_scope, _)| retained_scope == scope)
.count()
>= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
{
@ -436,8 +463,8 @@ impl CoordinatorService {
.insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
.insert(scope.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(scope) {
descriptor.online = true;
}
Ok(())

3
crates/clusterflux-coordinator/src/service/panels.rs
View file

@ -254,7 +254,8 @@ impl CoordinatorService {
.rev()
.find_map(|event| event.artifact_path.as_ref())
{
let artifact = artifact_id_from_path(path);
let artifact = artifact_id_from_path(path)
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
let context = clusterflux_core::AuthContext {
tenant,
project,

158
crates/clusterflux-coordinator/src/service/process_launch.rs
View file

@ -3,9 +3,9 @@ use std::collections::BTreeMap;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskBoundaryValue, TaskCheckpoint,
TaskDispatch, TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath,
WasmTaskInvocation,
NodeDescriptor, NodeId, Placement, PlacementError, PlacementRequest, ProcessId, ProjectId,
Scheduler, TaskBoundaryValue, TaskCheckpoint, TaskDispatch, TaskInstanceId, TaskSpec, TenantId,
VfsManifest, VfsObject, VfsPath, WasmTaskInvocation,
};
use crate::CoordinatorError;
@ -18,7 +18,76 @@ use super::{
use super::processes::*;
use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
fn select_task_placement(
candidates: &[Placement],
active_by_node: &BTreeMap<NodeId, usize>,
) -> Option<Placement> {
candidates
.iter()
.min_by(|left, right| {
right
.score
.cmp(&left.score)
.then_with(|| {
active_by_node
.get(&left.node)
.copied()
.unwrap_or_default()
.cmp(&active_by_node.get(&right.node).copied().unwrap_or_default())
})
.then_with(|| left.node.cmp(&right.node))
})
.cloned()
}
impl CoordinatorService {
fn place_workflow_task(
&self,
nodes: &[NodeDescriptor],
request: &PlacementRequest,
) -> Result<Placement, PlacementError> {
let candidates = nodes
.iter()
.filter_map(|node| {
DefaultScheduler
.place(std::slice::from_ref(node), request)
.ok()
})
.collect::<Vec<_>>();
if candidates.is_empty() {
return DefaultScheduler.place(nodes, request);
}
let active_by_node = self
.active_tasks
.iter()
.filter(|(tenant, project, _, _, _)| {
tenant == &request.tenant && project == &request.project
})
.fold(BTreeMap::<NodeId, usize>::new(), |mut counts, key| {
*counts.entry(key.3.clone()).or_default() += 1;
counts
});
let mut selected = select_task_placement(&candidates, &active_by_node)
.expect("one or more compatible task-placement candidates should remain");
let selected_load = active_by_node
.get(&selected.node)
.copied()
.unwrap_or_default();
if candidates.iter().any(|candidate| {
candidate.score == selected.score
&& active_by_node
.get(&candidate.node)
.copied()
.unwrap_or_default()
> selected_load
}) {
selected.reasons.push(format!(
"least active equal-locality node ({selected_load} active assignment(s))"
));
}
Ok(selected)
}
pub(super) fn capture_task_restart_checkpoint(
&mut self,
assignment: &TaskAssignment,
@ -49,14 +118,14 @@ impl CoordinatorService {
let mut objects = BTreeMap::new();
let mut missing_required_artifact = false;
for artifact in &task_spec.required_artifacts {
let Some(metadata) = self.artifact_registry.metadata(artifact) else {
let Some(metadata) =
self.artifact_registry
.metadata(&assignment.tenant, &assignment.project, artifact)
else {
missing_required_artifact = true;
continue;
};
if metadata.tenant != assignment.tenant
|| metadata.project != assignment.project
|| metadata.retaining_nodes.is_empty()
{
if metadata.retaining_nodes.is_empty() {
missing_required_artifact = true;
continue;
}
@ -414,17 +483,14 @@ impl CoordinatorService {
.validate()
.map_err(CoordinatorServiceError::Protocol)?;
for artifact in &task_spec.required_artifacts {
let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired"
))
})?;
if metadata.tenant != tenant || metadata.project != project {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is outside the task tenant/project scope"
))
.into());
}
let metadata = self
.artifact_registry
.metadata(&tenant, &project, artifact)
.ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired in this tenant/project scope"
))
})?;
if metadata.retaining_nodes.is_empty() {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} has no retaining node"
@ -456,7 +522,7 @@ impl CoordinatorService {
prefer_node: None,
};
let nodes = self.live_node_descriptors();
let placement = match DefaultScheduler.place(&nodes, &request) {
let placement = match self.place_workflow_task(&nodes, &request) {
Ok(placement) => placement,
Err(err) if wait_for_node => {
let reason = if err.message.is_empty() {
@ -479,13 +545,22 @@ impl CoordinatorService {
task_spec,
wasm_module_base64,
});
let queued_tasks = self
.pending_task_launches
.iter()
.filter(|pending| {
pending.tenant == tenant
&& pending.project == project
&& pending.process == process
})
.count();
return Ok(CoordinatorResponse::TaskQueued {
process,
task,
actor,
reason,
charged_spawns,
queued_tasks: self.pending_task_launches.len(),
queued_tasks,
});
}
Err(err) => return Err(err.into()),
@ -596,7 +671,9 @@ impl CoordinatorService {
));
};
self.task_attempts.remove(&removable);
self.task_terminal_states.remove(&removable);
}
self.task_terminal_states.remove(&key);
let attempts = self.task_attempts.entry(key).or_default();
for attempt in attempts.iter_mut() {
attempt.current = false;
@ -694,3 +771,42 @@ fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option<serde_json:
let mut descriptors = super::main_runtime::task_descriptors(&module).ok()?;
descriptors.remove(assignment.task_spec.task_definition.as_str())
}
#[cfg(test)]
mod placement_tests {
use super::*;
fn placement(node: &str, score: i64) -> Placement {
Placement {
node: NodeId::from(node),
score,
reasons: Vec::new(),
}
}
#[test]
fn equal_locality_prefers_the_least_active_node() {
let candidates = vec![placement("busy", 40), placement("idle", 40)];
let active = BTreeMap::from([(NodeId::from("busy"), 1)]);
assert_eq!(
select_task_placement(&candidates, &active)
.expect("one placement")
.node,
NodeId::from("idle")
);
}
#[test]
fn locality_score_remains_stronger_than_load_balancing() {
let candidates = vec![placement("warm", 50), placement("cold", 40)];
let active = BTreeMap::from([(NodeId::from("warm"), 2)]);
assert_eq!(
select_task_placement(&candidates, &active)
.expect("one placement")
.node,
NodeId::from("warm")
);
}
}

127
crates/clusterflux-coordinator/src/service/processes.rs
View file

@ -7,7 +7,7 @@ use clusterflux_core::{
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::keys::{process_control_key, task_control_key};
use super::{
@ -47,14 +47,10 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"task assignment poll is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let assignment_key = (tenant.clone(), project.clone(), node.clone());
let assignment = self
.task_assignments
@ -77,7 +73,11 @@ impl CoordinatorService {
project: &ProjectId,
node: &NodeId,
) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
let Some(descriptor) = self
.node_descriptors
.get(&NodeScopeKey::from_refs(tenant, project, node))
.cloned()
else {
return Ok(None);
};
let mut remaining = VecDeque::new();
@ -233,20 +233,18 @@ impl CoordinatorService {
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.node_identity(&tenant, &project, &node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"source preparation completion is outside the enrolled tenant/project scope"
.to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
debug_assert_eq!(identity.tenant, tenant);
debug_assert_eq!(identity.project, project);
let descriptor = self
.node_descriptors
.get_mut(&NodeScopeKey::from_refs(&tenant, &project, &node))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
if !descriptor.source_snapshots.contains(&source_snapshot)
&& descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
{
@ -273,6 +271,7 @@ impl CoordinatorService {
agent_signature: Option<AgentSignedRequest>,
request_payload_digest: Option<&Digest>,
process: String,
launch_attempt: Option<String>,
restart: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
@ -383,6 +382,10 @@ impl CoordinatorService {
|| attempt_project != &project
|| attempt_process != &process
});
self.task_terminal_states
.retain(|(task_tenant, task_project, task_process, _), _| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.restart_launches
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
attempt_tenant != &tenant
@ -392,10 +395,19 @@ impl CoordinatorService {
self.debug_audit_events.retain(|event| {
event.tenant != tenant || event.project != project || event.process != process
});
self.coordinator
.start_process(tenant, project, process.clone());
let active = self.coordinator.start_process_for_launch_attempt(
tenant.clone(),
project.clone(),
process.clone(),
launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
);
self.record_process_started(&tenant, &project, &process, now_epoch_seconds);
Ok(CoordinatorResponse::ProcessStarted {
process,
launch_attempt: active
.launch_attempt
.as_ref()
.map(|attempt| attempt.as_str().to_owned()),
epoch: self.coordinator.coordinator_epoch(),
actor,
charged_spawns,
@ -404,14 +416,18 @@ impl CoordinatorService {
pub(super) fn handle_reconnect_node(
&mut self,
tenant: String,
project: String,
node: String,
process: String,
epoch: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let process = ProcessId::new(process);
self.coordinator
.reconnect_node(&node, Some((&process, epoch)))?;
.reconnect_node(&tenant, &project, &node, Some((&process, epoch)))?;
Ok(CoordinatorResponse::NodeReconnected { node, process })
}
@ -502,6 +518,13 @@ impl CoordinatorService {
}
let process_key = process_control_key(&tenant, &project, &process);
if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
self.record_process_terminal(
&tenant,
&project,
&process,
super::ProcessFinalResult::Cancelled,
self.current_epoch_seconds()?,
);
self.coordinator
.abort_process(&tenant, &project, &process)?;
self.clear_operator_panel_state(&tenant, &project, &process);
@ -519,6 +542,7 @@ impl CoordinatorService {
project: String,
actor_user: String,
process: String,
launch_attempt: Option<String>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
@ -534,6 +558,17 @@ impl CoordinatorService {
})?;
debug_assert_eq!(active.tenant, tenant);
debug_assert_eq!(active.project, project);
let launch_attempt = launch_attempt.map(clusterflux_core::LaunchAttemptId::new);
if let Some(expected) = launch_attempt.as_ref() {
if active.launch_attempt.as_ref() != Some(expected) {
return Err(CoordinatorError::Unauthorized(format!(
"launch rollback denied: attempt {} does not own process {}",
expected.as_str(),
process.as_str()
))
.into());
}
}
let process_key = process_control_key(&tenant, &project, &process);
self.process_cancellations.remove(&process_key);
@ -575,8 +610,24 @@ impl CoordinatorService {
}
}
self.coordinator
.abort_process(&tenant, &project, &process)?;
self.record_process_terminal(
&tenant,
&project,
&process,
super::ProcessFinalResult::Cancelled,
self.current_epoch_seconds()?,
);
if let Some(launch_attempt) = launch_attempt.as_ref() {
self.coordinator.abort_process_for_launch_attempt(
&tenant,
&project,
&process,
launch_attempt,
)?;
} else {
self.coordinator
.abort_process(&tenant, &project, &process)?;
}
let active_restart_tasks = aborted_tasks
.iter()
.map(|target| target.task.clone())
@ -624,10 +675,18 @@ impl CoordinatorService {
.map(|active| {
let process_key = process_control_key(&active.tenant, &active.project, &active.id);
let main = self.main_runtime.controls.get(&process_key);
let stored = self.process_summaries.get(&process_key);
let stored_main_state = stored
.and_then(|summary| summary.main_terminal_state.as_ref())
.map(|state| match state {
super::TaskTerminalState::Completed => "completed",
super::TaskTerminalState::Failed => "failed",
super::TaskTerminalState::Cancelled => "cancelled",
});
let state = if self.process_cancellations.contains(&process_key) {
"cancelling"
} else {
main.map_or("running", |main| main.state.as_str())
"running"
};
let main_wait_state = main.and_then(|main| {
if main.state != "running" {
@ -652,9 +711,15 @@ impl CoordinatorService {
VirtualProcessStatus {
process: active.id,
state: state.to_owned(),
main_task_definition: main.map(|main| main.task_definition.clone()),
main_task_instance: main.map(|main| main.task_instance.clone()),
main_state: main.map(|main| main.state.clone()),
main_task_definition: main.map(|main| main.task_definition.clone()).or_else(
|| stored.and_then(|summary| summary.main_task_definition.clone()),
),
main_task_instance: main
.map(|main| main.task_instance.clone())
.or_else(|| stored.and_then(|summary| summary.main_task_instance.clone())),
main_state: main
.map(|main| main.state.clone())
.or_else(|| stored_main_state.map(str::to_owned)),
main_wait_state,
main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
connected_nodes: active.connected_nodes.into_iter().collect(),

1350
crates/clusterflux-coordinator/src/service/protocol.rs
View file

File diff suppressed because it is too large Load diff

168
crates/clusterflux-coordinator/src/service/protocol/responses.rs
View file

@ -183,6 +183,121 @@ pub struct VirtualProcessStatus {
pub coordinator_epoch: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeSummary {
pub id: NodeId,
pub display_name: String,
pub online: bool,
pub stale: bool,
pub last_seen_epoch_seconds: Option<u64>,
pub capabilities: NodeCapabilities,
pub direct_connectivity: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessLifecycleState {
Active,
RecentTerminal,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessActivityState {
Running,
WaitingForNode,
WaitingForTask,
AwaitingAction,
DebugEpochPartial,
Cancelling,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ProcessFinalResult {
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpochSummary {
pub epoch: u64,
pub command: String,
pub fully_frozen: bool,
pub partially_frozen: bool,
pub fully_resumed: bool,
pub failed: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProcessSummary {
pub process: ProcessId,
pub lifecycle: ProcessLifecycleState,
pub activity: ProcessActivityState,
pub main_wait_state: Option<String>,
pub started_at_epoch_seconds: u64,
pub ended_at_epoch_seconds: Option<u64>,
pub final_result: Option<ProcessFinalResult>,
pub connected_nodes: Vec<NodeId>,
pub current_debug_epoch: Option<DebugEpochSummary>,
pub order_cursor: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactAvailability {
Available,
NodeOffline,
Unavailable,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArtifactRetentionState {
NodeRetained,
ExplicitStorage,
Lost,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactSummary {
pub id: ArtifactId,
pub display_path: String,
pub display_name: String,
pub process: ProcessId,
pub producer_task: TaskInstanceId,
pub safe_node: Option<NodeId>,
pub digest: Digest,
pub size_bytes: u64,
pub availability: ArtifactAvailability,
pub downloadable_now: bool,
pub retention_state: ArtifactRetentionState,
pub explicit_storage: bool,
pub order_cursor: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskLogStream {
Stdout,
Stderr,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RecentLogEntry {
pub sequence: u64,
pub process: ProcessId,
pub task: TaskInstanceId,
pub stream: TaskLogStream,
pub text: String,
pub server_timestamp_epoch_seconds: u64,
pub truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SourcePreparationDisposition {
Pending { reason: String },
@ -213,7 +328,7 @@ pub enum CoordinatorResponse {
manual_review: bool,
sanitized_reason: Option<String>,
next_actions: Vec<String>,
private_moderation_details_exposed: bool,
sensitive_moderation_details_exposed: bool,
signup_failure_details_exposed: bool,
},
AdminStatus {
@ -282,6 +397,11 @@ pub enum CoordinatorResponse {
descriptors: Vec<NodeDescriptor>,
actor: UserId,
},
NodeSummaries {
nodes: Vec<NodeSummary>,
next_cursor: Option<String>,
actor: UserId,
},
NodeCredentialRevoked {
node: NodeId,
tenant: TenantId,
@ -344,6 +464,8 @@ pub enum CoordinatorResponse {
},
ProcessStarted {
process: ProcessId,
#[serde(default, skip_serializing_if = "Option::is_none")]
launch_attempt: Option<String>,
epoch: u64,
actor: WorkflowActor,
charged_spawns: u64,
@ -371,6 +493,11 @@ pub enum CoordinatorResponse {
processes: Vec<VirtualProcessStatus>,
actor: UserId,
},
ProcessSummaries {
processes: Vec<ProcessSummary>,
next_cursor: Option<String>,
actor: UserId,
},
QuotaStatus {
tenant: TenantId,
project: ProjectId,
@ -428,6 +555,7 @@ pub enum CoordinatorResponse {
DebugBreakpoints {
process: ProcessId,
actor: UserId,
revision: u64,
probe_symbols: Vec<String>,
hit_epoch: Option<u64>,
hit_task: Option<TaskInstanceId>,
@ -480,6 +608,17 @@ pub enum CoordinatorResponse {
stderr_tail: String,
backpressured: bool,
},
TaskLogChunkRecorded {
process: ProcessId,
task: TaskInstanceId,
sequence: Option<u64>,
next_offset: u64,
},
RecentLogs {
entries: Vec<RecentLogEntry>,
next_sequence: Option<u64>,
history_truncated: bool,
},
VfsMetadataRecorded {
process: ProcessId,
task: TaskInstanceId,
@ -516,6 +655,13 @@ pub enum CoordinatorResponse {
ArtifactDownloadLink {
link: DownloadLink,
},
Artifacts {
artifacts: Vec<ArtifactSummary>,
next_cursor: Option<String>,
},
Artifact {
artifact: ArtifactSummary,
},
ArtifactDownloadLinkRevoked {
link: DownloadLink,
},
@ -540,6 +686,24 @@ pub enum CoordinatorResponse {
artifact_size_bytes: u64,
},
Error {
message: String,
#[serde(flatten)]
error: clusterflux_core::ApiError,
},
}
impl CoordinatorResponse {
pub fn error(request_id: impl Into<String>, message: impl Into<String>) -> Self {
Self::Error {
error: clusterflux_core::ApiError::from_message(request_id, message),
}
}
pub fn service_error(
request_id: impl Into<String>,
error: &crate::service::CoordinatorServiceError,
) -> Self {
Self::Error {
error: error.api_error(request_id),
}
}
}

16
crates/clusterflux-coordinator/src/service/quota.rs
View file

@ -260,22 +260,6 @@ impl CoordinatorQuota {
self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
}
pub(super) fn can_charge_log_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), LimitError> {
self.can_charge(
tenant,
project,
LimitKind::LogBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn charge_log_bytes(
&mut self,
tenant: &TenantId,

154
crates/clusterflux-coordinator/src/service/relay.rs
View file

@ -65,6 +65,16 @@ pub struct ArtifactRelayUsage {
pub egress_bytes: u64,
pub abandoned_or_failed_bytes: u64,
pub reserved_bytes: u64,
pub lifetime_ingress_bytes: u64,
pub lifetime_egress_bytes: u64,
pub lifetime_abandoned_or_failed_bytes: u64,
pub completed_transfers: u64,
pub failed_transfers: u64,
pub cancelled_transfers: u64,
pub expired_transfers: u64,
pub tracked_scopes: usize,
pub max_active_global: usize,
pub max_tracked_scopes: usize,
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
@ -77,6 +87,20 @@ pub struct ArtifactRelayDurableState {
pub ingress_used: u64,
pub egress_used: u64,
pub abandoned_or_failed_used: u64,
#[serde(default)]
pub lifetime_ingress_bytes: u64,
#[serde(default)]
pub lifetime_egress_bytes: u64,
#[serde(default)]
pub lifetime_abandoned_or_failed_bytes: u64,
#[serde(default)]
pub completed_transfers: u64,
#[serde(default)]
pub failed_transfers: u64,
#[serde(default)]
pub cancelled_transfers: u64,
#[serde(default)]
pub expired_transfers: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@ -145,6 +169,13 @@ pub(super) struct ArtifactRelayLedger {
ingress_used: u64,
egress_used: u64,
abandoned_or_failed_used: u64,
lifetime_ingress_bytes: u64,
lifetime_egress_bytes: u64,
lifetime_abandoned_or_failed_bytes: u64,
completed_transfers: u64,
failed_transfers: u64,
cancelled_transfers: u64,
expired_transfers: u64,
}
impl Default for ArtifactRelayLedger {
@ -165,6 +196,13 @@ impl ArtifactRelayLedger {
ingress_used: 0,
egress_used: 0,
abandoned_or_failed_used: 0,
lifetime_ingress_bytes: 0,
lifetime_egress_bytes: 0,
lifetime_abandoned_or_failed_bytes: 0,
completed_transfers: 0,
failed_transfers: 0,
cancelled_transfers: 0,
expired_transfers: 0,
}
}
@ -218,6 +256,13 @@ impl ArtifactRelayLedger {
ingress_used: state.ingress_used,
egress_used: state.egress_used,
abandoned_or_failed_used: state.abandoned_or_failed_used,
lifetime_ingress_bytes: state.lifetime_ingress_bytes,
lifetime_egress_bytes: state.lifetime_egress_bytes,
lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
completed_transfers: state.completed_transfers,
failed_transfers: state.failed_transfers,
cancelled_transfers: state.cancelled_transfers,
expired_transfers: state.expired_transfers,
}
}
@ -260,6 +305,13 @@ impl ArtifactRelayLedger {
ingress_used: self.ingress_used,
egress_used: self.egress_used,
abandoned_or_failed_used: self.abandoned_or_failed_used,
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
lifetime_egress_bytes: self.lifetime_egress_bytes,
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
completed_transfers: self.completed_transfers,
failed_transfers: self.failed_transfers,
cancelled_transfers: self.cancelled_transfers,
expired_transfers: self.expired_transfers,
}
}
@ -491,9 +543,11 @@ impl ArtifactRelayLedger {
if ingress {
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
self.ingress_used = self.ingress_used.saturating_add(bytes);
self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
} else {
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
self.egress_used = self.egress_used.saturating_add(bytes);
self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
}
let project_key = (
reservation.scope.tenant.clone(),
@ -554,10 +608,29 @@ impl ArtifactRelayLedger {
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
if let Some(reservation) = self.reservations.remove(key) {
if reason != RelayFinishReason::Completed {
let abandoned_or_failed_bytes = reservation
.ingress_bytes
.saturating_add(reservation.egress_bytes);
self.abandoned_or_failed_used = self
.abandoned_or_failed_used
.saturating_add(reservation.ingress_bytes)
.saturating_add(reservation.egress_bytes);
.saturating_add(abandoned_or_failed_bytes);
self.lifetime_abandoned_or_failed_bytes = self
.lifetime_abandoned_or_failed_bytes
.saturating_add(abandoned_or_failed_bytes);
}
match reason {
RelayFinishReason::Completed => {
self.completed_transfers = self.completed_transfers.saturating_add(1);
}
RelayFinishReason::Failed => {
self.failed_transfers = self.failed_transfers.saturating_add(1);
}
RelayFinishReason::Cancelled => {
self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
}
RelayFinishReason::Expired => {
self.expired_transfers = self.expired_transfers.saturating_add(1);
}
}
}
}
@ -580,6 +653,18 @@ impl ArtifactRelayLedger {
ingress_bytes: self.ingress_used,
egress_bytes: self.egress_used,
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
lifetime_egress_bytes: self.lifetime_egress_bytes,
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
completed_transfers: self.completed_transfers,
failed_transfers: self.failed_transfers,
cancelled_transfers: self.cancelled_transfers,
expired_transfers: self.expired_transfers,
tracked_scopes: self.project_used.len()
+ self.tenant_used.len()
+ self.account_used.len(),
max_active_global: self.configuration.max_active_global,
max_tracked_scopes: self.configuration.max_tracked_scopes,
reserved_bytes: self
.reservations
.values()
@ -588,3 +673,68 @@ impl ArtifactRelayLedger {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
ledger
.reserve(
"transfer".to_owned(),
TenantId::from("tenant"),
ProjectId::from("project"),
UserId::from("user"),
16,
16,
100,
1,
)
.unwrap();
ledger.charge_ingress("transfer", 24, 1).unwrap();
ledger.charge_egress("transfer", 24, 1).unwrap();
ledger.finish("transfer", RelayFinishReason::Completed);
let usage = ledger.usage();
assert_eq!(usage.lifetime_ingress_bytes, 24);
assert_eq!(usage.lifetime_egress_bytes, 24);
assert_eq!(usage.completed_transfers, 1);
ledger.prepare_period(u64::MAX);
let usage = ledger.usage();
assert_eq!(usage.ingress_bytes, 0);
assert_eq!(usage.egress_bytes, 0);
assert_eq!(usage.lifetime_ingress_bytes, 24);
assert_eq!(usage.lifetime_egress_bytes, 24);
let restored = ArtifactRelayLedger::from_durable(
ArtifactRelayConfiguration::unlimited(),
ledger.durable_state(),
);
let usage = restored.usage();
assert_eq!(usage.lifetime_ingress_bytes, 24);
assert_eq!(usage.lifetime_egress_bytes, 24);
assert_eq!(usage.completed_transfers, 1);
}
#[test]
fn older_durable_relay_state_defaults_new_metrics() {
let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
"reservations": {},
"period": 1,
"project_used": [],
"tenant_used": [],
"account_used": [],
"ingress_used": 3,
"egress_used": 4,
"abandoned_or_failed_used": 2
}))
.unwrap();
assert_eq!(state.lifetime_ingress_bytes, 0);
assert_eq!(state.lifetime_egress_bytes, 0);
assert_eq!(state.completed_transfers, 0);
}
}

152
crates/clusterflux-coordinator/src/service/routing.rs
View file

@ -5,6 +5,9 @@ impl CoordinatorService {
&mut self,
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
request
.validate_external_identifiers()
.map_err(CoordinatorServiceError::Protocol)?;
self.pump_main_runtime_commands();
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
@ -42,7 +45,7 @@ impl CoordinatorService {
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
sensitive_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
@ -271,9 +274,17 @@ impl CoordinatorService {
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
tenant,
project,
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
} => self.handle_node_heartbeat(
tenant,
project,
node,
node_signature,
&request_payload_digest,
),
CoordinatorRequest::SignedNode {
node,
node_signature,
@ -287,6 +298,13 @@ impl CoordinatorService {
project,
actor_user,
} => self.handle_list_node_descriptors(tenant, project, actor_user),
CoordinatorRequest::ListNodeSummaries {
tenant,
project,
actor_user,
cursor,
limit,
} => self.handle_list_node_summaries(tenant, project, actor_user, cursor, limit),
CoordinatorRequest::RevokeNodeCredential {
tenant,
project,
@ -370,6 +388,7 @@ impl CoordinatorService {
agent_public_key_fingerprint,
agent_signature,
process,
launch_attempt,
restart,
} => self.handle_start_process(
tenant,
@ -380,6 +399,7 @@ impl CoordinatorService {
agent_signature,
Some(&request_payload_digest),
process,
launch_attempt,
restart,
),
CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
@ -401,12 +421,20 @@ impl CoordinatorService {
project,
actor_user,
process,
} => self.handle_abort_process(tenant, project, actor_user, process),
launch_attempt,
} => self.handle_abort_process(tenant, project, actor_user, process, launch_attempt),
CoordinatorRequest::ListProcesses {
tenant,
project,
actor_user,
} => self.handle_list_processes(tenant, project, actor_user),
CoordinatorRequest::ListProcessSummaries {
tenant,
project,
actor_user,
cursor,
limit,
} => self.handle_list_process_summaries(tenant, project, actor_user, cursor, limit),
CoordinatorRequest::QuotaStatus {
tenant,
project,
@ -452,7 +480,8 @@ impl CoordinatorService {
CoordinatorRequest::PollDebugCommand { .. }
| CoordinatorRequest::ReportDebugState { .. }
| CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportTaskLog { .. }
| CoordinatorRequest::ReportTaskLogChunk { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ListTaskEvents {
@ -467,6 +496,23 @@ impl CoordinatorService {
actor_user,
process,
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
CoordinatorRequest::ListRecentLogs {
tenant,
project,
actor_user,
process,
task,
after_sequence,
limit,
} => self.handle_list_recent_logs(
tenant,
project,
actor_user,
process,
task,
after_sequence,
limit,
),
CoordinatorRequest::JoinTask {
tenant,
project,
@ -513,6 +559,20 @@ impl CoordinatorService {
max_bytes,
ttl_seconds,
),
CoordinatorRequest::ListArtifacts {
tenant,
project,
actor_user,
process,
cursor,
limit,
} => self.handle_list_artifacts(tenant, project, actor_user, process, cursor, limit),
CoordinatorRequest::GetArtifact {
tenant,
project,
actor_user,
artifact,
} => self.handle_get_artifact(tenant, project, actor_user, artifact),
CoordinatorRequest::OpenArtifactDownloadStream {
tenant,
project,
@ -595,7 +655,7 @@ impl CoordinatorService {
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
sensitive_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
@ -682,6 +742,14 @@ impl CoordinatorService {
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::ListNodeSummaries { cursor, limit } => self
.handle_list_node_summaries(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
cursor,
limit,
),
AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
.handle_revoke_node_credential(
context.tenant.as_str().to_owned(),
@ -689,18 +757,22 @@ impl CoordinatorService {
actor.as_str().to_owned(),
node,
),
AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
.handle_start_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
Some(actor.as_str().to_owned()),
None,
None,
None,
None,
process,
restart,
),
AuthenticatedCoordinatorRequest::StartProcess {
process,
launch_attempt,
restart,
} => self.handle_start_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
Some(actor.as_str().to_owned()),
None,
None,
None,
None,
process,
launch_attempt,
restart,
),
request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
self.handle_authenticated_schedule_task(&context, request)
}
@ -714,17 +786,29 @@ impl CoordinatorService {
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
AuthenticatedCoordinatorRequest::AbortProcess {
process,
launch_attempt,
} => self.handle_abort_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
launch_attempt,
),
AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::ListProcessSummaries { cursor, limit } => self
.handle_list_process_summaries(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
cursor,
limit,
),
AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
@ -780,6 +864,20 @@ impl CoordinatorService {
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::ListRecentLogs {
process,
task,
after_sequence,
limit,
} => self.handle_list_recent_logs(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
after_sequence,
limit,
),
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
@ -799,6 +897,24 @@ impl CoordinatorService {
max_bytes,
ttl_seconds,
),
AuthenticatedCoordinatorRequest::ListArtifacts {
process,
cursor,
limit,
} => self.handle_list_artifacts(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
cursor,
limit,
),
AuthenticatedCoordinatorRequest::GetArtifact { artifact } => self.handle_get_artifact(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
),
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
artifact,
max_bytes,

198
crates/clusterflux-coordinator/src/service/signed_nodes.rs
View file

@ -1,6 +1,6 @@
use clusterflux_core::{NodeId, NodeSignedRequest};
use clusterflux_core::{NodeId, NodeSignedRequest, ProjectId, TenantId};
use crate::CoordinatorError;
use crate::{CoordinatorError, NodeScopeKey};
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
@ -12,22 +12,24 @@ impl CoordinatorService {
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request_kind = signed_node_request_kind(&request)?;
let request_node = signed_node_request_node(&request)?;
let request_scope = signed_node_request_scope(&request)?;
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize signed node request: {error}"
))
})?;
let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
let signed_node = NodeId::new(signed_node);
if request_node != signed_node {
let signed_node = NodeId::try_new(signed_node).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid signed node identifier: {error}"))
})?;
if request_scope.node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
)
.into());
}
self.authenticate_node_request(
&signed_node,
&request_scope,
Some(node_signature),
request_kind,
&payload_digest,
@ -144,11 +146,26 @@ impl CoordinatorService {
provider,
source_snapshot,
),
CoordinatorRequest::RequestRendezvous {
scope,
source,
destination,
direct_connectivity,
failure_reason,
} => self.handle_request_rendezvous(
scope,
source,
destination,
direct_connectivity,
failure_reason,
),
CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
process,
epoch,
} => self.handle_reconnect_node(node, process, epoch),
} => self.handle_reconnect_node(tenant, project, node, process, epoch),
CoordinatorRequest::PollTaskControl {
tenant,
project,
@ -236,6 +253,29 @@ impl CoordinatorService {
stderr_truncated,
backpressured,
),
CoordinatorRequest::ReportTaskLogChunk {
tenant,
project,
process,
node,
task,
stream,
offset,
source_bytes,
text,
truncated,
} => self.handle_report_task_log_chunk(
tenant,
project,
process,
node,
task,
stream,
offset,
source_bytes,
text,
truncated,
),
CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
@ -322,12 +362,14 @@ fn signed_node_request_kind(
CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
CoordinatorRequest::RequestRendezvous { .. } => Ok("request_rendezvous"),
CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
CoordinatorRequest::ReportTaskLogChunk { .. } => Ok("report_task_log_chunk"),
CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
_ => Err(CoordinatorError::Unauthorized(
@ -337,29 +379,135 @@ fn signed_node_request_kind(
}
}
fn signed_node_request_node(
fn signed_node_request_scope(
request: &CoordinatorRequest,
) -> Result<NodeId, CoordinatorServiceError> {
) -> Result<NodeScopeKey, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { node, .. }
| CoordinatorRequest::PollTaskAssignment { node, .. }
| CoordinatorRequest::PollArtifactTransfer { node, .. }
| CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
| CoordinatorRequest::FailArtifactTransfer { node, .. }
| CoordinatorRequest::LaunchChildTask { node, .. }
| CoordinatorRequest::JoinChildTask { node, .. }
| CoordinatorRequest::CompleteSourcePreparation { node, .. }
| CoordinatorRequest::ReconnectNode { node, .. }
| CoordinatorRequest::PollTaskControl { node, .. }
| CoordinatorRequest::PollDebugCommand { node, .. }
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
CoordinatorRequest::ReportNodeCapabilities {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollTaskAssignment {
tenant,
project,
node,
}
| CoordinatorRequest::PollArtifactTransfer {
tenant,
project,
node,
}
| CoordinatorRequest::UploadArtifactTransferChunk {
tenant,
project,
node,
..
}
| CoordinatorRequest::FailArtifactTransfer {
tenant,
project,
node,
..
}
| CoordinatorRequest::LaunchChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::JoinChildTask {
tenant,
project,
node,
..
}
| CoordinatorRequest::CompleteSourcePreparation {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReconnectNode {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollTaskControl {
tenant,
project,
node,
..
}
| CoordinatorRequest::PollDebugCommand {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugState {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportDebugProbeHit {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportTaskLog {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportTaskLogChunk {
tenant,
project,
node,
..
}
| CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
node,
..
}
| CoordinatorRequest::TaskCompleted {
tenant,
project,
node,
..
} => node_scope_from_strings(tenant, project, node),
CoordinatorRequest::RequestRendezvous { scope, source, .. } => Ok(NodeScopeKey::new(
scope.tenant.clone(),
scope.project.clone(),
source.node.clone(),
)),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}
fn node_scope_from_strings(
tenant: &str,
project: &str,
node: &str,
) -> Result<NodeScopeKey, CoordinatorServiceError> {
let tenant = TenantId::try_new(tenant.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped tenant identifier: {error}"))
})?;
let project = ProjectId::try_new(project.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped project identifier: {error}"))
})?;
let node = NodeId::try_new(node.to_owned()).map_err(|error| {
CoordinatorServiceError::Protocol(format!("invalid wrapped node identifier: {error}"))
})?;
Ok(NodeScopeKey::new(tenant, project, node))
}

565
crates/clusterflux-coordinator/src/service/summaries.rs Normal file
View file

@ -0,0 +1,565 @@
use std::collections::BTreeSet;
use std::time::Instant;
use clusterflux_core::{
ArtifactId, ArtifactMetadata, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId,
TenantId, UserId,
};
use super::keys::{process_control_key, ProcessControlKey};
use super::{
ArtifactAvailability, ArtifactRetentionState, ArtifactSummary, CoordinatorResponse,
CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugEpochSummary,
NodeSummary, ProcessActivityState, ProcessFinalResult, ProcessLifecycleState, ProcessSummary,
TaskAttemptState,
};
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct StoredProcessSummary {
pub(super) started_at_epoch_seconds: u64,
pub(super) ended_at_epoch_seconds: Option<u64>,
pub(super) final_result: Option<ProcessFinalResult>,
pub(super) connected_nodes: Vec<NodeId>,
pub(super) main_task_definition: Option<TaskDefinitionId>,
pub(super) main_task_instance: Option<TaskInstanceId>,
pub(super) main_terminal_state: Option<super::TaskTerminalState>,
pub(super) order: u64,
}
impl CoordinatorService {
pub(super) fn handle_list_node_summaries(
&mut self,
tenant: String,
project: String,
actor_user: String,
cursor: Option<String>,
limit: u32,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let cursor = cursor.as_deref();
let mut nodes = self
.node_descriptors
.iter()
.filter(|(scope, descriptor)| {
scope.tenant == tenant
&& scope.project == project
&& cursor.is_none_or(|cursor| descriptor.id.as_str() > cursor)
})
.map(|(scope, descriptor)| {
let online = self.node_is_live(scope);
NodeSummary {
id: descriptor.id.clone(),
display_name: descriptor.id.as_str().to_owned(),
online,
stale: !online,
last_seen_epoch_seconds: self.node_last_seen_epoch_seconds.get(scope).copied(),
capabilities: descriptor.capabilities.clone(),
direct_connectivity: descriptor.direct_connectivity,
}
})
.collect::<Vec<_>>();
nodes.sort_by(|left, right| left.id.cmp(&right.id));
let has_more = nodes.len() > limit as usize;
nodes.truncate(limit as usize);
let next_cursor = has_more
.then(|| nodes.last().map(|node| node.id.as_str().to_owned()))
.flatten();
Ok(CoordinatorResponse::NodeSummaries {
nodes,
next_cursor,
actor,
})
}
pub(super) fn record_process_started(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
now_epoch_seconds: u64,
) {
let key = process_control_key(tenant, project, process);
if let Some(logs) = self.recent_logs.get_mut(&(tenant.clone(), project.clone())) {
logs.retain(|entry| &entry.process != process);
if logs.is_empty() {
self.recent_logs.remove(&(tenant.clone(), project.clone()));
}
}
self.recent_log_dropped_through.remove(&key);
self.recent_log_accounted_bytes.retain(
|(entry_tenant, entry_project, entry_process, _, _), _| {
entry_tenant != tenant || entry_project != project || entry_process != process
},
);
self.recent_log_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, _, _)| {
entry_tenant != tenant || entry_project != project || entry_process != process
},
);
self.recent_log_quota_truncated_streams.retain(
|(entry_tenant, entry_project, entry_process, _, _)| {
entry_tenant != tenant || entry_project != project || entry_process != process
},
);
self.process_summary_order
.retain(|retained| retained != &key);
self.evict_process_summaries_for_project(tenant, project);
self.evict_process_summaries_total();
let order = self.next_process_summary_order;
self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
self.process_summaries.insert(
key.clone(),
StoredProcessSummary {
started_at_epoch_seconds: now_epoch_seconds,
ended_at_epoch_seconds: None,
final_result: None,
connected_nodes: Vec::new(),
main_task_definition: None,
main_task_instance: None,
main_terminal_state: None,
order,
},
);
self.process_summary_order.push_back(key);
}
pub(super) fn record_process_terminal(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
final_result: ProcessFinalResult,
now_epoch_seconds: u64,
) {
let key = process_control_key(tenant, project, process);
let connected_nodes = self
.coordinator
.active_process(tenant, project, process)
.map(|active| active.connected_nodes.iter().cloned().collect())
.unwrap_or_default();
let order = self.next_process_summary_order;
let entry = self
.process_summaries
.entry(key.clone())
.or_insert_with(|| {
self.next_process_summary_order = self.next_process_summary_order.saturating_add(1);
self.process_summary_order.push_back(key.clone());
StoredProcessSummary {
started_at_epoch_seconds: now_epoch_seconds,
ended_at_epoch_seconds: None,
final_result: None,
connected_nodes: Vec::new(),
main_task_definition: None,
main_task_instance: None,
main_terminal_state: None,
order,
}
});
entry.ended_at_epoch_seconds = Some(now_epoch_seconds);
entry.final_result = Some(final_result);
entry.connected_nodes = connected_nodes;
}
pub(super) fn record_main_terminal_state(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task_definition: TaskDefinitionId,
task_instance: TaskInstanceId,
terminal_state: super::TaskTerminalState,
) {
let key = process_control_key(tenant, project, process);
if !self.process_summaries.contains_key(&key) {
self.record_process_started(
tenant,
project,
process,
self.liveness_now_epoch_seconds(),
);
}
if let Some(summary) = self.process_summaries.get_mut(&key) {
summary.main_task_definition = Some(task_definition);
summary.main_task_instance = Some(task_instance);
summary.main_terminal_state = Some(terminal_state);
}
}
pub(super) fn handle_list_process_summaries(
&mut self,
tenant: String,
project: String,
actor_user: String,
cursor: Option<String>,
limit: u32,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let cursor = parse_order_cursor(cursor.as_deref(), "process")?;
let mut stored = self
.process_summaries
.iter()
.filter(|((entry_tenant, entry_project, _), summary)| {
entry_tenant == &tenant
&& entry_project == &project
&& cursor.is_none_or(|cursor| summary.order < cursor)
})
.map(|(key, summary)| (key.clone(), summary.clone()))
.collect::<Vec<_>>();
stored.sort_by(|(_, left), (_, right)| right.order.cmp(&left.order));
let has_more = stored.len() > limit as usize;
stored.truncate(limit as usize);
let processes = stored
.into_iter()
.map(|(key, stored)| self.process_summary_from_stored(&key, stored))
.collect::<Vec<_>>();
let next_cursor = has_more
.then(|| processes.last().map(|process| process.order_cursor.clone()))
.flatten();
Ok(CoordinatorResponse::ProcessSummaries {
processes,
next_cursor,
actor,
})
}
fn process_summary_from_stored(
&self,
key: &ProcessControlKey,
stored: StoredProcessSummary,
) -> ProcessSummary {
let (tenant, project, process) = key;
let active = self.coordinator.active_process(tenant, project, process);
let process_key = process_control_key(tenant, project, process);
let main_wait_state = active.and_then(|_| {
if self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
}) {
Some("waiting_for_node".to_owned())
} else if self
.main_runtime
.is_waiting_for_task(tenant, project, process)
{
Some("waiting_for_task".to_owned())
} else {
None
}
});
let current_debug_epoch = self.debug_epoch_summary(&process_key);
let awaiting_action = self.task_attempts.iter().any(
|((attempt_tenant, attempt_project, attempt_process, _), attempts)| {
attempt_tenant == tenant
&& attempt_project == project
&& attempt_process == process
&& attempts.iter().any(|attempt| {
attempt.current && attempt.state == TaskAttemptState::FailedAwaitingAction
})
},
);
let activity = if let Some(result) = &stored.final_result {
match result {
ProcessFinalResult::Completed => ProcessActivityState::Completed,
ProcessFinalResult::Failed => ProcessActivityState::Failed,
ProcessFinalResult::Cancelled => ProcessActivityState::Cancelled,
}
} else if self.process_cancellations.contains(&process_key) {
ProcessActivityState::Cancelling
} else if current_debug_epoch
.as_ref()
.is_some_and(|epoch| epoch.partially_frozen)
{
ProcessActivityState::DebugEpochPartial
} else if awaiting_action {
ProcessActivityState::AwaitingAction
} else {
match main_wait_state.as_deref() {
Some("waiting_for_node") => ProcessActivityState::WaitingForNode,
Some("waiting_for_task") => ProcessActivityState::WaitingForTask,
_ => ProcessActivityState::Running,
}
};
let connected_nodes = active
.map(|active| active.connected_nodes.iter().cloned().collect())
.unwrap_or(stored.connected_nodes);
ProcessSummary {
process: process.clone(),
lifecycle: if active.is_some() {
ProcessLifecycleState::Active
} else {
ProcessLifecycleState::RecentTerminal
},
activity,
main_wait_state,
started_at_epoch_seconds: stored.started_at_epoch_seconds,
ended_at_epoch_seconds: stored.ended_at_epoch_seconds,
final_result: stored.final_result,
connected_nodes,
current_debug_epoch,
order_cursor: format!("process:{}", stored.order),
}
}
fn debug_epoch_summary(&self, key: &ProcessControlKey) -> Option<DebugEpochSummary> {
let runtime = self.debug_epoch_runtime.get(key)?;
let acknowledgements = runtime.acknowledgements.values().collect::<Vec<_>>();
let all_acknowledged = !runtime.expected.is_empty()
&& runtime
.expected
.iter()
.all(|participant| runtime.acknowledgements.contains_key(participant));
let fully_frozen = runtime.command == "freeze"
&& all_acknowledged
&& acknowledgements
.iter()
.all(|ack| ack.state == DebugAcknowledgementState::Frozen);
let freeze_deadline_elapsed =
runtime.command == "freeze" && Instant::now() >= runtime.deadline;
let frozen_count = acknowledgements
.iter()
.filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
.count();
let partially_frozen = freeze_deadline_elapsed && frozen_count > 0 && !fully_frozen;
let fully_resumed = runtime.command == "resume"
&& all_acknowledged
&& acknowledgements
.iter()
.all(|ack| ack.state == DebugAcknowledgementState::Running);
let failed = acknowledgements
.iter()
.any(|ack| ack.state == DebugAcknowledgementState::Failed)
|| (freeze_deadline_elapsed
&& runtime
.expected
.iter()
.any(|participant| !runtime.acknowledgements.contains_key(participant)));
Some(DebugEpochSummary {
epoch: runtime.epoch,
command: runtime.command.clone(),
fully_frozen,
partially_frozen,
fully_resumed,
failed,
})
}
pub(super) fn handle_list_artifacts(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: Option<String>,
cursor: Option<String>,
limit: u32,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = process.map(ProcessId::new);
if let Some(process) = &process {
self.authorize_task_event_process_scope(&tenant, &project, process)?;
}
let cursor = parse_order_cursor(cursor.as_deref(), "artifact")?;
let mut metadata = self
.artifact_registry
.metadata_for_project(&tenant, &project)
.filter(|metadata| {
process
.as_ref()
.is_none_or(|process| &metadata.process == process)
&& cursor.is_none_or(|cursor| metadata.flushed_epoch < cursor)
})
.cloned()
.collect::<Vec<_>>();
metadata.sort_by(|left, right| right.flushed_epoch.cmp(&left.flushed_epoch));
let has_more = metadata.len() > limit as usize;
metadata.truncate(limit as usize);
let artifacts = metadata
.into_iter()
.map(|metadata| self.artifact_summary(metadata))
.collect::<Vec<_>>();
let next_cursor = has_more
.then(|| {
artifacts
.last()
.map(|artifact| artifact.order_cursor.clone())
})
.flatten();
Ok(CoordinatorResponse::Artifacts {
artifacts,
next_cursor,
})
}
pub(super) fn handle_get_artifact(
&mut self,
tenant: String,
project: String,
actor_user: String,
artifact: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let artifact = ArtifactId::new(artifact);
let metadata = self
.artifact_registry
.metadata(&tenant, &project, &artifact)
.cloned()
.ok_or(clusterflux_core::DownloadError::NotFound)?;
Ok(CoordinatorResponse::Artifact {
artifact: self.artifact_summary(metadata),
})
}
fn artifact_summary(&self, metadata: ArtifactMetadata) -> ArtifactSummary {
let live_retaining_nodes = metadata
.retaining_nodes
.iter()
.filter(|node| {
self.node_is_live(&crate::NodeScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
node,
))
})
.cloned()
.collect::<BTreeSet<_>>();
let safe_node = live_retaining_nodes
.iter()
.next()
.cloned()
.or_else(|| metadata.retaining_nodes.iter().next().cloned());
let explicit_storage = !metadata.explicit_locations.is_empty();
let downloadable_now = !live_retaining_nodes.is_empty() || explicit_storage;
let availability = if downloadable_now {
ArtifactAvailability::Available
} else if !metadata.retaining_nodes.is_empty() {
ArtifactAvailability::NodeOffline
} else {
ArtifactAvailability::Unavailable
};
let retention_state = if explicit_storage {
ArtifactRetentionState::ExplicitStorage
} else if !metadata.retaining_nodes.is_empty() {
ArtifactRetentionState::NodeRetained
} else {
ArtifactRetentionState::Lost
};
let display_suffix = metadata.id.as_str().replace(':', "/");
let display_path = format!("/vfs/artifacts/{display_suffix}");
let display_name = display_suffix
.rsplit('/')
.next()
.unwrap_or(metadata.id.as_str())
.to_owned();
ArtifactSummary {
id: metadata.id,
display_path,
display_name,
process: metadata.process,
producer_task: metadata.producer_task,
safe_node,
digest: metadata.digest,
size_bytes: metadata.size,
availability,
downloadable_now,
retention_state,
explicit_storage,
order_cursor: format!("artifact:{}", metadata.flushed_epoch),
}
}
fn evict_process_summaries_for_project(&mut self, tenant: &TenantId, project: &ProjectId) {
while self
.process_summaries
.keys()
.filter(|(entry_tenant, entry_project, _)| {
entry_tenant == tenant && entry_project == project
})
.count()
>= super::MAX_RECENT_PROCESS_SUMMARIES_PER_PROJECT
{
let candidate = self.process_summary_order.iter().find(|key| {
&key.0 == tenant
&& &key.1 == project
&& self
.process_summaries
.get(*key)
.is_some_and(|summary| summary.final_result.is_some())
});
let Some(candidate) = candidate.cloned() else {
break;
};
self.remove_process_summary_state(&candidate);
}
}
fn evict_process_summaries_total(&mut self) {
while self.process_summaries.len() >= super::MAX_RECENT_PROCESS_SUMMARIES_TOTAL {
let candidate = self.process_summary_order.iter().find(|key| {
self.process_summaries
.get(*key)
.is_some_and(|summary| summary.final_result.is_some())
});
let Some(candidate) = candidate.cloned() else {
break;
};
self.remove_process_summary_state(&candidate);
}
}
fn remove_process_summary_state(&mut self, key: &super::ProcessControlKey) {
self.process_summaries.remove(key);
self.recent_log_dropped_through.remove(key);
if let Some(logs) = self.recent_logs.get_mut(&(key.0.clone(), key.1.clone())) {
logs.retain(|entry| entry.process != key.2);
if logs.is_empty() {
self.recent_logs.remove(&(key.0.clone(), key.1.clone()));
}
}
self.recent_log_accounted_bytes
.retain(|(tenant, project, process, _, _), _| {
tenant != &key.0 || project != &key.1 || process != &key.2
});
self.recent_log_truncated_streams
.retain(|(tenant, project, process, _, _)| {
tenant != &key.0 || project != &key.1 || process != &key.2
});
self.recent_log_quota_truncated_streams
.retain(|(tenant, project, process, _, _)| {
tenant != &key.0 || project != &key.1 || process != &key.2
});
self.process_summary_order
.retain(|retained| retained != key);
}
}
fn parse_order_cursor(
cursor: Option<&str>,
expected_kind: &str,
) -> Result<Option<u64>, CoordinatorServiceError> {
cursor
.map(|cursor| {
let (kind, order) = cursor.split_once(':').ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"invalid {expected_kind} pagination cursor"
))
})?;
if kind != expected_kind {
return Err(CoordinatorServiceError::Protocol(format!(
"invalid {expected_kind} pagination cursor"
)));
}
order.parse::<u64>().map_err(|_| {
CoordinatorServiceError::Protocol(format!(
"invalid {expected_kind} pagination cursor"
))
})
})
.transpose()
}

164
crates/clusterflux-coordinator/src/service/tcp.rs
View file

@ -79,17 +79,15 @@ impl CoordinatorService {
continue;
}
let response = match decode_wire_request(&line) {
Ok(request) => match authorize_client_request(&request, authority_mode)
.and_then(|()| self.handle_request(request))
{
Ok(response) => response,
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
Ok((request_id, request)) => {
match authorize_client_request(&request, authority_mode)
.and_then(|()| self.handle_request(request))
{
Ok(response) => response,
Err(err) => CoordinatorResponse::service_error(request_id, &err),
}
}
Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
};
serde_json::to_writer(&mut writer, &response)?;
writer.write_all(b"\n")?;
@ -114,25 +112,19 @@ fn handle_shared_stream(
continue;
}
let response = match decode_wire_request(&line) {
Ok(request) => match authorize_client_request(&request, authority_mode) {
Ok((request_id, request)) => match authorize_client_request(&request, authority_mode) {
Ok(()) => match service.lock() {
Ok(mut service) => match service.handle_request(request) {
Ok(response) => response,
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(_) => CoordinatorResponse::Error {
message: "coordinator service lock poisoned".to_owned(),
Err(err) => CoordinatorResponse::service_error(request_id, &err),
},
Err(_) => {
CoordinatorResponse::error(request_id, "coordinator service lock poisoned")
}
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
Err(err) => CoordinatorResponse::service_error(request_id, &err),
},
Err(err) => CoordinatorResponse::service_error(wire_request_id_hint(&line), &err),
};
serde_json::to_writer(&mut writer, &response)?;
writer.write_all(b"\n")?;
@ -146,12 +138,27 @@ pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), Coordinato
Ok((listener, addr))
}
fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
fn decode_wire_request(
line: &str,
) -> Result<(String, CoordinatorRequest), CoordinatorServiceError> {
serde_json::from_str::<super::CoordinatorWireRequest>(line)?
.into_request()
.into_parts()
.map_err(CoordinatorServiceError::Protocol)
}
fn wire_request_id_hint(line: &str) -> String {
serde_json::from_str::<serde_json::Value>(line)
.ok()
.and_then(|value| {
value
.get("request_id")
.and_then(|value| value.as_str())
.map(str::to_owned)
})
.filter(|request_id| clusterflux_core::RequestId::try_new(request_id.clone()).is_ok())
.unwrap_or_else(|| "unavailable".to_owned())
}
fn authorize_client_request(
request: &CoordinatorRequest,
authority_mode: ClientAuthorityMode,
@ -180,15 +187,21 @@ fn authorize_client_request(
}
| CoordinatorRequest::AdminStatus { .. }
| CoordinatorRequest::SuspendTenant { .. } => Ok(()),
_ => Err(CoordinatorServiceError::Protocol(
_ => Err(crate::CoordinatorError::Unauthorized(
"strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
.to_owned(),
)),
)
.into()),
}
}
#[cfg(test)]
mod transport_boundary_tests {
use std::io::{BufRead as _, BufReader};
use clusterflux_core::{coordinator_wire_request, ProjectId, TenantId, UserId};
use serde_json::json;
use super::*;
#[test]
@ -197,4 +210,99 @@ mod transport_boundary_tests {
let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
assert!(error.to_string().contains("restricted to loopback"));
}
#[test]
fn malformed_identifiers_are_rejected_before_the_shared_service_lock_and_do_not_poison_it() {
let (listener, addr) = bind_listener("127.0.0.1:0").unwrap();
let mut coordinator = CoordinatorService::new(11);
coordinator
.issue_cli_session(
TenantId::from("tenant"),
ProjectId::from("project"),
UserId::from("user"),
"healthy-session",
None,
)
.unwrap();
let shared = Arc::new(Mutex::new(coordinator));
let server_shared = Arc::clone(&shared);
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
handle_shared_stream(server_shared, stream, ClientAuthorityMode::Strict).unwrap();
});
let mut stream = TcpStream::connect(addr).unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
for (index, malformed_process) in [
String::new(),
" ".to_owned(),
"bad\0process".to_owned(),
"bad process!".to_owned(),
"x".repeat(clusterflux_core::MAX_EXTERNAL_ID_BYTES + 1),
]
.into_iter()
.enumerate()
{
let malformed = coordinator_wire_request(
format!("malformed-{index}"),
json!({
"type": "authenticated",
"session_secret": "healthy-session",
"request": {
"type": "abort_process",
"process": malformed_process,
"launch_attempt": "valid-attempt"
}
}),
);
serde_json::to_writer(&mut stream, &malformed).unwrap();
stream.write_all(b"\n").unwrap();
stream.flush().unwrap();
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let CoordinatorResponse::Error { error } =
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
else {
panic!("malformed identifier request unexpectedly succeeded");
};
assert!(
error.message.contains("malformed external identifier")
&& error.message.contains("request.request.process"),
"unexpected malformed identifier response: {}",
error.message
);
assert_eq!(error.request_id, format!("malformed-{index}"));
assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
let valid = coordinator_wire_request(
format!("healthy-{index}"),
json!({
"type": "authenticated",
"session_secret": "healthy-session",
"request": { "type": "auth_status" }
}),
);
serde_json::to_writer(&mut stream, &valid).unwrap();
stream.write_all(b"\n").unwrap();
stream.flush().unwrap();
line.clear();
reader.read_line(&mut line).unwrap();
assert!(
matches!(
serde_json::from_str::<CoordinatorResponse>(&line).unwrap(),
CoordinatorResponse::AuthStatus {
authenticated: true,
..
}
),
"valid authenticated traffic failed after malformed request {index}"
);
}
stream.shutdown(std::net::Shutdown::Both).unwrap();
server.join().unwrap();
assert!(!shared.is_poisoned());
}
}

2762
crates/clusterflux-coordinator/src/service/tests.rs
View file

File diff suppressed because it is too large Load diff

20
crates/clusterflux-coordinator/src/service/wire_protocol.rs
View file

@ -1,4 +1,4 @@
use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
use serde::{Deserialize, Serialize};
use serde_json::Value;
@ -12,8 +12,12 @@ pub enum CoordinatorWireRequest {
impl CoordinatorWireRequest {
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
self.into_parts().map(|(_, request)| request)
}
pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
match self {
Self::Envelope(envelope) => envelope.into_request(),
Self::Envelope(envelope) => envelope.into_parts(),
}
}
}
@ -32,6 +36,10 @@ pub struct CoordinatorRequestEnvelope {
impl CoordinatorRequestEnvelope {
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
self.into_parts().map(|(_, request)| request)
}
pub fn into_parts(self) -> Result<(String, CoordinatorRequest), String> {
if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
return Err(format!(
"unsupported coordinator wire request type {}; expected {}",
@ -44,9 +52,9 @@ impl CoordinatorRequestEnvelope {
self.protocol_version, COORDINATOR_PROTOCOL_VERSION
));
}
if self.request_id.trim().is_empty() {
return Err("coordinator wire request_id must be non-empty".to_owned());
}
RequestId::try_new(self.request_id.clone())
.map_err(|error| format!("malformed coordinator wire request_id: {error}"))?;
self.payload.validate_external_identifiers()?;
let payload_operation = self.payload.operation()?;
if self.operation != payload_operation {
return Err(format!(
@ -54,6 +62,6 @@ impl CoordinatorRequestEnvelope {
self.operation, payload_operation
));
}
Ok(self.payload)
Ok((self.request_id, self.payload))
}
}

296
crates/clusterflux-core/src/api_error.rs Normal file
View file

@ -0,0 +1,296 @@
use serde::{Deserialize, Serialize};
use std::fmt;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ApiErrorCode {
Unauthenticated,
SessionExpired,
AccountSuspended,
Forbidden,
ValidationError,
NotFound,
Conflict,
ActiveProcessExists,
NodeOffline,
NoCapableNode,
TaskNotRestartable,
ArtifactUnavailable,
ArtifactLimitExceeded,
QuotaExceeded,
TemporaryCapacity,
DebugEpochPartial,
InternalError,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ApiErrorCategory {
Authentication,
Authorization,
Validation,
State,
Availability,
Resource,
Internal,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ApiError {
pub code: ApiErrorCode,
pub category: ApiErrorCategory,
pub message: String,
pub retryable: bool,
pub request_id: String,
}
impl fmt::Display for ApiError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"{} (code {:?}, request {})",
self.message, self.code, self.request_id
)
}
}
impl std::error::Error for ApiError {}
impl ApiError {
pub fn new(
code: ApiErrorCode,
category: ApiErrorCategory,
message: impl Into<String>,
retryable: bool,
request_id: impl Into<String>,
) -> Self {
Self {
code,
category,
message: message.into(),
retryable,
request_id: request_id.into(),
}
}
pub fn from_message(request_id: impl Into<String>, message: impl Into<String>) -> Self {
let request_id = request_id.into();
let message = message.into();
let normalized = message.to_ascii_lowercase();
let (code, category, retryable) = if normalized.contains("session credential has expired")
|| normalized.contains("session expired")
{
(
ApiErrorCode::SessionExpired,
ApiErrorCategory::Authentication,
false,
)
} else if normalized.contains("tenant is suspended")
|| normalized.contains("account is suspended")
|| normalized.contains("suspended by hosted")
{
(
ApiErrorCode::AccountSuspended,
ApiErrorCategory::Authorization,
false,
)
} else if normalized.contains("session credential")
|| normalized.contains("no authenticated")
|| normalized.contains("not authenticated")
|| normalized.contains("credential is not")
{
(
ApiErrorCode::Unauthenticated,
ApiErrorCategory::Authentication,
false,
)
} else if normalized.contains("project already has active virtual process") {
(
ApiErrorCode::ActiveProcessExists,
ApiErrorCategory::State,
false,
)
} else if normalized.contains("no capable node") {
(
ApiErrorCode::NoCapableNode,
ApiErrorCategory::Availability,
true,
)
} else if normalized.contains("node offline")
|| normalized.contains("node is not live")
|| normalized.contains("source node is not connected")
|| normalized.contains("direct connectivity unavailable")
{
(
ApiErrorCode::NodeOffline,
ApiErrorCategory::Availability,
true,
)
} else if normalized.contains("restart")
&& (normalized.contains("not restartable")
|| normalized.contains("requires whole")
|| normalized.contains("clean boundary"))
{
(
ApiErrorCode::TaskNotRestartable,
ApiErrorCategory::State,
false,
)
} else if normalized.contains("artifact")
&& (normalized.contains("exceeds download limit")
|| normalized.contains("download session limit")
|| normalized.contains("artifact limit"))
{
(
ApiErrorCode::ArtifactLimitExceeded,
ApiErrorCategory::Resource,
false,
)
} else if normalized.contains("artifact")
&& (normalized.contains("does not exist")
|| normalized.contains("not found")
|| normalized.contains("unknown artifact"))
{
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
} else if normalized.contains("artifact")
&& (normalized.contains("unavailable") || normalized.contains("retention"))
{
(
ApiErrorCode::ArtifactUnavailable,
ApiErrorCategory::Availability,
true,
)
} else if normalized.contains("resource limit")
|| normalized.contains("quota")
|| normalized.contains("limit exceeded")
{
(
ApiErrorCode::QuotaExceeded,
ApiErrorCategory::Resource,
true,
)
} else if normalized.contains("capacity")
|| normalized.contains("temporarily full")
|| normalized.contains("replay window is full")
{
(
ApiErrorCode::TemporaryCapacity,
ApiErrorCategory::Availability,
true,
)
} else if normalized.contains("partial debug epoch")
|| normalized.contains("debug epoch is partially")
{
(
ApiErrorCode::DebugEpochPartial,
ApiErrorCategory::State,
true,
)
} else if normalized.contains("malformed")
|| normalized.contains("invalid ")
|| normalized.contains("protocol")
|| normalized.contains("unknown field")
|| normalized.contains("missing field")
|| normalized.contains("must ")
{
(
ApiErrorCode::ValidationError,
ApiErrorCategory::Validation,
false,
)
} else if normalized.contains("outside")
|| normalized.contains("unauthorized")
|| normalized.contains("denied")
|| normalized.contains("requires an authenticated")
|| normalized.contains("may only")
{
(
ApiErrorCode::Forbidden,
ApiErrorCategory::Authorization,
false,
)
} else if normalized.contains("not found")
|| normalized.contains("does not exist")
|| normalized.contains("unknown ")
{
(ApiErrorCode::NotFound, ApiErrorCategory::State, false)
} else if normalized.contains("already")
|| normalized.contains("conflict")
|| normalized.contains("requires an active")
{
(ApiErrorCode::Conflict, ApiErrorCategory::State, false)
} else {
(
ApiErrorCode::InternalError,
ApiErrorCategory::Internal,
false,
)
};
Self::new(code, category, message, retryable, request_id)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn required_machine_codes_are_stably_serialized() {
let required = [
ApiErrorCode::Unauthenticated,
ApiErrorCode::SessionExpired,
ApiErrorCode::AccountSuspended,
ApiErrorCode::Forbidden,
ApiErrorCode::ValidationError,
ApiErrorCode::NotFound,
ApiErrorCode::Conflict,
ApiErrorCode::ActiveProcessExists,
ApiErrorCode::NodeOffline,
ApiErrorCode::NoCapableNode,
ApiErrorCode::TaskNotRestartable,
ApiErrorCode::ArtifactUnavailable,
ApiErrorCode::ArtifactLimitExceeded,
ApiErrorCode::QuotaExceeded,
ApiErrorCode::TemporaryCapacity,
ApiErrorCode::DebugEpochPartial,
];
let serialized = required
.into_iter()
.map(|code| serde_json::to_value(code).unwrap())
.collect::<Vec<_>>();
assert_eq!(
serialized,
vec![
"unauthenticated",
"session_expired",
"account_suspended",
"forbidden",
"validation_error",
"not_found",
"conflict",
"active_process_exists",
"node_offline",
"no_capable_node",
"task_not_restartable",
"artifact_unavailable",
"artifact_limit_exceeded",
"quota_exceeded",
"temporary_capacity",
"debug_epoch_partial",
]
);
}
#[test]
fn message_classification_keeps_request_identity() {
let error = ApiError::from_message(
"request-17",
"CLI session credential has expired; run login again",
);
assert_eq!(error.code, ApiErrorCode::SessionExpired);
assert_eq!(error.category, ApiErrorCategory::Authentication);
assert_eq!(error.request_id, "request-17");
assert!(!error.retryable);
}
}

749
crates/clusterflux-core/src/artifact.rs
View file

@ -10,7 +10,7 @@ use crate::{
const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
pub coordinator_has_large_bytes: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ArtifactScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub artifact: ArtifactId,
}
impl ArtifactScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
Self {
tenant,
project,
artifact,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
Self::new(tenant.clone(), project.clone(), artifact.clone())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactFlush {
pub id: ArtifactId,
@ -148,7 +169,7 @@ pub struct ArtifactFlush {
#[derive(Clone, Debug, Default)]
pub struct ArtifactRegistry {
artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
next_epoch: u64,
}
@ -162,42 +183,46 @@ impl ArtifactRegistry {
pub fn flush_metadata_bounded(
&mut self,
flush: ArtifactFlush,
pinned: &BTreeSet<ArtifactId>,
pinned: &BTreeSet<ArtifactScopeKey>,
) -> Result<ArtifactMetadata, String> {
let replacing_existing = self.artifacts.contains_key(&flush.id);
while !replacing_existing
&& self
.artifacts
.values()
.filter(|metadata| {
metadata.tenant == flush.tenant
&& metadata.project == flush.project
&& metadata.process == flush.process
})
.count()
>= MAX_ARTIFACT_METADATA_PER_PROCESS
{
let candidate = self
.artifacts
.values()
.filter(|metadata| {
metadata.tenant == flush.tenant
&& metadata.project == flush.project
&& metadata.process == flush.process
&& !pinned.contains(&metadata.id)
&& !self.issued_download_links.values().any(|issued| {
issued.link.artifact == metadata.id
})
})
.min_by_key(|metadata| metadata.flushed_epoch)
.map(|metadata| metadata.id.clone())
self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
}
pub fn flush_metadata_with_protected_processes(
&mut self,
flush: ArtifactFlush,
pinned: &BTreeSet<ArtifactScopeKey>,
protected_processes: &BTreeSet<ProcessId>,
) -> Result<ArtifactMetadata, String> {
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
let replacing = self.artifacts.contains_key(&key);
let retained_for_project = self
.artifacts
.values()
.filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
.count();
let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
None
} else {
let mut protected_processes = protected_processes.clone();
protected_processes.insert(flush.process.clone());
Some(
self.project_metadata_eviction_candidate(
&flush.tenant,
&flush.project,
pinned,
&protected_processes,
)
.ok_or_else(|| {
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
.to_owned()
})?;
self.artifacts.remove(&candidate);
}
self.next_epoch += 1;
format!(
"artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
exhausted by active or retained artifacts"
)
})?,
)
};
self.next_epoch = self.next_epoch.saturating_add(1);
let metadata = ArtifactMetadata {
id: flush.id.clone(),
tenant: flush.tenant,
@ -212,43 +237,137 @@ impl ArtifactRegistry {
explicit_locations: Vec::new(),
coordinator_has_large_bytes: false,
};
self.artifacts.insert(flush.id, metadata.clone());
if let Some(eviction) = eviction {
self.artifacts.remove(&eviction);
}
self.artifacts.insert(key, metadata.clone());
Ok(metadata)
}
pub fn enforce_project_metadata_limit(
&mut self,
tenant: &TenantId,
project: &ProjectId,
pinned: &BTreeSet<ArtifactScopeKey>,
protected_processes: &BTreeSet<ProcessId>,
) -> usize {
let mut evicted = 0;
while self
.artifacts
.values()
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
.count()
> MAX_ARTIFACT_METADATA_PER_PROJECT
{
let candidate = self.project_metadata_eviction_candidate(
tenant,
project,
pinned,
protected_processes,
);
let Some(candidate) = candidate else {
break;
};
self.artifacts.remove(&candidate);
evicted += 1;
}
evicted
}
fn project_metadata_eviction_candidate(
&self,
tenant: &TenantId,
project: &ProjectId,
pinned: &BTreeSet<ArtifactScopeKey>,
protected_processes: &BTreeSet<ProcessId>,
) -> Option<ArtifactScopeKey> {
self.artifacts
.values()
.filter(|metadata| {
&metadata.tenant == tenant
&& &metadata.project == project
&& !pinned.contains(&ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
))
&& !protected_processes.contains(&metadata.process)
&& metadata.explicit_locations.is_empty()
&& !self.issued_download_links.values().any(|issued| {
!issued.revoked
&& issued.link.tenant == metadata.tenant
&& issued.link.project == metadata.project
&& issued.link.artifact == metadata.id
})
})
.min_by_key(|metadata| metadata.flushed_epoch)
.map(|metadata| {
ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
})
}
pub fn sync_to_explicit_store(
&mut self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
location: impl Into<String>,
) -> Result<(), ArtifactUnavailable> {
let metadata = self
.artifacts
.get_mut(artifact)
.get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
.ok_or(ArtifactUnavailable)?;
metadata.explicit_locations.push(location.into());
Ok(())
}
pub fn garbage_collect_node(&mut self, node: &NodeId) {
for metadata in self.artifacts.values_mut() {
pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
for metadata in self
.artifacts
.values_mut()
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
{
metadata.retaining_nodes.remove(node);
}
}
pub fn reconcile_node_retention(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
retained_artifacts: &BTreeSet<ArtifactId>,
) {
for (artifact, metadata) in &mut self.artifacts {
if !retained_artifacts.contains(artifact) {
for (key, metadata) in &mut self.artifacts {
if &key.tenant != tenant || &key.project != project {
continue;
}
if retained_artifacts.contains(&key.artifact) {
metadata.retaining_nodes.insert(node.clone());
} else {
metadata.retaining_nodes.remove(node);
}
}
}
pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
self.artifacts.get(artifact)
pub fn metadata(
&self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
) -> Option<&ArtifactMetadata> {
self.artifacts
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
}
pub fn metadata_for_project<'a>(
&'a self,
tenant: &'a TenantId,
project: &'a ProjectId,
) -> impl Iterator<Item = &'a ArtifactMetadata> + 'a {
self.artifacts
.values()
.filter(move |metadata| &metadata.tenant == tenant && &metadata.project == project)
}
pub fn download_action(
@ -259,7 +378,11 @@ impl ArtifactRegistry {
) -> Result<DownloadAction, DownloadError> {
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let scope = Scope {
tenant: metadata.tenant.clone(),
@ -313,7 +436,11 @@ impl ArtifactRegistry {
self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
Ok(metadata.size)
}
@ -331,7 +458,11 @@ impl ArtifactRegistry {
if self
.issued_download_links
.values()
.filter(|issued| issued.link.artifact == *artifact)
.filter(|issued| {
issued.link.tenant == context.tenant
&& issued.link.project == context.project
&& issued.link.artifact == *artifact
})
.count()
>= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
{
@ -343,7 +474,11 @@ impl ArtifactRegistry {
let action = self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let policy_context_digest =
@ -396,7 +531,11 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact || issued.link.actor != context.actor {
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.actor != context.actor
{
return Err(DownloadError::InvalidToken);
}
self.download_action(
@ -432,7 +571,9 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.max_bytes != policy.max_bytes
|| issued.link.actor != context.actor
{
@ -450,7 +591,11 @@ impl ArtifactRegistry {
}
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
if download_policy_context_digest(metadata, &action.source, policy)
!= issued.link.policy_context_digest
@ -475,7 +620,11 @@ impl ArtifactRegistry {
) -> Result<(), DownloadError> {
let metadata = self
.artifacts
.get(&stream.link.artifact)
.get(&ArtifactScopeKey::from_refs(
&stream.link.tenant,
&stream.link.project,
&stream.link.artifact,
))
.ok_or(DownloadError::NotFound)?;
if !source_is_available(metadata, &stream.link.source) {
return Err(DownloadError::Unavailable);
@ -594,7 +743,13 @@ mod tests {
#[test]
fn flush_publishes_metadata_without_coordinator_bytes() {
let registry = registry_with_artifact();
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(!metadata.coordinator_has_large_bytes);
assert_eq!(metadata.id, ArtifactId::from("artifact"));
@ -613,7 +768,11 @@ mod tests {
#[test]
fn unsynced_node_loss_surfaces_as_unavailable() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -634,20 +793,68 @@ mod tests {
#[test]
fn signed_node_retention_inventory_removes_garbage_collected_location() {
let mut registry = registry_with_artifact();
registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
&BTreeSet::new(),
);
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(metadata.retaining_nodes.is_empty());
}
#[test]
fn signed_node_retention_inventory_restores_readvertised_location() {
let mut registry = registry_with_artifact();
let node = NodeId::from("node");
let artifact = ArtifactId::from("artifact");
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
);
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
&BTreeSet::from([artifact.clone()]),
);
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&artifact,
)
.unwrap();
assert!(metadata.retaining_nodes.contains(&node));
}
#[test]
fn explicit_user_storage_location_survives_node_retention_loss() {
let mut registry = registry_with_artifact();
registry
.sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
.sync_to_explicit_store(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
"s3://bucket/app",
)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -668,7 +875,11 @@ mod tests {
);
assert!(
!registry
.metadata(&ArtifactId::from("artifact"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap()
.coordinator_has_large_bytes
);
@ -699,7 +910,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
@ -719,13 +930,206 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
let artifact = ArtifactId::from("shared-artifact");
let tenant_a = TenantId::from("tenant-a");
let project_a = ProjectId::from("project-a");
let tenant_b = TenantId::from("tenant-b");
let project_b = ProjectId::from("project-b");
let project_c = ProjectId::from("project-c");
let node_a = NodeId::from("node-a");
let node_b = NodeId::from("node-b");
let node_c = NodeId::from("node-c");
let mut registry = ArtifactRegistry::default();
for (tenant, project, process, node, content, size) in [
(&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
(&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
(&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
] {
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::from(process),
producer_task: TaskInstanceId::from("producer"),
retaining_node: node.clone(),
digest: Digest::sha256(content),
size,
});
}
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
assert_eq!(
registry
.metadata(&tenant_a, &project_c, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-c")
);
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant_a.clone(),
project: project_a.clone(),
process: ProcessId::from("process-a"),
producer_task: TaskInstanceId::from("producer-repeat"),
retaining_node: node_a.clone(),
digest: Digest::sha256("bytes-a-repeat"),
size: 18,
});
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a-repeat")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
registry
.sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
.unwrap();
registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
let context_a = AuthContext {
tenant: tenant_a.clone(),
project: project_a.clone(),
actor: Actor::User(UserId::from("user")),
};
let context_b = AuthContext {
tenant: tenant_b.clone(),
project: project_b.clone(),
actor: Actor::User(UserId::from("user")),
};
let policy = DownloadPolicy { max_bytes: 100 };
assert_eq!(
registry
.download_action(&context_a, &artifact, &policy)
.unwrap()
.source,
StorageLocation::ExplicitStore("store://tenant-a".to_owned())
);
assert_eq!(
registry
.download_action(&context_b, &artifact, &policy)
.unwrap()
.source,
StorageLocation::RetainedNode(node_b)
);
let first_a = registry
.create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
let first_b = registry
.create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
assert_ne!(
first_a.scoped_token_digest, first_b.scoped_token_digest,
"the full artifact scope must contribute to download tokens"
);
for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
registry
.create_download_link(
&context_a,
&artifact,
&policy,
&format!("tenant-a-{index}"),
10,
60,
)
.unwrap();
}
assert!(matches!(
registry.create_download_link(
&context_a,
&artifact,
&policy,
"tenant-a-over-limit",
10,
60,
),
Err(DownloadError::Usage(_))
));
registry
.create_download_link(
&context_b,
&artifact,
&policy,
"tenant-b-still-independent",
10,
60,
)
.unwrap();
registry
.revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
.unwrap();
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
};
let mut meter = ResourceMeter::default();
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_b.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.expect("revoking tenant A's link must not revoke tenant B's link");
assert_eq!(
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_a.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.unwrap_err(),
DownloadError::InvalidToken
);
}
#[test]
fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -1002,51 +1406,200 @@ mod tests {
}
#[test]
fn artifact_metadata_is_bounded_without_evicting_pins() {
fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
let mut registry = ArtifactRegistry::default();
let mut pinned = BTreeSet::new();
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("artifact-0"),
tenant: TenantId::from("other-tenant"),
project: ProjectId::from("other-project"),
process: ProcessId::from("process"),
producer_task: TaskInstanceId::from("other-task"),
retaining_node: NodeId::from("other-node"),
digest: Digest::sha256("other-content"),
size: 1,
});
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
let id = ArtifactId::new(format!("artifact-{index}"));
pinned.insert(id.clone());
registry
.flush_metadata_bounded(
ArtifactFlush {
id,
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
producer_task: TaskInstanceId::new(format!("task-{index}")),
retaining_node: NodeId::from("node"),
digest: Digest::sha256(format!("content-{index}")),
size: 1,
},
&pinned,
)
.unwrap();
registry.flush_metadata(ArtifactFlush {
id,
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::new(if index == 3 {
"active-process-3".to_owned()
} else {
format!("completed-process-{index}")
}),
producer_task: TaskInstanceId::new(format!("task-{index}")),
retaining_node: NodeId::from("node"),
digest: Digest::sha256(format!("content-{index}")),
size: 1,
});
}
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
);
let pinned = BTreeSet::from([ArtifactScopeKey::new(
tenant.clone(),
project.clone(),
ArtifactId::from("artifact-0"),
)]);
registry
.sync_to_explicit_store(
&tenant,
&project,
&ArtifactId::from("artifact-1"),
"store://retained-export",
)
.unwrap();
let context = AuthContext {
tenant: tenant.clone(),
project: project.clone(),
actor: Actor::User(UserId::from("user")),
};
registry
.create_download_link(
&context,
&ArtifactId::from("artifact-2"),
&DownloadPolicy { max_bytes: 1 },
"active-download",
10,
60,
)
.unwrap();
let protected_processes = BTreeSet::from([
ProcessId::from("active-process-3"),
ProcessId::from("active-process"),
]);
let next = ArtifactFlush {
id: ArtifactId::from("artifact-next"),
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::from("active-process"),
producer_task: TaskInstanceId::from("task-next"),
retaining_node: NodeId::from("node"),
digest: Digest::sha256("next"),
size: 1,
};
registry
.flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
.unwrap();
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
);
for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
assert!(
registry
.metadata(&tenant, &project, &ArtifactId::from(id))
.is_some(),
"{id} was evicted despite being pinned, exported, downloaded, or live"
);
}
assert!(
registry
.metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
.is_none(),
"the oldest completed unprotected metadata should be evicted"
);
assert!(registry
.flush_metadata_bounded(next.clone(), &pinned)
.unwrap_err()
.contains("pinned"));
pinned.remove(&ArtifactId::from("artifact-0"));
registry.flush_metadata_bounded(next, &pinned).unwrap();
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
assert!(registry
.metadata(&ArtifactId::from("artifact-next"))
.metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
.is_some());
assert_eq!(
registry
.metadata(
&TenantId::from("other-tenant"),
&ProjectId::from("other-project"),
&ArtifactId::from("artifact-0"),
)
.unwrap()
.digest,
Digest::sha256("other-content"),
"eviction and pins in one scope must not affect an equal ID in another scope"
);
}
#[test]
fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
let mut registry = ArtifactRegistry::default();
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let active_process = ProcessId::from("active-process");
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
registry.flush_metadata(ArtifactFlush {
id: ArtifactId::new(format!("artifact-{index}")),
tenant: tenant.clone(),
project: project.clone(),
process: active_process.clone(),
producer_task: TaskInstanceId::new(format!("task-{index}")),
retaining_node: NodeId::from("node"),
digest: Digest::sha256(format!("content-{index}")),
size: 1,
});
}
let replacement = ArtifactFlush {
id: ArtifactId::from("artifact-0"),
tenant: tenant.clone(),
project: project.clone(),
process: active_process.clone(),
producer_task: TaskInstanceId::from("replacement-task"),
retaining_node: NodeId::from("node"),
digest: Digest::sha256("replacement"),
size: 2,
};
registry
.flush_metadata_with_protected_processes(
replacement,
&BTreeSet::new(),
&BTreeSet::from([active_process.clone()]),
)
.expect("replacement must remain possible at the metadata bound");
assert_eq!(
registry
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
.unwrap()
.digest,
Digest::sha256("replacement")
);
let epoch_before_rejection = registry.next_epoch;
let result = registry.flush_metadata_with_protected_processes(
ArtifactFlush {
id: ArtifactId::from("artifact-rejected"),
tenant: tenant.clone(),
project: project.clone(),
process: active_process.clone(),
producer_task: TaskInstanceId::from("rejected-task"),
retaining_node: NodeId::from("node"),
digest: Digest::sha256("rejected"),
size: 3,
},
&BTreeSet::new(),
&BTreeSet::from([active_process]),
);
assert!(result
.unwrap_err()
.contains("exhausted by active or retained artifacts"));
assert_eq!(registry.next_epoch, epoch_before_rejection);
assert_eq!(
registry.metadata_for_project(&tenant, &project).count(),
MAX_ARTIFACT_METADATA_PER_PROJECT
);
assert!(registry
.metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
.is_none());
assert_eq!(
registry
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
.unwrap()
.digest,
Digest::sha256("replacement"),
"rejection must not modify existing metadata"
);
}
#[test]
@ -1135,7 +1688,11 @@ mod tests {
registry
.stream_download_chunk(&mut stream, &limits, &mut meter, 16)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
assert_eq!(
registry

17
crates/clusterflux-core/src/auth.rs
View file

@ -364,7 +364,13 @@ pub fn agent_workflow_request_scope_from_payload(
.get("task_instance")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
(process, Some(TaskInstanceId::from(task)))
(
process,
Some(
TaskInstanceId::try_new(task)
.map_err(|error| format!("malformed launch_task task instance: {error}"))?,
),
)
}
_ => {
return Err(format!(
@ -373,10 +379,13 @@ pub fn agent_workflow_request_scope_from_payload(
}
};
AgentWorkflowRequestScope::new(
TenantId::from(tenant),
ProjectId::from(project),
TenantId::try_new(tenant)
.map_err(|error| format!("malformed agent workflow tenant: {error}"))?,
ProjectId::try_new(project)
.map_err(|error| format!("malformed agent workflow project: {error}"))?,
request_kind,
ProcessId::from(process),
ProcessId::try_new(process)
.map_err(|error| format!("malformed agent workflow process: {error}"))?,
task,
)
}

43
crates/clusterflux-core/src/execution.rs
View file

@ -300,6 +300,27 @@ impl TaskBoundaryValue {
_ => Vec::new(),
}
}
fn rebind_source_snapshot(&mut self, replacement: &Digest) -> usize {
match self {
Self::SourceSnapshot(snapshot) => {
*snapshot = replacement.clone();
1
}
Self::Structured(structured) => structured
.handles
.iter_mut()
.map(|handle| match handle {
TaskBoundaryHandle::SourceSnapshot(snapshot) => {
*snapshot = replacement.clone();
1
}
_ => 0,
})
.sum(),
_ => 0,
}
}
}
pub const WASM_TASK_ABI_VERSION: u32 = 1;
@ -846,6 +867,23 @@ impl TaskSpec {
}
Ok(())
}
pub fn rebind_source_snapshot(&mut self, replacement: Digest) -> Result<(), String> {
self.validate_boundary_authority()?;
if self.source_snapshot.is_none() {
return Err("task has no SourceSnapshot argument handle to rebind".to_owned());
}
let rebound = self
.args
.iter_mut()
.map(|argument| argument.rebind_source_snapshot(&replacement))
.sum::<usize>();
if rebound == 0 {
return Err("task source dependency has no canonical argument handle".to_owned());
}
self.source_snapshot = Some(replacement);
self.validate_boundary_authority()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
@ -1044,6 +1082,11 @@ mod tests {
bundle_digest: Some(Digest::sha256("bundle")),
};
spec.validate_boundary_authority().unwrap();
let replacement_source = Digest::sha256("replacement-source");
spec.rebind_source_snapshot(replacement_source.clone())
.unwrap();
assert_eq!(spec.source_snapshot, Some(replacement_source.clone()));
assert_eq!(spec.args[0].source_snapshots(), vec![replacement_source]);
spec.required_artifacts = vec![ArtifactId::from("forged.bin")];
assert!(spec
.validate_boundary_authority()

180
crates/clusterflux-core/src/ids.rs
View file

@ -1,18 +1,102 @@
#[cfg(not(target_arch = "wasm32"))]
use serde::{de::Error as _, Deserializer};
use serde::{Deserialize, Serialize};
pub const MAX_EXTERNAL_ID_BYTES: usize = 255;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OpaqueTokenError {
reason: String,
}
impl std::fmt::Display for OpaqueTokenError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.reason)
}
}
impl std::error::Error for OpaqueTokenError {}
pub fn validate_opaque_token(value: &str, max_bytes: usize) -> Result<(), OpaqueTokenError> {
let reason = if value.trim().is_empty() {
Some("value must not be empty or whitespace-only".to_owned())
} else if value.len() > max_bytes {
Some(format!("value exceeds the {max_bytes}-byte limit"))
} else if value.chars().any(char::is_control) {
Some("control characters are forbidden".to_owned())
} else {
None
};
reason.map_or(Ok(()), |reason| Err(OpaqueTokenError { reason }))
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct IdParseError {
id_type: &'static str,
reason: &'static str,
}
impl IdParseError {
fn new(id_type: &'static str, reason: &'static str) -> Self {
Self { id_type, reason }
}
}
impl std::fmt::Display for IdParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} is invalid: {}", self.id_type, self.reason)
}
}
impl std::error::Error for IdParseError {}
fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> {
if value.is_empty() || value.trim().is_empty() {
return Err(IdParseError::new(
id_type,
"value must not be empty or whitespace-only",
));
}
if value.len() > MAX_EXTERNAL_ID_BYTES {
return Err(IdParseError::new(
id_type,
"value exceeds the 255-byte limit",
));
}
if value.chars().any(char::is_control) {
return Err(IdParseError::new(
id_type,
"control characters are forbidden",
));
}
if !value.bytes().all(|byte| {
byte.is_ascii_alphanumeric()
|| matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+')
}) {
return Err(IdParseError::new(
id_type,
"only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed",
));
}
Ok(())
}
macro_rules! id_type {
($name:ident) => {
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[cfg_attr(target_arch = "wasm32", derive(Deserialize))]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct $name(String);
impl $name {
pub fn new(value: impl Into<String>) -> Self {
pub fn try_new(value: impl Into<String>) -> Result<Self, IdParseError> {
let value = value.into();
assert!(
!value.trim().is_empty(),
concat!(stringify!($name), " cannot be empty")
);
Self(value)
validate_id(&value, stringify!($name))?;
Ok(Self(value))
}
/// Constructs an identifier from a trusted, internally generated value.
pub fn new(value: impl Into<String>) -> Self {
Self::try_new(value).unwrap_or_else(|error| panic!("{error}"))
}
pub fn as_str(&self) -> &str {
@ -26,6 +110,17 @@ macro_rules! id_type {
}
}
#[cfg(not(target_arch = "wasm32"))]
impl<'de> Deserialize<'de> for $name {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let value = String::deserialize(deserializer)?;
Self::try_new(value).map_err(D::Error::custom)
}
}
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
@ -38,8 +133,79 @@ id_type!(AgentId);
id_type!(ArtifactId);
id_type!(NodeId);
id_type!(ProcessId);
id_type!(LaunchAttemptId);
id_type!(ProjectId);
id_type!(TaskDefinitionId);
id_type!(TaskInstanceId);
id_type!(TenantId);
id_type!(UserId);
pub type DebugSessionId = ProcessId;
pub type RequestId = LaunchAttemptId;
#[cfg(test)]
mod tests {
use super::*;
fn assert_hostile_values<T>(parse: impl Fn(String) -> Result<T, IdParseError>) {
for value in [
String::new(),
" ".to_owned(),
"bad\0id".to_owned(),
"bad id".to_owned(),
"x".repeat(MAX_EXTERNAL_ID_BYTES + 1),
] {
assert!(parse(value).is_err());
}
}
#[test]
fn every_external_identifier_type_rejects_hostile_values() {
assert_hostile_values(AgentId::try_new);
assert_hostile_values(ArtifactId::try_new);
assert_hostile_values(DebugSessionId::try_new);
assert_hostile_values(NodeId::try_new);
assert_hostile_values(ProcessId::try_new);
assert_hostile_values(LaunchAttemptId::try_new);
assert_hostile_values(ProjectId::try_new);
assert_hostile_values(RequestId::try_new);
assert_hostile_values(TaskDefinitionId::try_new);
assert_hostile_values(TaskInstanceId::try_new);
assert_hostile_values(TenantId::try_new);
assert_hostile_values(UserId::try_new);
}
#[test]
fn every_identifier_type_validates_during_deserialization() {
macro_rules! assert_deserialization {
($id:ty) => {
assert!(serde_json::from_str::<$id>(r#""valid-id""#).is_ok());
let error = serde_json::from_str::<$id>(r#""hostile id!""#).unwrap_err();
assert!(
error.to_string().contains("is invalid"),
"{} produced an unexpected error: {error}",
stringify!($id)
);
};
}
assert_deserialization!(AgentId);
assert_deserialization!(ArtifactId);
assert_deserialization!(NodeId);
assert_deserialization!(ProcessId);
assert_deserialization!(LaunchAttemptId);
assert_deserialization!(ProjectId);
assert_deserialization!(TaskDefinitionId);
assert_deserialization!(TaskInstanceId);
assert_deserialization!(TenantId);
assert_deserialization!(UserId);
}
#[test]
fn opaque_tokens_are_bounded_without_using_identifier_syntax() {
validate_opaque_token("opaque secret/+==", 64).unwrap();
assert!(validate_opaque_token("", 64).is_err());
assert!(validate_opaque_token("bad\0token", 64).is_err());
assert!(validate_opaque_token(&"x".repeat(65), 64).is_err());
}
}

11
crates/clusterflux-core/src/lib.rs
View file

@ -1,3 +1,4 @@
mod api_error;
pub mod artifact;
pub mod auth;
pub mod bundle;
@ -18,10 +19,11 @@ pub mod transport;
pub mod vfs;
pub mod wire;
pub use api_error::{ApiError, ApiErrorCategory, ApiErrorCode};
pub use artifact::{
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
DownloadStreamRequest, RetentionPolicy, StorageLocation,
ArtifactScopeKey, ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink,
DownloadPolicy, DownloadStreamRequest, RetentionPolicy, StorageLocation,
};
pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest,
@ -66,8 +68,9 @@ pub use execution::{
WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
};
pub use ids::{
AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
UserId,
validate_opaque_token, AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId,
NodeId, OpaqueTokenError, ProcessId, ProjectId, RequestId, TaskDefinitionId, TaskInstanceId,
TenantId, UserId, MAX_EXTERNAL_ID_BYTES,
};
pub use limits::{
LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,

91
crates/clusterflux-core/src/vfs.rs
View file

@ -1,18 +1,48 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde::{de::Error as _, Deserialize, Deserializer, Serialize};
use thiserror::Error;
use crate::{Digest, NodeId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub const MAX_VFS_PATH_BYTES: usize = 4096;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
pub struct VfsPath(String);
impl VfsPath {
pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
let path = path.into();
if !path.starts_with("/vfs/") {
return Err(VfsError::InvalidPath(path));
return Err(VfsError::invalid(path, "path must start with /vfs/"));
}
if path.len() > MAX_VFS_PATH_BYTES {
return Err(VfsError::invalid(
path,
"path exceeds the 4096-byte protocol limit",
));
}
if path.chars().any(char::is_control) {
return Err(VfsError::invalid(path, "control characters are forbidden"));
}
if path.contains('\\') {
return Err(VfsError::invalid(path, "backslashes are forbidden"));
}
let relative = &path["/vfs/".len()..];
if relative.is_empty() {
return Err(VfsError::invalid(
path,
"path after /vfs/ must not be empty",
));
}
if relative
.split('/')
.any(|component| component.is_empty() || matches!(component, "." | ".."))
{
return Err(VfsError::invalid(
path,
"empty, '.', and '..' path components are forbidden",
));
}
Ok(Self(path))
}
@ -22,6 +52,16 @@ impl VfsPath {
}
}
impl<'de> Deserialize<'de> for VfsPath {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let path = String::deserialize(deserializer)?;
Self::new(path).map_err(D::Error::custom)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsObject {
pub path: VfsPath,
@ -63,12 +103,18 @@ pub enum ReuseDecision {
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum VfsError {
#[error("VFS path must start with /vfs/: {0}")]
InvalidPath(String),
#[error("invalid VFS path {path:?}: {reason}")]
InvalidPath { path: String, reason: &'static str },
#[error("path is not visible in the published VFS manifest: {0}")]
NotVisible(String),
}
impl VfsError {
fn invalid(path: String, reason: &'static str) -> Self {
Self::InvalidPath { path, reason }
}
}
#[derive(Clone, Debug)]
pub struct VfsOverlay {
task: TaskInstanceId,
@ -238,4 +284,39 @@ mod tests {
assert_eq!(overlay.pending_len(), 0);
}
#[test]
fn vfs_path_rejects_the_complete_hostile_protocol_matrix() {
for invalid in [
"vfs/artifacts/app".to_owned(),
"/vfs/".to_owned(),
"/vfs/artifacts//app".to_owned(),
"/vfs/artifacts/app/".to_owned(),
"/vfs/artifacts/./app".to_owned(),
"/vfs/artifacts/../app".to_owned(),
"/vfs/artifacts\\app".to_owned(),
"/vfs/artifacts/bad\0app".to_owned(),
format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES)),
] {
assert!(
VfsPath::new(&invalid).is_err(),
"hostile VFS path unexpectedly passed: {invalid:?}"
);
assert!(
serde_json::from_value::<VfsPath>(serde_json::json!(invalid)).is_err(),
"hostile VFS path unexpectedly deserialized"
);
}
}
#[test]
fn vfs_path_accepts_the_4096_byte_boundary() {
let path = format!("/vfs/{}", "x".repeat(MAX_VFS_PATH_BYTES - "/vfs/".len()));
assert_eq!(path.len(), MAX_VFS_PATH_BYTES);
assert_eq!(VfsPath::new(&path).unwrap().as_str(), path);
let overlong = format!("{path}x");
assert_eq!(overlong.len(), MAX_VFS_PATH_BYTES + 1);
assert!(VfsPath::new(overlong).is_err());
}
}

986
crates/clusterflux-dap/src/adapter.rs
View file

File diff suppressed because it is too large Load diff

51
crates/clusterflux-dap/src/breakpoints.rs
View file

@ -231,10 +231,27 @@ pub(crate) fn stopped_thread_for_breakpoint(state: &AdapterState) -> i64 {
pub(crate) fn position_confirmed_breakpoint_stop(state: &mut AdapterState, stopped_thread: i64) {
let line = state
.breakpoints
.iter()
.copied()
.find(|line| thread_for_source_line(state, *line) == stopped_thread)
.stopped_probe_symbol
.as_deref()
.and_then(|symbol| symbol.strip_prefix("clusterflux.probe."))
.and_then(|function| {
state
.debug_probes
.iter()
.find(|probe| probe.function == function)
.and_then(|probe| {
state.breakpoints.iter().copied().find(|line| {
*line >= i64::from(probe.line_start) && *line <= i64::from(probe.line_end)
})
})
})
.or_else(|| {
state
.breakpoints
.iter()
.copied()
.find(|line| thread_for_source_line(state, *line) == stopped_thread)
})
.or_else(|| state.breakpoints.first().copied());
if let (Some(line), Some(thread)) = (line, state.threads.get_mut(&stopped_thread)) {
thread.line = line;
@ -258,11 +275,29 @@ pub(crate) fn next_breakpoint_after(state: &AdapterState, thread_id: i64) -> Opt
fn thread_for_source_line(state: &AdapterState, line: i64) -> i64 {
if let Some(probe) = debug_probe_for_line(state, line) {
if state.runtime_backend == crate::virtual_model::RuntimeBackend::Simulated {
let function = probe.function.to_ascii_lowercase();
if function.contains("linux") {
return LINUX_THREAD;
}
if function.contains("windows") {
return WINDOWS_THREAD;
}
if function.contains("package") {
return PACKAGE_THREAD;
}
return MAIN_THREAD;
}
return state
.threads
.values()
.find(|thread| thread.task_definition == probe.task)
.map(|thread| thread.id)
.stopped_task
.as_ref()
.and_then(|task| {
state
.threads
.values()
.find(|thread| &thread.task == task)
.map(|thread| thread.id)
})
.unwrap_or(MAIN_THREAD);
}

22
crates/clusterflux-dap/src/main.rs
View file

@ -22,7 +22,7 @@ use breakpoints::{
#[cfg(test)]
use dap_protocol::{initialize_capabilities, read_message};
#[cfg(test)]
use runtime_client::{client_user_request, parse_task_restart_response};
use runtime_client::{client_user_request, parse_task_restart_response, whole_process_status_code};
#[cfg(test)]
use variables::variables_response;
#[cfg(test)]
@ -36,6 +36,26 @@ use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
use virtual_model::{process_id, RuntimeBackend};
fn main() -> Result<()> {
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
match raw_args.as_slice() {
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
println!("clusterflux-debug-dap {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
println!(
"Clusterflux Debug Adapter Protocol server.\n\n\
Usage: clusterflux-debug-dap\n\n\
The adapter communicates over standard input and output.\n\n\
Options:\n \
-h, --help\n \
-V, --version"
);
return Ok(());
}
[] => {}
[argument, ..] => anyhow::bail!("unknown argument: {argument}"),
}
adapter::run_adapter()
}

1091
crates/clusterflux-dap/src/runtime_client.rs
View file

File diff suppressed because it is too large Load diff

4
crates/clusterflux-dap/src/runtime_client/debug_protocol.rs
View file

@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestart
restarted_task_instance: response
.get("restarted_task_instance")
.and_then(Value::as_str)
.map(TaskInstanceId::new),
.map(TaskInstanceId::try_new)
.transpose()
.map_err(|error| anyhow!("malformed restarted task instance: {error}"))?,
restarted_attempt_id: response
.get("restarted_attempt_id")
.and_then(Value::as_str)

51
crates/clusterflux-dap/src/runtime_client/transport.rs
View file

@ -21,18 +21,41 @@ pub(crate) fn client_user_request(state: &AdapterState, mut request: Value) -> V
})
}
pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
let mut session = ControlSession::connect(addr)?;
let wire_request = coordinator_wire_request("dap-1", request);
let response = session.request(&wire_request)?;
if response.get("type").and_then(Value::as_str) == Some("error") {
return Err(anyhow!(
"{}",
response
.get("message")
.and_then(Value::as_str)
.unwrap_or("coordinator returned an error")
));
}
Ok(response)
pub(super) struct CoordinatorSession {
session: ControlSession,
}
impl CoordinatorSession {
pub(super) fn connect(addr: &str) -> Result<Self> {
Ok(Self {
session: ControlSession::connect(addr)?,
})
}
pub(super) fn request(&mut self, request: Value) -> Result<Value> {
let response = self.request_allow_error(request)?;
if response.get("type").and_then(Value::as_str) == Some("error") {
return Err(anyhow!(
"{}",
response
.get("message")
.and_then(Value::as_str)
.unwrap_or("coordinator returned an error")
));
}
Ok(response)
}
pub(super) fn request_allow_error(&mut self, request: Value) -> Result<Value> {
let wire_request = coordinator_wire_request("dap-1", request);
Ok(self.session.request(&wire_request)?)
}
}
pub(super) fn coordinator_request(addr: &str, request: Value) -> Result<Value> {
CoordinatorSession::connect(addr)?.request(request)
}
pub(super) fn coordinator_request_allow_error(addr: &str, request: Value) -> Result<Value> {
CoordinatorSession::connect(addr)?.request_allow_error(request)
}

8
crates/clusterflux-dap/src/source.rs
View file

@ -7,12 +7,12 @@ use serde_json::{json, Value};
use crate::virtual_model::AdapterState;
pub(crate) fn infer_source_path(project: &str) -> String {
if Path::new(project).join("src/build.rs").is_file() {
"src/build.rs".to_owned()
if Path::new(project).join("src/lib.rs").is_file() {
"src/lib.rs".to_owned()
} else if Path::new(project).join("src/main.rs").is_file() {
"src/main.rs".to_owned()
} else if Path::new(project).join("src/lib.rs").is_file() {
"src/lib.rs".to_owned()
} else if Path::new(project).join("src/build.rs").is_file() {
"src/build.rs".to_owned()
} else {
"src/main.rs".to_owned()
}

372
crates/clusterflux-dap/src/tests.rs
View file

@ -45,6 +45,13 @@ fn initialize_capabilities_use_standard_dap_flags() {
.all(|key| key.starts_with("supports") && !key.contains("clusterflux")));
}
#[test]
fn stale_breakpoint_success_and_failure_completions_are_not_current() {
assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9));
assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9));
assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9));
}
#[test]
fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() {
assert_eq!(
@ -189,10 +196,19 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
state.apply_runtime_record(RuntimeLaunchRecord {
coordinator: "127.0.0.1:7999".to_owned(),
node: "node".to_owned(),
node_report: json!({ "terminal_event": {
"task": "compile-linux-1",
"task_definition": "compile-linux"
} }),
node_report: json!({
"task_snapshots": { "snapshots": [{
"task": "compile-linux-1",
"task_definition": "compile-linux",
"attempt_id": "attempt-1",
"current": true,
"state": "failed_awaiting_action"
}] },
"terminal_event": {
"task": "compile-linux-1",
"task_definition": "compile-linux"
}
}),
task_events: json!({ "events": [] }),
placed_task_launched: true,
status_code: Some(1),
@ -209,20 +225,25 @@ fn nonzero_local_runtime_record_marks_linux_task_failed() {
stopped_probe_symbol: None,
all_participants_frozen: false,
});
let linux = state
let exact_runtime_thread = state
.threads
.get(&LINUX_THREAD)
.expect("linux task should exist");
.values()
.find(|thread| thread.task == TaskInstanceId::from("compile-linux-1"))
.expect("the exact terminal task instance should have its own thread");
assert!(state.last_task_failed);
assert!(state
.command_status
.contains("failed through local services"));
assert!(matches!(linux.state, DebugRuntimeState::Failed(_)));
assert!(linux
assert!(matches!(
exact_runtime_thread.state,
DebugRuntimeState::Failed(_)
));
assert!(exact_runtime_thread
.recent_output
.iter()
.any(|line| line.contains("task failed")));
assert_eq!(state.threads.len(), 1);
}
#[test]
@ -275,7 +296,7 @@ fn breakpoint_resolution_uses_bundle_debug_probe_metadata() {
let mut state = AdapterState::default();
state.debug_probes = vec![BundleDebugProbe {
id: "probe-linux".to_owned(),
source_path: "src/build.rs".to_owned(),
source_path: "src/lib.rs".to_owned(),
line_start: 20,
line_end: 30,
function: "compile_linux".to_owned(),
@ -303,7 +324,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
let mut state = AdapterState::default();
state.project = project.path().to_string_lossy().into_owned();
state.source_path = "src/build.rs".to_owned();
state.source_path = "src/lib.rs".to_owned();
state.breakpoints = vec![1];
let requested_source = source_dir.join("main.rs");
@ -311,9 +332,7 @@ fn breakpoint_updates_from_another_source_do_not_replace_configured_breakpoints(
assert_eq!(resolved.len(), 1);
assert!(!resolved[0].verified);
assert!(resolved[0]
.message
.contains("configured for `src/build.rs`"));
assert!(resolved[0].message.contains("configured for `src/lib.rs`"));
assert_eq!(state.breakpoints, vec![1]);
}
@ -358,7 +377,11 @@ fn windows_thread_runtime_variables_share_virtual_process() {
variable["name"] == "virtual_process_id" && variable["value"] == state.process.to_string()
}));
assert!(variables.iter().any(|variable| {
variable["name"] == "virtual_thread" && variable["value"] == "compile windows"
variable["name"] == "virtual_thread" && variable["value"] == "compile-windows"
}));
assert!(variables.iter().any(|variable| {
variable["name"] == "task_attempt_id"
&& variable["value"] == format!("demo-attempt-{WINDOWS_THREAD}")
}));
}
@ -369,10 +392,19 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
state.apply_runtime_record(RuntimeLaunchRecord {
coordinator: "127.0.0.1:7999".to_owned(),
node: "node".to_owned(),
node_report: json!({ "terminal_event": {
"task": "compile-linux-1",
"task_definition": "compile-linux"
} }),
node_report: json!({
"task_snapshots": { "snapshots": [{
"task": "compile-linux-1",
"task_definition": "compile-linux",
"attempt_id": "attempt-1",
"current": true,
"state": "running"
}] },
"terminal_event": {
"task": "compile-linux-1",
"task_definition": "compile-linux"
}
}),
task_events: json!({ "events": [] }),
placed_task_launched: true,
status_code: Some(0),
@ -389,21 +421,28 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
stopped_probe_symbol: None,
all_participants_frozen: false,
});
let exact_thread_id = state
.threads
.values()
.find(|thread| thread.task == TaskInstanceId::from("compile-linux-1"))
.map(|thread| thread.id)
.expect("the exact terminal task instance should have its own thread");
{
let linux = state.threads.get_mut(&LINUX_THREAD).unwrap();
linux.runtime_task_args = vec![("source".to_owned(), "source://checkout".to_owned())];
linux.runtime_handles = vec![(
let runtime_thread = state.threads.get_mut(&exact_thread_id).unwrap();
runtime_thread.runtime_task_args =
vec![("source".to_owned(), "source://checkout".to_owned())];
runtime_thread.runtime_handles = vec![(
"artifact".to_owned(),
"artifact://runtime/output".to_owned(),
)];
linux.runtime_command_status = Some("frozen native command pid 42".to_owned());
runtime_thread.runtime_command_status = Some("frozen native command pid 42".to_owned());
}
let linux = state
let runtime_thread = state
.threads
.get(&LINUX_THREAD)
.expect("linux task should exist");
.get(&exact_thread_id)
.expect("exact runtime thread should exist");
let args = variables_response(&state, linux.args_ref);
let args = variables_response(&state, runtime_thread.args_ref);
let args = args["variables"]
.as_array()
.expect("variables response should be an array");
@ -418,7 +457,7 @@ fn variables_expose_mvp_runtime_handles_and_output_tails() {
&& variable["type"] == "runtime-handle"
}));
let runtime = variables_response(&state, linux.runtime_ref);
let runtime = variables_response(&state, runtime_thread.runtime_ref);
let runtime = runtime["variables"].as_array().unwrap();
let command_ref = runtime
.iter()
@ -634,14 +673,73 @@ fn terminal_events_do_not_resurrect_threads_absent_from_current_snapshots() {
}
#[test]
fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
fn terminal_record_without_snapshots_clears_active_threads() {
let mut state = AdapterState::default();
let project = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../examples/launch-build-demo")
.canonicalize()
state.runtime_backend = RuntimeBackend::LiveServices;
assert!(!state.threads.is_empty());
state.apply_runtime_record(RuntimeLaunchRecord {
coordinator: "127.0.0.1:9443".to_owned(),
node: "coordinator-main".to_owned(),
node_report: json!({ "terminal_event": { "terminal_state": "completed" } }),
task_events: json!({ "events": [] }),
placed_task_launched: true,
status_code: Some(0),
stdout_bytes: 0,
stderr_bytes: 0,
stdout_tail: String::new(),
stderr_tail: String::new(),
stdout_truncated: false,
stderr_truncated: false,
artifact_path: None,
event_count: 0,
debug_epoch: None,
stopped_task: None,
stopped_probe_symbol: None,
all_participants_frozen: false,
});
assert!(state.threads.is_empty());
}
#[test]
fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
let project = std::env::temp_dir().join(format!(
"clusterflux-dap-source-locals-{}",
std::process::id()
));
let src = project.join("src");
fs::create_dir_all(&src).unwrap();
fs::write(
src.join("lib.rs"),
r#"async fn run_build_workflow() {
let source = prepare_source_snapshot();
let linux = clusterflux::spawn::async_task_with_arg(source.clone(), compile_linux)
.name("compile linux")
.env(linux_env())
.start()
.await
.unwrap();
let linux_thread = linux.virtual_thread_id();
let linux_artifact = linux.join().await.unwrap();
let package = clusterflux::spawn::async_task_with_arg(
vec![linux_artifact.clone()],
package_release,
)
.name("package artifacts")
.env(linux_env())
.start()
.await
.unwrap();
let package_artifact = package.join().await.unwrap();
}
"#,
)
.unwrap();
let mut state = AdapterState::default();
state.project = project.to_string_lossy().into_owned();
state.source_path = "src/build.rs".to_owned();
state.source_path = "src/lib.rs".to_owned();
let source = fs::read_to_string(project.join(&state.source_path)).unwrap();
let inspection_line = source
.lines()
@ -675,6 +773,8 @@ fn source_locals_infer_clusterflux_api_values_from_runtime_state() {
variable["name"] == "unavailable-local-diagnostic"
&& variable["type"] == "unavailable-local"
}));
let _ = fs::remove_dir_all(project);
}
#[test]
@ -794,12 +894,21 @@ fn package_release(inputs: Vec<Artifact>) -> Artifact {
#[test]
fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
let project = std::env::temp_dir().join(format!(
"clusterflux-dap-wasm-locals-{}",
std::process::id()
));
let src = project.join("src");
fs::create_dir_all(&src).unwrap();
fs::write(
src.join("lib.rs"),
"pub extern \"C\" fn task_add_one(input: i32) -> i32 {\n input + 1\n}\n",
)
.unwrap();
let mut state = AdapterState::default();
state.project = Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../examples/launch-build-demo")
.to_string_lossy()
.into_owned();
state.source_path = "src/build.rs".to_owned();
state.project = project.to_string_lossy().into_owned();
state.source_path = "src/lib.rs".to_owned();
let source = fs::read_to_string(Path::new(&state.project).join(&state.source_path)).unwrap();
state.threads.get_mut(&MAIN_THREAD).unwrap().line = source
.lines()
@ -826,6 +935,8 @@ fn wasm_frame_locals_expose_only_values_from_the_node_snapshot() {
assert!(!locals
.iter()
.any(|variable| variable["name"] == "wasm-local-diagnostic"));
let _ = fs::remove_dir_all(project);
}
#[test]
@ -852,3 +963,188 @@ fn detects_current_failed_attempt_awaiting_operator_action() {
Some(("task-current", "attempt-current"))
);
}
#[test]
fn whole_process_terminal_status_covers_every_current_logical_task_attempt() {
let snapshots = |items: serde_json::Value| json!({ "snapshots": items });
assert_eq!(
whole_process_status_code(
Some(0),
&snapshots(json!([
{
"task": "child",
"current": true,
"state": "completed",
"status_code": 0
}
]))
),
Some(0),
"a successful main and final child must succeed"
);
for (state, status_code) in [("failed", 23), ("cancelled", 1)] {
assert_eq!(
whole_process_status_code(
Some(0),
&snapshots(json!([
{
"task": "child",
"current": true,
"state": state,
"status_code": status_code
}
]))
),
Some(status_code),
"a terminal child must determine the whole-process result"
);
}
assert_eq!(
whole_process_status_code(
Some(0),
&snapshots(json!([
{
"task": "accepted-failure",
"current": true,
"state": "failed",
"command_state": "failure_accepted",
"status_code": 17
}
]))
),
Some(17),
"accepting an AwaitOperator failure releases the process but does not turn failure into success"
);
assert_eq!(
whole_process_status_code(
Some(0),
&snapshots(json!([
{
"task": "restarted",
"attempt_id": "attempt-1",
"current": false,
"state": "failed",
"status_code": 7
},
{
"task": "restarted",
"attempt_id": "attempt-2",
"current": true,
"state": "completed",
"status_code": 0
}
]))
),
Some(0),
"a successful replacement attempt is authoritative over stale failed attempts"
);
assert_eq!(
whole_process_status_code(
Some(9),
&snapshots(json!([
{
"task": "child",
"current": true,
"state": "completed",
"status_code": 0
}
]))
),
Some(9),
"a failed coordinator main cannot be masked by successful children"
);
}
#[test]
fn exact_task_instance_ids_keep_stable_dap_threads_across_snapshot_reordering_and_retry() {
let mut state = AdapterState::default();
state.runtime_backend = RuntimeBackend::LiveServices;
let record = |snapshots: serde_json::Value| RuntimeLaunchRecord {
coordinator: "127.0.0.1:9443".to_owned(),
node: "node-a".to_owned(),
node_report: json!({
"task_snapshots": { "snapshots": snapshots },
"process_status": null
}),
task_events: json!({ "events": [] }),
placed_task_launched: true,
status_code: None,
stdout_bytes: 0,
stderr_bytes: 0,
stdout_tail: String::new(),
stderr_tail: String::new(),
stdout_truncated: false,
stderr_truncated: false,
artifact_path: None,
event_count: 0,
debug_epoch: None,
stopped_task: None,
stopped_probe_symbol: None,
all_participants_frozen: false,
};
state.apply_attach_record(record(json!([
{
"task": "lane-stable",
"task_definition": "build_lane",
"attempt_id": "stable-1",
"current": true,
"state": "running"
},
{
"task": "lane-recovering",
"task_definition": "build_lane",
"attempt_id": "recovering-1",
"current": true,
"state": "failed_awaiting_action"
}
])));
let stable_id = state
.threads
.values()
.find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
.unwrap()
.id;
let recovering_id = state
.threads
.values()
.find(|thread| thread.task == TaskInstanceId::from("lane-recovering"))
.unwrap()
.id;
assert_ne!(stable_id, recovering_id);
state.apply_runtime_record(record(json!([
{
"task": "lane-new",
"task_definition": "build_lane",
"attempt_id": "new-1",
"current": true,
"state": "running"
},
{
"task": "lane-stable",
"task_definition": "build_lane",
"attempt_id": "stable-2",
"current": true,
"state": "running"
}
])));
let stable = state
.threads
.values()
.find(|thread| thread.task == TaskInstanceId::from("lane-stable"))
.unwrap();
let new_lane = state
.threads
.values()
.find(|thread| thread.task == TaskInstanceId::from("lane-new"))
.unwrap();
assert_eq!(stable.id, stable_id);
assert_eq!(stable.attempt_id, "stable-2");
assert!(state
.threads
.values()
.all(|thread| thread.task != TaskInstanceId::from("lane-recovering")));
assert!(new_lane.id > recovering_id);
}

2
crates/clusterflux-dap/src/variables.rs
View file

@ -149,7 +149,7 @@ pub(crate) fn variables_response(state: &AdapterState, reference: i64) -> Value
},
{
"name": "virtual_thread",
"value": thread.name,
"value": thread.task.to_string(),
"variablesReference": 0
},
{

236
crates/clusterflux-dap/src/virtual_model.rs
View file

@ -66,11 +66,16 @@ pub(crate) struct AdapterState {
pub(crate) last_task_failed: bool,
pub(crate) runtime_artifact_path: Option<String>,
pub(crate) runtime_event_count: usize,
pub(crate) runtime_snapshot_fingerprint: String,
pub(crate) next_thread_id: i64,
pub(crate) coordinator_debug_epoch: Option<u64>,
pub(crate) debug_all_threads_stopped: bool,
pub(crate) stopped_task: Option<TaskInstanceId>,
pub(crate) stopped_probe_symbol: Option<String>,
pub(crate) debug_probes: Vec<BundleDebugProbe>,
pub(crate) breakpoints: Vec<i64>,
pub(crate) breakpoints_installed: bool,
pub(crate) breakpoint_revision: u64,
pub(crate) threads: BTreeMap<i64, VirtualThread>,
}
@ -107,9 +112,9 @@ impl Default for AdapterState {
threads: launch_threads(&entry),
entry,
project,
source_path: "src/build.rs".to_owned(),
source_path: "src/lib.rs".to_owned(),
runtime_backend: RuntimeBackend::Simulated,
coordinator_endpoint: "https://clusterflux.michelpaulissen.com".to_owned(),
coordinator_endpoint: "https://clusterflux.lesstuff.com".to_owned(),
tenant: TenantId::from("tenant"),
project_id: ProjectId::from("project"),
actor_user: UserId::from("dap"),
@ -120,11 +125,16 @@ impl Default for AdapterState {
last_task_failed: false,
runtime_artifact_path: None,
runtime_event_count: 0,
runtime_snapshot_fingerprint: String::new(),
next_thread_id: LINUX_THREAD,
coordinator_debug_epoch: None,
debug_all_threads_stopped: true,
stopped_task: None,
stopped_probe_symbol: None,
debug_probes: Vec::new(),
breakpoints: Vec::new(),
breakpoints_installed: false,
breakpoint_revision: 0,
}
}
}
@ -190,18 +200,24 @@ impl AdapterState {
}
self.coordinator_endpoint =
crate::view_state::normalize_coordinator_endpoint(&session.coordinator);
self.tenant = TenantId::new(session.tenant);
self.project_id = ProjectId::new(session.project);
self.actor_user = UserId::new(session.user);
self.tenant = TenantId::try_new(session.tenant)
.map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?;
self.project_id = ProjectId::try_new(session.project)
.map_err(|error| anyhow!("invalid project in CLI session: {error}"))?;
self.actor_user = UserId::try_new(session.user)
.map_err(|error| anyhow!("invalid user in CLI session: {error}"))?;
self.client_session_secret = Some(session.session_secret);
} else {
self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(
coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"),
);
if let Some(scope) = project_scope {
self.tenant = TenantId::new(scope.tenant);
self.project_id = ProjectId::new(scope.project);
self.actor_user = UserId::new(scope.user);
self.tenant = TenantId::try_new(scope.tenant)
.map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?;
self.project_id = ProjectId::try_new(scope.project)
.map_err(|error| anyhow!("invalid project in project scope: {error}"))?;
self.actor_user = UserId::try_new(scope.user)
.map_err(|error| anyhow!("invalid user in project scope: {error}"))?;
}
self.client_session_secret = None;
}
@ -211,13 +227,18 @@ impl AdapterState {
self.last_task_failed = false;
self.runtime_artifact_path = None;
self.runtime_event_count = 0;
self.runtime_snapshot_fingerprint.clear();
self.next_thread_id = LINUX_THREAD;
self.coordinator_debug_epoch = None;
self.debug_all_threads_stopped = true;
self.stopped_task = None;
self.stopped_probe_symbol = None;
self.debug_probes =
crate::breakpoints::load_bundle_debug_probes(&self.project, &self.source_path);
self.epoch = 0;
self.breakpoints.clear();
self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated;
self.breakpoint_revision = 0;
self.threads = if self.runtime_backend == RuntimeBackend::Simulated {
launch_threads(&self.entry)
} else {
@ -230,13 +251,18 @@ impl AdapterState {
self.coordinator_endpoint = record.coordinator.clone();
if self.runtime_backend != RuntimeBackend::Simulated {
if let Some(snapshots) = record.node_report.get("task_snapshots") {
self.threads = coordinator_threads_from_snapshots(
&self.entry,
snapshots,
record.node_report.get("process_status"),
);
self.reconcile_runtime_threads(snapshots, record.node_report.get("process_status"));
} else if record.node_report.get("terminal_event").is_some() {
self.threads.clear();
}
}
if let Some(fingerprint) = record
.node_report
.get("snapshot_fingerprint")
.and_then(Value::as_str)
{
self.runtime_snapshot_fingerprint = fingerprint.to_owned();
}
if !record.placed_task_launched {
self.command_status = format!(
"virtual process started through {}; no runtime task event observed yet",
@ -287,7 +313,11 @@ impl AdapterState {
self.debug_all_threads_stopped = record.all_participants_frozen;
self.epoch = record.debug_epoch.unwrap_or(self.epoch);
self.coordinator_debug_epoch = record.debug_epoch;
self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new);
self.stopped_task = record
.stopped_task
.as_deref()
.and_then(|task| TaskInstanceId::try_new(task).ok());
self.stopped_probe_symbol = record.stopped_probe_symbol.clone();
if let Some(acknowledgements) = record
.node_report
.pointer("/debug_epoch/acknowledgements")
@ -303,6 +333,12 @@ impl AdapterState {
else {
continue;
};
let Ok(task_id) = TaskInstanceId::try_new(task) else {
continue;
};
let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
continue;
};
let coordinator_main = acknowledgement.get("node").and_then(Value::as_str)
== Some("coordinator-main");
let thread_id = if coordinator_main {
@ -312,14 +348,19 @@ impl AdapterState {
.values()
.find(|thread| thread.task.as_str() == task)
.map(|thread| thread.id)
.unwrap_or_else(|| self.insert_runtime_thread(task, task_definition))
.unwrap_or_else(|| {
self.insert_runtime_thread(
task_id.clone(),
task_definition_id.clone(),
)
})
};
let thread = self
.threads
.get_mut(&thread_id)
.expect("runtime thread was found or inserted");
thread.task = TaskInstanceId::new(task);
thread.task_definition = TaskDefinitionId::new(task_definition);
thread.task = task_id;
thread.task_definition = task_definition_id;
thread.runtime_stack_frames = acknowledgement
.get("stack_frames")
.and_then(Value::as_array)
@ -362,23 +403,12 @@ impl AdapterState {
.and_then(Value::as_str)
.unwrap_or(self.entry.as_str())
.to_owned();
let terminal_task_definition = terminal_event
.get("task_definition")
.and_then(Value::as_str)
.unwrap_or(self.entry.as_str())
.to_owned();
let terminal_thread_id = self
.threads
.values()
.find(|thread| {
thread.task.as_str() == terminal_task.as_str()
|| thread.task_definition.as_str() == terminal_task_definition.as_str()
})
.map(|thread| thread.id)
.unwrap_or_else(|| {
self.insert_runtime_thread(&terminal_task, &terminal_task_definition)
});
if let Some(thread) = self.threads.get_mut(&terminal_thread_id) {
.find(|thread| thread.task.as_str() == terminal_task.as_str())
.map(|thread| thread.id);
if let Some(thread) = terminal_thread_id.and_then(|id| self.threads.get_mut(&id)) {
thread.stdout_bytes = record.stdout_bytes;
thread.stderr_bytes = record.stderr_bytes;
thread.stdout_tail = record.stdout_tail.clone();
@ -412,24 +442,22 @@ impl AdapterState {
let _ = crate::view_state::write_view_state(self, &record);
}
fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 {
let id = self
.threads
.keys()
.next_back()
.copied()
.unwrap_or(MAIN_THREAD)
+ 1;
fn insert_runtime_thread(
&mut self,
task: TaskInstanceId,
task_definition: TaskDefinitionId,
) -> i64 {
let id = self.allocate_runtime_thread_id();
let line = self
.debug_probes
.iter()
.find(|probe| {
probe.task.as_str() == task_definition || probe.function == task_definition
probe.task == task_definition || probe.function == task_definition.as_str()
})
.map(|probe| i64::from(probe.line_start))
.unwrap_or(1);
let definition_name = task_definition.replace(['_', '-'], " ");
let name = if task == task_definition {
let definition_name = task_definition.as_str().replace(['_', '-'], " ");
let name = if task.as_str() == task_definition.as_str() {
definition_name
} else {
format!("{definition_name} ({task})")
@ -447,9 +475,9 @@ impl AdapterState {
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::new(task),
task,
attempt_id: "unknown-attempt".to_owned(),
task_definition: TaskDefinitionId::new(task_definition),
task_definition,
name,
line,
state: DebugRuntimeState::Running,
@ -475,6 +503,51 @@ impl AdapterState {
id
}
fn allocate_runtime_thread_id(&mut self) -> i64 {
while self.threads.contains_key(&self.next_thread_id) || self.next_thread_id == MAIN_THREAD
{
self.next_thread_id += 1;
}
let id = self.next_thread_id;
self.next_thread_id += 1;
id
}
fn reconcile_runtime_threads(
&mut self,
task_snapshots: &Value,
process_status: Option<&Value>,
) {
let desired =
coordinator_threads_from_snapshots(&self.entry, task_snapshots, process_status);
let previous = std::mem::take(&mut self.threads);
let mut reconciled = BTreeMap::new();
for (_, mut thread) in desired {
let coordinator_main = process_status
.and_then(|status| status.get("main_task_instance"))
.and_then(Value::as_str)
.is_some_and(|task| task == thread.task.as_str());
let existing = if coordinator_main {
previous.get(&MAIN_THREAD)
} else {
previous
.values()
.find(|current| current.task == thread.task)
};
if let Some(existing) = existing {
preserve_thread_identity(&mut thread, existing);
} else if !coordinator_main {
thread.id = self.allocate_runtime_thread_id();
assign_thread_references(&mut thread);
} else {
thread.id = MAIN_THREAD;
assign_thread_references(&mut thread);
}
reconciled.insert(thread.id, thread);
}
self.threads = reconciled;
}
pub(crate) fn apply_attach_record(&mut self, record: RuntimeLaunchRecord) {
self.coordinator_endpoint = record.coordinator.clone();
self.command_status = format!(
@ -485,8 +558,7 @@ impl AdapterState {
self.last_task_failed = false;
self.runtime_artifact_path = None;
self.runtime_event_count = record.event_count;
self.threads = coordinator_threads_from_snapshots(
&self.entry,
self.reconcile_runtime_threads(
record
.node_report
.get("task_snapshots")
@ -507,6 +579,35 @@ impl AdapterState {
}
}
fn preserve_thread_identity(thread: &mut VirtualThread, existing: &VirtualThread) {
thread.id = existing.id;
thread.frame_id = existing.frame_id;
thread.locals_ref = existing.locals_ref;
thread.wasm_locals_ref = existing.wasm_locals_ref;
thread.args_ref = existing.args_ref;
thread.runtime_ref = existing.runtime_ref;
thread.output_ref = existing.output_ref;
thread.target_ref = existing.target_ref;
thread.vfs_ref = existing.vfs_ref;
thread.command_ref = existing.command_ref;
if thread.recent_output.is_empty() {
thread.recent_output = existing.recent_output.clone();
}
}
fn assign_thread_references(thread: &mut VirtualThread) {
let id = thread.id;
thread.frame_id = 1000 + id;
thread.locals_ref = 5000 + id;
thread.wasm_locals_ref = 9000 + id;
thread.args_ref = 2000 + id;
thread.runtime_ref = 3000 + id;
thread.output_ref = 4000 + id;
thread.target_ref = 6000 + id;
thread.vfs_ref = 7000 + id;
thread.command_ref = 8000 + id;
}
fn value_string_pairs(value: Option<&Value>) -> Vec<(String, String)> {
value
.and_then(Value::as_array)
@ -642,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId {
ProcessId::new(format!("vp-{suffix}"))
}
fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread {
fn coordinator_main_thread(
entry: &str,
task: TaskInstanceId,
task_definition: TaskDefinitionId,
) -> VirtualThread {
let name = format!("{entry} coordinator main ({task})");
VirtualThread {
id: MAIN_THREAD,
frame_id: 1_001,
@ -654,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi
target_ref: 4_501,
vfs_ref: 5_001,
command_ref: 5_501,
task: TaskInstanceId::from(task),
task,
attempt_id: "main:unknown".to_owned(),
task_definition: TaskDefinitionId::from(task_definition),
name: format!("{entry} coordinator main ({task})"),
task_definition,
name,
line: 0,
state: DebugRuntimeState::Running,
freeze_supported: true,
@ -693,10 +799,14 @@ fn coordinator_threads_from_events(
.expect("launch thread model should include main");
if let Some(status) = process_status {
if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) {
main.task = TaskInstanceId::from(task);
if let Ok(task) = TaskInstanceId::try_new(task) {
main.task = task;
}
}
if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) {
main.task_definition = TaskDefinitionId::from(definition);
if let Ok(definition) = TaskDefinitionId::try_new(definition) {
main.task_definition = definition;
}
}
main.name = format!("{entry} coordinator main ({})", main.task);
main.state = if status
@ -758,6 +868,12 @@ fn coordinator_threads_from_snapshots(
.get("main_task_definition")
.and_then(Value::as_str)
.unwrap_or(entry);
let (Ok(task), Ok(task_definition)) = (
TaskInstanceId::try_new(task),
TaskDefinitionId::try_new(task_definition),
) else {
return threads;
};
let mut main = coordinator_main_thread(entry, task, task_definition);
main.attempt_id = format!(
"main:{}",
@ -812,6 +928,12 @@ fn coordinator_threads_from_snapshots(
let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else {
continue;
};
let Ok(task_id) = TaskInstanceId::try_new(task) else {
continue;
};
let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
continue;
};
let attempt_id = snapshot
.get("attempt_id")
.and_then(Value::as_str)
@ -862,9 +984,9 @@ fn coordinator_threads_from_snapshots(
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::from(task),
task: task_id,
attempt_id,
task_definition: TaskDefinitionId::from(task_definition),
task_definition: task_definition_id,
name: format!("{display_name} ({task}, attempt {short_attempt})"),
line: snapshot
.get("source_line")
@ -911,6 +1033,8 @@ fn coordinator_threads_from_snapshots(
fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option<VirtualThread> {
let task = event.get("task").and_then(Value::as_str)?;
let task_definition = event.get("task_definition").and_then(Value::as_str)?;
let task_id = TaskInstanceId::try_new(task).ok()?;
let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?;
let definition_name = task_definition.replace(['_', '-'], " ");
let name = if task == task_definition {
definition_name
@ -956,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::from(task),
task: task_id,
attempt_id: event
.get("attempt_id")
.and_then(Value::as_str)
.unwrap_or("unknown-attempt")
.to_owned(),
task_definition: TaskDefinitionId::from(task_definition),
task_definition: task_definition_id,
name,
line: 0,
state,

166
crates/clusterflux-node/src/assignment_runner.rs
View file

@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::Read;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
@ -46,6 +46,40 @@ use validation::{
resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
};
#[derive(Debug)]
struct AssignmentExecutionError {
message: String,
stdout_source_bytes: u64,
stderr_source_bytes: u64,
}
impl std::fmt::Display for AssignmentExecutionError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for AssignmentExecutionError {}
fn execution_error_with_log_bytes(
message: impl Into<String>,
stdout_source_bytes: &AtomicU64,
stderr_source_bytes: &AtomicU64,
) -> Box<dyn std::error::Error> {
Box::new(AssignmentExecutionError {
message: message.into(),
stdout_source_bytes: stdout_source_bytes.load(Ordering::Relaxed),
stderr_source_bytes: stderr_source_bytes.load(Ordering::Relaxed),
})
}
pub(crate) fn assignment_error_log_bytes(error: &(dyn std::error::Error + 'static)) -> (u64, u64) {
error
.downcast_ref::<AssignmentExecutionError>()
.map(|error| (error.stdout_source_bytes, error.stderr_source_bytes))
.unwrap_or((0, 0))
}
pub(crate) fn run_verified_wasmtime_assignment(
args: &Args,
task: &RuntimeTask,
@ -95,6 +129,8 @@ pub(crate) fn run_verified_wasmtime_assignment(
return Err("Wasm entrypoint assignment omitted its descriptor export".into());
}
};
let command_stdout_source_bytes = Arc::new(AtomicU64::new(0));
let command_stderr_source_bytes = Arc::new(AtomicU64::new(0));
let (stdout, boundary_result) = match abi {
WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
let invocation = WasmTaskInvocation::new(
@ -102,18 +138,28 @@ pub(crate) fn run_verified_wasmtime_assignment(
task_spec.task_instance.clone(),
task_spec.args.clone(),
);
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&module,
expected_bundle_digest,
export,
&invocation,
Box::new(CoordinatorWasmTaskHost::new(
args,
task,
node_private_key,
let result = WasmtimeTaskRuntime::new()?
.run_task_export_verified_with_task_host(
&module,
)?),
)?;
expected_bundle_digest,
export,
&invocation,
Box::new(CoordinatorWasmTaskHost::new(
args,
task,
node_private_key,
&module,
Arc::clone(&command_stdout_source_bytes),
Arc::clone(&command_stderr_source_bytes),
)?),
)
.map_err(|error| {
execution_error_with_log_bytes(
error.to_string(),
&command_stdout_source_bytes,
&command_stderr_source_bytes,
)
})?;
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!(
"clusterflux debug control: Wasm assignment returned for task {}",
@ -122,17 +168,35 @@ pub(crate) fn run_verified_wasmtime_assignment(
}
match result.outcome {
WasmTaskOutcome::Completed => {
let boundary = result.result.ok_or("completed Wasm task omitted result")?;
let boundary = result.result.ok_or_else(|| {
execution_error_with_log_bytes(
"completed Wasm task omitted result",
&command_stdout_source_bytes,
&command_stderr_source_bytes,
)
})?;
(
format!("{}\n", serde_json::to_string(&boundary)?),
format!(
"{}\n",
serde_json::to_string(&boundary).map_err(|error| {
execution_error_with_log_bytes(
error.to_string(),
&command_stdout_source_bytes,
&command_stderr_source_bytes,
)
})?
),
Some(boundary),
)
}
WasmTaskOutcome::Failed => {
return Err(result
.error
.unwrap_or_else(|| "Wasm task failed without an error".to_owned())
.into())
return Err(execution_error_with_log_bytes(
result
.error
.unwrap_or_else(|| "Wasm task failed without an error".to_owned()),
&command_stdout_source_bytes,
&command_stderr_source_bytes,
))
}
}
}
@ -140,12 +204,18 @@ pub(crate) fn run_verified_wasmtime_assignment(
let task_id = TaskInstanceId::new(task.task.clone());
let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
let manifest = artifacts.flush();
let stdout_source_bytes = command_stdout_source_bytes
.load(Ordering::Relaxed)
.saturating_add(stdout.len() as u64);
let stderr_source_bytes = command_stderr_source_bytes.load(Ordering::Relaxed);
Ok((
CommandOutput {
virtual_thread: task_id,
status_code: Some(0),
stdout,
stderr: String::new(),
stdout_source_bytes,
stderr_source_bytes,
stdout_truncated: false,
stderr_truncated: false,
log_backpressured: false,
@ -176,6 +246,8 @@ struct CoordinatorWasmTaskHost {
next_handle_id: u64,
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
command_status: Arc<Mutex<Option<String>>>,
command_stdout_source_bytes: Arc<AtomicU64>,
command_stderr_source_bytes: Arc<AtomicU64>,
cancellation_requested: Arc<AtomicBool>,
abort_requested: Arc<AtomicBool>,
debug_control: Arc<WasmDebugControl>,
@ -188,6 +260,8 @@ impl CoordinatorWasmTaskHost {
parent: &RuntimeTask,
node_private_key: &str,
module: &[u8],
command_stdout_source_bytes: Arc<AtomicU64>,
command_stderr_source_bytes: Arc<AtomicU64>,
) -> Result<Self, Box<dyn std::error::Error>> {
let task_spec = parent
.task_spec
@ -268,6 +342,8 @@ impl CoordinatorWasmTaskHost {
next_handle_id: 1,
handles,
command_status,
command_stdout_source_bytes,
command_stderr_source_bytes,
cancellation_requested,
abort_requested,
debug_control,
@ -293,7 +369,16 @@ impl CoordinatorWasmTaskHost {
runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
let execution =
run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
let (
terminal_state,
status_code,
stdout,
stderr,
stdout_source_bytes,
stderr_source_bytes,
result,
retained,
) = match execution {
Ok((output, _manifest, result)) => {
let retained = retained_result_artifact(
self.args.project_root.as_deref(),
@ -306,20 +391,40 @@ impl CoordinatorWasmTaskHost {
output.status_code,
output.stdout,
output.stderr,
output.stdout_source_bytes,
output.stderr_source_bytes,
result,
retained,
),
Err(error) => ("failed", Some(1), String::new(), error, None, None),
Err(error) => (
"failed",
Some(1),
String::new(),
error.clone(),
output.stdout_source_bytes,
output
.stderr_source_bytes
.saturating_add(error.len() as u64),
None,
None,
),
}
}
Err(error) => (
"failed",
Some(1),
String::new(),
error.to_string(),
None,
None,
),
Err(error) => {
let (stdout_source_bytes, stderr_source_bytes) =
assignment_error_log_bytes(error.as_ref());
let error = error.to_string();
(
"failed",
Some(1),
String::new(),
error.clone(),
stdout_source_bytes,
stderr_source_bytes.saturating_add(error.len() as u64),
None,
None,
)
}
};
let artifact_path = retained
.as_ref()
@ -339,8 +444,8 @@ impl CoordinatorWasmTaskHost {
"task": runtime_task.task,
"terminal_state": terminal_state,
"status_code": status_code,
"stdout_bytes": stdout.len(),
"stderr_bytes": stderr.len(),
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": stdout,
"stderr_tail": stderr,
"stdout_truncated": false,
@ -676,6 +781,7 @@ impl WasmTaskHost for CoordinatorWasmTaskHost {
let mut runner = CoordinatorControlledProcessRunner::new(
self,
Duration::from_millis(request.timeout_ms),
configured_secrets.clone(),
);
let output = LinuxRootlessPodmanBackend
.execute_local_checkout_task(

290
crates/clusterflux-node/src/assignment_runner/process_runner.rs
View file

@ -1,4 +1,83 @@
use super::*;
use std::sync::mpsc::{self, Receiver, SyncSender};
struct LiveLogChunk {
stream: &'static str,
offset: u64,
source_bytes: u64,
bytes: Vec<u8>,
truncated: bool,
}
fn append_bounded_tail(tail: &mut Vec<u8>, bytes: &[u8], maximum: usize) {
if maximum == 0 {
tail.clear();
return;
}
if bytes.len() >= maximum {
tail.clear();
tail.extend_from_slice(&bytes[bytes.len() - maximum..]);
return;
}
let overflow = tail
.len()
.saturating_add(bytes.len())
.saturating_sub(maximum);
if overflow > 0 {
tail.drain(..overflow);
}
tail.extend_from_slice(bytes);
}
fn redact_safe_live_log_prefix(
pending: &[u8],
configured_secrets: &[String],
final_chunk: bool,
) -> Option<(usize, String)> {
if pending.is_empty() {
return None;
}
let secret_bytes = configured_secrets
.iter()
.filter(|secret| secret.len() >= 4)
.map(String::as_bytes)
.collect::<Vec<_>>();
let maximum_secret_bytes = secret_bytes
.iter()
.map(|secret| secret.len())
.max()
.unwrap_or(0);
if !final_chunk && pending.len() <= maximum_secret_bytes {
return None;
}
let mut consumed = if final_chunk {
pending.len()
} else {
pending.len() - maximum_secret_bytes
};
loop {
let previous = consumed;
for secret in &secret_bytes {
for start in 0..=pending.len().saturating_sub(secret.len()) {
let end = start + secret.len();
if start < consumed && end > consumed && &pending[start..end] == *secret {
consumed = end;
}
}
}
if consumed == previous {
break;
}
}
if consumed == 0 {
return None;
}
let text = redact_configured_values(
String::from_utf8_lossy(&pending[..consumed]).into_owned(),
configured_secrets,
);
Some((consumed, text))
}
pub(super) struct CoordinatorControlledProcessRunner {
pub(super) args: Args,
@ -7,13 +86,20 @@ pub(super) struct CoordinatorControlledProcessRunner {
pub(super) node_private_key: String,
pub(super) debug_control: Arc<WasmDebugControl>,
pub(super) command_status: Arc<Mutex<Option<String>>>,
pub(super) stdout_source_bytes: Arc<AtomicU64>,
pub(super) stderr_source_bytes: Arc<AtomicU64>,
pub(super) timeout: Duration,
pub(super) configured_secrets: Vec<String>,
}
impl CoordinatorControlledProcessRunner {
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
pub(super) fn new(
host: &CoordinatorWasmTaskHost,
timeout: Duration,
configured_secrets: Vec<String>,
) -> Self {
Self {
args: host.args.clone(),
process: host.process.clone(),
@ -21,7 +107,10 @@ impl CoordinatorControlledProcessRunner {
node_private_key: host.node_private_key.clone(),
debug_control: Arc::clone(&host.debug_control),
command_status: Arc::clone(&host.command_status),
stdout_source_bytes: Arc::clone(&host.command_stdout_source_bytes),
stderr_source_bytes: Arc::clone(&host.command_stderr_source_bytes),
timeout,
configured_secrets,
}
}
@ -201,10 +290,18 @@ impl CoordinatorControlledProcessRunner {
fn drain_bounded(
mut reader: impl Read + Send + 'static,
maximum: usize,
stream: &'static str,
sender: SyncSender<LiveLogChunk>,
configured_secrets: Vec<String>,
source_bytes_total: Arc<AtomicU64>,
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
thread::spawn(move || {
let mut captured = Vec::new();
let mut buffer = [0_u8; 16 * 1024];
let stream_base = source_bytes_total.load(Ordering::Relaxed);
let mut source_bytes_read = 0_u64;
let mut pending_offset = stream_base;
let mut pending = Vec::new();
loop {
let count = reader
.read(&mut buffer)
@ -212,12 +309,128 @@ impl CoordinatorControlledProcessRunner {
if count == 0 {
break;
}
let remaining = maximum.saturating_sub(captured.len());
captured.extend_from_slice(&buffer[..count.min(remaining)]);
let _ = source_bytes_total.fetch_update(
Ordering::Relaxed,
Ordering::Relaxed,
|current| Some(current.saturating_add(count as u64)),
);
source_bytes_read = source_bytes_read.saturating_add(count as u64);
append_bounded_tail(&mut captured, &buffer[..count], maximum);
pending.extend_from_slice(&buffer[..count]);
if let Some((consumed, text)) =
redact_safe_live_log_prefix(&pending, &configured_secrets, false)
{
let _ = sender.try_send(LiveLogChunk {
stream,
offset: pending_offset,
source_bytes: consumed as u64,
bytes: text.into_bytes(),
truncated: false,
});
pending.drain(..consumed);
pending_offset = pending_offset.saturating_add(consumed as u64);
}
}
if let Some((consumed, text)) =
redact_safe_live_log_prefix(&pending, &configured_secrets, true)
{
let _ = sender.try_send(LiveLogChunk {
stream,
offset: pending_offset,
source_bytes: consumed as u64,
bytes: text.into_bytes(),
truncated: false,
});
}
if source_bytes_read > maximum as u64 {
let _ = sender.try_send(LiveLogChunk {
stream,
offset: stream_base.saturating_add(source_bytes_read),
source_bytes: 0,
bytes: b"[log output truncated at node capture limit]".to_vec(),
truncated: true,
});
}
Ok(captured)
})
}
fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
let args = self.args.clone();
let process = self.process.clone();
let task = self.task.clone();
let node_private_key = self.node_private_key.clone();
let configured_secrets = self.configured_secrets.clone();
let command_status = Arc::clone(&self.command_status);
thread::spawn(move || {
let mut log_session = None;
let mut delivery_available = true;
while let Ok(chunk) = receiver.recv() {
if !delivery_available {
continue;
}
let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
text = redact_configured_values(text, &configured_secrets);
let mut delivered = false;
for _ in 0..2 {
if log_session.is_none() {
log_session = CoordinatorSession::connect_with_timeouts(
&args.coordinator,
Duration::from_millis(500),
Duration::from_millis(500),
)
.ok();
}
let Some(session) = log_session.as_mut() else {
continue;
};
let request = signed_node_request_json(
&args,
&node_private_key,
"report_task_log_chunk",
serde_json::json!({
"type": "report_task_log_chunk",
"tenant": &args.tenant,
"project": &args.project,
"process": &process,
"node": &args.node,
"task": &task,
"stream": chunk.stream,
"offset": chunk.offset,
"source_bytes": chunk.source_bytes,
"text": &text,
"truncated": chunk.truncated,
}),
);
let result = match request {
Ok(request) => session
.request(request)
.map(|_| ())
.map_err(|error| error.to_string()),
Err(error) => Err(error.to_string()),
};
match result {
Ok(()) => {
delivered = true;
break;
}
Err(_) => {
log_session = None;
}
}
}
if !delivered {
delivery_available = false;
if let Ok(mut current) = command_status.lock() {
*current = Some(
"live log delivery was interrupted; final bounded output remains available"
.to_owned(),
);
}
}
}
})
}
}
impl ProcessRunner for CoordinatorControlledProcessRunner {
@ -245,12 +458,17 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
command.program,
command.args.join(" ")
));
let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
let stdout = Self::drain_bounded(
child
.stdout
.take()
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
"stdout",
live_log_sender.clone(),
self.configured_secrets.clone(),
Arc::clone(&self.stdout_source_bytes),
);
let stderr = Self::drain_bounded(
child
@ -258,11 +476,19 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
.take()
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
"stderr",
live_log_sender,
self.configured_secrets.clone(),
Arc::clone(&self.stderr_source_bytes),
);
let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
Ok(session) => session,
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(format!(
"establish execution control channel: {error}"
)));
@ -277,6 +503,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Ok(None) => {}
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(error.to_string()));
}
}
@ -289,6 +518,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(format!(
"native command exceeded wall-clock timeout of {} ms",
self.timeout.as_millis()
@ -303,6 +533,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Cancelled(
"coordinator requested cancellation or abort".to_owned(),
));
@ -312,6 +543,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(error);
}
}
@ -360,6 +592,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
.join()
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
.map_err(BackendError::Command)?;
live_log_reporter
.join()
.map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
self.set_command_status(format!(
"native command exited with status {:?}",
status.code()
@ -371,3 +606,52 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
})
}
}
#[cfg(test)]
mod tests {
use super::{redact_safe_live_log_prefix, CoordinatorControlledProcessRunner};
use std::io::Cursor;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{mpsc, Arc};
#[test]
fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
let secrets = vec!["correct-horse".to_owned()];
let mut pending = b"prefix correct-".to_vec();
let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
pending.drain(..consumed);
pending.extend_from_slice(b"horse suffix");
let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
let combined = format!("{first}{second}");
assert_eq!(combined, "prefix [REDACTED] suffix");
assert!(!combined.contains("correct-"));
assert!(!combined.contains("horse"));
}
#[test]
fn bounded_capture_retains_the_real_tail_and_complete_source_byte_count() {
let source_bytes = Arc::new(AtomicU64::new(5));
let (sender, receiver) = mpsc::sync_channel(8);
let reader = CoordinatorControlledProcessRunner::drain_bounded(
Cursor::new(b"0123456789abcdefghijklmnopqrstuv".to_vec()),
8,
"stdout",
sender,
Vec::new(),
Arc::clone(&source_bytes),
);
let captured = reader.join().unwrap().unwrap();
let chunks = receiver.into_iter().collect::<Vec<_>>();
assert_eq!(captured, b"opqrstuv");
assert_eq!(source_bytes.load(Ordering::Relaxed), 37);
assert_eq!(
chunks.iter().map(|chunk| chunk.source_bytes).sum::<u64>(),
32
);
assert_eq!(chunks[0].offset, 5);
assert_eq!(chunks.last().unwrap().offset, 37);
assert!(chunks.iter().any(|chunk| chunk.truncated));
}
}

6
crates/clusterflux-node/src/assignment_runner/tests.rs
View file

@ -48,7 +48,10 @@ fn test_controlled_runner(
),
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
stdout_source_bytes: Arc::new(AtomicU64::new(0)),
stderr_source_bytes: Arc::new(AtomicU64::new(0)),
timeout,
configured_secrets: Vec::new(),
}
}
@ -89,7 +92,10 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
),
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
stdout_source_bytes: Arc::new(AtomicU64::new(0)),
stderr_source_bytes: Arc::new(AtomicU64::new(0)),
timeout: Duration::from_secs(30),
configured_secrets: Vec::new(),
};
let started = Instant::now();
let error = runner

34
crates/clusterflux-node/src/command_runner.rs
View file

@ -1,6 +1,6 @@
use clusterflux_core::{
CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
VfsOverlay, VfsPath,
CommandInvocation, Digest, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject, VfsOverlay,
VfsPath,
};
use serde::{Deserialize, Serialize};
@ -42,6 +42,8 @@ pub struct CommandOutput {
pub status_code: Option<i32>,
pub stdout: String,
pub stderr: String,
pub stdout_source_bytes: u64,
pub stderr_source_bytes: u64,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub log_backpressured: bool,
@ -83,6 +85,8 @@ impl LocalCommandExecutor {
&output.stderr,
max_log_bytes,
);
let stdout_source_bytes = output.stdout.len() as u64;
let stderr_source_bytes = output.stderr.len() as u64;
let staged_artifact = if let Some(path) = command.stage_stdout_as {
Some(overlay.write(
path,
@ -98,6 +102,8 @@ impl LocalCommandExecutor {
status_code: output.status.code(),
stdout: logs.stdout,
stderr: logs.stderr,
stdout_source_bytes,
stderr_source_bytes,
stdout_truncated: logs.stdout_truncated,
stderr_truncated: logs.stderr_truncated,
log_backpressured: logs.backpressured,
@ -107,24 +113,20 @@ impl LocalCommandExecutor {
}
pub(super) fn capture_command_logs(
task: &TaskInstanceId,
_task: &TaskInstanceId,
stdout: &[u8],
stderr: &[u8],
max_log_bytes: usize,
) -> CapturedCommandLogs {
let mut logs = LogBuffer::new(max_log_bytes);
logs.push(task.clone(), stdout);
logs.push(task.clone(), stderr);
let records = logs.records();
let stdout_record = &records[0];
let stderr_record = &records[1];
debug_assert_eq!(&stdout_record.task, task);
debug_assert_eq!(&stderr_record.task, task);
let stdout_truncated = stdout.len() > max_log_bytes;
let stderr_truncated = stderr.len() > max_log_bytes;
let stdout_start = stdout.len().saturating_sub(max_log_bytes);
let stderr_start = stderr.len().saturating_sub(max_log_bytes);
CapturedCommandLogs {
stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
stdout_truncated: stdout_record.truncated,
stderr_truncated: stderr_record.truncated,
backpressured: logs.backpressured(),
stdout: String::from_utf8_lossy(&stdout[stdout_start..]).into_owned(),
stderr: String::from_utf8_lossy(&stderr[stderr_start..]).into_owned(),
stdout_truncated,
stderr_truncated,
backpressured: stdout_truncated || stderr_truncated,
}
}

11
crates/clusterflux-node/src/coordinator_session.rs
View file

@ -3,6 +3,7 @@ use clusterflux_control::endpoint_identity;
use clusterflux_control::ControlSession;
use clusterflux_core::coordinator_wire_request;
use serde_json::Value;
use std::time::Duration;
pub(crate) struct CoordinatorSession {
inner: ControlSession,
@ -15,6 +16,16 @@ impl CoordinatorSession {
})
}
pub(crate) fn connect_with_timeouts(
addr: &str,
connect_timeout: Duration,
io_timeout: Duration,
) -> Result<Self, Box<dyn std::error::Error>> {
Ok(Self {
inner: ControlSession::connect_with_timeouts(addr, connect_timeout, io_timeout)?,
})
}
pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
let request_id = format!("node-{}", self.inner.requests() + 1);
let wire_request = coordinator_wire_request(request_id, value);

43
crates/clusterflux-node/src/daemon.rs
View file

@ -7,11 +7,11 @@ use std::time::{Duration, Instant};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
ProcessId, ProjectId, TaskInstanceId, TaskSpec, TenantId, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
};
use serde_json::{json, Value};
use crate::assignment_runner::run_verified_wasmtime_assignment;
use crate::assignment_runner::{assignment_error_log_bytes, run_verified_wasmtime_assignment};
#[cfg(test)]
use crate::coordinator_session::control_endpoint_identity;
use crate::coordinator_session::CoordinatorSession;
@ -84,6 +84,8 @@ pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
let heartbeat_request = json!({
"type": "node_heartbeat",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
});
let heartbeat_signature = sign_node_request(
@ -222,7 +224,7 @@ fn worker_loop(
let expiry = Instant::now()
.checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
.unwrap_or_else(Instant::now);
restart_pins.insert(ArtifactId::new(artifact), expiry);
restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry);
}
println!("{}", serde_json::to_string(&report)?);
std::io::stdout().flush()?;
@ -289,7 +291,7 @@ fn service_pending_artifact_transfer(
return Ok(false);
};
let transfer_id = required_string(transfer, "transfer_id")?;
let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?;
let expected_digest: Digest = serde_json::from_value(
transfer
.get("expected_digest")
@ -369,9 +371,11 @@ pub(crate) fn runtime_task_from_assignment(
.cloned()
.ok_or("task assignment missing task_spec")?,
)?;
let process = ProcessId::try_new(required_string(value, "process")?)?;
let task = TaskInstanceId::try_new(required_string(value, "task")?)?;
Ok(RuntimeTask {
process: required_string(value, "process")?,
task: required_string(value, "task")?,
process: process.to_string(),
task: task.to_string(),
epoch: value.get("epoch").and_then(Value::as_u64),
bundle_digest: task_spec.bundle_digest.clone(),
task_spec: Some(task_spec),
@ -418,6 +422,8 @@ fn run_runtime_task(
"reconnect_node",
json!({
"type": "reconnect_node",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"process": &task.process,
"epoch": epoch,
@ -458,6 +464,8 @@ fn run_runtime_task(
capability_report,
debug_command,
node_private_key,
0,
0,
);
}
@ -492,9 +500,13 @@ fn run_runtime_task(
debug_command,
node_private_key,
&error,
output.stdout_source_bytes,
output.stderr_source_bytes,
),
},
Err(error) => {
let (stdout_source_bytes, stderr_source_bytes) =
assignment_error_log_bytes(error.as_ref());
let error = error.to_string();
if error.contains("task execution cancelled:") {
record_cancelled_task(
@ -506,6 +518,8 @@ fn run_runtime_task(
capability_report,
debug_command,
node_private_key,
stdout_source_bytes,
stderr_source_bytes,
)
} else {
record_failed_task(
@ -518,6 +532,8 @@ fn run_runtime_task(
debug_command,
node_private_key,
&error,
stdout_source_bytes,
stderr_source_bytes,
)
}
}
@ -570,6 +586,12 @@ fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
}
}
TenantId::try_new(tenant.clone())?;
ProjectId::try_new(project.clone())?;
NodeId::try_new(node.clone())?;
if let Some(grant) = enrollment_grant.as_ref() {
clusterflux_core::validate_opaque_token(grant, 512)?;
}
Ok(Args {
coordinator: coordinator.ok_or("--coordinator is required")?,
tenant,
@ -622,13 +644,12 @@ mod tests {
#[test]
fn hosted_url_remains_an_https_control_endpoint() {
assert_eq!(
control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
control_endpoint_identity("https://clusterflux.lesstuff.com").unwrap(),
"https://clusterflux.lesstuff.com/api/v1/control"
);
assert_eq!(
control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
.unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
control_endpoint_identity("https://clusterflux.lesstuff.com/api/v1/control").unwrap(),
"https://clusterflux.lesstuff.com/api/v1/control"
);
assert_eq!(
control_endpoint_identity("127.0.0.1:7999").unwrap(),

21
crates/clusterflux-node/src/lib.rs
View file

@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use clusterflux_core::{
Capability, CommandBackendKind, CommandInvocation, CommandPlan, Digest, EnvironmentKind,
@ -9,6 +9,7 @@ use sha2::{Digest as ShaDigest, Sha256};
use thiserror::Error;
mod command_runner;
mod source_snapshot;
mod windows_dev;
pub use clusterflux_wasm_runtime::{
WasmDebugControl, WasmTaskError, WasmTaskHost, WasmtimeDebugProbe, WasmtimeTaskRuntime,
@ -20,6 +21,10 @@ pub use command_runner::{
};
pub use windows_dev::{WindowsCommandDevBackend, WindowsSandboxStubBackend};
pub fn snapshot_project_digest(project_root: &Path) -> Result<Digest, String> {
source_snapshot::snapshot_project(project_root).map(|snapshot| snapshot.digest)
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct MaterializedEnvironment {
pub name: String,
@ -736,7 +741,7 @@ mod tests {
}
#[test]
fn linux_backend_caps_logs_without_truncating_staged_artifact_bytes() {
fn linux_backend_retains_final_log_tail_without_truncating_staged_artifact_bytes() {
let invocation = CommandInvocation {
program: "cargo".to_owned(),
args: vec!["build".to_owned()],
@ -769,7 +774,7 @@ mod tests {
.unwrap();
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
assert_eq!(output.stdout, "abcd");
assert_eq!(output.stdout, "cdef");
assert!(output.stdout_truncated);
assert!(output.log_backpressured);
assert_eq!(output.staged_artifact.as_ref().unwrap().size, 6);
@ -987,7 +992,7 @@ mod tests {
#[cfg(unix)]
#[test]
fn local_command_executor_caps_logs_and_reports_backpressure_by_virtual_thread() {
fn local_command_executor_retains_each_stream_tail_and_reports_backpressure() {
let executor = LocalCommandExecutor {
node: clusterflux_core::NodeId::from("node"),
hosted_control_plane: false,
@ -1016,15 +1021,15 @@ mod tests {
.unwrap();
assert_eq!(output.virtual_thread, TaskInstanceId::from("compile-linux"));
assert_eq!(output.stdout, "abcd");
assert_eq!(output.stderr, "");
assert_eq!(output.stdout, "cdef");
assert_eq!(output.stderr, "err");
assert!(output.stdout_truncated);
assert!(output.stderr_truncated);
assert!(!output.stderr_truncated);
assert!(output.log_backpressured);
}
#[test]
fn public_node_crate_does_not_require_hosted_private_types() {
fn public_node_crate_does_not_require_hosted_service_types() {
let _tenant = TenantId::from("tenant");
let _project = ProjectId::from("project");
let _backend = LinuxRootlessPodmanBackend;

29
crates/clusterflux-node/src/main.rs
View file

@ -8,5 +8,34 @@ mod task_artifacts;
mod task_reports;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let raw_args = std::env::args().skip(1).collect::<Vec<_>>();
match raw_args.as_slice() {
[flag] if matches!(flag.as_str(), "--version" | "-V") => {
println!("clusterflux-node {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
[flag] if matches!(flag.as_str(), "--help" | "-h") => {
println!(
"Clusterflux node worker.\n\n\
Usage: clusterflux-node --coordinator <URL> [OPTIONS]\n\n\
Options:\n \
--coordinator <URL>\n \
--tenant <TENANT> [default: tenant]\n \
--project-id <PROJECT> [default: project]\n \
--node <NODE> [default: node]\n \
--project-root <PATH>\n \
--enrollment-grant <GRANT>\n \
--public-key <KEY>\n \
--worker\n \
--emit-ready\n \
--control-poll-ms <MILLISECONDS>\n \
--assignment-poll-ms <MILLISECONDS> [default: 500]\n \
-h, --help\n \
-V, --version"
);
return Ok(());
}
_ => {}
}
daemon::run()
}

90
crates/clusterflux-node/src/task_reports.rs
View file

@ -47,8 +47,8 @@ pub(crate) fn record_completed_task(
"process": &task.process,
"node": &args.node,
"task": &task.task,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_bytes": output.stdout_source_bytes,
"stderr_bytes": output.stderr_source_bytes,
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
@ -85,8 +85,8 @@ pub(crate) fn record_completed_task(
"node": &args.node,
"task": &task.task,
"status_code": output.status_code,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_bytes": output.stdout_source_bytes,
"stderr_bytes": output.stderr_source_bytes,
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
@ -123,8 +123,11 @@ pub(crate) fn record_failed_task(
debug_command: Value,
node_private_key: &str,
error: &str,
stdout_source_bytes: u64,
command_stderr_source_bytes: u64,
) -> Result<Value, Box<dyn std::error::Error>> {
let error = bounded_runtime_error(error);
let stderr_source_bytes = command_stderr_source_bytes.saturating_add(error.len() as u64);
let log_event = session.request(signed_node_request_json(
args,
node_private_key,
@ -136,8 +139,8 @@ pub(crate) fn record_failed_task(
"process": &task.process,
"node": &args.node,
"task": &task.task,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": "",
"stderr_tail": &error,
"stdout_truncated": false,
@ -175,8 +178,8 @@ pub(crate) fn record_failed_task(
"task": &task.task,
"terminal_state": "failed",
"status_code": -1,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": "",
"stderr_tail": &error,
"stdout_truncated": false,
@ -189,6 +192,8 @@ pub(crate) fn record_failed_task(
Ok(failed_node_report(
task,
&error,
stdout_source_bytes,
stderr_source_bytes,
registration,
heartbeat,
capability_report,
@ -210,6 +215,8 @@ pub(crate) fn record_cancelled_task(
capability_report: Value,
debug_command: Value,
node_private_key: &str,
stdout_source_bytes: u64,
stderr_source_bytes: u64,
) -> Result<Value, Box<dyn std::error::Error>> {
let recorded = session.request(signed_node_request_json(
args,
@ -224,12 +231,12 @@ pub(crate) fn record_cancelled_task(
"task": &task.task,
"terminal_state": "cancelled",
"status_code": null,
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": "",
"stderr_tail": "",
"stdout_truncated": false,
"stderr_truncated": false,
"stdout_truncated": stdout_source_bytes > 0,
"stderr_truncated": stderr_source_bytes > 0,
"artifact_path": null,
"artifact_digest": null,
"artifact_size_bytes": null,
@ -238,6 +245,8 @@ pub(crate) fn record_cancelled_task(
)?)?;
Ok(cancelled_node_report(
task,
stdout_source_bytes,
stderr_source_bytes,
registration,
heartbeat,
capability_report,
@ -281,8 +290,8 @@ pub(crate) fn completed_node_report(
"virtual_thread": output.virtual_thread,
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
"status_code": output.status_code,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_bytes": output.stdout_source_bytes,
"stderr_bytes": output.stderr_source_bytes,
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
@ -305,6 +314,8 @@ pub(crate) fn completed_node_report(
#[allow(clippy::too_many_arguments)]
pub(crate) fn cancelled_node_report(
task: &RuntimeTask,
stdout_source_bytes: u64,
stderr_source_bytes: u64,
registration_response: Value,
heartbeat_response: Value,
capability_response: Value,
@ -318,12 +329,12 @@ pub(crate) fn cancelled_node_report(
"virtual_thread": &task.task,
"terminal_state": "cancelled",
"status_code": null,
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": "",
"stderr_tail": "",
"stdout_truncated": false,
"stderr_truncated": false,
"stdout_truncated": stdout_source_bytes > 0,
"stderr_truncated": stderr_source_bytes > 0,
"log_backpressured": false,
"staged_artifact": null,
"large_bytes_uploaded": false,
@ -343,6 +354,8 @@ pub(crate) fn cancelled_node_report(
pub(crate) fn failed_node_report(
task: &RuntimeTask,
error: &str,
stdout_source_bytes: u64,
stderr_source_bytes: u64,
registration_response: Value,
heartbeat_response: Value,
capability_response: Value,
@ -357,8 +370,8 @@ pub(crate) fn failed_node_report(
"virtual_thread": &task.task,
"terminal_state": "failed",
"status_code": -1,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_bytes": stdout_source_bytes,
"stderr_bytes": stderr_source_bytes,
"stdout_tail": "",
"stderr_tail": error,
"stdout_truncated": false,
@ -377,3 +390,40 @@ pub(crate) fn failed_node_report(
"coordinator_response": coordinator_response,
})
}
#[cfg(test)]
mod tests {
use super::*;
use clusterflux_core::TaskInstanceId;
#[test]
fn completed_report_uses_source_counts_instead_of_bounded_tail_lengths() {
let report = completed_node_report(
CommandOutput {
virtual_thread: TaskInstanceId::from("task"),
status_code: Some(0),
stdout: "bounded tail".to_owned(),
stderr: String::new(),
stdout_source_bytes: 519,
stderr_source_bytes: 0,
stdout_truncated: true,
stderr_truncated: false,
log_backpressured: false,
staged_artifact: None,
},
false,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
Value::Null,
1,
);
assert_eq!(report["stdout_bytes"], 519);
assert_eq!(report["stdout_tail"], "bounded tail");
}
}

359
crates/clusterflux-sdk/src/lib.rs
View file

@ -1,3 +1,5 @@
extern crate self as clusterflux;
use serde::Serialize;
pub use clusterflux_core as core;
@ -10,6 +12,26 @@ pub use task_args::{
TaskArgBudget, TaskArgError, TaskArgKind, TaskArgValidation, Vfs,
};
pub use clusterflux_core::TaskFailurePolicy;
pub use sdk_runtime::TaskRuntimeError as Error;
pub type Result<T> = std::result::Result<T, Error>;
pub mod prelude {
pub use crate::{
command::Command, fs, source, Artifact, Result, SourceSnapshot, TaskFailurePolicy,
};
}
#[macro_export]
macro_rules! spawn {
($task:ident()) => {
$crate::spawn::__product_async_task($task, stringify!($task))
};
($task:ident($argument:expr)) => {
$crate::spawn::__product_async_task_with_arg($argument, $task, stringify!($task))
};
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
pub struct EntrypointDescriptor {
pub name: &'static str,
@ -63,6 +85,8 @@ impl RegisteredProgram {
pub mod __private;
pub mod spawn {
use std::future::{Future, IntoFuture};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, OnceLock};
@ -115,6 +139,273 @@ pub mod spawn {
runtime_threads().lock().unwrap().clone()
}
#[doc(hidden)]
pub fn __product_async_task<F, Fut, R>(
entry: F,
task_id: &'static str,
) -> ProductAsyncTaskBuilder<F, Fut, R>
where
F: FnOnce() -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg,
{
ProductAsyncTaskBuilder {
entry: Some(entry),
env: None,
name: task_id,
task_id,
failure_policy: TaskFailurePolicy::FailFast,
marker: std::marker::PhantomData,
}
}
#[doc(hidden)]
pub fn __product_async_task_with_arg<A, F, Fut, R>(
arg: A,
entry: F,
task_id: &'static str,
) -> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
where
A: TaskArg,
F: FnOnce(A) -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg,
{
ProductAsyncTaskWithArgBuilder {
arg: Some(arg),
entry: Some(entry),
env: None,
name: task_id,
task_id,
arg_budget: TaskArgBudget::default(),
failure_policy: TaskFailurePolicy::FailFast,
marker: std::marker::PhantomData,
}
}
pub struct ProductAsyncTaskBuilder<F, Fut, R>
where
F: FnOnce() -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg,
{
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
entry: Option<F>,
env: Option<EnvRef>,
name: &'static str,
task_id: &'static str,
failure_policy: TaskFailurePolicy,
marker: std::marker::PhantomData<fn() -> (Fut, R)>,
}
impl<F, Fut, R> ProductAsyncTaskBuilder<F, Fut, R>
where
F: FnOnce() -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg + DeserializeOwned,
{
pub fn on(mut self, env: EnvRef) -> Self {
self.env = Some(env);
self
}
pub fn name(mut self, name: &'static str) -> Self {
self.name = name;
self
}
pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
self.failure_policy = failure_policy;
self
}
pub async fn start(self) -> crate::Result<TaskHandle<R>> {
let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
let runtime_event = register_runtime_thread(id, self.name, self.env);
#[cfg(target_arch = "wasm32")]
{
let remote = start_guest_host_task(
clusterflux_core::TaskDefinitionId::from(self.task_id),
self.env,
Vec::new(),
self.failure_policy,
)?;
return Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: None,
remote: Some(remote),
});
}
#[cfg(not(target_arch = "wasm32"))]
{
if let Some(config) = ProductRuntimeConfig::from_env()? {
let remote = start_remote_task(
config,
self.task_id.to_owned(),
self.env,
Vec::new(),
self.failure_policy,
)?;
return Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: None,
remote: Some(remote),
});
}
let result = self.entry.expect("task entry used once")().await?;
Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: Some(result),
remote: None,
})
}
}
}
impl<F, Fut, R> IntoFuture for ProductAsyncTaskBuilder<F, Fut, R>
where
F: FnOnce() -> Fut + 'static,
Fut: Future<Output = crate::Result<R>> + 'static,
R: TaskArg + DeserializeOwned + 'static,
{
type Output = crate::Result<TaskHandle<R>>;
type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
fn into_future(self) -> Self::IntoFuture {
Box::pin(self.start())
}
}
pub struct ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
where
A: TaskArg,
F: FnOnce(A) -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg,
{
arg: Option<A>,
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
entry: Option<F>,
env: Option<EnvRef>,
name: &'static str,
task_id: &'static str,
arg_budget: TaskArgBudget,
failure_policy: TaskFailurePolicy,
marker: std::marker::PhantomData<fn() -> (Fut, R)>,
}
impl<A, F, Fut, R> ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
where
A: TaskArg,
F: FnOnce(A) -> Fut,
Fut: Future<Output = crate::Result<R>>,
R: TaskArg + DeserializeOwned,
{
pub fn on(mut self, env: EnvRef) -> Self {
self.env = Some(env);
self
}
pub fn name(mut self, name: &'static str) -> Self {
self.name = name;
self
}
pub fn arg_budget(mut self, arg_budget: TaskArgBudget) -> Self {
self.arg_budget = arg_budget;
self
}
pub fn failure_policy(mut self, failure_policy: TaskFailurePolicy) -> Self {
self.failure_policy = failure_policy;
self
}
pub async fn start(self) -> crate::Result<TaskHandle<R>> {
let arg = self.arg.expect("task argument used once");
validate_task_arg(&arg, self.arg_budget)
.map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
let id = NEXT_THREAD_ID.fetch_add(1, Ordering::SeqCst);
let runtime_event = register_runtime_thread(id, self.name, self.env);
#[cfg(target_arch = "wasm32")]
{
let boundary = arg
.task_boundary_value()
.map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
let remote = start_guest_host_task(
clusterflux_core::TaskDefinitionId::from(self.task_id),
self.env,
vec![boundary],
self.failure_policy,
)?;
return Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: None,
remote: Some(remote),
});
}
#[cfg(not(target_arch = "wasm32"))]
{
if let Some(config) = ProductRuntimeConfig::from_env()? {
let boundary = arg
.task_boundary_value()
.map_err(|error| TaskRuntimeError::Argument(error.to_string()))?;
let remote = start_remote_task(
config,
self.task_id.to_owned(),
self.env,
vec![boundary],
self.failure_policy,
)?;
return Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: None,
remote: Some(remote),
});
}
let result = self.entry.expect("task entry used once")(arg).await?;
Ok(TaskHandle {
virtual_thread_id: id,
name: self.name,
env: self.env,
debugger_visible: runtime_event.debugger_visible,
result: Some(result),
remote: None,
})
}
}
}
impl<A, F, Fut, R> IntoFuture for ProductAsyncTaskWithArgBuilder<A, F, Fut, R>
where
A: TaskArg + 'static,
F: FnOnce(A) -> Fut + 'static,
Fut: Future<Output = crate::Result<R>> + 'static,
R: TaskArg + DeserializeOwned + 'static,
{
type Output = crate::Result<TaskHandle<R>>;
type IntoFuture = Pin<Box<dyn Future<Output = Self::Output>>>;
fn into_future(self) -> Self::IntoFuture {
Box::pin(self.start())
}
}
pub fn task<F, R>(entry: F) -> TaskBuilder<F, R>
where
F: FnOnce() -> R,
@ -711,6 +1002,10 @@ pub mod command {
self
}
pub fn cwd(self, directory: impl Into<String>) -> Self {
self.current_dir(directory)
}
pub fn env(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.environment_variables.insert(name.into(), value.into());
self
@ -761,6 +1056,34 @@ pub mod command {
.to_owned(),
))
}
pub async fn run(self) -> Result<Output, TaskRuntimeError> {
let program = self.program.clone();
let output = self.output().await?;
if output.status_code == Some(0) {
return Ok(output);
}
Err(TaskRuntimeError::CommandFailed {
program,
status_code: output.status_code,
stdout: bounded_tail(output.stdout),
stderr: bounded_tail(output.stderr),
stdout_truncated: output.stdout_truncated,
stderr_truncated: output.stderr_truncated,
})
}
}
fn bounded_tail(value: String) -> String {
const LIMIT: usize = 4 * 1024;
if value.len() <= LIMIT {
return value;
}
let mut start = value.len() - LIMIT;
while !value.is_char_boundary(start) {
start += 1;
}
value[start..].to_owned()
}
}
@ -782,6 +1105,34 @@ pub mod source {
"source snapshots are produced only by a source-capable Clusterflux node".to_owned(),
))
}
pub struct CurrentProject;
pub fn current_project() -> CurrentProject {
CurrentProject
}
impl CurrentProject {
pub async fn snapshot(self) -> Result<SourceSnapshot, TaskRuntimeError> {
crate::spawn::__product_async_task(snapshot_current_project, "snapshot_current_project")
.await?
.join()
.await
}
}
#[clusterflux::task(capabilities = "source_filesystem")]
async fn snapshot_current_project() -> crate::Result<SourceSnapshot> {
snapshot().await
}
}
impl SourceSnapshot {
pub fn mount(&self) -> Result<&'static str> {
crate::task_args::parse_handle_digest(&self.digest)
.map_err(|error| Error::Argument(error.to_string()))?;
Ok("/workspace")
}
}
pub mod fs {
@ -814,6 +1165,10 @@ pub mod fs {
})
}
pub fn output(relative: impl Into<String>) -> Result<OutputPath, TaskRuntimeError> {
output_path(relative)
}
pub async fn flush(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
#[cfg(target_arch = "wasm32")]
{
@ -838,6 +1193,10 @@ pub mod fs {
}
}
pub async fn publish(path: &OutputPath) -> Result<Artifact, TaskRuntimeError> {
flush(path).await
}
pub async fn materialize(
artifact: &Artifact,
relative: impl Into<String>,

23
crates/clusterflux-sdk/src/sdk_runtime.rs
View file

@ -106,6 +106,14 @@ pub enum TaskRuntimeError {
waited: Duration,
},
ResultDecode(String),
CommandFailed {
program: String,
status_code: Option<i32>,
stdout: String,
stderr: String,
stdout_truncated: bool,
stderr_truncated: bool,
},
}
impl std::fmt::Display for TaskRuntimeError {
@ -124,6 +132,21 @@ impl std::fmt::Display for TaskRuntimeError {
);
}
Self::ResultDecode(message) => ("remote result", message),
Self::CommandFailed {
program,
status_code,
stdout,
stderr,
stdout_truncated,
stderr_truncated,
} => {
return write!(
formatter,
"command `{program}` failed with status {status_code:?}; stdout{}: {stdout:?}; stderr{}: {stderr:?}",
if *stdout_truncated { " (truncated)" } else { "" },
if *stderr_truncated { " (truncated)" } else { "" },
);
}
};
write!(formatter, "{kind} error: {message}")
}

37
crates/clusterflux-sdk/tests/product_api.rs Normal file
View file

@ -0,0 +1,37 @@
#[clusterflux::task]
async fn plus_one(value: i32) -> clusterflux::Result<i32> {
Ok(value + 1)
}
#[test]
fn unified_spawn_defaults_to_function_identity_and_unwraps_task_result() {
futures_executor::block_on(async {
let handle = clusterflux::spawn!(plus_one(41))
.on(clusterflux::env!("linux"))
.await
.unwrap();
assert_eq!(handle.name(), "plus_one");
assert_eq!(handle.env().unwrap().name, "linux");
assert_eq!(handle.join().await.unwrap(), 42);
});
}
#[test]
fn source_mount_and_command_failure_are_public_typed_contracts() {
let source = clusterflux::SourceSnapshot {
digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
.to_owned(),
};
assert_eq!(source.mount().unwrap(), "/workspace");
let error = clusterflux::Error::CommandFailed {
program: "cc".to_owned(),
status_code: Some(2),
stdout: "out".to_owned(),
stderr: "bad input".to_owned(),
stdout_truncated: false,
stderr_truncated: false,
};
assert!(error.to_string().contains("command"));
assert!(error.to_string().contains("cc"));
}

4
docs/artifacts.md
View file

@ -34,5 +34,5 @@ tenant, project, process, artifact, and policy context.
The reverse stream counts framing, base64 expansion, failed bytes, and abandoned
transfers. The CLI verifies the final digest. Hosted policy may impose
per-project, per-tenant, and per-account size, concurrency, and period limits.
Operators may disable relay traffic as an emergency safety control; one tenant's
period usage does not consume a shared customer byte quota.
There is no hosted global circuit breaker, and one tenant's period usage does
not consume a shared customer byte quota.

23
docs/debugging.md
View file

@ -1,14 +1,22 @@
# Debugging
The DAP runtime observer keeps one coordinator session open and reconnects it
with a 100 ms to 5 second exponential backoff. Polling is responsive around
debug actions (100 ms) and backs off to one observation cycle every 5 seconds
while idle. A cycle makes at most four coordinator requests, so steady-state
idle traffic is bounded at 0.8 requests per second per debug session.
The VS Code extension uses the Debug Adapter Protocol and the coordinator's
authoritative process, task, attempt, and Debug Epoch APIs.
## Launch
Open your project and start "Clusterflux: Launch Virtual Process". The adapter
builds the bundle, launches the selected entrypoint, and replaces its thread
view with coordinator task snapshots. Product mode does not infer threads from
completion events or demo fixtures.
acknowledges launch configuration immediately, then builds and starts the
selected entrypoint in the background. You can keep using the debug UI while a
long-running process is attached. The adapter replaces its thread view with
coordinator task snapshots as they arrive; product mode does not infer threads
from completion events or demo fixtures.
Each task view includes its logical task, attempt ID, definition, state, node,
environment, arguments, typed handles, command status, VFS checkpoint, probe
@ -19,6 +27,12 @@ source when known, and restart compatibility. Unknown source remains unknown.
When a breakpoint is hit, the coordinator asks every active participant to
freeze. A fully frozen epoch is a consistent all-participant view.
You do not need to configure a breakpoint before launch. Breakpoint changes are
sent to the running coordinator, and a stopped event is emitted only after an
executing Wasm probe reports a real hit. Pause, continue, inspection, and
disconnect requests remain responsive while the adapter observes the runtime in
the background. Attach does not fabricate an initial breakpoint.
If one or more participants cannot freeze within five seconds, the epoch becomes
partially frozen when at least one participant did freeze. The adapter warns
that running and frozen tasks do not form a consistent global snapshot. You may
@ -31,7 +45,8 @@ runtime did not report it.
## Continue and restart
Continue resumes only participants that acknowledged the freeze.
Continue responds immediately, then resumes only participants that acknowledged
the freeze while runtime observation continues in the background.
Restarting a terminal task rebuilds the bundle and checks its clean entry
checkpoint. A compatible restart creates a new attempt under the same logical

4
docs/environments.md
View file

@ -1,5 +1,9 @@
# Environments
Clusterflux disables network access while a task executes. Materializing an
environment for the first time can still fetch declared inputs; subsequent task
execution uses the materialized environment with networking disabled.
Declare environments in the bundle under:
~~~text

31
docs/getting-started.md
View file

@ -11,6 +11,9 @@ cargo install --path crates/clusterflux-node --bin clusterflux-node
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
~~~
On NixOS or another system with Nix, the equivalent package is available with
`nix profile install .#clusterflux-tools`.
Install rootless Podman on each Linux node that will execute container-backed
environments.
@ -52,7 +55,7 @@ same stored identity:
~~~bash
clusterflux-node \
--coordinator https://clusterflux.michelpaulissen.com \
--coordinator https://clusterflux.lesstuff.com \
--tenant "$TENANT" \
--project-id <hosted-project-id> \
--node workstation \
@ -81,6 +84,21 @@ clusterflux bundle inspect --project .
clusterflux run --project . build
~~~
From this checkout, the complete source-to-task-to-artifact path is:
~~~bash
clusterflux bundle inspect --project examples/hello-build
clusterflux run --project examples/hello-build build
clusterflux artifact list --process <process-id>
clusterflux artifact download <artifact-id> --to ./hello-clusterflux
chmod +x ./hello-clusterflux
./hello-clusterflux
~~~
The workflow snapshots `examples/hello-build`, starts its `compile` task in the
network-disabled Linux execution environment, and retains the real static executable returned by
that task.
Choose another entrypoint by replacing `build`. Clusterflux rejects an oversized
or invalid bundle before it creates the virtual process.
@ -101,12 +119,23 @@ Restart it as a new attempt under the same logical task identity:
clusterflux task restart <task-id> --process <process-id> --yes
~~~
`examples/recovery-build` demonstrates this without a synthetic trap. It starts
two instances of `build_lane`; one completes while the other runs a command that
exits with status 23. Edit that command to produce the recovering output, then
restart the failed task. Its original join resolves from the replacement
attempt.
## 6. Debug in VS Code
Open the project in VS Code and start `Clusterflux: Launch Virtual Process`.
Set a breakpoint on a generated probe location and use Threads, Stack,
Variables, Continue, Pause, and Restart.
Breakpoints remain unverified until the coordinator installs them. Thread IDs
are stable for an exact logical task instance across snapshot updates and retry
attempts. Observer reconnect diagnostics do not manufacture stopped events, and
Continue succeeds only after the coordinator acknowledges resume.
A fully frozen Debug Epoch gives a consistent all-participant view. If a
participant cannot freeze within five seconds, the adapter reports a partial
epoch. You may inspect frozen participants, but values across running and frozen

2
docs/nodes.md
View file

@ -23,7 +23,7 @@ key pair.
~~~bash
clusterflux-node \
--coordinator https://clusterflux.michelpaulissen.com \
--coordinator https://clusterflux.lesstuff.com \
--tenant "$TENANT" \
--project-id <project-id> \
--node workstation \

11
docs/task-abi.md
View file

@ -9,10 +9,13 @@ capability-bearing values use typed handles. A structured boundary value carries
both its JSON structure and the complete typed handle table required to
materialize it.
Supported handles include VFS and artifact identities with full tenant, project,
process, producer, digest, size, and path provenance. The SDK encodes handles as
explicit placeholders and the runtime replaces only placeholders that match the
declared type and metadata.
Portable handles carry the minimum stable value needed to cross the task ABI,
such as an artifact ID, digest, and size. Tenant, project, process, producer,
path, retaining location, and authorization state are coordinator-side
metadata; they are not portable authority fields supplied by task code. The SDK
encodes handles as explicit placeholders and the runtime replaces only
placeholders that match the declared type and authoritative coordinator
metadata.
A malformed, missing, extra, cross-scope, or type-mismatched handle fails closed.
The runtime does not reconstruct authority from a string path or caller-supplied

13
examples/hello-build/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "hello-build"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
publish = false
[lib]
crate-type = ["rlib", "cdylib"]
[dependencies]
clusterflux = { package = "clusterflux-sdk", path = "../../crates/clusterflux-sdk" }

18
examples/hello-build/README.md Normal file
View file

@ -0,0 +1,18 @@
# Hello build
The primary Clusterflux example snapshots this project, compiles a real static C
executable in the declared network-disabled Linux execution environment, and publishes the
executable as a retained artifact.
Run it with:
~~~bash
clusterflux bundle inspect --project examples/hello-build
clusterflux run --project examples/hello-build build
clusterflux artifact list --process <process-id>
clusterflux artifact download <artifact-id> --to ./hello-clusterflux
chmod +x ./hello-clusterflux
./hello-clusterflux
~~~
The final command prints hello from a real Clusterflux build.

0
examples/launch-build-demo/envs/linux/Containerfile → examples/hello-build/envs/linux/Containerfile
View file

0
examples/launch-build-demo/fixture/hello-clusterflux.c → examples/hello-build/fixture/hello-clusterflux.c
View file

30
examples/hello-build/src/lib.rs Normal file
View file

@ -0,0 +1,30 @@
use clusterflux::prelude::*;
#[clusterflux::task(capabilities = "command")]
pub async fn compile(source: SourceSnapshot) -> Result<Artifact> {
let executable = fs::output("hello-clusterflux")?;
Command::new("cc")
.args([
"-Os",
"-static",
"-s",
"fixture/hello-clusterflux.c",
"-o",
executable.as_str(),
])
.cwd(source.mount()?)
.env("SOURCE_DATE_EPOCH", "0")
.network_disabled()
.run()
.await?;
fs::publish(&executable).await
}
#[clusterflux::main]
pub async fn build() -> Result<Artifact> {
let source = source::current_project().snapshot().await?;
let compile = clusterflux::spawn!(compile(source))
.on(clusterflux::env!("linux"))
.await?;
compile.join().await
}

14
examples/launch-build-demo/README.md
View file

@ -1,14 +0,0 @@
# Launch Build Demo
This build workflow is expressed as Rust source code. It uses `env!("linux")`,
recognizes `env!("windows")` when a Windows development node is attached,
spawns debugger-visible virtual tasks, and returns portable artifact and source
handles instead of moving large bytes through task arguments.
The Linux environment is defined by `envs/linux/Containerfile`. The Windows environment is a user-attached development contract and does not claim secure managed Windows sandboxing.
Inspect the bundle metadata:
```bash
clusterflux bundle inspect --project examples/launch-build-demo
```

2
examples/launch-build-demo/envs/windows/Dockerfile
View file

@ -1,2 +0,0 @@
# User-attached Windows development execution contract.
# This is not a managed untrusted Windows sandbox.

33
examples/launch-build-demo/src/bin/sdk-product-runtime.rs
View file

@ -1,33 +0,0 @@
use std::error::Error;
use futures_executor::block_on;
fn main() -> Result<(), Box<dyn Error>> {
let handle = block_on(
clusterflux::spawn::task_with_arg(41_i32, |_| -> i32 {
panic!("product-mode spawn invoked its local Rust closure")
})
.task_id("task_add_one")
.name("SDK product Wasmtime task")
.env(clusterflux::env!("linux"))
.start(),
)?;
let task_spec = handle
.task_spec()
.cloned()
.ok_or("CLUSTERFLUX_SDK_COORDINATOR did not enable product runtime")?;
let virtual_thread_id = handle.virtual_thread_id();
let result = block_on(handle.join())?;
println!(
"{}",
serde_json::to_string(&serde_json::json!({
"kind": "clusterflux-sdk-product-runtime",
"virtual_thread_id": virtual_thread_id,
"task_spec": task_spec,
"remote_result": result,
"local_function_invoked_by_spawn": false,
}))?
);
Ok(())
}

Some files were not shown because too many files have changed in this diff Show more