michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public dry run dryrun-a43e907efd9d

Browse source 
Source commit: a43e907efd9d1561c23fe73499478e881f868355

Public tree identity: sha256:453dea30195485dd8939f575a69b39f4bb7acd84c4df23f8aa29b55d044f2673
This commit is contained in:
Clusterflux release dry run 2026-07-15 01:54:51 +02:00
commit 6f52bb46cd
210 changed files with 77158 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

7
.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
/target/
/.clusterflux/
**/.clusterflux/
/vscode-extension/node_modules/
/private/*/Cargo.lock
/private/*/target/
/scripts/containers-home/

22
CLUSTERFLUX_PUBLIC_TREE.json Normal file
View file

@ -0,0 +1,22 @@
{
"kind": "clusterflux-filtered-public-tree",
"source_commit": "a43e907efd9d1561c23fe73499478e881f868355",
"release_name": "dryrun-a43e907efd9d",
"filtered_out": [
"private/**",
"internal/**",
"experiments/**",
".git",
"target",
"git-ignored source paths",
"root/*.md except README.md",
"**/.clusterflux/**",
".forgejo/**"
],
"public_export": {
"host_neutral": true,
"include_forgejo_workflows": false
},
"forgejo_host": "git.michelpaulissen.com",
"default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
}

2924
Cargo.lock generated Normal file
View file

File diff suppressed because it is too large Load diff

44
Cargo.toml Normal file
View file

@ -0,0 +1,44 @@
[workspace]
resolver = "2"
members = [
"crates/clusterflux-cli",
"crates/clusterflux-control",
"crates/clusterflux-coordinator",
"crates/clusterflux-core",
"crates/clusterflux-dap",
"crates/clusterflux-macros",
"crates/clusterflux-node",
"crates/clusterflux-sdk",
"crates/clusterflux-wasm-runtime",
"examples/launch-build-demo",
]
[workspace.package]
edition = "2021"
license = "Apache-2.0 OR MIT"
repository = "https://git.michelpaulissen.com/michel/clusterflux"
[workspace.dependencies]
anyhow = "1.0"
base64 = "0.22"
clap = { version = "4.5", features = ["derive"] }
ed25519-dalek = "2"
futures-executor = "0.3"
getrandom = "0.3"
hex = "0.4"
libc = "0.2"
proc-macro2 = "1.0"
postgres = { version = "0.19", features = ["with-serde_json-1"] }
quinn = { version = "0.11.11", default-features = false, features = ["runtime-tokio", "rustls-ring"] }
quote = "1.0"
rcgen = "0.14"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.10"
syn = { version = "2.0", features = ["full"] }
tempfile = "3.10"
thiserror = "1.0"
ureq = { version = "2.12", default-features = false, features = ["tls"] }
tokio = { version = "1.52", features = ["io-util", "macros", "rt-multi-thread"] }
wasmtime = { version = "=43.0.2", default-features = false, features = ["async", "cranelift", "debug", "runtime", "std", "wat"] }
wasmparser = "0.245.1"

175
LICENSE-APACHE Normal file
View file

@ -0,0 +1,175 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work.
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS

21
LICENSE-MIT Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Clusterflux contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

80
README.md Normal file
View file

@ -0,0 +1,80 @@
# Clusterflux
Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
coordinator hosts the async main, while attached nodes execute Wasm tasks,
rootless containers, and native commands. Tasks exchange canonical values and
typed handles instead of sharing host memory.
Start with [Getting started](docs/getting-started.md). It takes you through
authentication, project setup, node enrollment, a run, debugging, task restart,
and artifact download.
## What you get
- One virtual process with distinct task instances and restart attempts.
- Bundle-declared environments resolved by digest.
- Native work only on nodes you attach.
- Metadata-first artifacts whose bytes remain on retaining nodes by default.
- VS Code debugging backed by coordinator task and attempt snapshots.
- Full and partial Debug Epochs with explicit consistency status.
- Human Authentik sessions plus scoped public-key identities for agents and nodes.
- A public coordinator, node runtime, CLI, SDK, and DAP adapter for self-hosting.
## Install from this checkout
~~~bash
cargo install --path crates/clusterflux-cli --bin clusterflux
cargo install --path crates/clusterflux-node --bin clusterflux-node
cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinator
cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
~~~
Rootless Podman is required on Linux nodes that build or run a declared
Containerfile environment. Install VS Code when you want the graphical debug
workflow.
## First run
For the hosted service:
~~~bash
clusterflux login --browser
clusterflux project init --new-project my-project --name "My Project" --yes
clusterflux node enroll --project-id my-project --json
~~~
Use the returned short-lived enrollment grant once with "clusterflux node
attach", then start "clusterflux-node --worker" from the project directory. See
[Nodes](docs/nodes.md) for the complete sequence.
Choose an entrypoint and run it:
~~~bash
clusterflux bundle inspect --project examples/launch-build-demo
clusterflux run --project examples/launch-build-demo build
~~~
Inspect the result:
~~~bash
clusterflux process status
clusterflux task list
clusterflux logs
clusterflux artifact list
clusterflux artifact download app.txt --to ./app.txt
~~~
To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md).
The hosted website is not required for self-hosted projects.
## Documentation
- [Getting started](docs/getting-started.md)
- [Architecture](docs/architecture.md)
- [Nodes](docs/nodes.md)
- [Environments](docs/environments.md)
- [Artifacts](docs/artifacts.md)
- [Debugging](docs/debugging.md)
- [Task ABI](docs/task-abi.md)
- [Self-hosting](docs/self-hosting.md)
- [Security](docs/security.md)

22
crates/clusterflux-cli/Cargo.toml Normal file
View file

@ -0,0 +1,22 @@
[package]
name = "clusterflux-cli"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[[bin]]
name = "clusterflux"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
base64.workspace = true
clap.workspace = true
clusterflux-core = { path = "../clusterflux-core" }
clusterflux-control = { path = "../clusterflux-control" }
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tempfile.workspace = true
wasmparser.workspace = true

236
crates/clusterflux-cli/src/admin.rs Normal file
View file

@ -0,0 +1,236 @@
use std::path::PathBuf;
use anyhow::Result;
use clusterflux_core::admin_request_proof;
use serde_json::{json, Value};
use crate::client::JsonLineSession;
use crate::project::project_init_report;
use crate::tools::{command_nonce, unix_timestamp_seconds};
use crate::{
confirmation_required_report, AdminBootstrapArgs, AdminStatusArgs, AdminSuspendTenantArgs,
ProjectInitArgs,
};
pub(crate) fn admin_status_report(args: AdminStatusArgs) -> Result<Value> {
if let Some(coordinator) = &args.scope.coordinator {
let tenant = args.scope.tenant.clone();
let user = args.scope.user.clone();
let admin_token = admin_token_for_request(args.admin_token.as_deref())?;
let admin_nonce = command_nonce("admin-status");
let issued_at_epoch_seconds = unix_timestamp_seconds();
let admin_proof = admin_request_proof(
&admin_token,
"admin_status",
&tenant,
&user,
&tenant,
&admin_nonce,
issued_at_epoch_seconds,
);
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(json!({
"type": "admin_status",
"tenant": tenant,
"actor_user": user,
"admin_proof": admin_proof,
"admin_nonce": admin_nonce,
"issued_at_epoch_seconds": issued_at_epoch_seconds,
}))?;
return Ok(json!({
"command": "admin status",
"coordinator": coordinator,
"tenant": tenant,
"user": user,
"suspended": response
.get("suspended")
.cloned()
.unwrap_or(json!(false)),
"response": response,
"safe_default": "read_only",
"private_website_required": false,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "admin status",
"mode": "self_hosted_local",
"safe_default": "read_only",
"private_website_required": false,
}))
}
pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) -> Result<Value> {
let scope = args.scope.clone();
let new_project = args.scope.project.clone();
let project_init = project_init_report(
ProjectInitArgs {
scope: args.scope,
new_project,
name: args.name,
yes: args.yes,
},
cwd,
)?;
let coordinator = scope
.coordinator
.clone()
.unwrap_or_else(|| "<local-coordinator>".to_owned());
let tenant = scope.tenant.clone();
let project = scope.project.clone();
let user = scope.user.clone();
Ok(json!({
"command": "admin bootstrap",
"mode": if scope.coordinator.is_some() { "public_coordinator_api" } else { "self_hosted_local" },
"tenant": tenant.clone(),
"project": project.clone(),
"user": user,
"coordinator": scope.coordinator,
"private_website_required": false,
"self_hosted_cli_only": true,
"project_config_written": project_init
.get("project_config_written")
.cloned()
.unwrap_or(json!(false)),
"project_init": project_init,
"admin_surfaces": {
"coordinator": "clusterflux-coordinator",
"project": "clusterflux project init/status/list/select",
"node": "clusterflux node enroll/list/status/revoke",
"process": "clusterflux run/process status/process restart/process cancel",
"logs": "clusterflux logs",
"artifacts": "clusterflux artifact list/download/export",
"quota": "clusterflux quota status",
"policy": "clusterflux admin status/suspend-tenant",
},
"bootstrap_sequence": [
{
"step": "start_self_hosted_coordinator",
"command": "clusterflux-coordinator --listen 127.0.0.1:0",
"private_website_required": false,
},
{
"step": "create_or_link_project",
"command": "clusterflux project init --yes",
"completed": true,
"private_website_required": false,
},
{
"step": "create_node_enrollment_grant",
"command": format!(
"clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project
),
"private_website_required": false,
},
{
"step": "attach_worker_node",
"command": format!(
"clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
tenant, project
),
"private_website_required": false,
},
{
"step": "run_process",
"command": format!(
"clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project
),
"private_website_required": false,
},
{
"step": "inspect_status_logs_artifacts",
"commands": [
"clusterflux process status",
"clusterflux task list",
"clusterflux logs",
"clusterflux artifact list",
"clusterflux quota status",
],
"private_website_required": false,
},
{
"step": "revoke_access",
"command": "clusterflux admin revoke-node --node <node-id> --yes",
"private_website_required": false,
}
],
}))
}
pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Result<Value> {
let tenant = args
.target_tenant
.unwrap_or_else(|| args.scope.tenant.clone());
if !args.yes {
return Ok(confirmation_required_report(
"admin suspend-tenant",
"suspend_tenant",
json!({
"coordinator": args.scope.coordinator,
"actor_tenant": args.scope.tenant,
"actor_user": args.scope.user,
"target_tenant": tenant,
}),
"clusterflux admin suspend-tenant --yes".to_owned(),
));
}
if let Some(coordinator) = &args.scope.coordinator {
let actor_tenant = args.scope.tenant.clone();
let actor_user = args.scope.user.clone();
let admin_token = admin_token_for_request(args.admin_token.as_deref())?;
let admin_nonce = command_nonce("admin-suspend-tenant");
let issued_at_epoch_seconds = unix_timestamp_seconds();
let admin_proof = admin_request_proof(
&admin_token,
"suspend_tenant",
&actor_tenant,
&actor_user,
&tenant,
&admin_nonce,
issued_at_epoch_seconds,
);
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(json!({
"type": "suspend_tenant",
"tenant": actor_tenant,
"actor_user": actor_user,
"target_tenant": tenant,
"admin_proof": admin_proof,
"admin_nonce": admin_nonce,
"issued_at_epoch_seconds": issued_at_epoch_seconds,
}))?;
return Ok(json!({
"command": "admin suspend-tenant",
"coordinator": coordinator,
"requires_confirmation": !args.yes,
"tenant": tenant,
"actor_tenant": actor_tenant,
"actor_user": actor_user,
"suspended": response.get("type").and_then(Value::as_str) == Some("tenant_suspended"),
"private_website_required": false,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "admin suspend-tenant",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"tenant": tenant,
"private_website_required": false,
}))
}
fn admin_token_for_request(explicit: Option<&str>) -> Result<String> {
explicit
.map(str::to_owned)
.or_else(|| std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok())
.filter(|token| !token.trim().is_empty())
.ok_or_else(|| {
anyhow::anyhow!(
"admin command requires --admin-token or CLUSTERFLUX_ADMIN_TOKEN for coordinator requests"
)
})
}

17
crates/clusterflux-cli/src/agent.rs Normal file
View file

@ -0,0 +1,17 @@
use clusterflux_core::Digest;
use serde::Serialize;
use crate::AgentEnrollArgs;
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct AgentEnrollmentPlan {
pub(crate) public_key_fingerprint: Digest,
pub(crate) browser_interaction_required_each_run: bool,
}
pub(crate) fn agent_enrollment_plan(args: AgentEnrollArgs) -> AgentEnrollmentPlan {
AgentEnrollmentPlan {
public_key_fingerprint: Digest::sha256(args.public_key),
browser_interaction_required_each_run: false,
}
}

547
crates/clusterflux-cli/src/artifact.rs Normal file
View file

@ -0,0 +1,547 @@
use anyhow::{Context, Result};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use serde_json::{json, Value};
use sha2::{Digest as _, Sha256};
use std::io::Write;
use std::path::Path;
use std::time::{Duration, Instant};
use crate::client::{
authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
JsonLineSession,
};
use crate::config::StoredCliSession;
use crate::errors::cli_error_summary_for_category;
use crate::process_events::{
artifact_download_grant_disclosures, artifact_download_session_summary,
artifact_export_plan_summary, artifact_response_machine_error, artifact_summaries,
};
use crate::{ArtifactDownloadArgs, ArtifactExportArgs, ArtifactListArgs};
pub(crate) const DEFAULT_ARTIFACT_EXPORT_MAX_BYTES: u64 = 64 * 1024 * 1024;
const ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES: u64 = 256 * 1024;
#[cfg(test)]
pub(crate) fn artifact_list_report(args: ArtifactListArgs) -> Result<Value> {
artifact_list_report_with_session(args, None)
}
pub(crate) fn artifact_list_report_with_session(
args: ArtifactListArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,
args.process.clone(),
stored_session,
)?;
let artifacts = artifact_summaries(events.as_ref());
Ok(json!({
"command": "artifact list",
"process": args.process,
"source": "task_events",
"artifacts": artifacts,
"default_durable_store_assumed": false,
"events": events,
}))
}
#[cfg(test)]
pub(crate) fn artifact_download_report(args: ArtifactDownloadArgs) -> Result<Value> {
artifact_download_report_with_session(args, None)
}
pub(crate) fn artifact_download_report_with_session(
args: ArtifactDownloadArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "create_artifact_download_link",
"artifact": args.artifact,
"max_bytes": args.max_bytes,
}),
json!({
"type": "create_artifact_download_link",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"artifact": args.artifact,
"max_bytes": args.max_bytes,
}),
)?)?;
let download_session = artifact_download_session_summary(&response);
let grant_disclosures = artifact_download_grant_disclosures(&response);
let local_download = match &args.to {
Some(path)
if response.get("type").and_then(Value::as_str)
== Some("artifact_download_link") =>
{
download_link_to_path(
&mut session,
coordinator,
&args.scope,
&args.artifact,
args.max_bytes,
path,
&response,
stored_session,
)?
}
Some(path) => json!({
"status": "download_link_failed",
"local_path": path,
"local_bytes_written_by_cli": false,
"machine_error": artifact_response_machine_error(
&response,
"coordinator rejected artifact download",
"connectivity"
),
}),
None => Value::Null,
};
return Ok(json!({
"command": "artifact download",
"coordinator": coordinator,
"artifact": args.artifact,
"max_bytes": args.max_bytes,
"to": args.to,
"download_session": download_session,
"local_download": local_download,
"grant_disclosures": grant_disclosures,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "artifact download",
"status": "requires_coordinator",
"artifact": args.artifact,
"max_bytes": args.max_bytes,
"to": args.to,
"download_session": {
"status": "requires_coordinator",
"link_issued": false,
"explicit_user_action_required": true,
"machine_error": cli_error_summary_for_category(
"connectivity",
"artifact download requires a coordinator"
),
},
"grant_disclosures": [],
}))
}
#[cfg(test)]
pub(crate) fn artifact_export_report(args: ArtifactExportArgs) -> Result<Value> {
artifact_export_report_with_session(args, None)
}
pub(crate) fn artifact_export_report_with_session(
args: ArtifactExportArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "export_artifact_to_node",
"artifact": args.artifact,
"receiver_node": args.receiver_node,
"direct_connectivity": true,
"failure_reason": "",
}),
json!({
"type": "export_artifact_to_node",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"artifact": args.artifact,
"receiver_node": args.receiver_node,
"direct_connectivity": true,
"failure_reason": "",
}),
)?)?;
let mut export_plan = artifact_export_plan_summary(&response, &args.to);
let local_export = if response.get("type").and_then(Value::as_str)
== Some("artifact_export_plan")
{
artifact_export_local_write_followup(&mut session, coordinator, &args, stored_session)?
} else {
json!({
"status": "skipped",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
"machine_error": artifact_response_machine_error(
&response,
"coordinator rejected artifact export",
"connectivity"
),
})
};
apply_local_export_summary(&mut export_plan, &local_export);
let grant_disclosures = local_export
.get("grant_disclosures")
.cloned()
.unwrap_or_else(|| json!([]));
return Ok(json!({
"command": "artifact export",
"coordinator": coordinator,
"artifact": args.artifact,
"to": args.to,
"receiver_node": args.receiver_node,
"export_plan": export_plan,
"local_export": local_export,
"grant_disclosures": grant_disclosures,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "artifact export",
"status": "requires_coordinator",
"artifact": args.artifact,
"to": args.to,
"receiver_node": args.receiver_node,
"export_plan": {
"status": "requires_coordinator",
"explicit_user_action": true,
"local_bytes_written_by_cli": false,
"default_durable_store_assumed": false,
"machine_error": cli_error_summary_for_category(
"connectivity",
"artifact export requires a coordinator"
),
},
"grant_disclosures": [],
}))
}
fn artifact_export_local_write_followup(
session: &mut JsonLineSession,
coordinator: &str,
args: &ArtifactExportArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let link_response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "create_artifact_download_link",
"artifact": args.artifact,
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
}),
json!({
"type": "create_artifact_download_link",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"artifact": args.artifact,
"max_bytes": DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
}),
)?)?;
if link_response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
return Ok(json!({
"status": "download_link_failed",
"explicit_user_action": true,
"local_path": &args.to,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"download_session": artifact_download_session_summary(&link_response),
"grant_disclosures": [],
"machine_error": artifact_response_machine_error(
&link_response,
"coordinator rejected artifact download for local export",
"connectivity"
),
"link_response": link_response,
}));
}
download_link_to_path(
session,
coordinator,
&args.scope,
&args.artifact,
DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
&args.to,
&link_response,
stored_session,
)
}
#[allow(clippy::too_many_arguments)]
fn download_link_to_path(
session: &mut JsonLineSession,
coordinator: &str,
scope: &crate::CliScopeArgs,
artifact: &str,
max_bytes: u64,
destination: &Path,
link_response: &Value,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let download_session = artifact_download_session_summary(link_response);
let grant_disclosures = artifact_download_grant_disclosures(link_response);
let token_digest = link_response
.pointer("/link/scoped_token_digest")
.cloned()
.context("artifact download link did not include a scoped token digest")?;
let expected_digest: clusterflux_core::Digest = serde_json::from_value(
link_response
.pointer("/link/artifact_digest")
.cloned()
.context("artifact download link omitted the published digest")?,
)?;
let expected_size = link_response
.pointer("/link/artifact_size_bytes")
.and_then(Value::as_u64)
.context("artifact download link omitted the published size")?;
if expected_size > max_bytes {
return Ok(json!({
"status": "download_limit_failed",
"local_path": destination,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"download_session": download_session,
"grant_disclosures": grant_disclosures,
"machine_error": cli_error_summary_for_category(
"quota",
&format!("artifact size {expected_size} exceeds requested limit {max_bytes}")
),
}));
}
let parent = destination
.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
let mut temporary = tempfile::Builder::new()
.prefix(".clusterflux-download-")
.tempfile_in(parent)
.with_context(|| {
format!(
"failed to create a bounded download in {}",
parent.display()
)
})?;
let started = Instant::now();
let mut received_bytes = 0_u64;
let mut hasher = Sha256::new();
let stream_response = loop {
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "open_artifact_download_stream",
"artifact": artifact,
"max_bytes": max_bytes,
"token_digest": token_digest,
"chunk_bytes": expected_size.clamp(1, ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES),
}),
json!({
"type": "open_artifact_download_stream",
"tenant": scope.tenant,
"project": scope.project,
"actor_user": scope.user,
"artifact": artifact,
"max_bytes": max_bytes,
"token_digest": token_digest,
"chunk_bytes": expected_size.clamp(1, ARTIFACT_DOWNLOAD_CONTROL_CHUNK_BYTES),
}),
)?)?;
if response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
break response;
}
if response
.get("content_bytes_available")
.and_then(Value::as_bool)
== Some(true)
{
let offset = response
.get("content_offset")
.and_then(Value::as_u64)
.unwrap_or(u64::MAX);
if offset != received_bytes {
break json!({
"type": "error",
"message": format!(
"artifact reverse stream returned offset {offset}, expected {received_bytes}"
),
});
}
let Some(content_base64) = response.get("content_base64").and_then(Value::as_str)
else {
break json!({
"type": "error",
"message": "artifact reverse stream marked bytes available without content",
});
};
let content = BASE64_STANDARD
.decode(content_base64)
.context("artifact download stream returned invalid base64 content")?;
received_bytes = received_bytes
.checked_add(content.len() as u64)
.context("artifact download byte count overflowed")?;
if received_bytes > expected_size || received_bytes > max_bytes {
break json!({
"type": "error",
"message": "artifact reverse stream exceeded its published size or CLI limit",
});
}
temporary
.write_all(&content)
.context("write bounded artifact download")?;
hasher.update(&content);
if response.get("content_eof").and_then(Value::as_bool) == Some(true) {
break response;
}
continue;
}
if started.elapsed() >= Duration::from_secs(120) {
break json!({
"type": "error",
"message": "timed out waiting for the retaining node to reverse-stream artifact bytes",
});
}
std::thread::sleep(Duration::from_millis(25));
};
if stream_response.get("type").and_then(Value::as_str) != Some("artifact_download_stream") {
return Ok(json!({
"status": "download_stream_failed",
"explicit_user_action": true,
"local_path": destination,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"download_session": download_session,
"grant_disclosures": grant_disclosures,
"machine_error": artifact_response_machine_error(
&stream_response,
"coordinator rejected artifact download stream",
"connectivity"
),
"stream": artifact_stream_summary(&stream_response),
}));
}
let actual_digest_hex = format!("{:x}", hasher.finalize());
let actual_digest = clusterflux_core::Digest::from_sha256_hex(&actual_digest_hex)
.map_err(anyhow::Error::msg)?;
if received_bytes != expected_size || actual_digest != expected_digest {
return Ok(json!({
"status": "download_integrity_failed",
"explicit_user_action": true,
"local_path": destination,
"local_bytes_written_by_cli": false,
"content_bytes_available": false,
"download_session": download_session,
"grant_disclosures": grant_disclosures,
"machine_error": cli_error_summary_for_category(
"program",
&format!(
"artifact download digest/size mismatch: received {received_bytes} bytes with {actual_digest}, expected {expected_size} bytes with {expected_digest}"
)
),
"stream": artifact_stream_summary(&stream_response),
}));
}
temporary
.as_file()
.sync_all()
.context("sync verified artifact download")?;
temporary.persist_noclobber(destination).map_err(|error| {
anyhow::anyhow!(
"refusing to replace artifact download destination {}: {}",
destination.display(),
error.error
)
})?;
Ok(json!({
"status": "local_bytes_written",
"explicit_user_action": true,
"local_path": destination,
"local_bytes_written_by_cli": true,
"bytes_written": received_bytes,
"verified_digest": actual_digest,
"published_digest": expected_digest,
"content_bytes_available": true,
"content_source": stream_response.get("content_source").cloned().unwrap_or(Value::Null),
"download_session": download_session,
"grant_disclosures": grant_disclosures,
"stream": artifact_stream_summary(&stream_response),
}))
}
pub(crate) fn artifact_stream_summary(response: &Value) -> Value {
let status = response
.get("type")
.and_then(Value::as_str)
.unwrap_or("coordinator_response");
let mut summary = json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"streamed_bytes": response.get("streamed_bytes").cloned().unwrap_or_else(|| json!(0)),
"charged_download_bytes": response.get("charged_download_bytes").cloned().unwrap_or_else(|| json!(0)),
"content_bytes_available": response.get("content_bytes_available").cloned().unwrap_or(json!(false)),
"content_offset": response.get("content_offset").cloned().unwrap_or(Value::Null),
"content_eof": response.get("content_eof").cloned().unwrap_or(json!(false)),
"content_source": response.get("content_source").cloned().unwrap_or(Value::Null),
"content_material_returned_in_report": false,
"error": response.get("message").cloned().unwrap_or(Value::Null),
});
if status != "artifact_download_stream" {
if let Some(object) = summary.as_object_mut() {
object.insert(
"machine_error".to_owned(),
artifact_response_machine_error(
response,
"coordinator rejected artifact download stream",
"connectivity",
),
);
}
}
summary
}
fn apply_local_export_summary(export_plan: &mut Value, local_export: &Value) {
let Some(object) = export_plan.as_object_mut() else {
return;
};
object.insert(
"local_bytes_written_by_cli".to_owned(),
local_export
.get("local_bytes_written_by_cli")
.cloned()
.unwrap_or(json!(false)),
);
object.insert(
"local_export_status".to_owned(),
local_export
.get("status")
.cloned()
.unwrap_or_else(|| json!("unknown")),
);
object.insert(
"content_bytes_available".to_owned(),
local_export
.get("content_bytes_available")
.cloned()
.unwrap_or(json!(false)),
);
if let Some(bytes_written) = local_export.get("bytes_written") {
object.insert("bytes_written".to_owned(), bytes_written.clone());
object.insert(
"writes_require_data_plane_followup".to_owned(),
json!(false),
);
}
}

780
crates/clusterflux-cli/src/auth.rs Normal file
View file

@ -0,0 +1,780 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
use serde::Serialize;
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, stored_session_for_coordinator, JsonLineSession,
};
use crate::config::{
default_hosted_coordinator_endpoint, effective_scope_value, read_cli_session,
read_project_config, write_cli_session, write_project_config, ProjectConfig, StoredCliSession,
};
use crate::errors::cli_error_summary;
use crate::run::{session_from_env, CliSession};
use crate::ConnectSelfHostedArgs;
use crate::{AuthStatusArgs, LoginArgs};
const DEFAULT_BROWSER_LOGIN_TRANSACTION_TIMEOUT_SECONDS: u64 = 300;
pub(crate) fn read_session_secret_from_stdin() -> Result<String> {
let mut secret = String::new();
std::io::stdin()
.read_line(&mut secret)
.context("failed to read self-hosted session secret from stdin")?;
let secret = secret.trim_end_matches(['\r', '\n']).to_owned();
if secret.trim().is_empty() {
anyhow::bail!("self-hosted session secret from stdin must not be empty");
}
Ok(secret)
}
pub(crate) fn connect_self_hosted_report(
args: ConnectSelfHostedArgs,
cwd: PathBuf,
session_secret: String,
) -> Result<Value> {
if !args.session_secret_stdin {
anyhow::bail!("self-hosted session configuration requires --session-secret-stdin");
}
let coordinator = args
.scope
.coordinator
.as_deref()
.filter(|value| !value.trim().is_empty())
.context("self-hosted session configuration requires --coordinator <host:port>")?;
if session_secret.trim().is_empty() {
anyhow::bail!("self-hosted session secret from stdin must not be empty");
}
let mut connection = JsonLineSession::connect(coordinator)?;
let response = connection.request(json!({
"type": "authenticated",
"session_secret": session_secret,
"request": { "type": "auth_status" },
}))?;
for (field, expected) in [
("tenant", args.scope.tenant.as_str()),
("project", args.scope.project.as_str()),
("actor", args.scope.user.as_str()),
] {
let actual = response.get(field).and_then(Value::as_str).unwrap_or("");
if actual != expected {
anyhow::bail!(
"self-hosted session {field} mismatch: coordinator returned {actual:?}, expected {expected:?}"
);
}
}
if response.get("authenticated").and_then(Value::as_bool) != Some(true) {
anyhow::bail!("self-hosted coordinator did not confirm the CLI session");
}
let stored = StoredCliSession {
kind: "self_hosted".to_owned(),
coordinator: coordinator.to_owned(),
tenant: args.scope.tenant,
project: args.scope.project,
user: args.scope.user,
cli_session_credential_kind: "CliDeviceSession".to_owned(),
session_secret: Some(session_secret),
token_expiry_posture: "configured_by_self_hosted_operator".to_owned(),
expires_at: None,
provider_tokens_exposed_to_cli: false,
provider_tokens_sent_to_nodes: false,
created_at_unix_seconds: unix_timestamp_seconds(),
};
let session_file = write_cli_session(&cwd, &stored)?;
Ok(json!({
"command": "auth connect-self-hosted",
"status": "connected",
"coordinator": coordinator,
"tenant": stored.tenant,
"project": stored.project,
"user": stored.user,
"session_file": session_file,
"session_secret_read_from_stdin": true,
"session_secret_exposed_in_report": false,
"provider_tokens_exposed_to_cli": false,
"provider_tokens_sent_to_nodes": false,
"coordinator_response": response,
"coordinator_session_requests": connection.requests(),
}))
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct LoginPlan {
pub(crate) coordinator: String,
pub(crate) human_flow: LoginFlowPlan,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub(crate) struct LoginCompletionReport {
pub(crate) plan: LoginPlan,
pub(crate) boundary: LoginCompletionBoundaryEvidence,
pub(crate) coordinator_response: Value,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct LoginCompletionBoundaryEvidence {
pub(crate) cli_contacted_coordinator: bool,
pub(crate) coordinator_address: String,
pub(crate) scoped_cli_session_received: bool,
pub(crate) local_cli_session_file_written: bool,
pub(crate) provider_tokens_persisted_locally: bool,
pub(crate) provider_tokens_exposed_to_cli: bool,
pub(crate) provider_tokens_sent_to_nodes: bool,
pub(crate) coordinator_session_requests: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) enum LoginFlowPlan {
Browser(HostedBrowserLoginPlan),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct HostedBrowserLoginPlan {
pub(crate) authorization_url: Option<String>,
pub(crate) requested_project: String,
pub(crate) server_owns_state: bool,
pub(crate) server_owns_nonce: bool,
pub(crate) pkce_required: bool,
pub(crate) hosted_callback: bool,
pub(crate) cli_receives_provider_authorization_code: bool,
pub(crate) cli_submits_identity_claims: bool,
}
pub(crate) fn auth_status_report(args: AuthStatusArgs, cwd: PathBuf) -> Result<Value> {
let config = read_project_config(&cwd)?;
let stored_session = read_cli_session(&cwd)?;
let configured_coordinator = args
.scope
.coordinator
.clone()
.or_else(|| {
config
.as_ref()
.and_then(|config| config.coordinator.clone())
})
.or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
let active_coordinator = configured_coordinator
.clone()
.unwrap_or_else(default_hosted_coordinator_endpoint);
let tenant = effective_scope_value(
&args.scope.tenant,
config
.as_ref()
.map(|config| config.tenant.as_str())
.or_else(|| {
stored_session
.as_ref()
.map(|session| session.tenant.as_str())
}),
"tenant",
);
let project = effective_scope_value(
&args.scope.project,
config
.as_ref()
.map(|config| config.project.as_str())
.or_else(|| {
stored_session
.as_ref()
.map(|session| session.project.as_str())
}),
"project",
);
let principal = effective_scope_value(
&args.scope.user,
config
.as_ref()
.map(|config| config.user.as_str())
.or_else(|| stored_session.as_ref().map(|session| session.user.as_str())),
"user",
);
let session_scope_mismatch = crate::auth_scope::session_scope_mismatch(
stored_session.as_ref(),
&active_coordinator,
&tenant,
&project,
&principal,
);
let session_matches_project = stored_session.is_some() && session_scope_mismatch.is_none();
let coordinator_account_status = configured_coordinator
.as_ref()
.filter(|_| session_scope_mismatch.is_none())
.map(|coordinator| {
coordinator_auth_status_summary(
coordinator,
&tenant,
&project,
&principal,
stored_session.as_ref(),
)
})
.unwrap_or_else(|| {
if let Some(fields) = &session_scope_mismatch {
return crate::auth_scope::session_scope_mismatch_status(fields);
}
json!({
"checked": false,
"reason": "no project or session coordinator configured",
"suspension_known": false,
"account_state_known": false,
"account_status": "unknown",
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
})
});
Ok(json!({
"command": "auth status",
"active_coordinator": active_coordinator,
"principal": principal,
"tenant": tenant,
"project": project,
"session": auth_state_value(&cwd)?,
"session_matches_current_project": session_matches_project,
"session_scope_mismatch": session_scope_mismatch,
"coordinator_account_status": coordinator_account_status,
"project_config": config,
}))
}
fn coordinator_auth_status_summary(
coordinator: &str,
tenant: &str,
project: &str,
principal: &str,
stored_session: Option<&StoredCliSession>,
) -> Value {
let mut session = match JsonLineSession::connect(coordinator) {
Ok(session) => session,
Err(error) => {
let message = error.to_string();
return json!({
"checked": true,
"reachable": false,
"source": "public_coordinator_api",
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
"next_actions": ["clusterflux doctor", "check coordinator status"],
"coordinator_session_requests": 0,
});
}
};
let request = match authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "auth_status",
}),
json!({
"type": "auth_status",
"tenant": tenant,
"project": project,
"actor_user": principal,
}),
) {
Ok(request) => request,
Err(error) => {
let message = error.to_string();
return json!({
"checked": false,
"reachable": "not_checked",
"source": "public_coordinator_api",
"authenticated_for_current_project": false,
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
"next_actions": ["clusterflux login --browser"],
"coordinator_session_requests": 0,
});
}
};
let response = match session.request_allow_error(request) {
Ok(response) => response,
Err(error) => {
let message = error.to_string();
return json!({
"checked": true,
"reachable": false,
"source": "public_coordinator_api",
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message),
"error": message,
"next_actions": ["clusterflux doctor", "check coordinator status"],
"coordinator_session_requests": session.requests(),
});
}
};
let coordinator_session_requests = session.requests();
if response.get("type").and_then(Value::as_str) == Some("error") {
let message = response
.get("message")
.and_then(Value::as_str)
.unwrap_or("coordinator rejected auth status");
return json!({
"checked": true,
"reachable": true,
"source": "public_coordinator_api",
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"machine_error": cli_error_summary(message),
"coordinator_response_type": "error",
"next_actions": ["clusterflux doctor", "clusterflux login --browser"],
"coordinator_session_requests": coordinator_session_requests,
});
}
let suspended = response
.get("suspended")
.and_then(Value::as_bool)
.unwrap_or(false);
let disabled = response
.get("disabled")
.and_then(Value::as_bool)
.unwrap_or(false);
let deleted = response
.get("deleted")
.and_then(Value::as_bool)
.unwrap_or(false);
let manual_review = response
.get("manual_review")
.and_then(Value::as_bool)
.unwrap_or(false);
let account_status = response
.get("account_status")
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| {
if deleted {
"deleted"
} else if disabled {
"disabled"
} else if suspended {
"suspended"
} else if manual_review {
"manual_review"
} else {
"active"
}
.to_owned()
});
let sanitized_reason = response.get("sanitized_reason").and_then(Value::as_str);
let next_actions = response
.get("next_actions")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default()
.into_iter()
.filter_map(|value| value.as_str().map(str::to_owned))
.collect::<Vec<_>>();
json!({
"checked": true,
"reachable": true,
"source": "public_coordinator_api",
"used_cli_session_credential": stored_session_for_coordinator(coordinator, stored_session).is_some(),
"account_status": account_status,
"suspension_known": true,
"account_state_known": true,
"suspended": suspended,
"disabled": disabled,
"deleted": deleted,
"manual_review": manual_review,
"sanitized_reason": sanitized_reason,
"next_actions": next_actions,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("auth_status"),
"coordinator_session_requests": coordinator_session_requests,
})
}
pub(crate) fn non_interactive_browser_login_report(args: &LoginArgs) -> Value {
let message =
"browser login requires an interactive browser, but non-interactive mode is enabled";
let next_actions = vec![
"rerun without --non-interactive to open the browser",
"clusterflux login --browser --plan",
"use CLUSTERFLUX_AGENT_PRIVATE_KEY for automation",
];
json!({
"command": "login",
"status": "authentication_required",
"coordinator": args.coordinator,
"non_interactive": true,
"browser_requested": true,
"browser_opened": false,
"safe_failure": true,
"message": message,
"next_actions": next_actions,
"machine_error": crate::auth_scope::non_interactive_auth_machine_error(message, next_actions),
})
}
pub(crate) fn auth_state_value(cwd: &Path) -> Result<Value> {
match session_from_env()? {
CliSession::Anonymous => {
if let Some(session) = read_cli_session(cwd)? {
return Ok(json!({
"kind": session.kind,
"authenticated": true,
"source": "session_file",
"coordinator": session.coordinator,
"tenant": session.tenant,
"project": session.project,
"principal": session.user,
"cli_session_credential_kind": session.cli_session_credential_kind,
"provider_tokens_exposed_to_cli": session.provider_tokens_exposed_to_cli,
"provider_tokens_exposed_to_nodes": session.provider_tokens_sent_to_nodes,
"expires_at": session.expires_at,
"token_expiry_posture": session.token_expiry_posture,
}));
}
Ok(json!({
"kind": "anonymous",
"authenticated": false,
"source": "environment",
"token_expiry_posture": "no_session",
}))
}
CliSession::HumanSession => {
let expires_at = std::env::var("CLUSTERFLUX_TOKEN_EXPIRES_AT").ok();
Ok(json!({
"kind": "human",
"authenticated": true,
"source": "CLUSTERFLUX_TOKEN",
"provider_tokens_exposed_to_nodes": false,
"expires_at": expires_at,
"token_expiry_posture": if expires_at.is_some() { "expires_at" } else { "unknown_env_token" },
}))
}
CliSession::AgentPublicKey {
agent,
public_key_fingerprint,
browser_interaction_required,
..
} => Ok(json!({
"kind": "agent_public_key",
"authenticated": true,
"agent": agent,
"source": "CLUSTERFLUX_AGENT_PRIVATE_KEY",
"public_key_fingerprint": public_key_fingerprint,
"browser_interaction_required": browser_interaction_required,
"token_expiry_posture": "not_applicable_public_key",
})),
}
}
pub(crate) fn login_plan(args: LoginArgs) -> LoginPlan {
let human_flow = LoginFlowPlan::Browser(HostedBrowserLoginPlan {
authorization_url: None,
requested_project: args.project.clone(),
server_owns_state: true,
server_owns_nonce: true,
pkce_required: true,
hosted_callback: true,
cli_receives_provider_authorization_code: false,
cli_submits_identity_claims: false,
});
LoginPlan {
coordinator: args.coordinator,
human_flow,
}
}
fn finalize_browser_login(
args: LoginArgs,
plan: LoginPlan,
coordinator_response: Value,
coordinator_session_requests: u64,
) -> Result<LoginCompletionReport> {
let coordinator = args.coordinator.clone();
let scoped_cli_session_received = coordinator_response
.pointer("/session/cli_session_credential_kind")
.and_then(Value::as_str)
== Some("CliDeviceSession");
let provider_tokens_sent_to_nodes = coordinator_response
.pointer("/session/provider_tokens_sent_to_nodes")
.and_then(Value::as_bool)
.unwrap_or(true);
let provider_tokens_exposed_to_cli = contains_provider_token_field(&coordinator_response);
let local_cli_session_file_written = if scoped_cli_session_received {
let cwd = std::env::current_dir()?;
let stored_session = stored_cli_session_from_login_response(
&coordinator,
&coordinator_response,
provider_tokens_exposed_to_cli,
provider_tokens_sent_to_nodes,
)?;
write_cli_session(&cwd, &stored_session)?;
write_project_config(
&cwd,
&ProjectConfig {
tenant: stored_session.tenant.clone(),
project: stored_session.project.clone(),
user: stored_session.user.clone(),
coordinator: Some(stored_session.coordinator.clone()),
},
)?;
true
} else {
false
};
Ok(LoginCompletionReport {
plan,
boundary: LoginCompletionBoundaryEvidence {
cli_contacted_coordinator: true,
coordinator_address: coordinator,
scoped_cli_session_received,
local_cli_session_file_written,
provider_tokens_persisted_locally: false,
provider_tokens_exposed_to_cli,
provider_tokens_sent_to_nodes,
coordinator_session_requests,
},
coordinator_response,
})
}
pub(crate) fn stored_cli_session_from_login_response(
coordinator: &str,
coordinator_response: &Value,
provider_tokens_exposed_to_cli: bool,
provider_tokens_sent_to_nodes: bool,
) -> Result<StoredCliSession> {
let session = coordinator_response.get("session").unwrap_or(&Value::Null);
let expires_at = session
.get("expires_at")
.or_else(|| session.get("token_expires_at"))
.and_then(Value::as_str)
.map(str::to_owned)
.or_else(|| {
session
.get("expires_at_epoch_seconds")
.and_then(Value::as_u64)
.map(|value| value.to_string())
});
let tenant = session
.get("tenant")
.and_then(Value::as_str)
.context("hosted login session omitted tenant")?;
let project = session
.get("project")
.and_then(Value::as_str)
.context("hosted login session omitted project")?;
let user = session
.get("user")
.and_then(Value::as_str)
.context("hosted login session omitted user")?;
let session_secret = session
.get("cli_session_secret")
.or_else(|| session.get("session_secret"))
.and_then(Value::as_str)
.context("hosted login session omitted CLI session secret")?;
Ok(StoredCliSession {
kind: "human".to_owned(),
coordinator: coordinator.to_owned(),
tenant: tenant.to_owned(),
project: project.to_owned(),
user: user.to_owned(),
cli_session_credential_kind: session
.get("cli_session_credential_kind")
.and_then(Value::as_str)
.unwrap_or("CliDeviceSession")
.to_owned(),
session_secret: Some(session_secret.to_owned()),
token_expiry_posture: if expires_at.is_some() {
"expires_at".to_owned()
} else {
"unknown_coordinator_session".to_owned()
},
expires_at,
provider_tokens_exposed_to_cli,
provider_tokens_sent_to_nodes,
created_at_unix_seconds: unix_timestamp_seconds(),
})
}
pub(crate) fn execute_interactive_browser_login(args: LoginArgs) -> Result<LoginCompletionReport> {
let coordinator = args.coordinator.clone();
let mut session = JsonLineSession::connect(&coordinator)?;
let started = session.request(json!({
"type": "begin_oidc_browser_login",
"requested_project": args.project,
}))?;
if started.get("type").and_then(Value::as_str) != Some("oidc_browser_login_started") {
anyhow::bail!("coordinator did not start a hosted browser login transaction");
}
let transaction_id = started
.get("transaction_id")
.and_then(Value::as_str)
.context("hosted login transaction omitted transaction id")?
.to_owned();
let polling_secret = started
.get("polling_secret")
.and_then(Value::as_str)
.context("hosted login transaction omitted polling secret")?
.to_owned();
let authorization_url = started
.get("authorization_url")
.and_then(Value::as_str)
.context("hosted login transaction omitted authorization URL")?
.to_owned();
let loopback_test_flow = coordinator.starts_with("http://")
&& crate::client::is_loopback_coordinator(&coordinator)
&& authorization_url.starts_with("http://")
&& crate::client::is_loopback_coordinator(&authorization_url);
if !authorization_url.starts_with("https://") && !loopback_test_flow {
anyhow::bail!(
"hosted login authorization URL must use HTTPS (plain HTTP is accepted only when both coordinator and identity provider are loopback test services)"
);
}
let plan = LoginPlan {
coordinator: coordinator.clone(),
human_flow: LoginFlowPlan::Browser(HostedBrowserLoginPlan {
authorization_url: Some(authorization_url.clone()),
requested_project: args.project.clone(),
server_owns_state: true,
server_owns_nonce: true,
pkce_required: true,
hosted_callback: true,
cli_receives_provider_authorization_code: false,
cli_submits_identity_claims: false,
}),
};
eprintln!("Opening Clusterflux browser login: {authorization_url}");
eprintln!("Waiting for the hosted login callback to complete.");
open_browser(&authorization_url)?;
let deadline = Instant::now() + browser_login_timeout();
loop {
let response = session.request(json!({
"type": "poll_oidc_browser_login",
"transaction_id": transaction_id,
"polling_secret": polling_secret,
}))?;
match response.get("type").and_then(Value::as_str) {
Some("oidc_browser_login_pending") => {
if Instant::now() >= deadline {
anyhow::bail!("timed out waiting for hosted browser login completion");
}
std::thread::sleep(Duration::from_millis(500));
}
Some("oidc_browser_session") => {
return finalize_browser_login(args, plan, response, session.requests());
}
_ => anyhow::bail!("coordinator returned an invalid hosted login status"),
}
}
}
pub(crate) fn print_browser_login_success(report: &LoginCompletionReport) {
let session = report.coordinator_response.get("session");
let tenant = session
.and_then(|value| value.get("tenant"))
.and_then(Value::as_str)
.unwrap_or("tenant");
let project = session
.and_then(|value| value.get("project"))
.and_then(Value::as_str)
.unwrap_or("project");
let user = session
.and_then(|value| value.get("user"))
.and_then(Value::as_str)
.unwrap_or("user");
println!(
"Signed in to {} as {user} for {tenant}/{project}.",
report.plan.coordinator
);
if report.boundary.scoped_cli_session_received {
println!("Received a scoped CLI session from the coordinator.");
}
}
fn open_browser(url: &str) -> Result<()> {
let mut command = if let Some(command) = std::env::var_os("CLUSTERFLUX_BROWSER_OPEN_COMMAND") {
Command::new(command)
} else {
platform_browser_command()
};
command
.arg(url)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null());
command
.spawn()
.with_context(|| format!("failed to start browser opener for {url}"))?;
Ok(())
}
#[cfg(target_os = "macos")]
fn platform_browser_command() -> Command {
Command::new("open")
}
#[cfg(target_os = "windows")]
fn platform_browser_command() -> Command {
let mut command = Command::new("cmd");
command.args(["/C", "start", ""]);
command
}
#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
fn platform_browser_command() -> Command {
Command::new("xdg-open")
}
fn browser_login_timeout() -> Duration {
let seconds = std::env::var("CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_BROWSER_LOGIN_TRANSACTION_TIMEOUT_SECONDS);
Duration::from_secs(seconds)
}
fn unix_timestamp_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
pub(crate) fn contains_provider_token_field(value: &Value) -> bool {
match value {
Value::Object(object) => object.iter().any(|(key, value)| {
matches!(
key.as_str(),
"access_token" | "refresh_token" | "id_token" | "provider_token" | "oauth_token"
) || contains_provider_token_field(value)
}),
Value::Array(items) => items.iter().any(contains_provider_token_field),
_ => false,
}
}

79
crates/clusterflux-cli/src/auth_scope.rs Normal file
View file

@ -0,0 +1,79 @@
use std::path::Path;
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::control_endpoint_identity;
use crate::config::{
effective_scope_value, read_project_config, StoredCliSession,
DEFAULT_HOSTED_COORDINATOR_ENDPOINT,
};
use crate::errors::cli_error_summary_for_category;
use crate::LoginArgs;
pub(crate) fn login_args_for_project(mut args: LoginArgs, cwd: &Path) -> Result<LoginArgs> {
let Some(config) = read_project_config(cwd)? else {
return Ok(args);
};
args.project = effective_scope_value(&args.project, Some(&config.project), "project");
if args.coordinator == DEFAULT_HOSTED_COORDINATOR_ENDPOINT {
if let Some(coordinator) = config.coordinator {
args.coordinator = coordinator;
}
}
Ok(args)
}
pub(crate) fn session_scope_mismatch(
session: Option<&StoredCliSession>,
active_coordinator: &str,
tenant: &str,
project: &str,
user: &str,
) -> Option<Vec<&'static str>> {
session.and_then(|session| {
let mut fields = Vec::new();
if control_endpoint_identity(&session.coordinator).ok()
!= control_endpoint_identity(active_coordinator).ok()
{
fields.push("coordinator");
}
if session.tenant != tenant {
fields.push("tenant");
}
if session.project != project {
fields.push("project");
}
if session.user != user {
fields.push("user");
}
(!fields.is_empty()).then_some(fields)
})
}
pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
json!({
"checked": false,
"reason": "stored CLI session does not match the current project",
"mismatched_fields": fields,
"authenticated_for_current_project": false,
"account_status": "unknown",
"suspension_known": false,
"account_state_known": false,
"private_moderation_details_exposed": false,
"signup_failure_details_exposed": false,
"next_actions": ["clusterflux login --browser"],
})
}
pub(crate) fn non_interactive_auth_machine_error(
message: &str,
next_actions: Vec<&'static str>,
) -> Value {
let mut machine_error = cli_error_summary_for_category("authentication", message);
if let Some(object) = machine_error.as_object_mut() {
object.insert("next_actions".to_owned(), json!(next_actions));
object.insert("browser_opened".to_owned(), json!(false));
}
machine_error
}

322
crates/clusterflux-cli/src/build.rs Normal file
View file

@ -0,0 +1,322 @@
use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::{bail, Context, Result};
use clusterflux_core::Digest;
use serde_json::{json, Value};
use wasmparser::{Parser, Payload};
use crate::errors::cli_error_summary_for_category;
use crate::{bundle_inspection, BuildArgs, BundleInspectArgs};
pub(crate) fn build_report(args: BuildArgs, cwd: PathBuf) -> Result<Value> {
let inspection = bundle_inspection(
BundleInspectArgs {
project: args.project.clone(),
source_provider: args.source_provider.clone(),
disabled_source_providers: args.disabled_source_providers.clone(),
json: true,
},
cwd,
)?;
let diagnostics = inspection.pre_schedule_diagnostics.clone();
let blocking_diagnostic = diagnostics
.iter()
.find(|diagnostic| diagnostic.severity == "error")
.cloned();
if let Some(diagnostic) = blocking_diagnostic {
let machine_category = match diagnostic.category.as_str() {
"environment" => "environment",
"source_provider" | "capability" => "capability",
_ => "unknown",
};
return Ok(json!({
"command": "build",
"status": "blocked_before_schedule",
"bundle": inspection,
"diagnostics": diagnostics,
"contains_full_repository_upload": false,
"content_addressed": false,
"debug_metadata_available": false,
"scheduled_work": false,
"machine_error": cli_error_summary_for_category(machine_category, &diagnostic.message),
}));
}
let mut wasm = compile_project_wasm(&inspection.project)?;
let environment_manifest = serde_json::to_vec(&inspection.metadata.environments)?;
append_custom_section(
&mut wasm.bytes,
"clusterflux.environments",
&environment_manifest,
);
let bundle_digest = Digest::sha256(&wasm.bytes);
let task_descriptors = descriptor_records(&wasm.bytes, "clusterflux.tasks")?;
let entrypoint_descriptors = descriptor_records(&wasm.bytes, "clusterflux.entrypoints")?;
if task_descriptors.is_empty() {
bail!(
"compiled Wasm module contains no #[clusterflux::task] descriptors; annotate at least one exported task"
);
}
if entrypoint_descriptors.is_empty() {
bail!(
"compiled Wasm module contains no #[clusterflux::main] descriptors; annotate at least one entrypoint"
);
}
let output = args.output.unwrap_or_else(|| {
inspection.project.join(".clusterflux/build").join(
bundle_digest
.as_str()
.trim_start_matches("sha256:")
.get(..16)
.unwrap_or("bundle"),
)
});
let bundle_artifact = write_bundle(
&output,
&wasm,
&bundle_digest,
&inspection,
&task_descriptors,
&entrypoint_descriptors,
)?;
Ok(json!({
"command": "build",
"status": "built",
"bundle": inspection,
"bundle_artifact": bundle_artifact,
"diagnostics": diagnostics,
"contains_full_repository_upload": false,
"content_addressed": true,
"debug_metadata_available": true,
"scheduled_work": false,
}))
}
fn append_custom_section(module: &mut Vec<u8>, name: &str, data: &[u8]) {
let mut section = Vec::new();
encode_unsigned_leb(name.len() as u64, &mut section);
section.extend_from_slice(name.as_bytes());
section.extend_from_slice(data);
module.push(0);
encode_unsigned_leb(section.len() as u64, module);
module.extend_from_slice(&section);
}
fn encode_unsigned_leb(mut value: u64, output: &mut Vec<u8>) {
loop {
let mut byte = (value & 0x7f) as u8;
value >>= 7;
if value != 0 {
byte |= 0x80;
}
output.push(byte);
if value == 0 {
break;
}
}
}
struct CompiledWasm {
bytes: Vec<u8>,
package: String,
target: String,
source_path: PathBuf,
}
fn compile_project_wasm(project: &Path) -> Result<CompiledWasm> {
let manifest = project.join("Cargo.toml");
let metadata_output = Command::new("cargo")
.args([
"metadata",
"--format-version",
"1",
"--no-deps",
"--manifest-path",
])
.arg(&manifest)
.output()
.with_context(|| format!("failed to run cargo metadata for {}", manifest.display()))?;
if !metadata_output.status.success() {
bail!(
"cargo metadata failed for {}: {}",
manifest.display(),
String::from_utf8_lossy(&metadata_output.stderr).trim()
);
}
let metadata: Value = serde_json::from_slice(&metadata_output.stdout)?;
let canonical_manifest = std::fs::canonicalize(&manifest)?;
let package = metadata["packages"]
.as_array()
.and_then(|packages| {
packages.iter().find(|package| {
package["manifest_path"]
.as_str()
.and_then(|path| std::fs::canonicalize(path).ok())
.as_ref()
== Some(&canonical_manifest)
})
})
.context("cargo metadata did not return the requested project package")?;
let package_name = package["name"]
.as_str()
.context("cargo package name missing")?;
let target = package["targets"]
.as_array()
.and_then(|targets| {
targets.iter().find(|target| {
target["crate_types"]
.as_array()
.is_some_and(|types| types.iter().any(|kind| kind == "cdylib"))
})
})
.context("Clusterflux project library must include crate-type = [\"cdylib\"]")?;
let target_name = target["name"]
.as_str()
.context("cargo target name missing")?;
let build = Command::new("cargo")
// The MVP transports one Wasm bundle in a bounded control frame. These are
// ordinary Cargo release-profile settings, applied only to the guest build,
// that keep the product SDK/runtime inside that accepted boundary.
.env("CARGO_PROFILE_RELEASE_OPT_LEVEL", "z")
.env("CARGO_PROFILE_RELEASE_LTO", "thin")
.env("CARGO_PROFILE_RELEASE_CODEGEN_UNITS", "1")
.args([
"build",
"--quiet",
"--release",
"--target",
"wasm32-unknown-unknown",
"--lib",
"--manifest-path",
])
.arg(&manifest)
.output()
.with_context(|| format!("failed to compile {} to Wasm", manifest.display()))?;
if !build.status.success() {
bail!(
"Wasm bundle compilation failed for {}: {}",
manifest.display(),
String::from_utf8_lossy(&build.stderr).trim()
);
}
let target_directory = metadata["target_directory"]
.as_str()
.context("cargo target_directory missing")?;
let source_path = Path::new(target_directory)
.join("wasm32-unknown-unknown/release")
.join(format!("{}.wasm", target_name.replace('-', "_")));
let bytes = std::fs::read(&source_path)
.with_context(|| format!("compiled Wasm module missing at {}", source_path.display()))?;
Ok(CompiledWasm {
bytes,
package: package_name.to_owned(),
target: target_name.to_owned(),
source_path,
})
}
fn descriptor_records(module: &[u8], section_name: &str) -> Result<Vec<Value>> {
let mut records: Vec<Value> = Vec::new();
for payload in Parser::new(0).parse_all(module) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != section_name {
continue;
}
for record in section
.data()
.split(|byte| *byte == b'\n' || *byte == 0)
.filter(|record| !record.is_empty())
{
records.push(serde_json::from_slice(record).with_context(|| {
format!("invalid descriptor record in Wasm custom section {section_name}")
})?);
}
}
records.sort_by(|left, right| left["name"].as_str().cmp(&right["name"].as_str()));
Ok(records)
}
fn write_bundle(
output: &Path,
wasm: &CompiledWasm,
bundle_digest: &Digest,
inspection: &crate::bundle::BundleInspection,
tasks: &[Value],
entrypoints: &[Value],
) -> Result<Value> {
std::fs::create_dir_all(output)?;
let module_path = output.join("module.wasm");
let task_path = output.join("task-descriptors.json");
let entrypoint_path = output.join("entrypoints.json");
let environment_path = output.join("environments.json");
let source_path = output.join("source-provider.json");
let vfs_path = output.join("vfs-seed.json");
let debug_path = output.join("debug-metadata.json");
let manifest_path = output.join("manifest.json");
std::fs::write(&module_path, &wasm.bytes)?;
write_json(&task_path, &json!(tasks))?;
write_json(&entrypoint_path, &json!(entrypoints))?;
write_json(&environment_path, &json!(inspection.metadata.environments))?;
write_json(&source_path, &json!(inspection.source_provider_manifest))?;
write_json(
&vfs_path,
&json!({
"epoch": 0,
"mounts": ["/vfs/artifacts", "/vfs/sources", "/vfs/blobs"],
"large_bytes_embedded": false,
}),
)?;
write_json(&debug_path, &json!(inspection.metadata.debug_metadata))?;
let manifest = json!({
"kind": "clusterflux-bundle",
"format_version": 1,
"package": wasm.package,
"target": wasm.target,
"bundle_digest": bundle_digest,
"module": "module.wasm",
"module_size_bytes": wasm.bytes.len(),
"task_descriptors": "task-descriptors.json",
"entrypoints": "entrypoints.json",
"environments": "environments.json",
"source_provider": "source-provider.json",
"vfs_seed": "vfs-seed.json",
"debug_metadata": "debug-metadata.json",
"required_capabilities": tasks.iter().flat_map(|task| {
task["required_capabilities"].as_array().into_iter().flatten().cloned()
}).collect::<Vec<_>>(),
"metadata_identity": inspection.metadata.identity,
"coordinator_receives_source_bytes_by_default": false,
"embeds_full_repository": false,
});
write_json(&manifest_path, &manifest)?;
Ok(json!({
"directory": output,
"manifest": manifest_path,
"module": module_path,
"compiled_module_source": wasm.source_path,
"bundle_digest": bundle_digest,
"module_size_bytes": wasm.bytes.len(),
"task_descriptor_count": tasks.len(),
"entrypoint_count": entrypoints.len(),
"files": [
"manifest.json",
"module.wasm",
"task-descriptors.json",
"entrypoints.json",
"environments.json",
"source-provider.json",
"vfs-seed.json",
"debug-metadata.json",
],
}))
}
fn write_json(path: &Path, value: &Value) -> Result<()> {
let mut bytes = serde_json::to_vec_pretty(value)?;
bytes.push(b'\n');
std::fs::write(path, bytes).with_context(|| format!("failed to write {}", path.display()))
}

379
crates/clusterflux-cli/src/bundle.rs Normal file
View file

@ -0,0 +1,379 @@
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use clusterflux_core::{
diagnose_environment_references, discover_source_debug_probes, BundleDebugProbe,
BundleIdentityInputs, BundleMetadata, Digest, EnvironmentReference, ProjectModel,
SelectedInput, SourceProviderKind, SourceProviderManifest,
};
use serde::Serialize;
use crate::BundleInspectArgs;
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct BundleInspection {
pub(crate) project: PathBuf,
pub(crate) default_source_providers: Vec<SourceProviderKind>,
pub(crate) source_provider_manifest: SourceProviderManifest,
pub(crate) source_provider_statuses: Vec<SourceProviderStatus>,
pub(crate) environment_diagnostics: Vec<EnvironmentDiagnosticReport>,
pub(crate) pre_schedule_diagnostics: Vec<CliDiagnostic>,
pub(crate) metadata: BundleMetadata,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct SourceProviderStatus {
pub(crate) provider: String,
pub(crate) status: String,
pub(crate) active: bool,
pub(crate) reason: String,
pub(crate) coordinator_checkout_required: bool,
pub(crate) coordinator_receives_source_bytes_by_default: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct EnvironmentDiagnosticReport {
pub(crate) path: String,
pub(crate) reference: EnvironmentReference,
pub(crate) message: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct CliDiagnostic {
pub(crate) severity: String,
pub(crate) category: String,
pub(crate) code: String,
pub(crate) message: String,
pub(crate) next_actions: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct SourceProviderSelection {
active_provider: SourceProviderKind,
statuses: Vec<SourceProviderStatus>,
}
pub(crate) fn discovered_environment_names(inspection: Option<&BundleInspection>) -> Vec<String> {
inspection
.map(|inspection| {
inspection
.metadata
.environments
.iter()
.map(|environment| environment.name.clone())
.collect()
})
.unwrap_or_default()
}
pub(crate) fn bundle_inspection(args: BundleInspectArgs, cwd: PathBuf) -> Result<BundleInspection> {
let project = args.project.unwrap_or(cwd);
let model = ProjectModel::discover_without_config(&project)?;
let selected_inputs = discover_selected_inputs(&project)?;
let debug_probes = discover_debug_probes(&project, &selected_inputs)?;
let provider_selection = source_provider_selection(
&project,
args.source_provider.as_deref(),
&args.disabled_source_providers,
);
let source_provider_manifest = source_provider_manifest(&provider_selection.active_provider);
source_provider_manifest.validate_public_mvp()?;
let environment_diagnostics =
environment_diagnostics_for_inputs(&project, &selected_inputs, &model.environments)?;
let identity_inputs = BundleIdentityInputs {
wasm_code: wasm_source_proxy_digest(&selected_inputs),
task_abi: task_abi_digest(&model),
entrypoints: model.entrypoints.keys().cloned().collect(),
default_entrypoint: model.default_entrypoint.clone(),
environments: model.environments.clone(),
source_provider_manifest: source_provider_manifest.digest.clone(),
source_transfer_policy: source_provider_manifest.transfer_policy.clone(),
selected_inputs,
};
let mut metadata = identity_inputs.inspectable_metadata();
metadata.debug_metadata.probes = debug_probes;
let pre_schedule_diagnostics = pre_schedule_diagnostics(
&provider_selection.statuses,
&environment_diagnostics,
&model.environments,
);
Ok(BundleInspection {
project,
default_source_providers: vec![SourceProviderKind::Filesystem, SourceProviderKind::Git],
source_provider_manifest,
source_provider_statuses: provider_selection.statuses,
environment_diagnostics,
pre_schedule_diagnostics,
metadata,
})
}
fn discover_selected_inputs(project: &Path) -> Result<Vec<SelectedInput>> {
let mut inputs = Vec::new();
for path in [
"Cargo.toml",
"Cargo.lock",
"src/main.rs",
"src/lib.rs",
"src/build.rs",
] {
let absolute = project.join(path);
if !absolute.is_file() {
continue;
}
let bytes = std::fs::read(&absolute)?;
inputs.push(SelectedInput {
path: path.to_owned(),
digest: Digest::from_parts([
b"bundle-selected-input:v1".as_slice(),
path.as_bytes(),
bytes.as_slice(),
]),
});
}
Ok(inputs)
}
fn source_provider_selection(
project: &Path,
requested_provider: Option<&str>,
disabled_providers: &[String],
) -> SourceProviderSelection {
let has_git_checkout = project.join(".git").exists();
let disabled = disabled_providers
.iter()
.map(|provider| provider.trim().to_ascii_lowercase())
.collect::<BTreeSet<_>>();
let requested = requested_provider.map(source_provider_kind_from_id);
let active_provider = requested.clone().unwrap_or_else(|| {
if has_git_checkout && !disabled.contains("git") {
SourceProviderKind::Git
} else {
SourceProviderKind::Filesystem
}
});
let active_provider_id = active_provider.provider_id().to_owned();
let mut statuses = vec![
builtin_source_provider_status(
SourceProviderKind::Filesystem,
true,
disabled.contains("filesystem"),
active_provider_id == "filesystem",
),
builtin_source_provider_status(
SourceProviderKind::Git,
has_git_checkout,
disabled.contains("git"),
active_provider_id == "git",
),
SourceProviderStatus {
provider: "custom".to_owned(),
status: "disabled".to_owned(),
active: false,
reason: "no custom source-provider module was configured for this project".to_owned(),
coordinator_checkout_required: false,
coordinator_receives_source_bytes_by_default: false,
},
];
if let Some(SourceProviderKind::Custom(provider)) = requested {
statuses.push(SourceProviderStatus {
provider,
status: "unsupported".to_owned(),
active: true,
reason: "custom source-provider modules must be supplied by public plugin/API code before use".to_owned(),
coordinator_checkout_required: false,
coordinator_receives_source_bytes_by_default: false,
});
}
SourceProviderSelection {
active_provider,
statuses,
}
}
fn source_provider_kind_from_id(provider: &str) -> SourceProviderKind {
match provider.trim().to_ascii_lowercase().as_str() {
"filesystem" => SourceProviderKind::Filesystem,
"git" => SourceProviderKind::Git,
other => SourceProviderKind::Custom(other.to_owned()),
}
}
fn builtin_source_provider_status(
kind: SourceProviderKind,
available: bool,
disabled: bool,
active: bool,
) -> SourceProviderStatus {
let provider = kind.provider_id().to_owned();
let (status, reason) = if disabled {
(
"disabled",
format!("source provider `{provider}` was disabled by CLI override"),
)
} else if available {
(
"enabled",
format!("source provider `{provider}` is available in this checkout"),
)
} else {
(
"missing",
format!("source provider `{provider}` is not available for this checkout"),
)
};
SourceProviderStatus {
provider,
status: status.to_owned(),
active,
reason,
coordinator_checkout_required: false,
coordinator_receives_source_bytes_by_default: false,
}
}
fn source_provider_manifest(kind: &SourceProviderKind) -> SourceProviderManifest {
SourceProviderManifest::local_first(
kind.clone(),
"default source provider manifest; snapshot creation can be scheduled as a node task",
)
}
fn environment_diagnostics_for_inputs(
project: &Path,
selected_inputs: &[SelectedInput],
environments: &[clusterflux_core::EnvironmentResource],
) -> Result<Vec<EnvironmentDiagnosticReport>> {
let mut diagnostics = Vec::new();
for input in selected_inputs {
if !input.path.ends_with(".rs") {
continue;
}
let source = std::fs::read_to_string(project.join(&input.path))
.with_context(|| format!("failed to read {}", project.join(&input.path).display()))?;
diagnostics.extend(
diagnose_environment_references(&source, environments)
.into_iter()
.map(|diagnostic| EnvironmentDiagnosticReport {
path: input.path.clone(),
reference: diagnostic.reference,
message: diagnostic.message,
}),
);
}
Ok(diagnostics)
}
fn discover_debug_probes(
project: &Path,
selected_inputs: &[SelectedInput],
) -> Result<Vec<BundleDebugProbe>> {
let mut probes = Vec::new();
for input in selected_inputs {
if !input.path.ends_with(".rs") {
continue;
}
let source = std::fs::read_to_string(project.join(&input.path))
.with_context(|| format!("failed to read {}", project.join(&input.path).display()))?;
probes.extend(discover_source_debug_probes(&input.path, &source));
}
Ok(probes)
}
fn pre_schedule_diagnostics(
source_provider_statuses: &[SourceProviderStatus],
environment_diagnostics: &[EnvironmentDiagnosticReport],
environments: &[clusterflux_core::EnvironmentResource],
) -> Vec<CliDiagnostic> {
let mut diagnostics = Vec::new();
diagnostics.extend(
environment_diagnostics
.iter()
.map(|diagnostic| CliDiagnostic {
severity: "error".to_owned(),
category: "environment".to_owned(),
code: "missing_environment".to_owned(),
message: format!("{} at {}", diagnostic.message, diagnostic.path),
next_actions: vec![
"create the missing envs/<name>/Containerfile or envs/<name>/Dockerfile"
.to_owned(),
"rerun clusterflux inspect".to_owned(),
],
}),
);
diagnostics.extend(
source_provider_statuses
.iter()
.filter(|status| {
status.active
&& matches!(
status.status.as_str(),
"missing" | "disabled" | "unsupported"
)
})
.map(|status| CliDiagnostic {
severity: "error".to_owned(),
category: "source_provider".to_owned(),
code: format!("source_provider_{}", status.status),
message: format!(
"active source provider `{}` is {}: {}",
status.provider, status.status, status.reason
),
next_actions: vec![
"choose an available source provider with --source-provider".to_owned(),
"rerun clusterflux inspect --json".to_owned(),
],
}),
);
for environment in environments {
if environment.requirements.capabilities.is_empty() {
continue;
}
let mut capabilities = environment
.requirements
.capabilities
.iter()
.map(|capability| format!("{capability:?}"))
.collect::<Vec<_>>();
capabilities.sort();
diagnostics.push(CliDiagnostic {
severity: "info".to_owned(),
category: "capability".to_owned(),
code: "environment_capability_requirements".to_owned(),
message: format!(
"environment `{}` requires node capabilities: {}",
environment.name,
capabilities.join(", ")
),
next_actions: vec![
"attach a node that reports these capabilities before scheduling work".to_owned(),
],
});
}
diagnostics
}
fn wasm_source_proxy_digest(selected_inputs: &[SelectedInput]) -> Digest {
let mut parts = vec![b"wasm-source-proxy:v1".to_vec()];
for input in selected_inputs {
parts.push(input.path.as_bytes().to_vec());
parts.push(input.digest.as_str().as_bytes().to_vec());
}
Digest::from_parts(parts)
}
pub(crate) fn task_abi_digest(model: &ProjectModel) -> Digest {
let mut parts = vec![b"task-abi:v1".to_vec()];
for entrypoint in model.entrypoints.values() {
parts.push(entrypoint.name.as_bytes().to_vec());
parts.push(entrypoint.function.as_bytes().to_vec());
}
Digest::from_parts(parts)
}

183
crates/clusterflux-cli/src/client.rs Normal file
View file

@ -0,0 +1,183 @@
use anyhow::{Context, Result};
use clusterflux_control::{endpoint_identity, endpoint_is_loopback, ControlSession};
use clusterflux_core::coordinator_wire_request;
use serde_json::{json, Value};
use crate::config::StoredCliSession;
use crate::errors::cli_error_summary;
use crate::CliScopeArgs;
pub(crate) struct JsonLineSession {
inner: ControlSession,
}
impl JsonLineSession {
pub(crate) fn connect(addr: &str) -> Result<Self> {
let inner = ControlSession::connect(addr)
.with_context(|| format!("failed to connect to coordinator {addr}"))?;
Ok(Self { inner })
}
pub(crate) fn request(&mut self, value: Value) -> Result<Value> {
let response = self.request_allow_error(value)?;
if response.get("type").and_then(Value::as_str) == Some("error") {
let message = response
.get("message")
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| response.to_string());
let machine_error = cli_error_summary(&message);
let category = machine_error
.get("category")
.and_then(Value::as_str)
.unwrap_or("unknown");
let exit_code = machine_error
.get("stable_exit_code")
.and_then(Value::as_i64)
.unwrap_or(1);
anyhow::bail!("coordinator error ({category}, exit {exit_code}): {response}");
}
Ok(response)
}
pub(crate) fn request_allow_error(&mut self, value: Value) -> Result<Value> {
let request_id = format!("cli-{}", self.inner.requests() + 1);
let wire_request = coordinator_wire_request(request_id, value);
self.inner
.request(&wire_request)
.map_err(anyhow::Error::from)
}
pub(crate) fn requests(&self) -> u64 {
self.inner.requests()
}
}
pub(crate) fn control_endpoint_identity(endpoint: &str) -> Result<String> {
endpoint_identity(endpoint).map_err(anyhow::Error::from)
}
pub(crate) fn authenticated_or_local_trusted_request(
coordinator: &str,
stored_session: Option<&StoredCliSession>,
authenticated_request: Value,
local_trusted_request: Value,
) -> Result<Value> {
if let Some(session_secret) = stored_session_for_coordinator(coordinator, stored_session)
.and_then(|session| session.session_secret.as_ref())
{
Ok(json!({
"type": "authenticated",
"session_secret": session_secret,
"request": authenticated_request,
}))
} else if is_loopback_coordinator(coordinator) {
Ok(local_trusted_request)
} else {
anyhow::bail!(
"no authenticated CLI session matches coordinator {coordinator}; run `clusterflux login --browser` from the current project"
)
}
}
pub(crate) fn is_loopback_coordinator(coordinator: &str) -> bool {
endpoint_is_loopback(coordinator)
}
pub(crate) fn stored_session_for_coordinator<'a>(
coordinator: &str,
stored_session: Option<&'a StoredCliSession>,
) -> Option<&'a StoredCliSession> {
stored_session.filter(|session| {
session.session_secret.is_some()
&& control_endpoint_identity(&session.coordinator).ok()
== control_endpoint_identity(coordinator).ok()
})
}
pub(crate) fn list_task_events_if_available_with_session(
coordinator: Option<&str>,
scope: &CliScopeArgs,
process: Option<String>,
stored_session: Option<&StoredCliSession>,
) -> Result<Option<Value>> {
let Some(coordinator) = coordinator else {
return Ok(None);
};
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "list_task_events",
"process": process,
}),
json!({
"type": "list_task_events",
"tenant": scope.tenant,
"project": scope.project,
"actor_user": scope.user,
"process": process,
}),
)?)?;
Ok(Some(json!({
"coordinator": coordinator,
"response": response,
"coordinator_session_requests": session.requests(),
})))
}
pub(crate) fn list_attached_nodes_if_available_with_session(
coordinator: Option<&str>,
scope: &CliScopeArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let Some(coordinator) = coordinator else {
return Ok(json!({
"checked": false,
"source": "no_coordinator",
"count": 0,
"online": 0,
"response": null,
}));
};
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "list_node_descriptors",
}),
json!({
"type": "list_node_descriptors",
"tenant": scope.tenant,
"project": scope.project,
"actor_user": scope.user,
}),
)?)?;
let descriptors = response
.get("descriptors")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let online = descriptors
.iter()
.filter(|descriptor| {
descriptor
.get("online")
.and_then(Value::as_bool)
.unwrap_or(false)
})
.count();
Ok(json!({
"checked": true,
"source": "coordinator",
"coordinator": coordinator,
"count": descriptors.len(),
"online": online,
"response": response,
"coordinator_session_requests": session.requests(),
}))
}

150
crates/clusterflux-cli/src/config.rs Normal file
View file

@ -0,0 +1,150 @@
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::CliScopeArgs;
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
"https://clusterflux.michelpaulissen.com";
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct ProjectConfig {
pub(crate) tenant: String,
pub(crate) project: String,
pub(crate) user: String,
pub(crate) coordinator: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct StoredCliSession {
pub(crate) kind: String,
pub(crate) coordinator: String,
pub(crate) tenant: String,
pub(crate) project: String,
pub(crate) user: String,
pub(crate) cli_session_credential_kind: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub(crate) session_secret: Option<String>,
pub(crate) token_expiry_posture: String,
pub(crate) expires_at: Option<String>,
pub(crate) provider_tokens_exposed_to_cli: bool,
pub(crate) provider_tokens_sent_to_nodes: bool,
pub(crate) created_at_unix_seconds: u64,
}
pub(crate) fn default_hosted_coordinator_endpoint() -> String {
DEFAULT_HOSTED_COORDINATOR_ENDPOINT.to_owned()
}
pub(crate) fn project_config_file(project: &Path) -> PathBuf {
project.join(".clusterflux").join("project.json")
}
pub(crate) fn session_config_file(project: &Path) -> PathBuf {
project.join(".clusterflux").join("session.json")
}
pub(crate) fn read_project_config(project: &Path) -> Result<Option<ProjectConfig>> {
let file = project_config_file(project);
if !file.exists() {
return Ok(None);
}
let bytes =
std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
let config = serde_json::from_slice(&bytes)
.with_context(|| format!("failed to parse {}", file.display()))?;
Ok(Some(config))
}
pub(crate) fn write_project_config(project: &Path, config: &ProjectConfig) -> Result<()> {
let file = project_config_file(project);
if let Some(parent) = file.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
std::fs::write(&file, serde_json::to_vec_pretty(config)?)
.with_context(|| format!("failed to write {}", file.display()))
}
pub(crate) fn read_cli_session(project: &Path) -> Result<Option<StoredCliSession>> {
let file = session_config_file(project);
if !file.exists() {
return Ok(None);
}
let bytes =
std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
let session = serde_json::from_slice(&bytes)
.with_context(|| format!("failed to parse {}", file.display()))?;
Ok(Some(session))
}
pub(crate) fn write_cli_session(project: &Path, session: &StoredCliSession) -> Result<PathBuf> {
let file = session_config_file(project);
if let Some(parent) = file.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
}
let bytes = serde_json::to_vec_pretty(session)?;
let mut options = std::fs::OpenOptions::new();
options.create(true).truncate(true).write(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
use std::io::Write;
let mut output = options
.open(&file)
.with_context(|| format!("failed to open {}", file.display()))?;
output
.write_all(&bytes)
.with_context(|| format!("failed to write {}", file.display()))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600))
.with_context(|| format!("failed to secure {}", file.display()))?;
}
Ok(file)
}
pub(crate) fn effective_project_scope(
scope: &CliScopeArgs,
config: Option<&ProjectConfig>,
) -> CliScopeArgs {
CliScopeArgs {
coordinator: scope
.coordinator
.clone()
.or_else(|| config.and_then(|config| config.coordinator.clone())),
tenant: effective_scope_value(
&scope.tenant,
config.map(|config| config.tenant.as_str()),
"tenant",
),
project: effective_scope_value(
&scope.project,
config.map(|config| config.project.as_str()),
"project",
),
user: effective_scope_value(
&scope.user,
config.map(|config| config.user.as_str()),
"user",
),
json: scope.json,
}
}
pub(crate) fn effective_scope_value(
cli_value: &str,
config_value: Option<&str>,
default_value: &str,
) -> String {
if cli_value == default_value {
config_value.unwrap_or(cli_value).to_owned()
} else {
cli_value.to_owned()
}
}

35
crates/clusterflux-cli/src/confirm.rs Normal file
View file

@ -0,0 +1,35 @@
use serde_json::{json, Value};
use crate::errors::cli_error_summary_for_category;
pub(crate) fn confirmation_required_report(
command: &str,
operation: &str,
target: Value,
confirm_command: String,
) -> Value {
let message = format!("{command} requires --yes before {operation}");
let next_actions = json!([
confirm_command,
"review the command target before confirming",
"rerun with --json to inspect the safe failure"
]);
let mut machine_error = cli_error_summary_for_category("policy", &message);
if let Some(object) = machine_error.as_object_mut() {
object.insert("confirmation_required".to_owned(), json!(true));
object.insert("next_actions".to_owned(), next_actions.clone());
}
json!({
"command": command,
"status": "confirmation_required",
"operation": operation,
"target": target,
"requires_confirmation": true,
"confirmation_required": true,
"explicit_user_action_required": true,
"coordinator_request_sent": false,
"safe_failure": true,
"next_actions": next_actions,
"machine_error": machine_error,
})
}

120
crates/clusterflux-cli/src/debug.rs Normal file
View file

@ -0,0 +1,120 @@
use std::process::Command;
use anyhow::{Context, Result};
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, stored_session_for_coordinator, JsonLineSession,
};
use crate::config::StoredCliSession;
use crate::tools::dap_binary_path;
use crate::{DapArgs, DebugAttachArgs};
pub(crate) fn dap_plan(args: DapArgs) -> Result<Value> {
Ok(json!({
"command": "dap",
"adapter": dap_binary_path()?.display().to_string(),
"args": args.args,
"private_website_required": false,
}))
}
pub(crate) fn exec_dap(args: DapArgs) -> Result<()> {
let status = Command::new(dap_binary_path()?)
.args(args.args)
.status()
.context("failed to launch clusterflux-debug-dap")?;
if !status.success() {
anyhow::bail!("clusterflux-debug-dap exited with {status}");
}
Ok(())
}
#[cfg(test)]
pub(crate) fn debug_attach_report_with_dap(args: DebugAttachArgs, dap: String) -> Result<Value> {
debug_attach_report_with_dap_and_session(args, dap, None)
}
pub(crate) fn debug_attach_report_with_dap_and_session(
mut args: DebugAttachArgs,
dap: String,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
if args.scope.coordinator.is_none() {
args.scope.coordinator = stored_session
.filter(|session| session.session_secret.is_some())
.map(|session| session.coordinator.clone());
}
if let Some(bound_session) = args
.scope
.coordinator
.as_deref()
.and_then(|coordinator| stored_session_for_coordinator(coordinator, stored_session))
{
args.scope.tenant = bound_session.tenant.clone();
args.scope.project = bound_session.project.clone();
args.scope.user = bound_session.user.clone();
}
if let Some(coordinator) = &args.scope.coordinator {
let tenant = args.scope.tenant.clone();
let project = args.scope.project.clone();
let user = args.scope.user.clone();
let process = args.process.clone();
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "debug_attach",
"process": process,
}),
json!({
"type": "debug_attach",
"tenant": tenant,
"project": project,
"actor_user": user,
"process": process,
}),
)?)?;
let authorization = response.get("authorization").cloned().unwrap_or_else(
|| json!({"allowed": false, "reason": "missing authorization response"}),
);
return Ok(json!({
"command": "debug attach",
"process": process,
"coordinator": coordinator,
"tenant": tenant,
"project": project,
"user": user,
"dap": dap,
"authorized": authorization
.get("allowed")
.cloned()
.unwrap_or(json!(false)),
"authorization": authorization,
"audit_event": response.get("audit_event").cloned().unwrap_or(Value::Null),
"charged_debug_read_bytes": response
.get("charged_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"used_debug_read_bytes": response
.get("used_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"debug_reads_quota_limited": true,
"private_website_required": false,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "debug attach",
"process": args.process,
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"dap": dap,
"authorized": "unknown_without_coordinator",
"debug_reads_quota_limited": "unknown_without_coordinator",
"private_website_required": false,
}))
}

343
crates/clusterflux-cli/src/dispatch.rs Normal file
View file

@ -0,0 +1,343 @@
use anyhow::Result;
use clap::Parser;
use super::*;
pub(crate) fn run_cli() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Commands::Doctor(args) => {
let json_output = args.scope.json;
let report = doctor_report(args, std::env::current_dir()?)?;
emit_report(&report, json_output)?;
}
Commands::Login(args) => {
let args = crate::auth_scope::login_args_for_project(args, &std::env::current_dir()?)?;
let json_output = args.json;
if args.non_interactive && !args.plan {
let report = non_interactive_browser_login_report(&args);
emit_report(&report, json_output)?;
} else if !args.plan {
let report = execute_interactive_browser_login(args)?;
if json_output {
emit_report(&report, true)?;
} else {
print_browser_login_success(&report);
}
} else {
let plan = login_plan(args);
emit_report(&plan, json_output)?;
}
}
Commands::Logout(args) => {
let json_output = args.scope.json;
let report = logout_report(args, std::env::current_dir()?, "logout")?;
emit_report(&report, json_output)?;
}
Commands::Auth { command } => {
let (report, json_output) = match command {
AuthCommands::Status(args) => {
let json_output = args.scope.json;
(
auth_status_report(args, std::env::current_dir()?)?,
json_output,
)
}
AuthCommands::ConnectSelfHosted(args) => {
let json_output = args.scope.json;
let secret = read_session_secret_from_stdin()?;
(
connect_self_hosted_report(args, std::env::current_dir()?, secret)?,
json_output,
)
}
AuthCommands::Logout(args) => {
let json_output = args.scope.json;
(
auth_logout_report(args, std::env::current_dir()?)?,
json_output,
)
}
};
emit_report(&report, json_output)?;
}
Commands::Agent {
command: AgentCommands::Enroll(args),
} => {
let json_output = args.json;
let plan = agent_enrollment_plan(args);
emit_report(&plan, json_output)?;
}
Commands::Key { command } => {
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
let (report, json_output) = match command {
KeyCommands::Add(args) => {
let json_output = args.scope.json;
(
key_add_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
KeyCommands::List(args) => {
let json_output = args.scope.json;
(
key_list_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
KeyCommands::Revoke(args) => {
let json_output = args.scope.json;
(
key_revoke_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
};
emit_report(&report, json_output)?;
}
Commands::Project { command } => {
let cwd = std::env::current_dir()?;
let (report, json_output) = match command {
ProjectCommands::Init(args) => {
let json_output = args.scope.json;
(project_init_report(args, cwd)?, json_output)
}
ProjectCommands::Status(args) => {
let json_output = args.scope.json;
(project_status_report(args, cwd)?, json_output)
}
ProjectCommands::List(args) => {
let json_output = args.scope.json;
(project_list_report(args, cwd)?, json_output)
}
ProjectCommands::Select(args) => {
let json_output = args.scope.json;
(project_select_report(args, cwd)?, json_output)
}
};
emit_report(&report, json_output)?;
}
Commands::Inspect(args) => {
let json_output = args.json;
let inspection = bundle_inspection(args, std::env::current_dir()?)?;
emit_report(&inspection, json_output)?;
}
Commands::Build(args) => {
let json_output = args.json;
let report = build_report(args, std::env::current_dir()?)?;
emit_report(&report, json_output)?;
}
Commands::Bundle {
command: BundleCommands::Inspect(args),
} => {
let json_output = args.json;
let inspection = bundle_inspection(args, std::env::current_dir()?)?;
emit_report(&inspection, json_output)?;
}
Commands::Run(args) => {
let json_output = args.json;
let cwd = std::env::current_dir()?;
let session_project = args.project.clone().unwrap_or_else(|| cwd.clone());
let session = session_from_sources(&session_project)?;
let report = run_report(args, cwd, session)?;
emit_report(&report, json_output)?;
}
Commands::Node {
command: NodeCommands::Attach(args),
} => {
let json_output = args.json;
if args.coordinator.is_some() {
let report = execute_node_attach(args)?;
emit_report(&report, json_output)?;
} else {
let plan = attach_plan(args);
emit_report(&plan, json_output)?;
}
}
Commands::Node { command } => {
let cwd = std::env::current_dir()?;
let (report, json_output) = match command {
NodeCommands::Enroll(args) => {
let json_output = args.scope.json;
(node_enroll_report(args, cwd.clone())?, json_output)
}
NodeCommands::List(args) => {
let json_output = args.scope.json;
(node_list_report(args, cwd.clone())?, json_output)
}
NodeCommands::Status(args) => {
let json_output = args.scope.json;
(node_status_report(args, cwd.clone())?, json_output)
}
NodeCommands::Revoke(args) => {
let json_output = args.scope.json;
(node_revoke_report(args, cwd)?, json_output)
}
NodeCommands::Attach(_) => unreachable!("node attach is handled above"),
};
emit_report(&report, json_output)?;
}
Commands::Process { command } => {
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
let (report, json_output) = match command {
ProcessCommands::List(args) => {
let json_output = args.scope.json;
(
process_list_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ProcessCommands::Status(args) => {
let json_output = args.scope.json;
(
process_status_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ProcessCommands::Restart(args) => {
let json_output = args.scope.json;
(
process_restart_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ProcessCommands::Cancel(args) => {
let json_output = args.scope.json;
(
process_cancel_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ProcessCommands::Abort(args) => {
let json_output = args.scope.json;
(
process_abort_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
};
emit_report(&report, json_output)?;
}
Commands::Task { command } => {
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
let (report, json_output) = match command {
TaskCommands::List(args) => {
let json_output = args.scope.json;
(
task_list_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
TaskCommands::Restart(args) => {
let json_output = args.scope.json;
(
task_restart_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
};
emit_report(&report, json_output)?;
}
Commands::Logs(args) => {
let json_output = args.scope.json;
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
emit_report(
&logs_report_with_session(args, stored_session.as_ref())?,
json_output,
)?;
}
Commands::Artifact { command } => {
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
let (report, json_output) = match command {
ArtifactCommands::List(args) => {
let json_output = args.scope.json;
(
artifact_list_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ArtifactCommands::Download(args) => {
let json_output = args.scope.json;
(
artifact_download_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
ArtifactCommands::Export(args) => {
let json_output = args.scope.json;
(
artifact_export_report_with_session(args, stored_session.as_ref())?,
json_output,
)
}
};
emit_report(&report, json_output)?;
}
Commands::Dap(args) => {
let json_output = args.json;
if args.plan {
emit_report(&dap_plan(args)?, json_output)?;
} else {
return exec_dap(args);
}
}
Commands::Debug {
command: DebugCommands::Attach(args),
} => {
let json_output = args.scope.json;
let cwd = std::env::current_dir()?;
let stored_session = read_cli_session(&cwd)?;
let dap = dap_binary_path()?.display().to_string();
emit_report(
&debug_attach_report_with_dap_and_session(args, dap, stored_session.as_ref())?,
json_output,
)?;
}
Commands::Quota {
command: QuotaCommands::Status(args),
} => {
let json_output = args.scope.json;
emit_report(
&quota_status_report(args, std::env::current_dir()?)?,
json_output,
)?;
}
Commands::Admin { command } => {
let (report, json_output) = match command {
AdminCommands::Status(args) => {
let json_output = args.scope.json;
(admin_status_report(args)?, json_output)
}
AdminCommands::Bootstrap(args) => {
let json_output = args.scope.json;
(
admin_bootstrap_report(args, std::env::current_dir()?)?,
json_output,
)
}
AdminCommands::RevokeNode(args) => {
let json_output = args.scope.json;
(
node_revoke_report(args, std::env::current_dir()?)?,
json_output,
)
}
AdminCommands::StopProcess(args) => {
let json_output = args.scope.json;
(process_cancel_report(args)?, json_output)
}
AdminCommands::SuspendTenant(args) => {
let json_output = args.scope.json;
(admin_suspend_tenant_report(args)?, json_output)
}
};
emit_report(&report, json_output)?;
}
}
Ok(())
}

163
crates/clusterflux-cli/src/doctor.rs Normal file
View file

@ -0,0 +1,163 @@
use std::path::PathBuf;
use std::time::Duration;
use anyhow::{Context, Result};
use clusterflux_control::ControlSession;
use clusterflux_core::{coordinator_wire_request, Capability, NodeCapabilities};
use serde_json::{json, Value};
use crate::auth::auth_state_value;
use crate::config::read_project_config;
use crate::tools::{command_available, sibling_binary};
use crate::DoctorArgs;
const DOCTOR_COORDINATOR_TIMEOUT: Duration = Duration::from_millis(500);
pub(crate) fn doctor_report(args: DoctorArgs, cwd: PathBuf) -> Result<Value> {
let config = read_project_config(&cwd)?;
let coordinator = args.scope.coordinator.or_else(|| {
config
.as_ref()
.and_then(|config| config.coordinator.clone())
});
let coordinator_reachability = coordinator_reachability(coordinator.as_deref());
let dependencies = json!({
"cargo": command_available("cargo"),
"git": command_available("git"),
"podman": command_available("podman"),
"clusterflux-node": command_available("clusterflux-node") || sibling_binary("clusterflux-node").is_some(),
"clusterflux-coordinator": command_available("clusterflux-coordinator") || sibling_binary("clusterflux-coordinator").is_some(),
"clusterflux-debug-dap": command_available("clusterflux-debug-dap") || sibling_binary("clusterflux-debug-dap").is_some(),
});
let node_readiness = NodeCapabilities::detect_current();
let node_readiness_summary = node_readiness_summary(&node_readiness, &dependencies);
Ok(json!({
"command": "doctor",
"cwd": cwd,
"coordinator": coordinator,
"coordinator_reachability": coordinator_reachability,
"auth": auth_state_value(&cwd)?,
"project": config,
"dependencies": dependencies,
"node_readiness": node_readiness,
"node_readiness_summary": node_readiness_summary,
"next_actions": [
"clusterflux login --browser",
"clusterflux project init",
"clusterflux node attach",
"clusterflux run"
]
}))
}
fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value) -> Value {
let has_command = capabilities.capabilities.contains(&Capability::Command);
let has_vfs_artifacts = capabilities
.capabilities
.contains(&Capability::VfsArtifacts);
let has_source_filesystem = capabilities
.capabilities
.contains(&Capability::SourceFilesystem);
let has_container_backend = capabilities
.environment_backends
.contains(&clusterflux_core::EnvironmentBackend::Container);
let podman_available = dependencies
.get("podman")
.and_then(Value::as_bool)
.unwrap_or(false);
let git_available = dependencies
.get("git")
.and_then(Value::as_bool)
.unwrap_or(false);
let clusterflux_node_available = dependencies
.get("clusterflux-node")
.and_then(Value::as_bool)
.unwrap_or(false);
let mut missing = Vec::new();
if has_container_backend && !podman_available {
missing.push("podman");
}
if capabilities.source_providers.contains("git") && !git_available {
missing.push("git");
}
if !clusterflux_node_available {
missing.push("clusterflux-node");
}
let basic_runtime_ready = true;
let local_dependencies_ready = missing.is_empty();
let status = if basic_runtime_ready && local_dependencies_ready {
"ready_to_attach"
} else if basic_runtime_ready {
"local_dependencies_missing"
} else {
"limited_capabilities"
};
let next_actions = if status == "ready_to_attach" {
vec![
"clusterflux node enroll",
"clusterflux node attach",
"clusterflux-node --worker",
]
} else {
vec![
"install missing local dependencies",
"rerun clusterflux doctor",
]
};
json!({
"status": status,
"explicit_attach_required": true,
"command_execution_capability": has_command,
"wasm_runtime": "wasmtime",
"wasm_runtime_is_baseline": true,
"artifact_capability": has_vfs_artifacts,
"source_filesystem_capability": has_source_filesystem,
"container_backend_reported": has_container_backend,
"podman_binary_available": podman_available,
"git_binary_available": git_available,
"node_binary_available": clusterflux_node_available,
"missing_local_dependencies": missing,
"source_providers": capabilities.source_providers.iter().collect::<Vec<_>>(),
"next_actions": next_actions,
})
}
fn coordinator_reachability(coordinator: Option<&str>) -> Value {
let Some(coordinator) = coordinator else {
return json!({
"checked": false,
"status": "not_configured",
"next_action": "run clusterflux login --browser or pass --coordinator"
});
};
match ping_coordinator(coordinator, DOCTOR_COORDINATOR_TIMEOUT) {
Ok(response) => json!({
"checked": true,
"status": "reachable",
"coordinator": coordinator,
"response": response
}),
Err(err) => json!({
"checked": true,
"status": "unreachable",
"coordinator": coordinator,
"error": err.to_string(),
"next_action": "check the coordinator URL, network, and service status"
}),
}
}
fn ping_coordinator(coordinator: &str, timeout: Duration) -> Result<Value> {
let mut session = ControlSession::connect_with_timeouts(coordinator, timeout, timeout)
.with_context(|| format!("failed to connect to coordinator {coordinator}"))?;
session
.request(&coordinator_wire_request(
"doctor-1",
json!({ "type": "ping" }),
))
.with_context(|| format!("coordinator ping failed for {coordinator}"))
}

258
crates/clusterflux-cli/src/errors.rs Normal file
View file

@ -0,0 +1,258 @@
use serde_json::{json, Value};
pub(crate) fn cli_error_summary(message: &str) -> Value {
let category = classify_cli_error_message(message);
cli_error_summary_for_category(category, message)
}
pub(crate) fn cli_error_summary_with_default(
message: &str,
default_category: &'static str,
) -> Value {
let category = classify_cli_error_message(message);
let category = if category == "unknown" {
default_category
} else {
category
};
cli_error_summary_for_category(category, message)
}
pub(crate) fn cli_error_summary_for_category(category: &'static str, message: &str) -> Value {
let mut summary = json!({
"category": category,
"stable_exit_code": cli_error_exit_code(category),
"process_exit_code_applied": false,
"retryable_after_user_action": cli_error_retryable_after_user_action(category),
"message": message,
"safe_failure": true,
"next_actions": cli_error_next_actions(category),
});
if category == "quota" {
if let Some(object) = summary.as_object_mut() {
object.insert(
"resource_category".to_owned(),
json!(
quota_error_resource_category(message).unwrap_or_else(|| "unknown".to_owned())
),
);
object.insert("community_tier_language".to_owned(), json!(true));
object.insert("community_tier_label".to_owned(), json!("community tier"));
object.insert("private_abuse_heuristics_exposed".to_owned(), json!(false));
}
}
summary
}
fn quota_error_resource_category(message: &str) -> Option<String> {
let lower = message.to_ascii_lowercase();
for marker in [
"resource limit exceeded for ",
"quota unavailable for ",
"quota exceeded for ",
"limit exceeded for ",
"metering limit exceeded for ",
"quota limit for ",
"resource category ",
"resource=",
"resource:",
] {
if let Some(index) = lower.find(marker) {
let start = index + marker.len();
let rest = &message[start..];
let value: String = rest
.chars()
.skip_while(|ch| ch.is_ascii_whitespace())
.take_while(|ch| {
ch.is_ascii_alphanumeric()
|| *ch == '_'
|| *ch == '-'
|| *ch == '.'
|| *ch == ':'
})
.collect();
let value = value.trim_matches(|ch: char| ch == '.' || ch == ':' || ch == ',');
if !value.is_empty() {
return Some(value.to_owned());
}
}
}
None
}
fn classify_cli_error_message(message: &str) -> &'static str {
let message = message.to_ascii_lowercase();
if message.contains("already has active virtual process") || message.contains("active process")
{
return "active_process";
}
if message_mentions_locality_failure(&message) {
return "connectivity";
}
if message.contains("no capable node")
|| message.contains("missing capability")
|| message.contains("capability")
|| message.contains("placement failed")
|| message.contains("cmd.run")
|| message.contains("node required")
{
return "capability";
}
if message.contains("quota")
|| message.contains("resource limit")
|| message.contains("limit exceeded")
|| message.contains("metering")
{
return "quota";
}
if message.contains("policy")
|| message.contains("disallowed")
|| message.contains("not allowed")
|| message.contains("suspended")
|| message.contains("blocked by")
|| message.contains("denied")
{
return "policy";
}
if message.contains("unauthenticated")
|| message.contains("not authenticated")
|| message.contains("login required")
|| message.contains("browser login required")
|| message.contains("missing session")
|| message.contains("no cli session")
|| message.contains("session credential has expired")
|| message.contains("session credential has been revoked")
|| message.contains("401")
{
return "authentication";
}
if message.contains("unauthorized")
|| message.contains("not authorized")
|| message.contains("forbidden")
|| message.contains("permission")
|| message.contains("tenant mismatch")
|| message.contains("project mismatch")
{
return "authorization";
}
if message.contains("failed to connect")
|| message.contains("connection refused")
|| message.contains("closed session")
|| message.contains("timed out")
|| message.contains("timeout")
|| message.contains("name resolution")
|| message.contains("network unreachable")
|| message.contains("dns")
{
return "connectivity";
}
if message.contains("missing environment")
|| message.contains("environment mismatch")
|| message.contains("incompatible environment")
|| message.contains("containerfile")
|| message.contains("dockerfile")
|| message.contains("envs/")
{
return "environment";
}
if message.contains("task exited")
|| message.contains("exit status")
|| message.contains("status code")
|| message.contains("stderr")
|| message.contains("panic")
|| message.contains("program error")
|| message.contains("command failed")
{
return "program";
}
"unknown"
}
pub(crate) fn message_mentions_locality_failure(message: &str) -> bool {
message.contains("direct connectivity unavailable")
|| message.contains("direct transfer")
|| message.contains("source snapshot unavailable")
|| (message.contains("required artifact")
&& message.contains("unavailable")
&& message.contains("direct connectivity"))
|| message.contains("locality assumption")
}
fn cli_error_exit_code(category: &str) -> i64 {
match category {
"authentication" => 20,
"authorization" => 21,
"quota" => 22,
"policy" => 23,
"capability" => 24,
"connectivity" => 25,
"environment" => 26,
"program" => 27,
"active_process" => 28,
_ => 1,
}
}
fn cli_error_retryable_after_user_action(category: &str) -> bool {
matches!(
category,
"authentication"
| "authorization"
| "quota"
| "policy"
| "capability"
| "connectivity"
| "environment"
| "program"
| "active_process"
)
}
fn cli_error_next_actions(category: &str) -> Vec<&'static str> {
match category {
"authentication" => vec!["clusterflux login --browser", "clusterflux auth status"],
"authorization" => vec![
"clusterflux auth status",
"check tenant/project selection",
"ask an admin to grant access",
],
"quota" => vec![
"clusterflux quota status",
"reduce concurrent work or wait for usage to fall",
],
"policy" => vec![
"clusterflux doctor",
"check coordinator policy for this action",
],
"capability" => vec![
"clusterflux node list",
"attach a node with the required capabilities",
"check tenant/project on the attached node",
],
"connectivity" => vec![
"clusterflux doctor",
"check the coordinator endpoint and network reachability",
],
"environment" => vec![
"clusterflux inspect",
"check envs/<name>/Containerfile or envs/<name>/Dockerfile",
],
"program" => vec![
"clusterflux logs",
"fix the program or task command and rerun",
],
"active_process" => vec![
"clusterflux process list",
"clusterflux process status",
"clusterflux debug attach",
"clusterflux process restart --yes",
"clusterflux process cancel --yes",
"clusterflux process abort --yes",
"use another Coordinator Project",
],
_ => vec![
"clusterflux doctor",
"rerun with --json for machine-readable details",
],
}
}

271
crates/clusterflux-cli/src/key.rs Normal file
View file

@ -0,0 +1,271 @@
use anyhow::Result;
use clusterflux_core::Digest;
use serde_json::{json, Value};
use crate::client::{authenticated_or_local_trusted_request, JsonLineSession};
use crate::config::StoredCliSession;
use crate::{confirmation_required_report, KeyAddArgs, KeyListArgs, KeyRevokeArgs};
#[cfg(test)]
pub(crate) fn key_add_report(args: KeyAddArgs) -> Result<Value> {
key_add_report_with_session(args, None)
}
pub(crate) fn key_add_report_with_session(
args: KeyAddArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
if let Some(coordinator) = &coordinator {
let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
&session.tenant
});
let project = effective_session_value(stored_session, &args.scope.project, |session| {
&session.project
});
let user =
effective_session_value(stored_session, &args.scope.user, |session| &session.user);
let agent = args.agent.clone();
let public_key_fingerprint = Digest::sha256(&args.public_key);
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "register_agent_public_key",
"agent": agent,
"public_key": args.public_key,
}),
json!({
"type": "register_agent_public_key",
"tenant": tenant,
"project": project,
"user": user,
"agent": agent,
"public_key": args.public_key,
}),
)?)?;
let record = response.get("record").cloned().unwrap_or_else(|| {
json!({
"tenant": tenant,
"project": project,
"user": user,
"agent": agent,
"public_key_fingerprint": public_key_fingerprint,
"scopes": ["project:read", "project:run"],
"human_account_creation_privilege": false,
"browser_interaction_required_each_run": false,
})
});
return Ok(json!({
"command": "key add",
"coordinator": coordinator,
"tenant": tenant,
"project": project,
"user": user,
"agent": agent,
"public_key_fingerprint": record
.get("public_key_fingerprint")
.cloned()
.unwrap_or_else(|| json!(public_key_fingerprint)),
"credential_scope": {
"tenant": tenant,
"project": project,
"actions": record
.get("scopes")
.cloned()
.unwrap_or_else(|| json!(["project:read", "project:run"])),
"human_account_creation_privilege": record
.get("human_account_creation_privilege")
.cloned()
.unwrap_or(json!(false)),
},
"browser_interaction_required_each_run": record
.get("browser_interaction_required_each_run")
.cloned()
.unwrap_or(json!(false)),
"attribution": {
"registered_by_user": user,
"agent": agent,
"credential_kind": "public_key",
},
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "key add",
"status": "planned_without_coordinator",
"agent": args.agent,
"public_key_fingerprint": Digest::sha256(args.public_key),
"browser_interaction_required_each_run": false,
}))
}
#[cfg(test)]
pub(crate) fn key_list_report(args: KeyListArgs) -> Result<Value> {
key_list_report_with_session(args, None)
}
pub(crate) fn key_list_report_with_session(
args: KeyListArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
if let Some(coordinator) = &coordinator {
let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
&session.tenant
});
let project = effective_session_value(stored_session, &args.scope.project, |session| {
&session.project
});
let user =
effective_session_value(stored_session, &args.scope.user, |session| &session.user);
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "list_agent_public_keys",
}),
json!({
"type": "list_agent_public_keys",
"tenant": tenant,
"project": project,
"user": user,
}),
)?)?;
return Ok(json!({
"command": "key list",
"coordinator": coordinator,
"tenant": tenant,
"project": project,
"user": user,
"records": response
.get("records")
.cloned()
.unwrap_or_else(|| json!([])),
"credential_scope": {
"tenant": tenant,
"project": project,
"listed_for_user": user,
"human_account_creation_privilege": false,
},
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "key list",
"status": "requires_coordinator",
"records": [],
}))
}
#[cfg(test)]
pub(crate) fn key_revoke_report(args: KeyRevokeArgs) -> Result<Value> {
key_revoke_report_with_session(args, None)
}
pub(crate) fn key_revoke_report_with_session(
args: KeyRevokeArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
if !args.yes {
return Ok(confirmation_required_report(
"key revoke",
"revoke_agent_public_key",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"user": args.scope.user,
"agent": args.agent,
}),
format!("clusterflux key revoke --agent {} --yes", args.agent),
));
}
let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);
if let Some(coordinator) = &coordinator {
let tenant = effective_session_value(stored_session, &args.scope.tenant, |session| {
&session.tenant
});
let project = effective_session_value(stored_session, &args.scope.project, |session| {
&session.project
});
let user =
effective_session_value(stored_session, &args.scope.user, |session| &session.user);
let agent = args.agent.clone();
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "revoke_agent_public_key",
"agent": agent,
}),
json!({
"type": "revoke_agent_public_key",
"tenant": tenant,
"project": project,
"user": user,
"agent": agent,
}),
)?)?;
return Ok(json!({
"command": "key revoke",
"coordinator": coordinator,
"requires_confirmation": !args.yes,
"tenant": tenant,
"project": project,
"user": user,
"agent": agent,
"revoked": response
.pointer("/record/revoked")
.cloned()
.unwrap_or(json!(true)),
"credential_scope": {
"tenant": tenant,
"project": project,
"revoked_for_user": user,
"human_account_creation_privilege": false,
},
"attribution": {
"revoked_by_user": user,
"agent": agent,
"credential_kind": "public_key",
},
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "key revoke",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"agent": args.agent,
}))
}
fn effective_coordinator(
requested: &Option<String>,
stored_session: Option<&StoredCliSession>,
) -> Option<String> {
requested.clone().or_else(|| {
stored_session
.filter(|session| session.session_secret.is_some())
.map(|session| session.coordinator.clone())
})
}
fn effective_session_value(
stored_session: Option<&StoredCliSession>,
requested: &str,
value: impl FnOnce(&StoredCliSession) -> &String,
) -> String {
stored_session
.filter(|session| session.session_secret.is_some())
.map(value)
.cloned()
.unwrap_or_else(|| requested.to_owned())
}

127
crates/clusterflux-cli/src/logout.rs Normal file
View file

@ -0,0 +1,127 @@
use std::path::PathBuf;
use anyhow::{Context, Result};
use serde_json::{json, Value};
use crate::client::JsonLineSession;
use crate::config::{read_cli_session, session_config_file, StoredCliSession};
use crate::confirm::confirmation_required_report;
use crate::errors::cli_error_summary;
use crate::AuthLogoutArgs;
pub(crate) fn auth_logout_report(args: AuthLogoutArgs, cwd: PathBuf) -> Result<Value> {
logout_report(args, cwd, "auth logout")
}
pub(crate) fn logout_report(args: AuthLogoutArgs, cwd: PathBuf, command: &str) -> Result<Value> {
let session_file = session_config_file(&cwd);
if !args.yes {
return Ok(confirmation_required_report(
command,
"delete_cli_session",
json!({
"session_file": session_file,
"node_credentials_untouched": true,
}),
format!("clusterflux {command} --yes"),
));
}
let stored_session = read_cli_session(&cwd).ok().flatten();
let coordinator_revocation = revoke_stored_cli_session_if_possible(stored_session.as_ref());
let existed = session_file.exists();
if existed {
std::fs::remove_file(&session_file)
.with_context(|| format!("failed to remove {}", session_file.display()))?;
}
Ok(json!({
"command": command,
"requires_confirmation": !args.yes,
"removed_cli_session_file": existed,
"server_session_revocation": coordinator_revocation,
"node_credentials_untouched": true,
"session_file": session_file,
}))
}
fn revoke_stored_cli_session_if_possible(stored_session: Option<&StoredCliSession>) -> Value {
let Some(session) = stored_session else {
return json!({
"attempted": false,
"revoked": false,
"reason": "no_parseable_cli_session",
});
};
let Some(session_secret) = session.session_secret.as_deref() else {
return json!({
"attempted": false,
"revoked": false,
"reason": "no_cli_session_secret",
"coordinator": session.coordinator,
});
};
// A CLI session credential is authority issued by one coordinator. Never
// redirect it to a command-line endpoint override.
let coordinator = session.coordinator.as_str();
let mut coordinator_session = match JsonLineSession::connect(coordinator) {
Ok(session) => session,
Err(error) => {
let message = error.to_string();
return json!({
"attempted": true,
"revoked": false,
"reachable": false,
"coordinator": coordinator,
"error": message,
"machine_error": cli_error_summary(&message),
"next_actions": ["clusterflux login --browser"],
});
}
};
let response = match coordinator_session.request_allow_error(json!({
"type": "authenticated",
"session_secret": session_secret,
"request": {
"type": "revoke_cli_session",
},
})) {
Ok(response) => response,
Err(error) => {
let message = error.to_string();
return json!({
"attempted": true,
"revoked": false,
"reachable": false,
"coordinator": coordinator,
"error": message,
"machine_error": cli_error_summary(&message),
"coordinator_session_requests": coordinator_session.requests(),
"next_actions": ["clusterflux login --browser"],
});
}
};
let revoked = response.get("type").and_then(Value::as_str) == Some("cli_session_revoked");
if !revoked {
let message = response
.get("message")
.and_then(Value::as_str)
.unwrap_or("coordinator did not revoke CLI session");
return json!({
"attempted": true,
"revoked": false,
"reachable": true,
"coordinator": coordinator,
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("unknown"),
"machine_error": cli_error_summary(message),
"coordinator_session_requests": coordinator_session.requests(),
"next_actions": ["clusterflux login --browser"],
});
}
json!({
"attempted": true,
"revoked": true,
"reachable": true,
"coordinator": coordinator,
"coordinator_response_type": "cli_session_revoked",
"coordinator_session_requests": coordinator_session.requests(),
})
}

34
crates/clusterflux-cli/src/logs.rs Normal file
View file

@ -0,0 +1,34 @@
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::list_task_events_if_available_with_session;
use crate::config::StoredCliSession;
use crate::process_events::log_entries;
use crate::LogsArgs;
#[cfg(test)]
pub(crate) fn logs_report(args: LogsArgs) -> Result<Value> {
logs_report_with_session(args, None)
}
pub(crate) fn logs_report_with_session(
args: LogsArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,
args.process.clone(),
stored_session,
)?;
let log_entries = log_entries(events.as_ref(), args.task.as_deref());
Ok(json!({
"command": "logs",
"process": args.process,
"task": args.task,
"log_entries": log_entries,
"logs_are_capped": true,
"secret_redaction_policy": "configured-redaction-boundary",
"events": events,
}))
}

689
crates/clusterflux-cli/src/main.rs Normal file
View file

@ -0,0 +1,689 @@
#[cfg(test)]
use std::io::{BufRead, BufReader, Write};
#[cfg(test)]
use std::net::TcpListener;
#[cfg(test)]
use std::path::Path;
use std::path::PathBuf;
use clap::{Args, Parser, Subcommand};
#[cfg(test)]
use clusterflux_core::{Capability, Digest, ProjectModel, SourceProviderKind};
#[cfg(test)]
use serde_json::json;
use serde_json::Value;
mod admin;
mod agent;
mod artifact;
mod auth;
mod auth_scope;
mod build;
mod bundle;
mod client;
mod config;
mod confirm;
mod debug;
mod dispatch;
mod doctor;
mod errors;
mod key;
mod logout;
mod logs;
mod node;
mod output;
mod process;
mod process_events;
mod project;
mod quota;
mod run;
mod task;
mod tools;
use admin::{admin_bootstrap_report, admin_status_report, admin_suspend_tenant_report};
use agent::agent_enrollment_plan;
#[cfg(test)]
use artifact::{
artifact_download_report, artifact_export_report, artifact_list_report,
artifact_stream_summary, DEFAULT_ARTIFACT_EXPORT_MAX_BYTES,
};
use artifact::{
artifact_download_report_with_session, artifact_export_report_with_session,
artifact_list_report_with_session,
};
use auth::{
auth_status_report, connect_self_hosted_report, execute_interactive_browser_login, login_plan,
non_interactive_browser_login_report, print_browser_login_success,
read_session_secret_from_stdin,
};
#[cfg(test)]
use auth::{contains_provider_token_field, stored_cli_session_from_login_response, LoginFlowPlan};
#[cfg(test)]
use auth_scope::login_args_for_project;
use build::build_report;
#[cfg(test)]
use bundle::task_abi_digest;
pub(crate) use bundle::{bundle_inspection, discovered_environment_names};
#[cfg(test)]
use client::control_endpoint_identity;
use config::{default_hosted_coordinator_endpoint, read_cli_session};
#[cfg(test)]
use config::{
read_project_config, session_config_file, write_cli_session, write_project_config,
ProjectConfig, StoredCliSession, DEFAULT_HOSTED_COORDINATOR_ENDPOINT,
};
pub(crate) use confirm::confirmation_required_report;
#[cfg(test)]
use debug::debug_attach_report_with_dap;
use debug::{dap_plan, debug_attach_report_with_dap_and_session, exec_dap};
use doctor::doctor_report;
use errors::cli_error_summary;
#[cfg(test)]
use errors::cli_error_summary_for_category;
#[cfg(test)]
use key::{key_add_report, key_list_report, key_revoke_report};
use key::{
key_add_report_with_session, key_list_report_with_session, key_revoke_report_with_session,
};
use logout::{auth_logout_report, logout_report};
#[cfg(test)]
use logs::logs_report;
use logs::logs_report_with_session;
use node::{
attach_plan, execute_node_attach, node_enroll_report, node_list_report, node_revoke_report,
node_status_report,
};
use output::emit_report;
#[cfg(test)]
use output::{apply_command_report_exit_code, human_report};
use process::{
process_abort_report_with_session, process_cancel_report, process_cancel_report_with_session,
process_list_report_with_session, process_restart_report_with_session,
process_status_report_with_session,
};
#[cfg(test)]
use process::{process_restart_report, process_status_report};
#[cfg(test)]
use process_events::task_summaries;
#[cfg(test)]
use process_events::{
artifact_download_session_summary, artifact_export_plan_summary, log_entries,
};
use project::{
project_init_report, project_list_report, project_select_report, project_status_report,
};
use quota::quota_status_report;
#[cfg(test)]
use run::{
agent_session_from_keys, run_plan, run_start_summary, should_execute_local_node, CliSession,
CoordinatorSelection,
};
use run::{run_report, session_from_sources};
#[cfg(test)]
use task::{task_list_report, task_restart_report};
use task::{task_list_report_with_session, task_restart_report_with_session};
use tools::dap_binary_path;
#[derive(Clone, Debug, Parser)]
#[command(
name = "clusterflux",
version,
arg_required_else_help = true,
about = "Clusterflux distributed Wasm runtime CLI.",
after_help = "Primary workflow:
1. clusterflux login --browser
2. clusterflux project init
3. clusterflux node enroll; clusterflux node attach; clusterflux-node --worker
4. clusterflux run [entry] --project <path>
5. Debug with VS Code \"Clusterflux: Launch Virtual Process\" or clusterflux dap
6. Inspect with clusterflux process list, clusterflux process status, task list, logs, and artifact list
7. Request cooperative shutdown with process cancel; force termination with process abort
Use --json on primary commands for scriptable output. Hosted account creation happens in the browser login flow."
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Clone, Debug, Subcommand)]
enum Commands {
Doctor(DoctorArgs),
Login(LoginArgs),
Logout(AuthLogoutArgs),
Auth {
#[command(subcommand)]
command: AuthCommands,
},
Agent {
#[command(subcommand)]
command: AgentCommands,
},
Key {
#[command(subcommand)]
command: KeyCommands,
},
Project {
#[command(subcommand)]
command: ProjectCommands,
},
Inspect(BundleInspectArgs),
Build(BuildArgs),
Bundle {
#[command(subcommand)]
command: BundleCommands,
},
Run(RunArgs),
Node {
#[command(subcommand)]
command: NodeCommands,
},
Process {
#[command(subcommand)]
command: ProcessCommands,
},
Task {
#[command(subcommand)]
command: TaskCommands,
},
Logs(LogsArgs),
Artifact {
#[command(subcommand)]
command: ArtifactCommands,
},
Dap(DapArgs),
Debug {
#[command(subcommand)]
command: DebugCommands,
},
Quota {
#[command(subcommand)]
command: QuotaCommands,
},
Admin {
#[command(subcommand)]
command: AdminCommands,
},
}
#[derive(Clone, Debug, Parser)]
struct DoctorArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct LoginArgs {
#[arg(long = "browser")]
_browser: bool,
#[arg(long)]
non_interactive: bool,
#[arg(long)]
plan: bool,
#[arg(long)]
json: bool,
#[arg(long, default_value_t = default_hosted_coordinator_endpoint())]
coordinator: String,
#[arg(long = "project-id", default_value = "project")]
project: String,
}
#[derive(Clone, Debug, Subcommand)]
enum AuthCommands {
Status(AuthStatusArgs),
ConnectSelfHosted(ConnectSelfHostedArgs),
Logout(AuthLogoutArgs),
}
#[derive(Clone, Debug, Parser)]
struct AuthStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct ConnectSelfHostedArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, required = true)]
session_secret_stdin: bool,
}
#[derive(Clone, Debug, Parser)]
struct AuthLogoutArgs {
#[arg(long)]
yes: bool,
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Subcommand)]
enum AgentCommands {
Enroll(AgentEnrollArgs),
}
#[derive(Clone, Debug, Subcommand)]
enum KeyCommands {
Add(KeyAddArgs),
List(KeyListArgs),
Revoke(KeyRevokeArgs),
}
#[derive(Clone, Debug, Subcommand)]
enum BundleCommands {
Inspect(BundleInspectArgs),
}
#[derive(Clone, Debug, Parser)]
struct AgentEnrollArgs {
#[arg(long)]
json: bool,
#[arg(long = "public-key")]
public_key: String,
}
#[derive(Clone, Debug, Parser)]
struct KeyAddArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "agent")]
agent: String,
#[arg(long = "public-key")]
public_key: String,
}
#[derive(Clone, Debug, Parser)]
struct KeyListArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct KeyRevokeArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "agent")]
agent: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Subcommand)]
enum ProjectCommands {
Init(ProjectInitArgs),
Status(ProjectStatusArgs),
List(ProjectListArgs),
Select(ProjectSelectArgs),
}
#[derive(Clone, Debug, Parser)]
struct ProjectInitArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "project")]
new_project: String,
#[arg(long, default_value = "Clusterflux Project")]
name: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Parser)]
struct ProjectStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct ProjectListArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct ProjectSelectArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(value_name = "PROJECT")]
selected_project: String,
}
#[derive(Clone, Debug, Parser)]
struct BundleInspectArgs {
#[arg(long)]
project: Option<PathBuf>,
#[arg(long = "source-provider")]
source_provider: Option<String>,
#[arg(long = "disable-source-provider")]
disabled_source_providers: Vec<String>,
#[arg(long)]
json: bool,
}
#[derive(Clone, Debug, Parser)]
struct BuildArgs {
#[arg(long)]
project: Option<PathBuf>,
#[arg(long = "source-provider")]
source_provider: Option<String>,
#[arg(long = "disable-source-provider")]
disabled_source_providers: Vec<String>,
#[arg(long)]
output: Option<PathBuf>,
#[arg(long)]
json: bool,
}
#[derive(Clone, Debug, Parser)]
struct RunArgs {
entry: Option<String>,
#[arg(long)]
project: Option<PathBuf>,
#[arg(long)]
coordinator: Option<String>,
#[arg(long)]
local: bool,
#[arg(long)]
non_interactive: bool,
#[arg(long)]
json: bool,
}
#[derive(Clone, Debug, Subcommand)]
enum NodeCommands {
Attach(AttachArgs),
Enroll(NodeEnrollArgs),
List(NodeListArgs),
Status(NodeStatusArgs),
Revoke(NodeRevokeArgs),
}
#[derive(Clone, Debug, Parser)]
struct AttachArgs {
#[arg(long)]
coordinator: Option<String>,
#[arg(long, default_value = "tenant")]
tenant: String,
#[arg(long = "project-id", default_value = "project")]
project: String,
#[arg(long)]
node: Option<String>,
#[arg(long = "cap")]
caps: Vec<String>,
#[arg(long = "enrollment-grant")]
enrollment_grant: Option<String>,
#[arg(long = "public-key")]
public_key: Option<String>,
#[arg(long)]
json: bool,
}
#[derive(Clone, Debug, Parser)]
struct NodeEnrollArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value_t = 900)]
ttl_seconds: u64,
}
#[derive(Clone, Debug, Parser)]
struct NodeListArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct NodeStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
node: Option<String>,
}
#[derive(Clone, Debug, Parser)]
struct NodeRevokeArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
node: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Subcommand)]
enum ProcessCommands {
List(ProcessListArgs),
Status(ProcessStatusArgs),
Restart(ProcessRestartArgs),
Cancel(ProcessCancelArgs),
Abort(ProcessAbortArgs),
}
#[derive(Clone, Debug, Parser)]
struct ProcessListArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Parser)]
struct ProcessStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "vp-current")]
process: String,
}
#[derive(Clone, Debug, Parser)]
struct ProcessRestartArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "vp-current")]
process: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Parser)]
struct ProcessCancelArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "vp-current")]
process: String,
#[arg(long)]
node: Option<String>,
#[arg(long)]
task: Option<String>,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Parser)]
struct ProcessAbortArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "vp-current")]
process: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Subcommand)]
enum TaskCommands {
List(TaskListArgs),
Restart(TaskRestartArgs),
}
#[derive(Clone, Debug, Parser)]
struct TaskListArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
process: Option<String>,
}
#[derive(Clone, Debug, Parser)]
struct TaskRestartArgs {
#[command(flatten)]
scope: CliScopeArgs,
task: String,
#[arg(long, default_value = "vp-current")]
process: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Parser)]
struct LogsArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
process: Option<String>,
#[arg(long)]
task: Option<String>,
}
#[derive(Clone, Debug, Subcommand)]
enum ArtifactCommands {
List(ArtifactListArgs),
Download(ArtifactDownloadArgs),
Export(ArtifactExportArgs),
}
#[derive(Clone, Debug, Parser)]
struct ArtifactListArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
process: Option<String>,
}
#[derive(Clone, Debug, Parser)]
struct ArtifactDownloadArgs {
#[command(flatten)]
scope: CliScopeArgs,
artifact: String,
/// Write the verified artifact bytes to this local path.
#[arg(long)]
to: Option<PathBuf>,
#[arg(long, default_value_t = 64 * 1024 * 1024)]
max_bytes: u64,
}
#[derive(Clone, Debug, Parser)]
struct ArtifactExportArgs {
#[command(flatten)]
scope: CliScopeArgs,
artifact: String,
#[arg(long)]
to: PathBuf,
#[arg(long, default_value = "node-local")]
receiver_node: String,
}
#[derive(Clone, Debug, Parser)]
struct DapArgs {
#[arg(long)]
plan: bool,
#[arg(long)]
json: bool,
#[arg(last = true)]
args: Vec<String>,
}
#[derive(Clone, Debug, Subcommand)]
enum DebugCommands {
Attach(DebugAttachArgs),
}
#[derive(Clone, Debug, Parser)]
struct DebugAttachArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "vp-current")]
process: String,
}
#[derive(Clone, Debug, Subcommand)]
enum QuotaCommands {
Status(QuotaStatusArgs),
}
#[derive(Clone, Debug, Parser)]
struct QuotaStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
}
#[derive(Clone, Debug, Subcommand)]
enum AdminCommands {
Status(AdminStatusArgs),
Bootstrap(AdminBootstrapArgs),
RevokeNode(NodeRevokeArgs),
StopProcess(ProcessCancelArgs),
SuspendTenant(AdminSuspendTenantArgs),
}
#[derive(Clone, Debug, Parser)]
struct AdminStatusArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long)]
admin_token: Option<String>,
}
#[derive(Clone, Debug, Parser)]
struct AdminBootstrapArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long, default_value = "Clusterflux Project")]
name: String,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Parser)]
struct AdminSuspendTenantArgs {
#[command(flatten)]
scope: CliScopeArgs,
#[arg(long = "target-tenant")]
target_tenant: Option<String>,
#[arg(long)]
admin_token: Option<String>,
#[arg(long)]
yes: bool,
}
#[derive(Clone, Debug, Args)]
struct CliScopeArgs {
#[arg(long)]
coordinator: Option<String>,
#[arg(long, default_value = "tenant")]
tenant: String,
#[arg(long = "project-id", default_value = "project")]
project: String,
#[arg(long, default_value = "user")]
user: String,
#[arg(long)]
json: bool,
}
fn main() {
if let Err(error) = dispatch::run_cli() {
let message = error.to_string();
let machine_error = cli_error_summary(&message);
let category = machine_error
.get("category")
.and_then(Value::as_str)
.unwrap_or("unknown");
let exit_code = machine_error
.get("stable_exit_code")
.and_then(Value::as_i64)
.and_then(|code| i32::try_from(code).ok())
.unwrap_or(1);
eprintln!("Error ({category}, exit {exit_code}): {error:#}");
std::process::exit(exit_code);
}
}
#[cfg(test)]
mod tests;

837
crates/clusterflux-cli/src/node.rs Normal file
View file

@ -0,0 +1,837 @@
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use clusterflux_core::{
generate_ed25519_private_key, node_ed25519_public_key_from_private_key, sign_node_request,
signed_request_payload_digest, Capability, Digest, EnvironmentBackend, NodeCapabilities,
NodeId,
};
use serde::Serialize;
use serde_json::{json, Value};
use crate::client::{authenticated_or_local_trusted_request, JsonLineSession};
use crate::config::{effective_scope_value, read_cli_session, StoredCliSession};
use crate::tools::{command_available, command_nonce, unix_timestamp_seconds};
use crate::{
confirmation_required_report, AttachArgs, CliScopeArgs, NodeEnrollArgs, NodeListArgs,
NodeRevokeArgs, NodeStatusArgs,
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct NodeAttachPlan {
pub(crate) node: String,
pub(crate) coordinator: Option<String>,
pub(crate) capabilities: NodeCapabilities,
pub(crate) detection: NodeAttachDetectionEvidence,
pub(crate) grant_disclosures: Vec<CapabilityGrantDisclosure>,
pub(crate) enrollment: Option<NodeEnrollmentPlan>,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub(crate) struct NodeAttachReport {
pub(crate) command: String,
pub(crate) node: String,
pub(crate) plan: NodeAttachPlan,
pub(crate) grant_disclosures: Vec<CapabilityGrantDisclosure>,
pub(crate) boundary: NodeAttachBoundaryEvidence,
pub(crate) coordinator_response: Value,
pub(crate) heartbeat_response: Value,
pub(crate) capability_response: Value,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct NodeAttachBoundaryEvidence {
pub(crate) cli_contacted_coordinator: bool,
pub(crate) coordinator_address: String,
pub(crate) used_enrollment_exchange: bool,
pub(crate) coordinator_session_requests: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct CapabilityGrantDisclosure {
pub(crate) capability: Capability,
pub(crate) grant: String,
pub(crate) description: String,
pub(crate) risk: String,
pub(crate) coordinator_policy_limited: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct NodeAttachDetectionEvidence {
pub(crate) auto_detected: bool,
pub(crate) os: clusterflux_core::Os,
pub(crate) arch: String,
pub(crate) command_backend: String,
pub(crate) command_backend_available: bool,
pub(crate) container_backend: Option<String>,
pub(crate) container_backend_reported: bool,
pub(crate) container_backend_available: bool,
pub(crate) source_provider_backends: Vec<SourceProviderBackendStatus>,
pub(crate) manual_capability_overrides_allowed: bool,
pub(crate) manual_capability_overrides: Vec<String>,
pub(crate) recognized_capability_overrides: Vec<Capability>,
pub(crate) unrecognized_capability_overrides: Vec<String>,
pub(crate) os_arch_capabilities_require_manual_flags: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct SourceProviderBackendStatus {
pub(crate) provider: String,
pub(crate) detected: bool,
pub(crate) available: bool,
pub(crate) reason: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct NodeEnrollmentPlan {
pub(crate) grant: String,
pub(crate) public_key_fingerprint: Digest,
pub(crate) exchanges_short_lived_grant_for_long_lived_node_identity: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
struct StoredNodeCredential {
kind: String,
node: String,
private_key: String,
public_key: String,
credential_scope: String,
}
pub(crate) fn node_enroll_report(args: NodeEnrollArgs, cwd: PathBuf) -> Result<Value> {
let stored_session = read_cli_session(&cwd)?;
let coordinator = args.scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
if let Some(coordinator) = &coordinator {
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let project = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.project,
|session| session.project.as_str(),
"project",
);
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.user,
|session| session.user.as_str(),
"user",
);
let ttl_seconds = args.ttl_seconds;
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "create_node_enrollment_grant",
"ttl_seconds": ttl_seconds,
}),
json!({
"type": "create_node_enrollment_grant",
"tenant": tenant.clone(),
"project": project.clone(),
"actor_user": user.clone(),
"ttl_seconds": ttl_seconds,
}),
)?;
let response = session.request(request)?;
let enrollment_grant =
node_enrollment_grant_summary(&response, &tenant, &project, &user, ttl_seconds)?;
return Ok(json!({
"command": "node enroll",
"status": "created",
"coordinator": coordinator,
"tenant": tenant,
"project": project,
"user": user,
"private_website_required": false,
"enrollment_grant": enrollment_grant,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "node enroll",
"status": "requires_coordinator",
"private_website_required": false,
"requested_ttl_seconds": args.ttl_seconds,
"enrollment_grant": null,
"reason": "enrollment grants are generated by the coordinator and cannot be planned client-side",
}))
}
fn node_enrollment_grant_summary(
response: &Value,
tenant: &str,
project: &str,
user: &str,
ttl_seconds: u64,
) -> Result<Value> {
let grant = response
.get("grant")
.and_then(Value::as_str)
.context("coordinator did not return its generated enrollment grant")?;
Ok(json!({
"grant": grant,
"tenant": response
.get("tenant")
.cloned()
.unwrap_or_else(|| json!(tenant)),
"project": response
.get("project")
.cloned()
.unwrap_or_else(|| json!(project)),
"user": user,
"scope": response
.get("scope")
.cloned()
.unwrap_or_else(|| json!("node:attach")),
"ttl_seconds": ttl_seconds,
"expires_at_epoch_seconds": response
.get("expires_at_epoch_seconds")
.cloned()
.unwrap_or(Value::Null),
"short_lived": true,
"exchange_for_persistent_node_identity": true,
"node_credentials_separate_from_user_session": true,
}))
}
pub(crate) fn node_list_report(args: NodeListArgs, cwd: PathBuf) -> Result<Value> {
node_descriptors_report("node list", args.scope, None, cwd)
}
pub(crate) fn node_status_report(args: NodeStatusArgs, cwd: PathBuf) -> Result<Value> {
node_descriptors_report("node status", args.scope, args.node, cwd)
}
fn node_descriptors_report(
command: &str,
scope: CliScopeArgs,
node: Option<String>,
cwd: PathBuf,
) -> Result<Value> {
let stored_session = read_cli_session(&cwd)?;
let coordinator = scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
if let Some(coordinator) = &coordinator {
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let project = session_or_effective_scope_value(
stored_session.as_ref(),
&scope.project,
|session| session.project.as_str(),
"project",
);
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&scope.user,
|session| session.user.as_str(),
"user",
);
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "list_node_descriptors",
}),
json!({
"type": "list_node_descriptors",
"tenant": tenant.clone(),
"project": project.clone(),
"actor_user": user.clone(),
}),
)?;
let response = session.request(request)?;
return Ok(json!({
"command": command,
"coordinator": coordinator,
"node": node,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": command,
"status": "local_capability_snapshot",
"node": node.unwrap_or_else(default_node_id),
"capabilities": NodeCapabilities::detect_current(),
}))
}
pub(crate) fn node_revoke_report(args: NodeRevokeArgs, cwd: PathBuf) -> Result<Value> {
if !args.yes {
return Ok(confirmation_required_report(
"node revoke",
"revoke_node_credential",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"user": args.scope.user,
"node": args.node,
}),
format!("clusterflux node revoke --node {} --yes", args.node),
));
}
let stored_session = read_cli_session(&cwd)?;
let coordinator = args.scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
if let Some(coordinator) = &coordinator {
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let project = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.project,
|session| session.project.as_str(),
"project",
);
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.user,
|session| session.user.as_str(),
"user",
);
let node = args.node.clone();
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "revoke_node_credential",
"node": node.clone(),
}),
json!({
"type": "revoke_node_credential",
"tenant": tenant.clone(),
"project": project.clone(),
"actor_user": user.clone(),
"node": node.clone(),
}),
)?;
let response = session.request(request)?;
return Ok(json!({
"command": "node revoke",
"coordinator": coordinator,
"requires_confirmation": !args.yes,
"tenant": tenant,
"project": project,
"user": user,
"node": node,
"credential_revoked": response.get("type").and_then(Value::as_str) == Some("node_credential_revoked"),
"descriptor_removed": response
.get("descriptor_removed")
.cloned()
.unwrap_or(json!(false)),
"queued_assignments_removed": response
.get("queued_assignments_removed")
.cloned()
.unwrap_or_else(|| json!(0)),
"node_credentials_separate_from_user_session": true,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "node revoke",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"node": args.node,
}))
}
fn session_or_effective_scope_value(
stored_session: Option<&StoredCliSession>,
cli_value: &str,
session_value: impl FnOnce(&StoredCliSession) -> &str,
default_value: &str,
) -> String {
if let Some(session) = stored_session.filter(|session| session.session_secret.is_some()) {
session_value(session).to_owned()
} else {
effective_scope_value(cli_value, stored_session.map(session_value), default_value)
}
}
pub(crate) fn attach_plan(args: AttachArgs) -> NodeAttachPlan {
let mut capabilities = NodeCapabilities::detect_current();
let mut recognized_capability_overrides = Vec::new();
let mut unrecognized_capability_overrides = Vec::new();
for cap in &args.caps {
if let Some(parsed) = parse_capability(cap) {
recognized_capability_overrides.push(parsed.clone());
capabilities.capabilities.insert(parsed);
} else {
unrecognized_capability_overrides.push(cap.clone());
}
}
recognized_capability_overrides.sort();
recognized_capability_overrides.dedup();
let node = args.node.unwrap_or_else(default_node_id);
let public_key = args
.public_key
.unwrap_or_else(|| default_node_public_key_for_plan(&node));
let enrollment = args.enrollment_grant.map(|grant| NodeEnrollmentPlan {
grant,
public_key_fingerprint: Digest::sha256(public_key),
exchanges_short_lived_grant_for_long_lived_node_identity: true,
});
let detection = node_attach_detection_evidence(
&capabilities,
args.caps,
recognized_capability_overrides,
unrecognized_capability_overrides,
);
let grant_disclosures = capability_grant_disclosures(&capabilities);
NodeAttachPlan {
node,
coordinator: args.coordinator,
capabilities,
detection,
grant_disclosures,
enrollment,
}
}
fn node_attach_detection_evidence(
capabilities: &NodeCapabilities,
manual_capability_overrides: Vec<String>,
recognized_capability_overrides: Vec<Capability>,
unrecognized_capability_overrides: Vec<String>,
) -> NodeAttachDetectionEvidence {
let command_backend_available = capabilities.capabilities.contains(&Capability::Command);
let container_backend_reported = capabilities
.environment_backends
.contains(&EnvironmentBackend::Container);
let container_backend = container_backend_reported.then(|| "rootless-podman".to_owned());
let container_backend_available = container_backend_reported
&& capabilities
.capabilities
.contains(&Capability::RootlessPodman)
&& command_available("podman");
NodeAttachDetectionEvidence {
auto_detected: true,
os: capabilities.os.clone(),
arch: capabilities.arch.clone(),
command_backend: if command_backend_available {
"native-command".to_owned()
} else {
"unavailable".to_owned()
},
command_backend_available,
container_backend,
container_backend_reported,
container_backend_available,
source_provider_backends: source_provider_backend_statuses(capabilities),
manual_capability_overrides_allowed: true,
manual_capability_overrides,
recognized_capability_overrides,
unrecognized_capability_overrides,
os_arch_capabilities_require_manual_flags: false,
}
}
fn source_provider_backend_statuses(
capabilities: &NodeCapabilities,
) -> Vec<SourceProviderBackendStatus> {
let mut statuses = BTreeMap::new();
for provider in &capabilities.source_providers {
let available = match provider.as_str() {
"filesystem" => capabilities
.capabilities
.contains(&Capability::SourceFilesystem),
"git" => command_available("git"),
_ => true,
};
statuses.insert(
provider.clone(),
SourceProviderBackendStatus {
provider: provider.clone(),
detected: true,
available,
reason: if available {
"detected by local node capability probe".to_owned()
} else {
format!(
"source provider `{provider}` was detected but its local helper is missing"
)
},
},
);
}
statuses.into_values().collect()
}
fn capability_grant_disclosures(capabilities: &NodeCapabilities) -> Vec<CapabilityGrantDisclosure> {
let mut disclosures = Vec::new();
let mut push = |capability: Capability, grant: &str, description: &str, risk: &str| {
if capabilities.capabilities.contains(&capability) {
disclosures.push(CapabilityGrantDisclosure {
capability,
grant: grant.to_owned(),
description: description.to_owned(),
risk: risk.to_owned(),
coordinator_policy_limited: true,
});
}
};
push(
Capability::Command,
"native_command_execution",
"placed tasks may run native commands on this node",
"local process execution under the node account",
);
push(
Capability::WindowsCommandDev,
"native_command_execution",
"placed tasks may run Windows developer commands on this node",
"local process execution under the node account",
);
push(
Capability::SourceFilesystem,
"source_access",
"placed tasks may read the local project/source checkout exposed by this node",
"broad local source visibility",
);
push(
Capability::SourceGit,
"source_access",
"placed tasks may use Git-backed source access exposed by this node",
"source-provider visibility",
);
push(
Capability::Network,
"network_access",
"placed tasks may use outbound network access from this node",
"network egress from the node environment",
);
push(
Capability::HostFilesystem,
"host_filesystem_access",
"placed tasks may access configured host filesystem mounts",
"host file visibility outside the project checkout",
);
push(
Capability::Secrets,
"secret_access",
"placed tasks may receive configured secret material",
"secret exposure to authorized task code",
);
push(
Capability::InboundPorts,
"inbound_ports",
"placed tasks may expose inbound ports from this node",
"network service exposure from the node environment",
);
push(
Capability::ArbitrarySyscalls,
"arbitrary_syscalls",
"placed tasks may use broader host syscall surface",
"reduced host isolation",
);
disclosures.sort_by(|left, right| {
left.grant
.cmp(&right.grant)
.then_with(|| left.capability.cmp(&right.capability))
});
disclosures
}
pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport> {
let coordinator = args
.coordinator
.clone()
.context("node attach execution requires --coordinator")?;
let tenant = args.tenant.clone();
let project = args.project.clone();
let node = args.node.clone().unwrap_or_else(default_node_id);
let node_private_key = node_private_key_for_attach(&node)?;
let derived_public_key =
node_ed25519_public_key_from_private_key(&node_private_key).map_err(anyhow::Error::msg)?;
let public_key = args
.public_key
.clone()
.unwrap_or(derived_public_key.clone());
if public_key != derived_public_key {
anyhow::bail!(
"node attach --public-key must match CLUSTERFLUX_NODE_PRIVATE_KEY or the stored local node credential"
);
}
let mut plan = attach_plan(args);
if let Some(enrollment) = &mut plan.enrollment {
enrollment.public_key_fingerprint = Digest::sha256(&public_key);
}
let mut session = JsonLineSession::connect(&coordinator)?;
let used_enrollment_exchange = plan.enrollment.is_some();
let coordinator_response = if let Some(enrollment) = &plan.enrollment {
session.request(json!({
"type": "exchange_node_enrollment_grant",
"tenant": &tenant,
"project": &project,
"node": &node,
"public_key": &public_key,
"enrollment_grant": enrollment.grant,
}))?
} else {
session.request(json!({
"type": "attach_node",
"tenant": &tenant,
"project": &project,
"node": &node,
"public_key": &public_key,
}))?
};
let heartbeat_request = json!({
"type": "node_heartbeat",
"node": &plan.node,
});
let heartbeat_signature = sign_node_request(
&node_private_key,
&NodeId::from(plan.node.as_str()),
"node_heartbeat",
&signed_request_payload_digest(&heartbeat_request),
command_nonce("node-heartbeat"),
unix_timestamp_seconds(),
)
.map_err(anyhow::Error::msg)?;
let mut heartbeat_request = heartbeat_request;
heartbeat_request["node_signature"] = json!(heartbeat_signature);
let heartbeat_response = session.request(heartbeat_request)?;
let capability_response = session.request(signed_node_request_json(
&node_private_key,
&plan.node,
"report_node_capabilities",
json!({
"type": "report_node_capabilities",
"tenant": &tenant,
"project": &project,
"node": &plan.node,
"capabilities": &plan.capabilities,
"cached_environment_digests": [],
"dependency_cache_digests": [],
"source_snapshots": [],
"artifact_locations": [],
"direct_connectivity": true,
"online": false,
}),
)?)?;
Ok(NodeAttachReport {
command: "node attach".to_owned(),
node: plan.node.clone(),
grant_disclosures: plan.grant_disclosures.clone(),
plan,
boundary: NodeAttachBoundaryEvidence {
cli_contacted_coordinator: true,
coordinator_address: coordinator,
used_enrollment_exchange,
coordinator_session_requests: session.requests(),
},
coordinator_response,
heartbeat_response,
capability_response,
})
}
fn node_private_key_for_attach(node: &str) -> Result<String> {
if let Ok(private_key) = std::env::var("CLUSTERFLUX_NODE_PRIVATE_KEY") {
return Ok(private_key);
}
load_or_create_local_node_credential(&std::env::current_dir()?, node)
}
pub(crate) fn load_or_create_local_node_credential(project: &Path, node: &str) -> Result<String> {
let file = local_node_credential_file(project, node);
if credential_file_exists_without_symlink(&file)? {
let bytes =
std::fs::read(&file).with_context(|| format!("failed to read {}", file.display()))?;
let credential: StoredNodeCredential = serde_json::from_slice(&bytes)
.with_context(|| format!("failed to parse {}", file.display()))?;
if credential.node != node {
anyhow::bail!(
"stored node credential {} belongs to node `{}` instead of `{}`",
file.display(),
credential.node,
node
);
}
let public_key = node_ed25519_public_key_from_private_key(&credential.private_key)
.map_err(anyhow::Error::msg)?;
if public_key != credential.public_key {
anyhow::bail!(
"stored node credential {} has a public key that does not match its private key",
file.display()
);
}
return Ok(credential.private_key);
}
let private_key = generate_ed25519_private_key().map_err(anyhow::Error::msg)?;
let public_key =
node_ed25519_public_key_from_private_key(&private_key).map_err(anyhow::Error::msg)?;
let credential = StoredNodeCredential {
kind: "clusterflux_node_credential".to_owned(),
node: node.to_owned(),
private_key: private_key.clone(),
public_key,
credential_scope: "local_project_node_identity".to_owned(),
};
persist_node_credential(&file, &credential)?;
Ok(private_key)
}
fn credential_file_exists_without_symlink(file: &Path) -> Result<bool> {
match std::fs::symlink_metadata(file) {
Ok(metadata) if metadata.file_type().is_symlink() => anyhow::bail!(
"refusing to read node credential through symbolic link {}",
file.display()
),
Ok(metadata) if !metadata.is_file() => anyhow::bail!(
"node credential path {} is not a regular file",
file.display()
),
Ok(_) => Ok(true),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(error).with_context(|| format!("failed to inspect {}", file.display())),
}
}
fn persist_node_credential(file: &Path, credential: &StoredNodeCredential) -> Result<()> {
use std::io::Write;
let parent = file
.parent()
.with_context(|| format!("node credential path {} has no parent", file.display()))?;
std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?;
if std::fs::symlink_metadata(parent)?.file_type().is_symlink() {
anyhow::bail!(
"refusing to store node credential through symbolic-link directory {}",
parent.display()
);
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))
.with_context(|| format!("failed to secure {}", parent.display()))?;
}
let mut temporary = tempfile::NamedTempFile::new_in(parent).with_context(|| {
format!(
"failed to create temporary credential in {}",
parent.display()
)
})?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
temporary
.as_file()
.set_permissions(std::fs::Permissions::from_mode(0o600))?;
}
temporary.write_all(&serde_json::to_vec_pretty(credential)?)?;
temporary.as_file().sync_all()?;
temporary.persist_noclobber(file).map_err(|error| {
anyhow::anyhow!(
"refusing to overwrite node credential {}: {}",
file.display(),
error.error
)
})?;
Ok(())
}
pub(crate) fn local_node_credential_file(project: &Path, node: &str) -> PathBuf {
let digest = Digest::sha256(node);
let file_stem = digest.as_str().trim_start_matches("sha256:");
project
.join(".clusterflux")
.join("nodes")
.join(format!("{file_stem}.json"))
}
fn default_node_public_key_for_plan(node: &str) -> String {
let private_key = generate_ed25519_private_key()
.unwrap_or_else(|_| format!("unavailable-random-node-plan-key:{node}"));
node_ed25519_public_key_from_private_key(&private_key)
.unwrap_or_else(|_| format!("{node}-public-key"))
}
fn signed_node_request_json(
node_private_key: &str,
node: &str,
request_kind: &str,
request: Value,
) -> Result<Value> {
let payload_digest = signed_request_payload_digest(&request);
let node_signature = sign_node_request(
node_private_key,
&NodeId::from(node),
request_kind,
&payload_digest,
command_nonce(request_kind),
unix_timestamp_seconds(),
)
.map_err(anyhow::Error::msg)?;
Ok(json!({
"type": "signed_node",
"node": node,
"node_signature": node_signature,
"request": request,
}))
}
pub(crate) fn default_node_id() -> String {
std::env::var("CLUSTERFLUX_NODE_ID")
.or_else(|_| std::env::var("HOSTNAME"))
.or_else(|_| std::env::var("COMPUTERNAME"))
.unwrap_or_else(|_| "node-local".to_owned())
}
fn parse_capability(cap: &str) -> Option<Capability> {
match cap {
"command" => Some(Capability::Command),
"containers" => Some(Capability::Containers),
"rootless-podman" => Some(Capability::RootlessPodman),
"source-filesystem" => Some(Capability::SourceFilesystem),
"source-git" => Some(Capability::SourceGit),
"host-filesystem" => Some(Capability::HostFilesystem),
"network" => Some(Capability::Network),
"secrets" => Some(Capability::Secrets),
"inbound-ports" => Some(Capability::InboundPorts),
"arbitrary-syscalls" => Some(Capability::ArbitrarySyscalls),
"vfs-artifacts" => Some(Capability::VfsArtifacts),
"windows-command-dev" => Some(Capability::WindowsCommandDev),
"quic-direct" => Some(Capability::QuicDirect),
_ => None,
}
}

638
crates/clusterflux-cli/src/output.rs Normal file
View file

@ -0,0 +1,638 @@
use anyhow::Result;
use serde::Serialize;
use serde_json::{json, Value};
pub(crate) fn emit_report<T: Serialize>(report: &T, json_output: bool) -> Result<()> {
let mut value = serde_json::to_value(report)?;
let exit_code = apply_command_report_exit_code(&mut value);
if json_output {
println!("{}", serde_json::to_string_pretty(&value)?);
} else {
println!("{}", human_report(&value));
}
if let Some(exit_code) = exit_code {
std::process::exit(exit_code);
}
Ok(())
}
pub(crate) fn human_report(value: &Value) -> String {
let mut lines = Vec::new();
let title = value
.get("command")
.and_then(Value::as_str)
.map(|command| format!("Clusterflux {command}"))
.or_else(|| {
if value.get("human_flow").is_some() {
Some("Clusterflux login".to_owned())
} else if value.get("metadata").is_some()
&& value.get("source_provider_manifest").is_some()
{
Some("Clusterflux bundle inspect".to_owned())
} else if value.get("entry").is_some() && value.get("session").is_some() {
Some("Clusterflux run".to_owned())
} else if value.get("capabilities").is_some() && value.get("node").is_some() {
Some("Clusterflux node attach".to_owned())
} else if value.get("public_key_fingerprint").is_some() {
Some("Clusterflux agent enroll".to_owned())
} else {
None
}
})
.unwrap_or_else(|| "Clusterflux report".to_owned());
lines.push(title);
push_string_field(&mut lines, value, "status", "status");
push_string_field(&mut lines, value, "coordinator", "coordinator");
push_string_field(&mut lines, value, "active_coordinator", "coordinator");
push_string_field(&mut lines, value, "tenant", "tenant");
push_string_field(&mut lines, value, "project", "project");
push_string_field(&mut lines, value, "process", "process");
push_string_field(&mut lines, value, "active_process", "active process");
push_string_field(&mut lines, value, "node", "node");
push_string_field(&mut lines, value, "artifact", "artifact");
push_string_field(&mut lines, value, "entry", "entry");
push_string_field(&mut lines, value, "quota_tier", "quota tier");
push_string_field(&mut lines, value, "config_file", "config");
push_string_field(&mut lines, value, "adapter", "adapter");
if let Some(project) = value.get("project").and_then(Value::as_str) {
if !lines
.iter()
.any(|line| line == &format!("project: {project}"))
{
lines.push(format!("project: {project}"));
}
}
if let Some(project_config) = value.get("project_config").or_else(|| value.get("project")) {
if project_config.is_object() {
push_nested_string_field(&mut lines, project_config, "tenant", "tenant");
push_nested_string_field(&mut lines, project_config, "project", "project");
push_nested_string_field(&mut lines, project_config, "coordinator", "coordinator");
}
}
if let Some(link) = value.get("current_directory_link") {
if link
.get("links_current_directory")
.and_then(Value::as_bool)
.unwrap_or(false)
{
lines.push("current directory linked: true".to_owned());
}
push_nested_string_field(&mut lines, link, "config_file", "current directory config");
}
if let Some(metadata) = value.get("metadata") {
push_nested_string_field(&mut lines, metadata, "identity", "bundle");
if let Some(environments) = metadata.get("environments").and_then(Value::as_array) {
let names = environments
.iter()
.filter_map(|env| env.get("name").and_then(Value::as_str))
.collect::<Vec<_>>();
if !names.is_empty() {
lines.push(format!("environments: {}", names.join(", ")));
}
}
}
if let Some(bundle) = value.get("bundle") {
if let Some(metadata) = bundle.get("metadata") {
push_nested_string_field(&mut lines, metadata, "identity", "bundle");
}
}
if let Some(environments) = value
.get("discovered_environments")
.and_then(Value::as_array)
{
let names = environments
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
if !names.is_empty() {
lines.push(format!("environments: {}", names.join(", ")));
}
}
if let Some(attached_nodes) = value.get("attached_nodes") {
if let Some(count) = attached_nodes.get("count").and_then(Value::as_u64) {
let online = attached_nodes
.get("online")
.and_then(Value::as_u64)
.unwrap_or(0);
lines.push(format!("attached nodes: {count} ({online} online)"));
}
}
if let Some(current_usage) = value.pointer("/quota_posture/current_usage") {
lines.push(format!("quota usage: {}", compact_json(current_usage)));
}
if let Some(current_task_count) = value.get("current_task_count").and_then(Value::as_u64) {
lines.push(format!("tasks: {current_task_count}"));
}
if let Some(tasks) = value.get("current_tasks").and_then(Value::as_array) {
push_task_placement_reasons(&mut lines, tasks);
push_task_locality_failures(&mut lines, tasks);
}
if let Some(tasks) = value.get("tasks").and_then(Value::as_array) {
lines.push(format!("tasks: {}", tasks.len()));
push_task_placement_reasons(&mut lines, tasks);
push_task_locality_failures(&mut lines, tasks);
}
if let Some(log_entries) = value.get("log_entries").and_then(Value::as_array) {
lines.push(format!("log entries: {}", log_entries.len()));
}
if let Some(artifacts) = value.get("artifacts").and_then(Value::as_array) {
lines.push(format!("artifacts: {}", artifacts.len()));
}
if let Some(statuses) = value
.get("source_provider_statuses")
.and_then(Value::as_array)
{
push_source_provider_statuses(&mut lines, statuses);
}
if let Some(bundle_statuses) = value
.pointer("/bundle/source_provider_statuses")
.and_then(Value::as_array)
{
push_source_provider_statuses(&mut lines, bundle_statuses);
}
if let Some(diagnostics) = value
.get("pre_schedule_diagnostics")
.or_else(|| value.get("diagnostics"))
.and_then(Value::as_array)
{
push_cli_diagnostics(&mut lines, diagnostics);
}
if let Some(bundle_diagnostics) = value
.pointer("/bundle/pre_schedule_diagnostics")
.and_then(Value::as_array)
{
push_cli_diagnostics(&mut lines, bundle_diagnostics);
}
if let Some(current_usage) = value.get("current_usage") {
lines.push(format!("quota usage: {}", compact_json(current_usage)));
}
if let Some(next_blocked_action) = value.get("next_blocked_action") {
if !next_blocked_action.is_null() {
lines.push(format!(
"next blocked action: {}",
compact_json(next_blocked_action)
));
}
}
push_machine_error_line(&mut lines, value.get("machine_error"));
push_machine_error_line(&mut lines, value.pointer("/run_start/machine_error"));
push_machine_error_line(&mut lines, value.pointer("/download_session/machine_error"));
push_machine_error_line(&mut lines, value.pointer("/export_plan/machine_error"));
push_machine_error_line(&mut lines, value.pointer("/local_export/machine_error"));
push_machine_error_line(
&mut lines,
value.pointer("/local_export/download_session/machine_error"),
);
push_machine_error_line(
&mut lines,
value.pointer("/local_export/stream/machine_error"),
);
if let Some(flow) = value.get("human_flow") {
if let Some(browser) = flow.get("Browser") {
lines.push("flow: browser".to_owned());
push_nested_string_field(&mut lines, browser, "authorization_url", "open");
}
}
if let Some(session) = value.get("session") {
lines.push(format!("session: {}", compact_json(session)));
}
if let Some(account) = value.get("coordinator_account_status") {
if let Some(checked) = account.get("checked").and_then(Value::as_bool) {
lines.push(format!("account status checked: {checked}"));
}
push_nested_string_field(&mut lines, account, "account_status", "account status");
if let Some(suspended) = account.get("suspended").and_then(Value::as_bool) {
lines.push(format!("account suspended: {suspended}"));
}
if let Some(disabled) = account.get("disabled").and_then(Value::as_bool) {
lines.push(format!("account disabled: {disabled}"));
}
if let Some(deleted) = account.get("deleted").and_then(Value::as_bool) {
lines.push(format!("account deleted: {deleted}"));
}
if let Some(manual_review) = account.get("manual_review").and_then(Value::as_bool) {
lines.push(format!("account manual review: {manual_review}"));
}
push_nested_string_field(&mut lines, account, "sanitized_reason", "account reason");
if let Some(exposed) = account
.get("private_moderation_details_exposed")
.and_then(Value::as_bool)
{
lines.push(format!("private moderation details exposed: {exposed}"));
}
}
if let Some(coordinator_selection) = value.get("coordinator") {
if coordinator_selection.is_object() {
lines.push(format!(
"coordinator: {}",
compact_json(coordinator_selection)
));
}
}
if let Some(reachability) = value.get("coordinator_reachability") {
if let Some(status) = reachability.get("status").and_then(Value::as_str) {
lines.push(format!("coordinator reachability: {status}"));
}
if let Some(error) = reachability.get("error").and_then(Value::as_str) {
lines.push(format!("coordinator error: {error}"));
}
if let Some(response_type) = reachability
.pointer("/response/type")
.and_then(Value::as_str)
{
lines.push(format!("coordinator ping: {response_type}"));
}
}
if let Some(response) = value
.get("response")
.or_else(|| value.get("coordinator_response"))
{
if let Some(response_type) = response.get("type").and_then(Value::as_str) {
lines.push(format!("coordinator response: {response_type}"));
}
}
if let Some(events) = value
.pointer("/events/response/events")
.and_then(Value::as_array)
{
lines.push(format!("events: {}", events.len()));
}
if let Some(events) = value
.pointer("/coordinator_response/response/events")
.and_then(Value::as_array)
{
lines.push(format!("events: {}", events.len()));
}
if let Some(requires_confirmation) = value.get("requires_confirmation").and_then(Value::as_bool)
{
lines.push(format!(
"confirmation: {}",
if requires_confirmation {
"required"
} else {
"confirmed"
}
));
}
if let Some(flag) = value
.get("private_website_required")
.and_then(Value::as_bool)
{
lines.push(format!("private website required: {flag}"));
}
if let Some(next_actions) = value.get("next_actions").and_then(Value::as_array) {
let actions = next_actions
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
if !actions.is_empty() {
lines.push(format!("next: {}", actions.join("; ")));
}
}
if let Some(auth) = value.get("auth").or_else(|| value.get("session")) {
if let Some(kind) = auth.get("kind").and_then(Value::as_str) {
lines.push(format!("auth: {kind}"));
}
if let Some(authenticated) = auth.get("authenticated").and_then(Value::as_bool) {
lines.push(format!("authenticated: {authenticated}"));
}
if let Some(expires) = auth.get("expires_at").and_then(Value::as_str) {
lines.push(format!("token expiry: {expires}"));
} else if let Some(posture) = auth.get("token_expiry_posture").and_then(Value::as_str) {
lines.push(format!("token expiry: {posture}"));
} else if auth.get("authenticated").is_some() {
lines.push("token expiry: unavailable".to_owned());
}
}
if let Some(dependencies) = value.get("dependencies").and_then(Value::as_object) {
let mut entries = dependencies
.iter()
.map(|(name, available)| {
format!(
"{name}={}",
if available.as_bool().unwrap_or(false) {
"ok"
} else {
"missing"
}
)
})
.collect::<Vec<_>>();
entries.sort();
if !entries.is_empty() {
lines.push(format!("dependencies: {}", entries.join(", ")));
}
}
if let Some(readiness) = value.get("node_readiness") {
push_nested_string_field(&mut lines, readiness, "os", "node os");
push_nested_string_field(&mut lines, readiness, "arch", "node arch");
if let Some(capabilities) = readiness.get("capabilities").and_then(Value::as_array) {
let caps = capabilities
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
if !caps.is_empty() {
lines.push(format!("node capabilities: {}", caps.join(", ")));
}
}
}
if let Some(summary) = value.get("node_readiness_summary") {
push_nested_string_field(&mut lines, summary, "status", "node readiness");
if let Some(missing) = summary
.get("missing_local_dependencies")
.and_then(Value::as_array)
{
let missing = missing.iter().filter_map(Value::as_str).collect::<Vec<_>>();
if !missing.is_empty() {
lines.push(format!("node missing dependencies: {}", missing.join(", ")));
}
}
if let Some(actions) = summary.get("next_actions").and_then(Value::as_array) {
let actions = actions.iter().filter_map(Value::as_str).collect::<Vec<_>>();
if !actions.is_empty() {
lines.push(format!("node next: {}", actions.join("; ")));
}
}
}
if let Some(detection) = value
.pointer("/plan/detection")
.or_else(|| value.get("detection"))
{
push_node_attach_detection(&mut lines, detection);
}
if let Some(disclosures) = value
.get("grant_disclosures")
.and_then(Value::as_array)
.filter(|disclosures| !disclosures.is_empty())
{
for disclosure in disclosures {
let grant = disclosure
.get("grant")
.and_then(Value::as_str)
.unwrap_or("capability");
let description = disclosure
.get("description")
.and_then(Value::as_str)
.unwrap_or("capability grant");
let policy = if disclosure
.get("coordinator_policy_limited")
.and_then(Value::as_bool)
.unwrap_or(false)
{
"policy-limited"
} else {
"unbounded"
};
lines.push(format!("grant {grant}: {description} ({policy})"));
}
}
lines.dedup();
lines.join("\n")
}
fn push_node_attach_detection(lines: &mut Vec<String>, detection: &Value) {
push_nested_string_field(lines, detection, "os", "node os");
push_nested_string_field(lines, detection, "arch", "node arch");
push_nested_string_field(lines, detection, "command_backend", "command backend");
if let Some(backend) = detection.get("container_backend").and_then(Value::as_str) {
let available = detection
.get("container_backend_available")
.and_then(Value::as_bool)
.unwrap_or(false);
lines.push(format!(
"container backend: {backend} ({})",
if available { "available" } else { "reported" }
));
} else if detection
.get("container_backend_reported")
.and_then(Value::as_bool)
.unwrap_or(false)
{
lines.push("container backend: reported".to_owned());
}
if let Some(providers) = detection
.get("source_provider_backends")
.and_then(Value::as_array)
{
let entries = providers
.iter()
.filter_map(|provider| {
let name = provider.get("provider").and_then(Value::as_str)?;
let state = if provider
.get("available")
.and_then(Value::as_bool)
.unwrap_or(false)
{
"available"
} else {
"detected"
};
Some(format!("{name}={state}"))
})
.collect::<Vec<_>>();
if !entries.is_empty() {
lines.push(format!("source providers: {}", entries.join(", ")));
}
}
if let Some(overrides) = detection
.get("manual_capability_overrides")
.and_then(Value::as_array)
{
let overrides = overrides
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
if !overrides.is_empty() {
lines.push(format!("capability overrides: {}", overrides.join(", ")));
}
}
}
fn push_string_field(lines: &mut Vec<String>, value: &Value, key: &str, label: &str) {
if let Some(text) = value.get(key).and_then(Value::as_str) {
lines.push(format!("{label}: {text}"));
} else if let Some(path) = value.get(key).and_then(|value| {
value
.as_object()
.and_then(|object| object.get("display"))
.and_then(Value::as_str)
}) {
lines.push(format!("{label}: {path}"));
}
}
fn push_task_placement_reasons(lines: &mut Vec<String>, tasks: &[Value]) {
for task in tasks {
let Some(placement) = task.get("node_placement") else {
continue;
};
let Some(reasons) = placement.get("reasons").and_then(Value::as_array) else {
continue;
};
let reasons = reasons.iter().filter_map(Value::as_str).collect::<Vec<_>>();
if reasons.is_empty() {
continue;
}
let task_name = task
.get("task")
.and_then(Value::as_str)
.unwrap_or("unknown");
let node = placement
.get("node")
.and_then(Value::as_str)
.unwrap_or("unknown");
lines.push(format!(
"placement {task_name}: {node} ({})",
reasons.join(", ")
));
}
}
fn push_task_locality_failures(lines: &mut Vec<String>, tasks: &[Value]) {
for task in tasks {
let Some(locality) = task.get("locality_failure") else {
continue;
};
if locality.is_null() {
continue;
}
let task_name = task
.get("task")
.and_then(Value::as_str)
.unwrap_or("unknown");
let affected = locality
.get("affected_data")
.and_then(Value::as_str)
.unwrap_or("direct_transfer");
let reason = locality
.get("reason")
.and_then(Value::as_str)
.unwrap_or("direct transfer or locality failed");
lines.push(format!("locality {task_name}: {affected} ({reason})"));
let actions = locality
.get("safe_next_actions")
.and_then(Value::as_array)
.map(|actions| actions.iter().filter_map(Value::as_str).collect::<Vec<_>>())
.unwrap_or_default();
if !actions.is_empty() {
lines.push(format!("locality next {task_name}: {}", actions.join("; ")));
}
}
}
fn push_nested_string_field(lines: &mut Vec<String>, value: &Value, key: &str, label: &str) {
if let Some(text) = value.get(key).and_then(Value::as_str) {
lines.push(format!("{label}: {text}"));
}
}
fn push_machine_error_line(lines: &mut Vec<String>, machine_error: Option<&Value>) {
let Some(machine_error) = machine_error else {
return;
};
if machine_error.is_null() {
return;
}
let category = machine_error
.get("category")
.and_then(Value::as_str)
.unwrap_or("unknown");
let exit_code = machine_error
.get("stable_exit_code")
.and_then(Value::as_i64)
.unwrap_or(1);
lines.push(format!("error category: {category} (exit {exit_code})"));
if let Some(tier) = machine_error
.get("community_tier_label")
.and_then(Value::as_str)
{
lines.push(format!("quota tier: {tier}"));
}
if let Some(next_actions) = machine_error.get("next_actions").and_then(Value::as_array) {
let actions = next_actions
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>();
if !actions.is_empty() {
lines.push(format!("error next: {}", actions.join("; ")));
}
}
}
fn push_source_provider_statuses(lines: &mut Vec<String>, statuses: &[Value]) {
let entries = statuses
.iter()
.filter_map(|status| {
let provider = status.get("provider").and_then(Value::as_str)?;
let state = status.get("status").and_then(Value::as_str)?;
let active = status
.get("active")
.and_then(Value::as_bool)
.unwrap_or(false);
Some(format!(
"{provider}={state}{}",
if active { "(active)" } else { "" }
))
})
.collect::<Vec<_>>();
if !entries.is_empty() {
lines.push(format!("source providers: {}", entries.join(", ")));
}
}
fn push_cli_diagnostics(lines: &mut Vec<String>, diagnostics: &[Value]) {
if diagnostics.is_empty() {
return;
}
lines.push(format!("diagnostics: {}", diagnostics.len()));
for diagnostic in diagnostics.iter().take(3) {
let severity = diagnostic
.get("severity")
.and_then(Value::as_str)
.unwrap_or("info");
let message = diagnostic
.get("message")
.and_then(Value::as_str)
.unwrap_or("diagnostic");
lines.push(format!("diagnostic {severity}: {message}"));
}
}
fn compact_json(value: &Value) -> String {
serde_json::to_string(value).unwrap_or_else(|_| "<unprintable>".to_owned())
}
pub(crate) fn apply_command_report_exit_code(value: &mut Value) -> Option<i32> {
for pointer in [
"/machine_error",
"/run_start/machine_error",
"/restart_request/machine_error",
"/cancel_request/machine_error",
"/task_restart/machine_error",
"/download_session/machine_error",
"/export_plan/machine_error",
"/local_export/machine_error",
"/local_export/download_session/machine_error",
"/local_export/stream/machine_error",
] {
let Some(machine_error) = value.pointer_mut(pointer) else {
continue;
};
let Some(exit_code) = machine_error
.get("stable_exit_code")
.and_then(Value::as_i64)
.and_then(|code| i32::try_from(code).ok())
.filter(|code| *code != 0)
else {
continue;
};
if let Some(object) = machine_error.as_object_mut() {
object.insert("process_exit_code_applied".to_owned(), json!(true));
}
return Some(exit_code);
}
None
}

339
crates/clusterflux-cli/src/process.rs Normal file
View file

@ -0,0 +1,339 @@
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
stored_session_for_coordinator, JsonLineSession,
};
use crate::config::StoredCliSession;
use crate::process_events::{
process_cancel_request_summary, process_restart_request_summary, process_state_from_tasks,
task_summaries,
};
use crate::{
confirmation_required_report, CliScopeArgs, ProcessAbortArgs, ProcessCancelArgs,
ProcessListArgs, ProcessRestartArgs, ProcessStatusArgs,
};
fn hydrate_process_scope(scope: &mut CliScopeArgs, stored_session: Option<&StoredCliSession>) {
if scope.coordinator.is_none() {
scope.coordinator = stored_session
.filter(|session| session.session_secret.is_some())
.map(|session| session.coordinator.clone());
}
if let Some(bound_session) = scope
.coordinator
.as_deref()
.and_then(|coordinator| stored_session_for_coordinator(coordinator, stored_session))
{
scope.tenant = bound_session.tenant.clone();
scope.project = bound_session.project.clone();
scope.user = bound_session.user.clone();
}
}
pub(crate) fn process_list_report_with_session(
mut args: ProcessListArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
let Some(coordinator) = &args.scope.coordinator else {
return Ok(json!({
"command": "process list",
"status": "requires_coordinator",
"processes": [],
}));
};
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({ "type": "list_processes" }),
json!({
"type": "list_processes",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
}),
)?)?;
let processes = response
.get("processes")
.cloned()
.unwrap_or_else(|| json!([]));
Ok(json!({
"command": "process list",
"status": "ok",
"coordinator": coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"user": args.scope.user,
"processes": processes,
"response": response,
"coordinator_session_requests": session.requests(),
}))
}
#[cfg(test)]
pub(crate) fn process_status_report(args: ProcessStatusArgs) -> Result<Value> {
process_status_report_with_session(args, None)
}
pub(crate) fn process_status_report_with_session(
mut args: ProcessStatusArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
let live = process_list_report_with_session(
ProcessListArgs {
scope: args.scope.clone(),
},
stored_session,
)?;
let live_process = live
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
processes.iter().find(|process| {
process.get("process").and_then(Value::as_str) == Some(args.process.as_str())
})
})
.cloned();
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,
Some(args.process.clone()),
stored_session,
)?;
let current_tasks = task_summaries(events.as_ref());
let current_task_count = current_tasks.as_array().map(Vec::len).unwrap_or(0);
let historical_state = process_state_from_tasks(events.as_ref());
let state = live_process
.as_ref()
.and_then(|process| process.get("state"))
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| {
if live.get("status").and_then(Value::as_str) == Some("ok") {
"not_active".to_owned()
} else if events.is_some() {
historical_state.to_owned()
} else {
"unknown_without_coordinator".to_owned()
}
});
Ok(json!({
"command": "process status",
"process": args.process,
"state": state,
"live_process": live_process,
"started_entrypoint": "unknown_from_task_events",
"current_task_count": current_task_count,
"current_tasks": current_tasks,
"debug_state": {
"frozen": null,
"status": "unknown_from_cli_task_events"
},
"events": events,
}))
}
#[cfg(test)]
pub(crate) fn process_restart_report(args: ProcessRestartArgs) -> Result<Value> {
process_restart_report_with_session(args, None)
}
pub(crate) fn process_restart_report_with_session(
mut args: ProcessRestartArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
if !args.yes {
return Ok(confirmation_required_report(
"process restart",
"restart_virtual_process",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"process": args.process,
}),
format!(
"clusterflux process restart --process {} --yes",
args.process
),
));
}
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "start_process",
"process": args.process,
"restart": true,
}),
json!({
"type": "start_process",
"tenant": args.scope.tenant,
"project": args.scope.project,
"process": args.process,
"restart": true,
}),
)?)?;
let restart_request = process_restart_request_summary(&response, !args.yes);
return Ok(json!({
"command": "process restart",
"coordinator": coordinator,
"process": args.process,
"requires_confirmation": !args.yes,
"restart_request": restart_request,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "process restart",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"process": args.process,
"restart_request": {
"status": "requires_coordinator",
"operation": "restart_virtual_process",
"explicit_user_action": true,
},
}))
}
pub(crate) fn process_cancel_report(args: ProcessCancelArgs) -> Result<Value> {
process_cancel_report_with_session(args, None)
}
pub(crate) fn process_cancel_report_with_session(
mut args: ProcessCancelArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
if !args.yes {
return Ok(confirmation_required_report(
"process cancel",
"cancel_virtual_process",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"process": args.process,
"node": args.node,
"task": args.task,
}),
format!(
"clusterflux process cancel --process {} --yes",
args.process
),
));
}
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "cancel_process",
"process": args.process,
}),
json!({
"type": "cancel_process",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"process": args.process,
}),
)?)?;
let cancel_request = process_cancel_request_summary(&response, !args.yes);
return Ok(json!({
"command": "process cancel",
"coordinator": coordinator,
"requires_confirmation": !args.yes,
"process": args.process,
"node": args.node,
"task": args.task,
"cancel_request": cancel_request,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "process cancel",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"process": args.process,
"node": args.node,
"task": args.task,
"cancel_request": {
"status": "requires_coordinator",
"operation": "cancel_virtual_process",
"whole_process_cancel_available": true,
"explicit_user_action": true,
},
}))
}
pub(crate) fn process_abort_report_with_session(
mut args: ProcessAbortArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
hydrate_process_scope(&mut args.scope, stored_session);
if !args.yes {
return Ok(confirmation_required_report(
"process abort",
"abort_virtual_process",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"process": args.process,
}),
format!("clusterflux process abort --process {} --yes", args.process),
));
}
let Some(coordinator) = &args.scope.coordinator else {
return Ok(json!({
"command": "process abort",
"status": "requires_coordinator",
"process": args.process,
"requires_confirmation": false,
}));
};
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "abort_process",
"process": args.process,
}),
json!({
"type": "abort_process",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"process": args.process,
}),
)?)?;
Ok(json!({
"command": "process abort",
"status": "aborted",
"coordinator": coordinator,
"process": args.process,
"requires_confirmation": false,
"abort_request": {
"accepted": response.get("type").and_then(Value::as_str) == Some("process_aborted"),
"operation": "abort_virtual_process",
"forced": true,
"cooperative": false,
"process_slot_released": true,
},
"response": response,
"coordinator_session_requests": session.requests(),
}))
}

666
crates/clusterflux-cli/src/process_events.rs Normal file
View file

@ -0,0 +1,666 @@
use std::path::Path;
use serde_json::{json, Value};
use crate::errors::{
cli_error_summary, cli_error_summary_for_category, cli_error_summary_with_default,
message_mentions_locality_failure,
};
fn task_event_values(task_events: Option<&Value>) -> Vec<&Value> {
task_events
.and_then(|task_events| task_events.pointer("/response/events"))
.and_then(Value::as_array)
.map(|events| events.iter().collect())
.unwrap_or_default()
}
fn event_string(event: &Value, field: &str) -> Option<String> {
event.get(field).and_then(Value::as_str).map(str::to_owned)
}
fn event_u64(event: &Value, field: &str) -> Option<u64> {
event.get(field).and_then(Value::as_u64)
}
pub(crate) fn task_summaries(task_events: Option<&Value>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.map(|event| {
let task = event_string(event, "task").unwrap_or_else(|| "unknown".to_owned());
let terminal_state =
event_string(event, "terminal_state").unwrap_or_else(|| "unknown".to_owned());
let node = event_string(event, "node");
let placement = event.get("placement");
let placement_reasons = placement
.and_then(|placement| placement.get("reasons"))
.cloned()
.unwrap_or_else(|| json!([]));
let placement_score = placement
.and_then(|placement| placement.get("score"))
.cloned()
.unwrap_or(Value::Null);
let failure_reason = task_failure_reason(event);
let locality_failure = task_locality_failure_from_reason(&failure_reason);
let machine_error = task_failure_machine_error_from_reason(event, &failure_reason);
json!({
"process": event_string(event, "process"),
"task": task,
"state": terminal_state,
"environment": event.get("environment").cloned().unwrap_or_else(|| json!("unknown_from_task_event")),
"environment_digest": event.get("environment_digest").cloned().unwrap_or(Value::Null),
"node_placement": {
"node": node,
"source": "coordinator_task_event",
"score": placement_score,
"reasons": placement_reasons,
"explanation_available": placement.is_some(),
},
"failure_reason": failure_reason,
"locality_failure": locality_failure,
"machine_error": machine_error,
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
})
})
.collect(),
)
}
fn task_failure_reason(event: &Value) -> Value {
match event.get("terminal_state").and_then(Value::as_str) {
Some("failed") => {
if let Some(stderr) = event.get("stderr_tail").and_then(Value::as_str) {
if !stderr.is_empty() {
return json!(redact_secret_like_text(stderr).0);
}
}
if let Some(status_code) = event.get("status_code").and_then(Value::as_i64) {
return json!(format!("task exited with status {status_code}"));
}
json!("task failed")
}
Some("cancelled") => json!("task cancelled"),
_ => Value::Null,
}
}
fn task_failure_machine_error_from_reason(event: &Value, reason: &Value) -> Value {
match event.get("terminal_state").and_then(Value::as_str) {
Some("failed") => {
let reason = reason.as_str().unwrap_or("task failed").to_owned();
let mut summary = cli_error_summary_with_default(&reason, "program");
if let Some(object) = summary.as_object_mut() {
if message_mentions_locality_failure(&reason.to_ascii_lowercase()) {
object.insert("locality_failure".to_owned(), json!(true));
object.insert(
"next_actions".to_owned(),
json!(locality_failure_next_actions(&reason)),
);
}
}
summary
}
Some("cancelled") => cli_error_summary_for_category("program", "task cancelled"),
_ => Value::Null,
}
}
fn task_locality_failure_from_reason(reason: &Value) -> Value {
let Some(reason) = reason.as_str() else {
return Value::Null;
};
let lower = reason.to_ascii_lowercase();
if !message_mentions_locality_failure(&lower) {
return Value::Null;
}
let affected_data = if lower.contains("source snapshot") {
"source_snapshot"
} else if lower.contains("artifact") {
"artifact"
} else {
"direct_transfer"
};
json!({
"category": "connectivity",
"affected_data": affected_data,
"reason": reason,
"coordinator_bulk_relay_used": false,
"safe_failure": true,
"safe_next_actions": locality_failure_next_actions(reason),
})
}
fn locality_failure_next_actions(reason: &str) -> Vec<&'static str> {
let lower = reason.to_ascii_lowercase();
if lower.contains("source snapshot") {
return vec![
"attach or select a node that already has the required source snapshot",
"rerun source preparation on an attached node",
"restore direct node-to-node connectivity and retry",
"do not rely on coordinator bulk source relay",
];
}
if lower.contains("artifact") {
return vec![
"attach or select a node that already has the required artifact",
"explicitly export or download the artifact before retrying",
"restore direct node-to-node connectivity and retry",
"do not rely on coordinator bulk artifact relay",
];
}
vec![
"check node direct connectivity and NAT traversal",
"attach a node with the needed source/artifact locality",
"retry after node connectivity is restored",
"do not rely on coordinator bulk relay",
]
}
pub(crate) fn process_state_from_tasks(task_events: Option<&Value>) -> &'static str {
let events = task_event_values(task_events);
if events.is_empty() {
return "no_tasks_observed";
}
if events.iter().any(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "failed")
}) {
return "has_failed_tasks";
}
if events.iter().any(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "cancelled")
}) {
return "has_cancelled_tasks";
}
if events.iter().all(|event| {
event
.get("terminal_state")
.and_then(Value::as_str)
.is_some_and(|state| state == "completed")
}) {
return "completed_tasks_observed";
}
"tasks_observed"
}
pub(crate) fn log_entries(task_events: Option<&Value>, task_filter: Option<&str>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.filter(|event| {
task_filter.is_none_or( |task_filter| {
event
.get("task")
.and_then(Value::as_str)
.is_some_and(|task| task == task_filter)
})
})
.map(|event| {
let stdout_tail = event
.get("stdout_tail")
.and_then(Value::as_str)
.unwrap_or("");
let stderr_tail = event
.get("stderr_tail")
.and_then(Value::as_str)
.unwrap_or("");
let (stdout_tail, stdout_tail_redacted) = redact_secret_like_text(stdout_tail);
let (stderr_tail, stderr_tail_redacted) = redact_secret_like_text(stderr_tail);
json!({
"process": event_string(event, "process"),
"task": event_string(event, "task"),
"node": event_string(event, "node"),
"stdout_bytes": event_u64(event, "stdout_bytes").unwrap_or(0),
"stderr_bytes": event_u64(event, "stderr_bytes").unwrap_or(0),
"stdout_tail": stdout_tail,
"stderr_tail": stderr_tail,
"stdout_truncated": event.get("stdout_truncated").and_then(Value::as_bool).unwrap_or(false),
"stderr_truncated": event.get("stderr_truncated").and_then(Value::as_bool).unwrap_or(false),
"capped": true,
"secret_like_values_redacted": stdout_tail_redacted || stderr_tail_redacted,
"redacted_fields": redacted_log_fields(stdout_tail_redacted, stderr_tail_redacted),
})
})
.collect(),
)
}
fn redacted_log_fields(
stdout_tail_redacted: bool,
stderr_tail_redacted: bool,
) -> Vec<&'static str> {
let mut fields = Vec::new();
if stdout_tail_redacted {
fields.push("stdout_tail");
}
if stderr_tail_redacted {
fields.push("stderr_tail");
}
fields
}
fn redact_secret_like_text(text: &str) -> (String, bool) {
let markers = [
"access_token=",
"access_token:",
"refresh_token=",
"refresh_token:",
"id_token=",
"id_token:",
"api_key=",
"api_key:",
"api-key=",
"api-key:",
"token=",
"token:",
"secret=",
"secret:",
"password=",
"password:",
"passwd=",
"passwd:",
"bearer ",
];
let mut output = text.to_owned();
let mut redacted = false;
for marker in markers {
let (updated, changed) = redact_marker_values(output, marker);
output = updated;
redacted |= changed;
}
(output, redacted)
}
fn redact_marker_values(mut text: String, marker: &str) -> (String, bool) {
let mut changed = false;
let mut search_start = 0;
loop {
let lower = text.to_ascii_lowercase();
let Some(relative) = lower[search_start..].find(marker) else {
break;
};
let value_start = search_start + relative + marker.len();
let value_end = text[value_start..]
.char_indices()
.find_map(|(offset, character)| {
(character.is_whitespace()
|| matches!(
character,
'&' | '"' | '\'' | '`' | '<' | '>' | ',' | ';' | ')' | ']'
))
.then_some(value_start + offset)
})
.unwrap_or(text.len());
if value_start == value_end {
search_start = value_end;
if search_start >= text.len() {
break;
}
continue;
}
if text[value_start..value_end].starts_with("[redacted") {
search_start = value_end;
if search_start >= text.len() {
break;
}
continue;
}
text.replace_range(value_start..value_end, "[redacted]");
changed = true;
search_start = value_start + "[redacted]".len();
if search_start >= text.len() {
break;
}
}
(text, changed)
}
pub(crate) fn artifact_summaries(task_events: Option<&Value>) -> Value {
Value::Array(
task_event_values(task_events)
.into_iter()
.filter_map(|event| {
let path = event.get("artifact_path").and_then(Value::as_str)?;
let node = event_string(event, "node");
Some(json!({
"artifact": artifact_name_from_path(path),
"path": path,
"producer_task": event_string(event, "task"),
"producer_node": node,
"process": event_string(event, "process"),
"digest": event.get("artifact_digest").cloned().unwrap_or(Value::Null),
"size_bytes": event.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
"state": if event.get("artifact_digest").is_some() { "metadata_flushed" } else { "metadata_without_digest" },
"known_locations": node.into_iter().collect::<Vec<_>>(),
"durable_storage": false,
}))
})
.collect(),
)
}
fn artifact_name_from_path(path: &str) -> String {
path.rsplit('/').next().unwrap_or(path).to_owned()
}
fn response_error_message(response: &Value, fallback: &str) -> String {
response
.get("message")
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| fallback.to_owned())
}
pub(crate) fn artifact_response_machine_error(
response: &Value,
fallback: &str,
default_category: &'static str,
) -> Value {
let message = response_error_message(response, fallback);
cli_error_summary_with_default(&message, default_category)
}
pub(crate) fn process_restart_request_summary(
response: &Value,
requires_confirmation: bool,
) -> Value {
if response.get("type").and_then(Value::as_str) != Some("process_started") {
let message = response_error_message(response, "coordinator rejected process restart");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "restart_virtual_process",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
json!({
"status": "process_started",
"operation": "restart_virtual_process",
"accepted": true,
"process": response.get("process").cloned().unwrap_or(Value::Null),
"coordinator_epoch": response.get("epoch").cloned().unwrap_or(Value::Null),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"website_required": false,
"single_active_process_boundary": true,
})
}
pub(crate) fn process_cancel_request_summary(
response: &Value,
requires_confirmation: bool,
) -> Value {
if response.get("type").and_then(Value::as_str) != Some("process_cancellation_requested") {
let message = response_error_message(response, "coordinator rejected process cancel");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "cancel_virtual_process",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"whole_process_cancel_available": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
let cancelled_tasks = response
.get("cancelled_tasks")
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0);
json!({
"status": "process_cancellation_requested",
"operation": "cancel_virtual_process",
"accepted": true,
"process": response.get("process").cloned().unwrap_or(Value::Null),
"cancelled_task_count": cancelled_tasks,
"cancelled_tasks": response.get("cancelled_tasks").cloned().unwrap_or_else(|| json!([])),
"affected_nodes": response.get("affected_nodes").cloned().unwrap_or_else(|| json!([])),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"website_required": false,
"whole_process_cancel_available": true,
"node_must_poll_task_control": true,
"new_task_launches_blocked": true,
"surviving_state_visibility": "task and artifact state remains visible after terminal task events are reported",
})
}
pub(crate) fn task_restart_request_summary(response: &Value, requires_confirmation: bool) -> Value {
if response.get("type").and_then(Value::as_str) != Some("task_restart") {
let message = response_error_message(response, "coordinator rejected task restart");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"operation": "restart_selected_task",
"accepted": false,
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"clean_boundary_required": true,
"error": message,
"machine_error": cli_error_summary(&message),
});
}
json!({
"status": "task_restart",
"operation": "restart_selected_task",
"accepted": response.get("accepted").cloned().unwrap_or(json!(false)),
"process": response.get("process").cloned().unwrap_or(Value::Null),
"task": response.get("task").cloned().unwrap_or(Value::Null),
"requires_confirmation": requires_confirmation,
"explicit_user_action": true,
"clean_boundary_required": true,
"clean_boundary_available": response
.get("clean_boundary_available")
.cloned()
.unwrap_or(json!(false)),
"active_task": response
.get("active_task")
.cloned()
.unwrap_or(json!(false)),
"completed_event_observed": response
.get("completed_event_observed")
.cloned()
.unwrap_or(json!(false)),
"requires_whole_process_restart": response
.get("requires_whole_process_restart")
.cloned()
.unwrap_or(json!(true)),
"message": response.get("message").cloned().unwrap_or(Value::Null),
"audit_event": response.get("audit_event").cloned().unwrap_or(Value::Null),
"charged_debug_read_bytes": response
.get("charged_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"used_debug_read_bytes": response
.get("used_debug_read_bytes")
.cloned()
.unwrap_or_else(|| json!(0)),
"debug_reads_quota_limited": true,
"website_required": false,
})
}
pub(crate) fn artifact_download_session_summary(response: &Value) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
let message = response_error_message(response, "coordinator rejected artifact download");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"link_issued": false,
"explicit_user_action_required": true,
"error": message.clone(),
"machine_error": cli_error_summary_with_default(&message, "connectivity"),
});
}
let link = response.get("link").unwrap_or(&Value::Null);
json!({
"status": "download_link_issued",
"link_issued": true,
"explicit_user_action_required": true,
"coordinator_preflight": "completed_before_link_issued",
"tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
"project": link.get("project").cloned().unwrap_or(Value::Null),
"process": link.get("process").cloned().unwrap_or(Value::Null),
"artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
"actor": link.get("actor").cloned().unwrap_or(Value::Null),
"source": link.get("source").cloned().unwrap_or(Value::Null),
"url_path": link.get("url_path").cloned().unwrap_or(Value::Null),
"expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
"max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
"token_material_returned": false,
"scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
"policy_context_digest_present": link.get("policy_context_digest").is_some(),
"authorization_required": true,
"short_lived": link.get("expires_at_epoch_seconds").is_some(),
"guessable_public_url": false,
"cross_tenant_usable": false,
"unauthorized_project_usable": false,
"default_durable_store_assumed": false,
})
}
pub(crate) fn artifact_download_grant_disclosures(response: &Value) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_download_link") {
return json!([]);
}
let link = response.get("link").unwrap_or(&Value::Null);
json!([{
"grant": "artifact_download",
"description": "download scoped artifact bytes to the requesting machine",
"risk": "authorized artifact bytes leave the retaining node or explicit storage through an explicit download/export operation",
"coordinator_policy_limited": true,
"authorization_required": true,
"explicit_user_action_required": true,
"tenant": link.get("tenant").cloned().unwrap_or(Value::Null),
"project": link.get("project").cloned().unwrap_or(Value::Null),
"process": link.get("process").cloned().unwrap_or(Value::Null),
"artifact": link.get("artifact").cloned().unwrap_or(Value::Null),
"actor": link.get("actor").cloned().unwrap_or(Value::Null),
"source": link.get("source").cloned().unwrap_or(Value::Null),
"max_bytes": link.get("max_bytes").cloned().unwrap_or(Value::Null),
"expires_at_epoch_seconds": link.get("expires_at_epoch_seconds").cloned().unwrap_or(Value::Null),
"short_lived": link.get("expires_at_epoch_seconds").is_some(),
"scoped_token_digest_present": link.get("scoped_token_digest").is_some(),
"token_material_returned": false,
"guessable_public_url": false,
"cross_tenant_reuse_allowed": false,
"unauthorized_project_reuse_allowed": false,
"default_durable_store_assumed": false,
"private_website_required": false,
}])
}
pub(crate) fn artifact_export_plan_summary(response: &Value, to: &Path) -> Value {
if response.get("type").and_then(Value::as_str) != Some("artifact_export_plan") {
let message = response_error_message(response, "coordinator rejected artifact export");
return json!({
"status": response.get("type").and_then(Value::as_str).unwrap_or("coordinator_response"),
"explicit_user_action": true,
"local_path": to,
"local_bytes_written_by_cli": false,
"default_durable_store_assumed": false,
"error": message.clone(),
"machine_error": cli_error_summary_with_default(&message, "connectivity"),
});
}
let plan = response.get("plan").unwrap_or(&Value::Null);
json!({
"status": "transfer_plan_created",
"explicit_user_action": true,
"local_path": to,
"local_bytes_written_by_cli": false,
"writes_require_data_plane_followup": true,
"default_durable_store_assumed": false,
"artifact_size_bytes": response.get("artifact_size_bytes").cloned().unwrap_or(Value::Null),
"source_node": response.get("source_node").cloned().unwrap_or(Value::Null),
"receiver_node": response.get("receiver_node").cloned().unwrap_or(Value::Null),
"transport": plan.get("transport").cloned().unwrap_or(Value::Null),
"artifact": plan.pointer("/scope/object/Artifact").cloned().unwrap_or(Value::Null),
"tenant": plan.pointer("/scope/tenant").cloned().unwrap_or(Value::Null),
"project": plan.pointer("/scope/project").cloned().unwrap_or(Value::Null),
"process": plan.pointer("/scope/process").cloned().unwrap_or(Value::Null),
"coordinator_assisted_rendezvous": plan.get("coordinator_assisted_rendezvous").cloned().unwrap_or(Value::Null),
"coordinator_bulk_relay_allowed": plan.get("coordinator_bulk_relay_allowed").cloned().unwrap_or(Value::Null),
"authorization_digest_present": plan.get("authorization_digest").is_some(),
})
}
pub(crate) fn task_event_count(task_events: Option<&Value>) -> usize {
task_events
.and_then(|task_events| task_events.pointer("/response/events"))
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0)
}
pub(crate) fn project_quota_posture(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
let current_usage = quota_current_usage(attached_nodes, task_events);
let next_blocked_action = quota_next_blocked_action(&current_usage);
json!({
"source": "cli_project_status_summary",
"current_usage": current_usage,
"limits": quota_limits_value(),
"next_blocked_action": next_blocked_action,
"private_abuse_heuristics_exposed": false,
})
}
pub(crate) fn quota_current_usage(attached_nodes: &Value, task_events: Option<&Value>) -> Value {
let attached_node_count = attached_nodes
.get("count")
.and_then(Value::as_u64)
.unwrap_or(0);
let online_node_count = attached_nodes
.get("online")
.and_then(Value::as_u64)
.unwrap_or(0);
json!({
"attached_nodes": attached_node_count,
"online_nodes": online_node_count,
"observed_task_events": task_event_count(task_events),
"artifact_download_bytes": 0,
"rendezvous_attempts": 0,
"hosted_wasm_processes": 0,
})
}
pub(crate) fn quota_limits_value() -> Value {
json!({
"source": "coordinator",
"configured": false,
"message": "connect to the selected coordinator to read its scoped quota configuration",
})
}
pub(crate) fn quota_next_blocked_action(current_usage: &Value) -> Value {
if current_usage
.get("online_nodes")
.and_then(Value::as_u64)
.unwrap_or(0)
== 0
{
return json!({
"action": "node_work_requires_online_attached_node",
"category": "capability",
"quota_related": false,
"message": "no online attached node is visible for work that requires a user node",
"machine_error": cli_error_summary_for_category(
"capability",
"no online attached node is visible for work that requires a user node"
)
});
}
Value::Null
}

381
crates/clusterflux-cli/src/project.rs Normal file
View file

@ -0,0 +1,381 @@
use std::path::PathBuf;
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, list_attached_nodes_if_available_with_session,
list_task_events_if_available_with_session, stored_session_for_coordinator, JsonLineSession,
};
use crate::config::{
effective_project_scope, effective_scope_value, project_config_file, read_cli_session,
read_project_config, write_project_config, ProjectConfig, StoredCliSession,
};
use crate::process::process_list_report_with_session;
use crate::process_events::project_quota_posture;
use crate::{
bundle_inspection, discovered_environment_names, BundleInspectArgs, ProcessListArgs,
ProjectInitArgs, ProjectListArgs, ProjectSelectArgs, ProjectStatusArgs,
};
pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result<Value> {
let stored_session = read_cli_session(&cwd)?;
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let project = args.new_project.clone();
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.user,
|session| session.user.as_str(),
"user",
);
let coordinator = args.scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
let name = args.name.clone();
let config = ProjectConfig {
tenant: tenant.clone(),
project: project.clone(),
user: user.clone(),
coordinator: coordinator.clone(),
};
let config_file = project_config_file(&cwd);
if config_file.exists() && !args.yes {
anyhow::bail!(
"{} already exists; rerun with --yes to update the project link",
config_file.display()
);
}
let mut coordinator_session_requests = 0;
let coordinator_response = if let Some(coordinator) = &coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "create_project",
"project": project,
"name": name,
}),
json!({
"type": "create_project",
"tenant": tenant,
"actor_user": user,
"project": project,
"name": name,
}),
)?;
let response = session.request(request)?;
coordinator_session_requests = session.requests();
Some(response)
} else {
None
};
write_project_config(&cwd, &config)?;
let created_or_linked_project = coordinator_response
.as_ref()
.and_then(|response| response.get("project"))
.cloned()
.unwrap_or_else(|| {
json!({
"id": config.project.clone(),
"tenant": config.tenant.clone(),
"name": args.name.clone(),
})
});
Ok(json!({
"command": "project init",
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
"private_website_required": false,
"project_config_written": true,
"project_config_write_after_coordinator_acceptance": coordinator.is_some(),
"coordinator_create_before_local_write": coordinator.is_some(),
"coordinator_session_requests": coordinator_session_requests,
"created_or_linked_project": created_or_linked_project,
"current_directory_link": {
"cwd": cwd,
"config_file": config_file,
"config_format": "clusterflux_project_config_v1",
"links_current_directory": true,
"writes_current_directory_only": true,
"private_website_required": false,
},
"safe_defaults": {
"tenant": config.tenant.clone(),
"project": config.project.clone(),
"user": config.user.clone(),
"coordinator": config.coordinator.clone(),
"project_name": args.name.clone(),
"default_project_id_used": args.new_project == "project",
"default_project_name_used": args.name == "Clusterflux Project",
"browser_interaction_required": false,
"private_website_required": false,
},
"project_config": config,
"config_file": project_config_file(&cwd),
"coordinator_response": coordinator_response,
}))
}
pub(crate) fn project_status_report(args: ProjectStatusArgs, cwd: PathBuf) -> Result<Value> {
let config = read_project_config(&cwd)?;
let stored_session = read_cli_session(&cwd)?;
let mut effective_scope = effective_project_scope(&args.scope, config.as_ref());
if effective_scope.coordinator.is_none() {
effective_scope.coordinator = stored_session
.as_ref()
.filter(|session| session.session_secret.is_some())
.map(|session| session.coordinator.clone());
}
if let (Some(config), Some(session)) = (config.as_ref(), stored_session.as_ref()) {
let same_coordinator = effective_scope
.coordinator
.as_deref()
.is_some_and(|coordinator| {
crate::client::control_endpoint_identity(coordinator).ok()
== crate::client::control_endpoint_identity(&session.coordinator).ok()
});
if same_coordinator
&& (session.tenant != config.tenant
|| session.project != config.project
|| session.user != config.user)
{
anyhow::bail!(
"stored CLI session is for {}/{}/{} but this workspace is configured for {}/{}/{}; run `clusterflux login --browser` from this workspace",
session.tenant,
session.project,
session.user,
config.tenant,
config.project,
config.user,
);
}
}
if let Some(bound_session) = effective_scope
.coordinator
.as_deref()
.and_then(|coordinator| {
stored_session_for_coordinator(coordinator, stored_session.as_ref())
})
{
effective_scope.tenant = bound_session.tenant.clone();
effective_scope.project = bound_session.project.clone();
effective_scope.user = bound_session.user.clone();
}
let inspection = bundle_inspection(
BundleInspectArgs {
project: Some(cwd.clone()),
source_provider: None,
disabled_source_providers: Vec::new(),
json: true,
},
cwd.clone(),
)
.ok();
let coordinator = effective_scope.coordinator.clone();
let attached_nodes = list_attached_nodes_if_available_with_session(
coordinator.as_deref(),
&effective_scope,
stored_session.as_ref(),
)?;
let coordinator_response = list_task_events_if_available_with_session(
coordinator.as_deref(),
&effective_scope,
None,
stored_session.as_ref(),
)?;
let process_report = process_list_report_with_session(
ProcessListArgs {
scope: effective_scope.clone(),
},
stored_session.as_ref(),
)?;
let discovered_environments = discovered_environment_names(inspection.as_ref());
let active_process = process_report
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| processes.first())
.and_then(|process| process.get("process"))
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| {
if process_report.get("status").and_then(Value::as_str) == Some("ok") {
"none".to_owned()
} else {
"unknown_without_coordinator".to_owned()
}
});
let quota_posture = project_quota_posture(&attached_nodes, coordinator_response.as_ref());
Ok(json!({
"command": "project status",
"cwd": cwd,
"tenant": effective_scope.tenant,
"project": effective_scope.project,
"user": effective_scope.user,
"coordinator": coordinator,
"project_identity": {
"tenant": effective_scope.tenant,
"project": effective_scope.project,
"user": effective_scope.user,
"source": if config.is_some() { "project_config_with_cli_overrides" } else { "cli_scope" }
},
"project_config": config,
"bundle": inspection,
"discovered_environments": discovered_environments,
"active_process": active_process,
"processes": process_report.get("processes").cloned().unwrap_or_else(|| json!([])),
"attached_nodes": attached_nodes,
"quota_posture": quota_posture,
"coordinator_response": coordinator_response,
}))
}
pub(crate) fn project_list_report(args: ProjectListArgs, cwd: PathBuf) -> Result<Value> {
let stored_session = read_cli_session(&cwd)?;
let coordinator = args.scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.user,
|session| session.user.as_str(),
"user",
);
if let Some(coordinator) = &coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "list_projects",
}),
json!({
"type": "list_projects",
"tenant": tenant,
"actor_user": user,
}),
)?;
let response = session.request(request)?;
let projects = response
.get("projects")
.cloned()
.unwrap_or_else(|| json!([]));
let project_count = projects.as_array().map(Vec::len).unwrap_or(0);
return Ok(json!({
"command": "project list",
"source": "public_coordinator_api",
"coordinator": coordinator,
"tenant": tenant,
"user": user,
"projects": projects,
"project_count": project_count,
"private_website_required": false,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
let projects = read_project_config(&cwd)?.into_iter().collect::<Vec<_>>();
let project_count = projects.len();
Ok(json!({
"command": "project list",
"source": "local_project_config",
"projects": projects,
"project_count": project_count,
"private_website_required": false,
}))
}
pub(crate) fn project_select_report(args: ProjectSelectArgs, cwd: PathBuf) -> Result<Value> {
let stored_session = read_cli_session(&cwd)?;
let tenant = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.tenant,
|session| session.tenant.as_str(),
"tenant",
);
let user = session_or_effective_scope_value(
stored_session.as_ref(),
&args.scope.user,
|session| session.user.as_str(),
"user",
);
let coordinator = args.scope.coordinator.clone().or_else(|| {
stored_session
.as_ref()
.map(|session| session.coordinator.clone())
});
let config = ProjectConfig {
tenant: tenant.clone(),
project: args.selected_project.clone(),
user: user.clone(),
coordinator: coordinator.clone(),
};
let coordinator_response = if let Some(coordinator) = &coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let request = authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({
"type": "select_project",
"project": args.selected_project,
}),
json!({
"type": "select_project",
"tenant": tenant,
"actor_user": user,
"project": args.selected_project,
}),
)?;
Some(session.request(request)?)
} else {
None
};
write_project_config(&cwd, &config)?;
let selected_project = coordinator_response
.as_ref()
.and_then(|response| response.get("project"))
.cloned()
.unwrap_or_else(|| {
json!({
"id": config.project.clone(),
"tenant": config.tenant.clone(),
"name": config.project.clone(),
})
});
Ok(json!({
"command": "project select",
"source": if coordinator.is_some() { "public_coordinator_api" } else { "local_project_config" },
"selected_project": selected_project,
"project_config_written": true,
"private_website_required": false,
"project_config": config,
"coordinator_response": coordinator_response,
}))
}
fn session_or_effective_scope_value(
stored_session: Option<&StoredCliSession>,
cli_value: &str,
session_value: impl FnOnce(&StoredCliSession) -> &str,
default_value: &str,
) -> String {
if let Some(session) = stored_session.filter(|session| session.session_secret.is_some()) {
session_value(session).to_owned()
} else {
effective_scope_value(cli_value, stored_session.map(session_value), default_value)
}
}

111
crates/clusterflux-cli/src/quota.rs Normal file
View file

@ -0,0 +1,111 @@
use std::path::PathBuf;
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, list_attached_nodes_if_available_with_session,
list_task_events_if_available_with_session, stored_session_for_coordinator, JsonLineSession,
};
use crate::config::{effective_project_scope, read_cli_session, read_project_config};
use crate::process_events::{quota_current_usage, quota_limits_value, quota_next_blocked_action};
use crate::QuotaStatusArgs;
pub(crate) fn quota_status_report(args: QuotaStatusArgs, cwd: PathBuf) -> Result<Value> {
let config = read_project_config(&cwd)?;
let stored_session = read_cli_session(&cwd)?;
let mut effective_scope = effective_project_scope(&args.scope, config.as_ref());
if effective_scope.coordinator.is_none() {
effective_scope.coordinator = stored_session
.as_ref()
.filter(|session| session.session_secret.is_some())
.map(|session| session.coordinator.clone());
}
if let Some(bound_session) = effective_scope
.coordinator
.as_deref()
.and_then(|coordinator| {
stored_session_for_coordinator(coordinator, stored_session.as_ref())
})
{
effective_scope.tenant = bound_session.tenant.clone();
effective_scope.project = bound_session.project.clone();
effective_scope.user = bound_session.user.clone();
}
let coordinator = effective_scope.coordinator.clone();
let attached_nodes = list_attached_nodes_if_available_with_session(
coordinator.as_deref(),
&effective_scope,
stored_session.as_ref(),
)?;
let task_events = list_task_events_if_available_with_session(
coordinator.as_deref(),
&effective_scope,
None,
stored_session.as_ref(),
)?;
let quota_status = if let Some(coordinator) = coordinator.as_deref() {
let mut session = JsonLineSession::connect(coordinator)?;
Some(session.request(authenticated_or_local_trusted_request(
coordinator,
stored_session.as_ref(),
json!({ "type": "quota_status" }),
json!({
"type": "quota_status",
"tenant": effective_scope.tenant,
"project": effective_scope.project,
"actor_user": effective_scope.user,
}),
)?)?)
} else {
None
};
let mut current_usage = quota_current_usage(&attached_nodes, task_events.as_ref());
if let (Some(object), Some(status)) = (current_usage.as_object_mut(), quota_status.as_ref()) {
object.insert(
"scoped_resource_usage".to_owned(),
status.get("usage").cloned().unwrap_or(Value::Null),
);
object.insert(
"window_started_epoch_seconds".to_owned(),
status
.get("window_started_epoch_seconds")
.cloned()
.unwrap_or(Value::Null),
);
}
let limits = quota_status
.as_ref()
.and_then(|status| status.pointer("/limits/limits"))
.cloned()
.unwrap_or_else(quota_limits_value);
let window_seconds = quota_status
.as_ref()
.and_then(|status| status.get("window_seconds"))
.cloned()
.unwrap_or(Value::Null);
let quota_tier = quota_status
.as_ref()
.and_then(|status| status.get("policy_label"))
.cloned()
.unwrap_or(Value::Null);
Ok(json!({
"command": "quota status",
"tenant": effective_scope.tenant,
"project": effective_scope.project,
"user": effective_scope.user,
"coordinator": coordinator,
"project_config": config,
"policy_surface": "generic public quota categories; hosted tuning remains private policy",
"limits": limits,
"window_seconds": window_seconds,
"current_usage": current_usage,
"attached_nodes": attached_nodes,
"task_events": task_events,
"next_blocked_action": quota_next_blocked_action(&current_usage),
"quota_configuration_source": if quota_status.is_some() { "coordinator" } else { "unavailable_offline" },
"quota_tier": quota_tier,
"private_abuse_heuristics_exposed": false,
"quota_response": quota_status,
}))
}

1066
crates/clusterflux-cli/src/run.rs Normal file
View file

File diff suppressed because it is too large Load diff

177
crates/clusterflux-cli/src/run/local_services.rs Normal file
View file

@ -0,0 +1,177 @@
use std::io::{BufRead, BufReader};
use std::path::Path;
use std::process::{Child, Command, Stdio};
use anyhow::{Context, Result};
use serde_json::Value;
pub(super) struct LocalNodeWorker {
pub(super) process_id: u32,
child: Option<Child>,
}
impl LocalNodeWorker {
pub(super) fn start(coordinator: &str, project: &Path, enrollment_grant: &str) -> Result<Self> {
let mut command = node_command()?;
command.args([
"--coordinator",
coordinator,
"--tenant",
"tenant",
"--project-id",
"project",
"--node",
"node-cli-local",
"--enrollment-grant",
enrollment_grant,
"--worker",
"--project-root",
]);
command.arg(project);
command.args(["--assignment-poll-ms", "20"]);
command.stdout(Stdio::null());
command.stderr(Stdio::inherit());
let child = command.spawn().context("failed to spawn node process")?;
let process_id = child.id();
Ok(Self {
process_id,
child: Some(child),
})
}
pub(super) fn stop(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
impl Drop for LocalNodeWorker {
fn drop(&mut self) {
self.stop();
}
}
pub(super) struct LocalCoordinator {
pub(super) address: String,
pub(super) process_id: Option<u32>,
child: Option<Child>,
}
impl LocalCoordinator {
pub(super) fn external(address: &str) -> Self {
Self {
address: address.to_owned(),
process_id: None,
child: None,
}
}
pub(super) fn start_ephemeral() -> Result<Self> {
let mut command = coordinator_command()?;
command.args(["--listen", "127.0.0.1:0", "--allow-local-trusted-loopback"]);
command.stdout(Stdio::piped());
command.stderr(Stdio::inherit());
let mut child = command
.spawn()
.context("failed to spawn local coordinator process")?;
let process_id = child.id();
let address = match read_coordinator_ready_address(&mut child) {
Ok(address) => address,
Err(error) => {
let _ = child.kill();
let _ = child.wait();
return Err(error);
}
};
Ok(Self {
address,
process_id: Some(process_id),
child: Some(child),
})
}
}
impl Drop for LocalCoordinator {
fn drop(&mut self) {
if let Some(mut child) = self.child.take() {
let _ = child.kill();
let _ = child.wait();
}
}
}
fn read_coordinator_ready_address(child: &mut Child) -> Result<String> {
let stdout = child
.stdout
.take()
.context("local coordinator stdout was not captured")?;
let mut ready_line = String::new();
BufReader::new(stdout)
.read_line(&mut ready_line)
.context("failed to read local coordinator ready line")?;
if ready_line.trim().is_empty() {
anyhow::bail!("local coordinator exited before reporting its listen address");
}
let ready: Value =
serde_json::from_str(&ready_line).context("local coordinator ready line was not JSON")?;
ready
.get("listen")
.and_then(Value::as_str)
.map(str::to_owned)
.context("local coordinator did not report a listen address")
}
fn node_command() -> Result<Command> {
if let Some(path) = std::env::var_os("CLUSTERFLUX_NODE_BIN") {
return Ok(Command::new(path));
}
let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
sibling.set_file_name(format!("clusterflux-node{}", std::env::consts::EXE_SUFFIX));
if sibling.is_file() {
return Ok(Command::new(sibling));
}
let mut command = Command::new("cargo");
command.args([
"run",
"-q",
"-p",
"clusterflux-node",
"--bin",
"clusterflux-node",
"--",
]);
Ok(command)
}
fn coordinator_command() -> Result<Command> {
if let Some(path) = std::env::var_os("CLUSTERFLUX_COORDINATOR_BIN") {
return Ok(Command::new(path));
}
let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
sibling.set_file_name(format!(
"clusterflux-coordinator{}",
std::env::consts::EXE_SUFFIX
));
if sibling.is_file() {
return Ok(Command::new(sibling));
}
let mut command = Command::new("cargo");
command.args([
"run",
"-q",
"-p",
"clusterflux-coordinator",
"--bin",
"clusterflux-coordinator",
"--",
]);
Ok(command)
}

106
crates/clusterflux-cli/src/task.rs Normal file
View file

@ -0,0 +1,106 @@
use anyhow::Result;
use serde_json::{json, Value};
use crate::client::{
authenticated_or_local_trusted_request, list_task_events_if_available_with_session,
JsonLineSession,
};
use crate::config::StoredCliSession;
use crate::process_events::{task_restart_request_summary, task_summaries};
use crate::{confirmation_required_report, TaskListArgs, TaskRestartArgs};
#[cfg(test)]
pub(crate) fn task_list_report(args: TaskListArgs) -> Result<Value> {
task_list_report_with_session(args, None)
}
pub(crate) fn task_list_report_with_session(
args: TaskListArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
let events = list_task_events_if_available_with_session(
args.scope.coordinator.as_deref(),
&args.scope,
args.process.clone(),
stored_session,
)?;
let tasks = task_summaries(events.as_ref());
Ok(json!({
"command": "task list",
"process": args.process,
"tasks": tasks,
"events": events,
}))
}
#[cfg(test)]
pub(crate) fn task_restart_report(args: TaskRestartArgs) -> Result<Value> {
task_restart_report_with_session(args, None)
}
pub(crate) fn task_restart_report_with_session(
args: TaskRestartArgs,
stored_session: Option<&StoredCliSession>,
) -> Result<Value> {
if !args.yes {
return Ok(confirmation_required_report(
"task restart",
"restart_selected_task",
json!({
"coordinator": args.scope.coordinator,
"tenant": args.scope.tenant,
"project": args.scope.project,
"process": args.process,
"task": args.task,
}),
format!(
"clusterflux task restart {} --process {} --yes",
args.task, args.process
),
));
}
if let Some(coordinator) = &args.scope.coordinator {
let mut session = JsonLineSession::connect(coordinator)?;
let response = session.request_allow_error(authenticated_or_local_trusted_request(
coordinator,
stored_session,
json!({
"type": "restart_task",
"process": args.process,
"task": args.task,
}),
json!({
"type": "restart_task",
"tenant": args.scope.tenant,
"project": args.scope.project,
"actor_user": args.scope.user,
"process": args.process,
"task": args.task,
}),
)?)?;
let restart_request = task_restart_request_summary(&response, !args.yes);
return Ok(json!({
"command": "task restart",
"coordinator": coordinator,
"process": args.process,
"task": args.task,
"requires_confirmation": !args.yes,
"restart_request": restart_request,
"response": response,
"coordinator_session_requests": session.requests(),
}));
}
Ok(json!({
"command": "task restart",
"status": "requires_coordinator",
"requires_confirmation": !args.yes,
"process": args.process,
"task": args.task,
"restart_request": {
"status": "requires_coordinator",
"operation": "restart_selected_task",
"explicit_user_action": true,
"clean_boundary_required": true,
},
}))
}

4762
crates/clusterflux-cli/src/tests.rs Normal file
View file

File diff suppressed because it is too large Load diff

63
crates/clusterflux-cli/src/tools.rs Normal file
View file

@ -0,0 +1,63 @@
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::time::{SystemTime, UNIX_EPOCH};
use anyhow::Result;
pub(crate) fn command_available(command: &str) -> bool {
Command::new(command)
.arg("--version")
.stdout(Stdio::null())
.stderr(Stdio::null())
.status()
.is_ok()
}
pub(crate) fn command_nonce(prefix: &str) -> String {
let now = unix_timestamp_nanos();
format!("{prefix}-{now}-{}", std::process::id())
}
pub(crate) fn unix_timestamp_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or_default()
}
fn unix_timestamp_nanos() -> u128 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or_default()
}
pub(crate) fn sibling_binary(name: &str) -> Option<PathBuf> {
let mut sibling = std::env::current_exe().ok()?;
sibling.set_file_name(format!("{name}{}", std::env::consts::EXE_SUFFIX));
sibling.is_file().then_some(sibling)
}
pub(crate) fn dap_binary_path() -> Result<PathBuf> {
if let Some(path) = std::env::var_os("CLUSTERFLUX_DAP_BIN") {
return Ok(PathBuf::from(path));
}
if let Some(path) = sibling_binary("clusterflux-debug-dap") {
return Ok(path);
}
let release = PathBuf::from("target/release").join(format!(
"clusterflux-debug-dap{}",
std::env::consts::EXE_SUFFIX
));
if release.is_file() {
return Ok(release);
}
let debug = PathBuf::from("target/debug").join(format!(
"clusterflux-debug-dap{}",
std::env::consts::EXE_SUFFIX
));
if debug.is_file() {
return Ok(debug);
}
anyhow::bail!("could not locate clusterflux-debug-dap; set CLUSTERFLUX_DAP_BIN")
}

13
crates/clusterflux-control/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "clusterflux-control"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
serde_json.workspace = true
thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
ureq.workspace = true

266
crates/clusterflux-control/src/lib.rs Normal file
View file

@ -0,0 +1,266 @@
#[cfg(not(target_arch = "wasm32"))]
use std::io::{BufRead, BufReader, Read, Write};
#[cfg(not(target_arch = "wasm32"))]
use std::net::TcpStream;
use std::net::{IpAddr, ToSocketAddrs};
use std::time::Duration;
use serde_json::Value;
use thiserror::Error;
pub const CONTROL_API_PATH: &str = "/api/v1/control";
pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024;
#[derive(Debug, Error)]
pub enum ControlTransportError {
#[error("invalid coordinator endpoint: {0}")]
InvalidEndpoint(String),
#[error("insecure remote coordinator endpoint is forbidden: {0}")]
InsecureRemote(String),
#[error("coordinator transport I/O failed: {0}")]
Io(#[from] std::io::Error),
#[error("coordinator transport JSON failed: {0}")]
Json(#[from] serde_json::Error),
#[error("coordinator HTTP request failed: {0}")]
Http(String),
#[error("coordinator control frame exceeds {MAX_CONTROL_FRAME_BYTES} bytes")]
FrameTooLarge,
#[error("coordinator closed the local control session without a response")]
Closed,
#[error("coordinator network transport is unavailable inside a Wasm guest")]
UnavailableInWasm,
}
enum ControlTransport {
#[cfg(not(target_arch = "wasm32"))]
Https { agent: ureq::Agent, url: String },
#[cfg(not(target_arch = "wasm32"))]
LoopbackJsonLine {
writer: TcpStream,
reader: BufReader<TcpStream>,
},
#[cfg(target_arch = "wasm32")]
#[allow(dead_code)]
Unavailable,
}
pub struct ControlSession {
transport: ControlTransport,
requests: u64,
}
impl ControlSession {
pub fn connect(endpoint: &str) -> Result<Self, ControlTransportError> {
Self::connect_with_timeouts(endpoint, Duration::from_secs(10), Duration::from_secs(30))
}
pub fn connect_with_timeouts(
endpoint: &str,
connect_timeout: Duration,
io_timeout: Duration,
) -> Result<Self, ControlTransportError> {
#[cfg(target_arch = "wasm32")]
{
let _ = (endpoint, connect_timeout, io_timeout);
return Err(ControlTransportError::UnavailableInWasm);
}
#[cfg(not(target_arch = "wasm32"))]
{
let endpoint = endpoint.trim();
if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
let url = control_api_url(endpoint)?;
if endpoint.starts_with("http://") && !endpoint_is_loopback(endpoint) {
return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
}
let agent = ureq::AgentBuilder::new()
.timeout_connect(connect_timeout)
.timeout_read(io_timeout)
.timeout_write(io_timeout)
.build();
return Ok(Self {
transport: ControlTransport::Https { agent, url },
requests: 0,
});
}
let loopback_address = endpoint
.strip_prefix("clusterflux+tcp://")
.unwrap_or(endpoint);
if !endpoint_is_loopback(loopback_address) {
return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
}
let writer = TcpStream::connect(loopback_address)?;
writer.set_read_timeout(Some(io_timeout))?;
writer.set_write_timeout(Some(io_timeout))?;
let reader = BufReader::new(writer.try_clone()?);
Ok(Self {
transport: ControlTransport::LoopbackJsonLine { writer, reader },
requests: 0,
})
}
}
pub fn request(&mut self, value: &Value) -> Result<Value, ControlTransportError> {
#[cfg(target_arch = "wasm32")]
{
let _ = (&self.transport, self.requests, value);
return Err(ControlTransportError::UnavailableInWasm);
}
#[cfg(not(target_arch = "wasm32"))]
{
let encoded = serde_json::to_vec(value)?;
if encoded.len() > MAX_CONTROL_FRAME_BYTES {
return Err(ControlTransportError::FrameTooLarge);
}
let response = match &mut self.transport {
#[cfg(not(target_arch = "wasm32"))]
ControlTransport::Https { agent, url } => {
let response = agent
.post(url)
.set("Content-Type", "application/json")
.set("Accept", "application/json")
.send_bytes(&encoded)
.map_err(|error| ControlTransportError::Http(error.to_string()))?;
if response.status() != 200 {
return Err(ControlTransportError::Http(format!(
"coordinator returned HTTP {} {}",
response.status(),
response.status_text()
)));
}
let mut bytes = Vec::new();
response
.into_reader()
.take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
.read_to_end(&mut bytes)?;
if bytes.len() > MAX_CONTROL_FRAME_BYTES {
return Err(ControlTransportError::FrameTooLarge);
}
serde_json::from_slice(&bytes)?
}
ControlTransport::LoopbackJsonLine { writer, reader } => {
writer.write_all(&encoded)?;
writer.write_all(b"\n")?;
writer.flush()?;
let mut bytes = Vec::new();
reader
.take((MAX_CONTROL_FRAME_BYTES + 1) as u64)
.read_until(b'\n', &mut bytes)?;
if bytes.is_empty() {
return Err(ControlTransportError::Closed);
}
if bytes.len() > MAX_CONTROL_FRAME_BYTES {
return Err(ControlTransportError::FrameTooLarge);
}
serde_json::from_slice(&bytes)?
}
};
self.requests += 1;
Ok(response)
}
}
pub fn requests(&self) -> u64 {
self.requests
}
}
pub fn control_api_url(endpoint: &str) -> Result<String, ControlTransportError> {
let endpoint = endpoint.trim().trim_end_matches('/');
if !(endpoint.starts_with("https://") || endpoint.starts_with("http://")) {
return Err(ControlTransportError::InvalidEndpoint(endpoint.to_owned()));
}
if endpoint.ends_with(CONTROL_API_PATH) {
Ok(endpoint.to_owned())
} else {
Ok(format!("{endpoint}{CONTROL_API_PATH}"))
}
}
pub fn endpoint_identity(endpoint: &str) -> Result<String, ControlTransportError> {
let endpoint = endpoint.trim();
if endpoint.starts_with("https://") || endpoint.starts_with("http://") {
if endpoint.starts_with("http://") && !endpoint_is_loopback(endpoint) {
return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
}
return control_api_url(endpoint);
}
let loopback_address = endpoint
.strip_prefix("clusterflux+tcp://")
.unwrap_or(endpoint);
if endpoint_is_loopback(loopback_address) {
return Ok(format!("clusterflux+tcp://{loopback_address}"));
}
Err(ControlTransportError::InsecureRemote(endpoint.to_owned()))
}
pub fn endpoint_is_loopback(endpoint: &str) -> bool {
let authority = endpoint
.trim()
.strip_prefix("https://")
.or_else(|| endpoint.trim().strip_prefix("http://"))
.or_else(|| endpoint.trim().strip_prefix("clusterflux+tcp://"))
.unwrap_or(endpoint.trim())
.split('/')
.next()
.unwrap_or_default();
let host = if authority.starts_with('[') {
authority
.strip_prefix('[')
.and_then(|value| value.split_once(']'))
.map(|(host, _)| host)
.unwrap_or(authority)
} else {
authority
.rsplit_once(':')
.map(|(host, _)| host)
.unwrap_or(authority)
};
if host.eq_ignore_ascii_case("localhost") {
return true;
}
if host
.parse::<IpAddr>()
.is_ok_and(|address| address.is_loopback())
{
return true;
}
authority
.to_socket_addrs()
.ok()
.is_some_and(|mut addresses| addresses.all(|address| address.ip().is_loopback()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hosted_endpoints_are_real_https_api_urls() {
assert_eq!(
control_api_url("https://clusterflux.example").unwrap(),
"https://clusterflux.example/api/v1/control"
);
assert_eq!(
endpoint_identity("https://clusterflux.example/api/v1/control").unwrap(),
"https://clusterflux.example/api/v1/control"
);
}
#[test]
fn plaintext_transport_is_restricted_to_loopback() {
assert!(endpoint_is_loopback("127.0.0.1:7999"));
assert!(endpoint_is_loopback("clusterflux+tcp://127.0.0.1:7999"));
assert!(endpoint_is_loopback("http://[::1]:7999"));
assert!(matches!(
ControlSession::connect("http://example.com:7999"),
Err(ControlTransportError::InsecureRemote(_))
));
assert!(matches!(
ControlSession::connect("example.com:7999"),
Err(ControlTransportError::InsecureRemote(_))
));
}
}

18
crates/clusterflux-coordinator/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "clusterflux-coordinator"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
clusterflux-core = { path = "../clusterflux-core" }
clusterflux-wasm-runtime = { path = "../clusterflux-wasm-runtime" }
postgres.workspace = true
serde.workspace = true
serde_json.workspace = true
sha2.workspace = true
tempfile.workspace = true
thiserror.workspace = true
wasmparser.workspace = true

174
crates/clusterflux-coordinator/src/agents.rs Normal file
View file

@ -0,0 +1,174 @@
use clusterflux_core::{
verify_agent_workflow_signature, Actor, AgentId, AgentSignedRequest, AgentWorkflowScope,
AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId,
};
use crate::{
AgentPublicKeyRecord, Coordinator, CoordinatorError, CredentialRecord, ProjectPermissionRecord,
};
impl Coordinator {
pub fn grant_project_debug(&mut self, tenant: TenantId, project: ProjectId, user: UserId) {
self.durable.project_permissions.insert(
(tenant.clone(), project.clone(), user.clone()),
ProjectPermissionRecord {
tenant,
project,
user,
can_debug: true,
},
);
}
pub fn register_agent_public_key(
&mut self,
tenant: TenantId,
project: ProjectId,
user: UserId,
agent: AgentId,
public_key: impl Into<String>,
) -> AgentPublicKeyRecord {
let key = (tenant.clone(), project.clone(), agent.clone());
let version = self
.durable
.agent_public_keys
.get(&key)
.map_or(1, |record| record.version.saturating_add(1));
let public_key = public_key.into();
let public_key_fingerprint = Digest::sha256(&public_key);
let record = AgentPublicKeyRecord {
tenant: tenant.clone(),
project: project.clone(),
user: user.clone(),
agent: agent.clone(),
public_key,
public_key_fingerprint: public_key_fingerprint.clone(),
version,
revoked: false,
scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
human_account_creation_privilege: false,
browser_interaction_required_each_run: false,
};
self.durable.agent_public_keys.insert(key, record.clone());
let subject = format!("agent:{tenant}:{project}:{agent}");
self.durable.credentials.insert(
subject.clone(),
CredentialRecord {
subject,
tenant,
project: Some(project),
kind: CredentialKind::PublicKey,
public_key_fingerprint: Some(public_key_fingerprint),
},
);
record
}
pub fn list_agent_public_keys(&self, context: &AuthContext) -> Vec<AgentPublicKeyRecord> {
self.durable
.agent_public_keys
.values()
.filter(|record| record.tenant == context.tenant && record.project == context.project)
.filter(|record| match &context.actor {
Actor::User(user) => &record.user == user,
Actor::Agent(agent) => &record.agent == agent,
Actor::Node(_) | Actor::Task(_) => false,
})
.cloned()
.collect()
}
pub fn revoke_agent_public_key(
&mut self,
context: &AuthContext,
agent: &AgentId,
) -> Result<AgentPublicKeyRecord, CoordinatorError> {
let key = (
context.tenant.clone(),
context.project.clone(),
agent.clone(),
);
let record = self
.durable
.agent_public_keys
.get_mut(&key)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"agent public key is not registered for this project".to_owned(),
)
})?;
match &context.actor {
Actor::User(user) if &record.user == user => {}
Actor::User(_) => {
return Err(CoordinatorError::Unauthorized(
"agent public key is outside the signed-in user scope".to_owned(),
));
}
_ => {
return Err(CoordinatorError::Unauthorized(
"agent public-key revocation requires a user identity".to_owned(),
));
}
}
record.revoked = true;
let subject = format!(
"agent:{}:{}:{}",
context.tenant, context.project, record.agent
);
self.durable.credentials.remove(&subject);
Ok(record.clone())
}
pub fn authorize_agent_project_run(
&self,
scope: AgentWorkflowScope<'_>,
public_key_fingerprint: Option<&Digest>,
payload_digest: &Digest,
signature: &AgentSignedRequest,
now_epoch_seconds: u64,
) -> Result<AgentPublicKeyRecord, CoordinatorError> {
let record = self
.durable
.agent_public_keys
.get(&(
scope.tenant.clone(),
scope.project.clone(),
scope.agent.clone(),
))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"agent public key is not registered for this tenant/project".to_owned(),
)
})?;
if record.revoked {
return Err(CoordinatorError::Unauthorized(
"agent public key has been revoked".to_owned(),
));
}
if let Some(public_key_fingerprint) = public_key_fingerprint {
if &record.public_key_fingerprint != public_key_fingerprint {
return Err(CoordinatorError::Unauthorized(
"agent public key fingerprint does not match the registered key".to_owned(),
));
}
}
let max_signature_skew_seconds = 300;
if signature
.issued_at_epoch_seconds
.abs_diff(now_epoch_seconds)
> max_signature_skew_seconds
{
return Err(CoordinatorError::Unauthorized(
"agent signed request is expired or outside the allowed clock skew".to_owned(),
));
}
if !record.scopes.iter().any(|scope| scope == "project:run") {
return Err(CoordinatorError::Unauthorized(
"agent public key is not scoped for project runs".to_owned(),
));
}
verify_agent_workflow_signature(&record.public_key, scope, payload_digest, signature)
.map_err(CoordinatorError::Unauthorized)?;
Ok(record.clone())
}
}

145
crates/clusterflux-coordinator/src/durable.rs Normal file
View file

@ -0,0 +1,145 @@
use std::collections::BTreeMap;
use clusterflux_core::{
AgentId, CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TenantRecord {
pub id: TenantId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct UserRecord {
pub id: UserId,
pub tenant: TenantId,
pub credential_kind: CredentialKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProjectRecord {
pub id: ProjectId,
pub tenant: TenantId,
pub name: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeIdentityRecord {
pub id: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub public_key: String,
pub enrollment_scope: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CredentialRecord {
pub subject: String,
pub tenant: TenantId,
pub project: Option<ProjectId>,
pub kind: CredentialKind,
pub public_key_fingerprint: Option<Digest>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CliSessionRecord {
pub session_digest: Digest,
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub credential_kind: CredentialKind,
pub expires_at_epoch_seconds: Option<u64>,
pub revoked: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceProviderConfigRecord {
pub tenant: TenantId,
pub project: ProjectId,
pub provider: SourceProviderKind,
pub manifest_digest: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ServicePolicyRecord {
pub tenant: TenantId,
pub name: String,
pub digest: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AccountPolicyState {
pub account_status: String,
pub suspended: bool,
pub disabled: bool,
pub deleted: bool,
pub manual_review: bool,
pub sanitized_reason: Option<String>,
pub next_actions: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProjectPermissionRecord {
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub can_debug: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgentPublicKeyRecord {
pub tenant: TenantId,
pub project: ProjectId,
pub user: UserId,
pub agent: AgentId,
pub public_key: String,
pub public_key_fingerprint: Digest,
pub version: u64,
pub revoked: bool,
pub scopes: Vec<String>,
pub human_account_creation_privilege: bool,
pub browser_interaction_required_each_run: bool,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct DurableState {
pub tenants: BTreeMap<TenantId, TenantRecord>,
pub users: BTreeMap<UserId, UserRecord>,
pub projects: BTreeMap<ProjectId, ProjectRecord>,
pub node_identities: BTreeMap<NodeId, NodeIdentityRecord>,
pub credentials: BTreeMap<String, CredentialRecord>,
pub cli_sessions: BTreeMap<Digest, CliSessionRecord>,
pub source_provider_configs:
BTreeMap<(TenantId, ProjectId, String), SourceProviderConfigRecord>,
pub service_policy_records: BTreeMap<(TenantId, String), ServicePolicyRecord>,
pub project_permissions: BTreeMap<(TenantId, ProjectId, UserId), ProjectPermissionRecord>,
pub agent_public_keys: BTreeMap<(TenantId, ProjectId, AgentId), AgentPublicKeyRecord>,
}
pub trait DurableStore {
fn load(&self) -> DurableState;
fn save(&mut self, state: DurableState);
}
pub trait FallibleDurableStore {
type Error;
fn load_state(&mut self) -> Result<DurableState, Self::Error>;
fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error>;
}
#[derive(Clone, Debug, Default)]
pub struct InMemoryDurableStore {
state: DurableState,
}
impl DurableStore for InMemoryDurableStore {
fn load(&self) -> DurableState {
self.state.clone()
}
fn save(&mut self, state: DurableState) {
self.state = state;
}
}

1047
crates/clusterflux-coordinator/src/lib.rs Normal file
View file

File diff suppressed because it is too large Load diff

62
crates/clusterflux-coordinator/src/main.rs Normal file
View file

@ -0,0 +1,62 @@
use std::io::Write;
use clusterflux_coordinator::{service::bind_listener, CoordinatorService};
use clusterflux_core::{ProjectId, TenantId, UserId};
use serde_json::json;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut listen = "127.0.0.1:0".to_owned();
let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
.ok()
.as_deref()
== Some("1");
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
if arg == "--listen" {
listen = args.next().ok_or("--listen requires an address")?;
} else if arg == "--allow-local-trusted-loopback" {
allow_local_trusted = true;
}
}
let (listener, addr) = bind_listener(&listen)?;
let database_url = std::env::var("DATABASE_URL").ok();
let mut service = CoordinatorService::new_with_database_url(1, database_url.as_deref())?;
let self_hosted_session_secret = std::env::var("CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET")
.ok()
.filter(|secret| !secret.trim().is_empty());
if let Some(session_secret) = self_hosted_session_secret.as_deref() {
service.issue_cli_session(
TenantId::new(
std::env::var("CLUSTERFLUX_SELF_HOSTED_TENANT")
.unwrap_or_else(|_| "tenant".to_owned()),
),
ProjectId::new(
std::env::var("CLUSTERFLUX_SELF_HOSTED_PROJECT")
.unwrap_or_else(|_| "project".to_owned()),
),
UserId::new(
std::env::var("CLUSTERFLUX_SELF_HOSTED_USER").unwrap_or_else(|_| "user".to_owned()),
),
session_secret,
None,
)?;
}
println!(
"{}",
json!({
"listen": addr.to_string(),
"client_authority": if allow_local_trusted { "local_trusted_loopback" } else { "strict" },
"self_hosted_session_bootstrapped": self_hosted_session_secret.is_some(),
"durable_store": service.durable_store_kind(),
})
);
std::io::stdout().flush()?;
if allow_local_trusted {
service.serve_tcp_local_trusted(listener)?;
} else {
service.serve_tcp(listener)?;
}
Ok(())
}

576
crates/clusterflux-coordinator/src/postgres_store.rs Normal file
View file

@ -0,0 +1,576 @@
use postgres::{Client, NoTls};
use serde::{de::DeserializeOwned, Serialize};
use thiserror::Error;
use crate::{
AgentPublicKeyRecord, CliSessionRecord, CredentialRecord, DurableState, FallibleDurableStore,
NodeIdentityRecord, ProjectPermissionRecord, ProjectRecord, ServicePolicyRecord,
SourceProviderConfigRecord, TenantRecord, UserRecord,
};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PostgresTable {
pub name: &'static str,
pub durable_record: &'static str,
pub restart_surviving: bool,
}
pub const POSTGRES_DURABLE_TABLES: &[PostgresTable] = &[
PostgresTable {
name: "clusterflux_tenants",
durable_record: "tenants",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_users",
durable_record: "users",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_projects",
durable_record: "projects",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_node_identities",
durable_record: "node identities",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_credentials",
durable_record: "credentials",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_cli_sessions",
durable_record: "CLI sessions",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_agent_public_keys",
durable_record: "agent public keys",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_source_provider_configs",
durable_record: "source-provider configuration",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_service_policy_records",
durable_record: "durable service policy records",
restart_surviving: true,
},
PostgresTable {
name: "clusterflux_project_permissions",
durable_record: "explicit project permissions",
restart_surviving: true,
},
];
#[derive(Debug, Error)]
pub enum PostgresStoreError {
#[error("postgres durable store error: {0}")]
Postgres(#[from] postgres::Error),
#[error("durable state serialization error: {0}")]
Serialization(#[from] serde_json::Error),
}
pub struct PostgresDurableStore {
client: Client,
}
impl PostgresDurableStore {
pub fn connect(connection_string: &str) -> Result<Self, PostgresStoreError> {
let mut store = Self {
client: Client::connect(connection_string, NoTls)?,
};
store.migrate()?;
Ok(store)
}
pub fn from_client(client: Client) -> Result<Self, PostgresStoreError> {
let mut store = Self { client };
store.migrate()?;
Ok(store)
}
pub fn schema_sql() -> &'static str {
POSTGRES_SCHEMA_SQL
}
pub fn durable_tables() -> &'static [PostgresTable] {
POSTGRES_DURABLE_TABLES
}
pub fn migrate(&mut self) -> Result<(), PostgresStoreError> {
self.client.batch_execute(Self::schema_sql())?;
Ok(())
}
fn query_records<T: DeserializeOwned>(
&mut self,
sql: &str,
) -> Result<Vec<T>, PostgresStoreError> {
self.client
.query(sql, &[])?
.into_iter()
.map(|row| {
let value: serde_json::Value = row.get("record");
Ok(serde_json::from_value(value)?)
})
.collect()
}
fn record_value(record: &impl Serialize) -> Result<serde_json::Value, PostgresStoreError> {
Ok(serde_json::to_value(record)?)
}
}
impl FallibleDurableStore for PostgresDurableStore {
type Error = PostgresStoreError;
fn load_state(&mut self) -> Result<DurableState, Self::Error> {
let mut state = DurableState::default();
for record in self.query_records::<TenantRecord>(
"SELECT record FROM clusterflux_tenants ORDER BY tenant_id",
)? {
state.tenants.insert(record.id.clone(), record);
}
for record in self
.query_records::<UserRecord>("SELECT record FROM clusterflux_users ORDER BY user_id")?
{
state.users.insert(record.id.clone(), record);
}
for record in self.query_records::<ProjectRecord>(
"SELECT record FROM clusterflux_projects ORDER BY project_id",
)? {
state.projects.insert(record.id.clone(), record);
}
for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM clusterflux_node_identities ORDER BY node_id",
)? {
state.node_identities.insert(record.id.clone(), record);
}
for record in self.query_records::<CredentialRecord>(
"SELECT record FROM clusterflux_credentials ORDER BY subject",
)? {
state.credentials.insert(record.subject.clone(), record);
}
for record in self.query_records::<CliSessionRecord>(
"SELECT record FROM clusterflux_cli_sessions ORDER BY session_digest",
)? {
state
.cli_sessions
.insert(record.session_digest.clone(), record);
}
for record in self.query_records::<AgentPublicKeyRecord>(
"SELECT record FROM clusterflux_agent_public_keys ORDER BY tenant_id, project_id, agent_id",
)? {
state.agent_public_keys.insert(
(
record.tenant.clone(),
record.project.clone(),
record.agent.clone(),
),
record,
);
}
for record in self.query_records::<SourceProviderConfigRecord>(
"SELECT record FROM clusterflux_source_provider_configs ORDER BY tenant_id, project_id, provider_key",
)? {
let provider_key = format!("{:?}", record.provider);
state.source_provider_configs.insert(
(record.tenant.clone(), record.project.clone(), provider_key),
record,
);
}
for record in self.query_records::<ServicePolicyRecord>(
"SELECT record FROM clusterflux_service_policy_records ORDER BY tenant_id, name",
)? {
state
.service_policy_records
.insert((record.tenant.clone(), record.name.clone()), record);
}
for record in self.query_records::<ProjectPermissionRecord>(
"SELECT record FROM clusterflux_project_permissions ORDER BY tenant_id, project_id, user_id",
)? {
state.project_permissions.insert(
(
record.tenant.clone(),
record.project.clone(),
record.user.clone(),
),
record,
);
}
Ok(state)
}
fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
let mut tx = self.client.transaction()?;
tx.batch_execute(
"
DELETE FROM clusterflux_project_permissions;
DELETE FROM clusterflux_service_policy_records;
DELETE FROM clusterflux_source_provider_configs;
DELETE FROM clusterflux_agent_public_keys;
DELETE FROM clusterflux_cli_sessions;
DELETE FROM clusterflux_credentials;
DELETE FROM clusterflux_node_identities;
DELETE FROM clusterflux_projects;
DELETE FROM clusterflux_users;
DELETE FROM clusterflux_tenants;
",
)?;
for record in state.tenants.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_tenants (tenant_id, record) VALUES ($1, $2)",
&[&record.id.as_str(), &value],
)?;
}
for record in state.users.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_users (user_id, tenant_id, record) VALUES ($1, $2, $3)",
&[&record.id.as_str(), &record.tenant.as_str(), &value],
)?;
}
for record in state.projects.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_projects (project_id, tenant_id, record) VALUES ($1, $2, $3)",
&[&record.id.as_str(), &record.tenant.as_str(), &value],
)?;
}
for record in state.node_identities.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_node_identities (node_id, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
&[
&record.id.as_str(),
&record.tenant.as_str(),
&record.project.as_str(),
&value,
],
)?;
}
for record in state.credentials.values() {
let value = Self::record_value(record)?;
let project_id = record.project.as_ref().map(|project| project.as_str());
tx.execute(
"INSERT INTO clusterflux_credentials (subject, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
&[&record.subject.as_str(), &record.tenant.as_str(), &project_id, &value],
)?;
}
for record in state.cli_sessions.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_cli_sessions (session_digest, tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4, $5)",
&[
&record.session_digest.as_str(),
&record.tenant.as_str(),
&record.project.as_str(),
&record.user.as_str(),
&value,
],
)?;
}
for record in state.agent_public_keys.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_agent_public_keys (tenant_id, project_id, user_id, agent_id, record) VALUES ($1, $2, $3, $4, $5)",
&[
&record.tenant.as_str(),
&record.project.as_str(),
&record.user.as_str(),
&record.agent.as_str(),
&value,
],
)?;
}
for ((_, _, provider_key), record) in &state.source_provider_configs {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_source_provider_configs (tenant_id, project_id, provider_key, record) VALUES ($1, $2, $3, $4)",
&[
&record.tenant.as_str(),
&record.project.as_str(),
&provider_key.as_str(),
&value,
],
)?;
}
for record in state.service_policy_records.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_service_policy_records (tenant_id, name, record) VALUES ($1, $2, $3)",
&[&record.tenant.as_str(), &record.name.as_str(), &value],
)?;
}
for record in state.project_permissions.values() {
let value = Self::record_value(record)?;
tx.execute(
"INSERT INTO clusterflux_project_permissions (tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4)",
&[
&record.tenant.as_str(),
&record.project.as_str(),
&record.user.as_str(),
&value,
],
)?;
}
tx.commit()?;
Ok(())
}
}
const POSTGRES_SCHEMA_SQL: &str = r#"
CREATE TABLE IF NOT EXISTS clusterflux_tenants (
tenant_id TEXT PRIMARY KEY,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_users (
user_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_projects (
project_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
record JSONB NOT NULL
);
CREATE TABLE IF NOT EXISTS clusterflux_agent_public_keys (
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
agent_id TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, agent_id)
);
CREATE TABLE IF NOT EXISTS clusterflux_source_provider_configs (
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
provider_key TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, provider_key)
);
CREATE TABLE IF NOT EXISTS clusterflux_service_policy_records (
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
name TEXT NOT NULL,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, name)
);
CREATE TABLE IF NOT EXISTS clusterflux_project_permissions (
tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, user_id)
);
"#;
#[cfg(test)]
mod tests {
use clusterflux_core::{
CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
};
use super::*;
use crate::{Coordinator, DurableStore, FallibleDurableStore, InMemoryDurableStore};
#[test]
fn postgres_schema_contains_only_restart_surviving_durable_tables() {
let names = PostgresDurableStore::durable_tables()
.iter()
.map(|table| table.name)
.collect::<Vec<_>>();
assert_eq!(names.len(), 10);
assert!(names.contains(&"clusterflux_tenants"));
assert!(names.contains(&"clusterflux_users"));
assert!(names.contains(&"clusterflux_projects"));
assert!(names.contains(&"clusterflux_node_identities"));
assert!(names.contains(&"clusterflux_credentials"));
assert!(names.contains(&"clusterflux_cli_sessions"));
assert!(names.contains(&"clusterflux_agent_public_keys"));
assert!(names.contains(&"clusterflux_source_provider_configs"));
assert!(names.contains(&"clusterflux_service_policy_records"));
assert!(names.contains(&"clusterflux_project_permissions"));
assert!(PostgresDurableStore::durable_tables()
.iter()
.all(|table| table.restart_surviving));
for runtime_only in [
"active_process",
"virtual_thread",
"scheduler_state",
"debug_epoch",
"vfs_manifest",
"transient_artifact_location",
] {
assert!(
!PostgresDurableStore::schema_sql().contains(runtime_only),
"{runtime_only} must remain outside Postgres durable state"
);
}
}
#[test]
fn fallible_store_boot_uses_durable_state_and_still_drops_live_processes() {
#[derive(Default)]
struct FallibleMemoryStore {
inner: InMemoryDurableStore,
}
impl FallibleDurableStore for FallibleMemoryStore {
type Error = std::convert::Infallible;
fn load_state(&mut self) -> Result<DurableState, Self::Error> {
Ok(self.inner.load())
}
fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
self.inner.save(state.clone());
Ok(())
}
}
let mut store = FallibleMemoryStore::default();
let mut first = Coordinator::try_boot(&mut store, 1).unwrap();
first.upsert_tenant(TenantId::from("tenant"));
first.upsert_user(
TenantId::from("tenant"),
UserId::from("user"),
CredentialKind::CliDeviceSession,
);
first.upsert_project(TenantId::from("tenant"), ProjectId::from("project"), "demo");
first.enroll_node(
TenantId::from("tenant"),
ProjectId::from("project"),
NodeId::from("node"),
"public-key",
"node:attach",
);
first.upsert_source_provider_config(
TenantId::from("tenant"),
ProjectId::from("project"),
SourceProviderKind::Git,
Digest::sha256("git-manifest"),
);
first.start_process(
TenantId::from("tenant"),
ProjectId::from("project"),
clusterflux_core::ProcessId::from("process"),
);
first.try_persist(&mut store).unwrap();
let restarted = Coordinator::try_boot(&mut store, 2).unwrap();
assert!(restarted.project(&ProjectId::from("project")).is_some());
assert!(restarted.node_identity(&NodeId::from("node")).is_some());
assert_eq!(restarted.active_process_count(), 0);
}
#[test]
fn postgres_round_trip_runs_when_dsn_is_configured() {
let Ok(dsn) = std::env::var("CLUSTERFLUX_TEST_POSTGRES") else {
return;
};
let mut store = PostgresDurableStore::connect(&dsn).unwrap();
let mut state = DurableState::default();
state.tenants.insert(
TenantId::from("tenant"),
TenantRecord {
id: TenantId::from("tenant"),
},
);
state.projects.insert(
ProjectId::from("project"),
ProjectRecord {
id: ProjectId::from("project"),
tenant: TenantId::from("tenant"),
name: "demo".to_owned(),
},
);
state.users.insert(
UserId::from("user"),
UserRecord {
id: UserId::from("user"),
tenant: TenantId::from("tenant"),
credential_kind: CredentialKind::CliDeviceSession,
},
);
let session_digests = [
Digest::sha256("postgres-round-trip-session-one"),
Digest::sha256("postgres-round-trip-session-two"),
];
for session_digest in &session_digests {
state.cli_sessions.insert(
session_digest.clone(),
CliSessionRecord {
session_digest: session_digest.clone(),
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
user: UserId::from("user"),
credential_kind: CredentialKind::CliDeviceSession,
expires_at_epoch_seconds: None,
revoked: false,
},
);
let subject = format!("cli-session:{}", session_digest.as_str());
state.credentials.insert(
subject.clone(),
CredentialRecord {
subject,
tenant: TenantId::from("tenant"),
project: Some(ProjectId::from("project")),
kind: CredentialKind::CliDeviceSession,
public_key_fingerprint: None,
},
);
}
store.save_state(&state).unwrap();
let loaded = store.load_state().unwrap();
assert!(loaded.projects.contains_key(&ProjectId::from("project")));
assert!(session_digests
.iter()
.all(|session_digest| loaded.cli_sessions.contains_key(session_digest)));
assert_eq!(loaded.credentials.len(), 2);
}
}

489
crates/clusterflux-coordinator/src/service.rs Normal file
View file

@ -0,0 +1,489 @@
// Request handlers intentionally spell out their deserialized protocol fields.
// Keeping those authority and payload values explicit at this boundary is safer
// than passing an unvalidated wire request deeper into the service.
#![allow(clippy::too_many_arguments)]
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
RateLimit, TenantId, TransportError, UserId,
};
use thiserror::Error;
use crate::{Coordinator, CoordinatorError};
mod admin;
mod artifacts;
mod authenticated;
mod authorization;
mod debug;
mod debug_requests;
mod durable_runtime;
mod keys;
mod logs;
mod main_runtime;
mod nodes;
mod panels;
mod process_launch;
mod processes;
mod protocol;
mod quota;
mod relay;
mod routing;
mod signed_nodes;
mod tcp;
mod wire_protocol;
use authorization::authorize_authenticated_user_operation;
use durable_runtime::RuntimeDurableStore;
use keys::{
artifact_id_from_path, enrollment_grant_key, EnrollmentGrantKey, PanelStopKey,
ProcessControlKey, TaskAssignmentKey, TaskControlKey, TaskRestartKey,
};
pub use protocol::{
ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
TaskTerminalState, VirtualProcessStatus, WorkflowActor,
};
pub use quota::CoordinatorQuotaConfiguration;
pub use relay::{ArtifactRelayConfiguration, ArtifactRelayError, ArtifactRelayUsage};
pub use tcp::{bind_listener, ClientAuthorityMode};
pub use wire_protocol::CoordinatorWireRequest;
const MAX_TASK_LOG_TAIL_BYTES: usize = 256 * 1024;
const DEBUG_CONTROL_READ_BYTES: u64 = 1024;
const MAX_REPLAY_NONCES_PER_AUTHORITY: usize = 1_024;
const NODE_SIGNATURE_WINDOW_SECONDS: u64 = 30;
const MAX_NODE_REPLAY_NONCES_PER_AUTHORITY: usize = 4_096;
const MAX_ENROLLMENT_GRANTS_PER_PROJECT: usize = 64;
const MAX_TASK_EVENTS_PER_PROCESS: usize = 128;
const MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS: usize = 256;
const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
requested.clamp(1, maximum)
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorMainRuntimeConfiguration {
pub fuel_units_per_second: u64,
pub fuel_burst_seconds: u64,
pub memory_bytes: usize,
pub nested_join_timeout_ms: u64,
}
impl Default for CoordinatorMainRuntimeConfiguration {
fn default() -> Self {
Self {
fuel_units_per_second: 10_000_000,
fuel_burst_seconds: 60,
memory_bytes: 256 * 1024 * 1024,
nested_join_timeout_ms: 24 * 60 * 60 * 1_000,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorAdmission {
pub workflow_placement_allowed: bool,
pub max_node_enrollment_ttl_seconds: u64,
pub max_artifact_download_ttl_seconds: u64,
}
impl Default for CoordinatorAdmission {
fn default() -> Self {
Self {
workflow_placement_allowed: true,
max_node_enrollment_ttl_seconds: 15 * 60,
max_artifact_download_ttl_seconds: 15 * 60,
}
}
}
#[derive(Debug, Error)]
pub enum CoordinatorServiceError {
#[error("coordinator protocol I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("coordinator protocol JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("coordinator protocol error: {0}")]
Protocol(String),
#[error("coordinator request failed: {0}")]
Coordinator(#[from] CoordinatorError),
#[error("artifact download request failed: {0}")]
Download(#[from] clusterflux_core::DownloadError),
#[error("scheduler placement failed: {0}")]
Scheduler(#[from] clusterflux_core::PlacementError),
#[error("transport request failed: {0}")]
Transport(#[from] TransportError),
#[error("resource limit failed: {0}")]
Resource(#[from] LimitError),
#[error("operator panel request failed: {0}")]
Panel(#[from] clusterflux_core::PanelError),
#[error("invalid node capability report: {0}")]
CapabilityReport(#[from] CapabilityReportError),
#[error("invalid VFS artifact path reported by node: {0}")]
InvalidArtifactPath(String),
#[error("invalid task log tail reported by node: {0}")]
InvalidTaskLogTail(String),
#[error("durable coordinator state failed: {0}")]
Durable(String),
}
pub struct CoordinatorService {
coordinator: Coordinator,
store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
task_events: VecDeque<TaskCompletionEvent>,
process_scope_history: VecDeque<ProcessControlKey>,
debug_audit_events: VecDeque<DebugAuditEvent>,
debug_epochs: BTreeMap<ProcessControlKey, u64>,
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
debug_breakpoints: BTreeMap<ProcessControlKey, debug::DebugBreakpointPlan>,
debug_commands: BTreeMap<TaskControlKey, debug::DebugPendingCommand>,
task_assignments: BTreeMap<TaskAssignmentKey, VecDeque<TaskAssignment>>,
task_restart_checkpoints: BTreeMap<TaskRestartKey, processes::TaskRestartCheckpoint>,
task_restart_checkpoint_order: VecDeque<TaskRestartKey>,
task_attempts: BTreeMap<TaskRestartKey, Vec<protocol::TaskAttemptSnapshot>>,
restart_launches: BTreeSet<TaskRestartKey>,
main_runtime: main_runtime::CoordinatorMainRuntime,
pending_task_launches: VecDeque<processes::PendingTaskLaunch>,
task_placements: BTreeMap<TaskControlKey, Placement>,
active_tasks: BTreeSet<TaskControlKey>,
task_cancellations: BTreeSet<TaskControlKey>,
task_aborts: BTreeSet<TaskControlKey>,
process_cancellations: BTreeSet<ProcessControlKey>,
process_aborts: BTreeSet<ProcessControlKey>,
agent_replay_nonces: BTreeMap<(TenantId, ProjectId, AgentId, String), u64>,
node_replay_nonces: BTreeMap<(NodeId, String), u64>,
panel_snapshots: BTreeMap<PanelStopKey, PanelState>,
stopped_panels: BTreeSet<PanelStopKey>,
panel_event_limits: BTreeMap<(TenantId, ProjectId, ProcessId, String), RateLimit>,
artifact_registry: ArtifactRegistry,
artifact_reverse_transfers: BTreeMap<String, artifacts::ArtifactReverseTransfer>,
artifact_transfer_by_token: BTreeMap<Digest, String>,
artifact_relay: relay::ArtifactRelayLedger,
transport: NativeQuicTransport,
quota: quota::CoordinatorQuota,
admission: CoordinatorAdmission,
#[cfg(test)]
server_time_override: Option<u64>,
admin_token_digest: Option<Digest>,
admin_replay_nonces: BTreeMap<String, u64>,
}
impl CoordinatorService {
pub fn configure_artifact_relay(
&mut self,
configuration: ArtifactRelayConfiguration,
) -> Result<(), ArtifactRelayError> {
self.artifact_relay.configure(configuration)
}
pub fn artifact_relay_usage(&self) -> ArtifactRelayUsage {
self.artifact_relay.usage()
}
pub fn set_debug_freeze_timeout(&mut self, timeout: std::time::Duration) {
self.debug_freeze_timeout = timeout.max(std::time::Duration::from_millis(1));
}
pub fn configure_coordinator_main_runtime(
&mut self,
configuration: CoordinatorMainRuntimeConfiguration,
) -> Result<(), CoordinatorServiceError> {
self.main_runtime.configure(configuration)
}
pub(super) fn authorize_node_for_process_or_termination(
&self,
node: &NodeId,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorServiceError> {
let process_key = keys::process_control_key(tenant, project, process);
if self.process_cancellations.contains(&process_key)
|| self.process_aborts.contains(&process_key)
{
let identity = self
.coordinator
.node_identity(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node process-control request is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
return Ok(());
}
self.coordinator
.authorize_node_for_process(node, tenant, project, process)?;
Ok(())
}
pub fn new(coordinator_epoch: u64) -> Self {
Self::new_with_optional_admin_token_and_admission(
coordinator_epoch,
std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
CoordinatorAdmission::default(),
)
}
pub fn new_with_admin_token(coordinator_epoch: u64, admin_token: impl Into<String>) -> Self {
Self::new_with_optional_admin_token_and_admission(
coordinator_epoch,
Some(admin_token.into()),
CoordinatorAdmission::default(),
)
}
pub fn new_with_admission(coordinator_epoch: u64, admission: CoordinatorAdmission) -> Self {
Self::new_with_optional_admin_token_and_admission(
coordinator_epoch,
std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
admission,
)
}
fn new_with_optional_admin_token_and_admission(
coordinator_epoch: u64,
admin_token: Option<String>,
admission: CoordinatorAdmission,
) -> Self {
Self::try_new_with_optional_admin_token_admission_and_database_url(
coordinator_epoch,
admin_token,
admission,
None,
CoordinatorQuotaConfiguration::default(),
)
.expect("in-memory durable coordinator store initialization cannot fail")
}
pub fn new_with_database_url(
coordinator_epoch: u64,
database_url: Option<&str>,
) -> Result<Self, CoordinatorServiceError> {
Self::try_new_with_optional_admin_token_admission_and_database_url(
coordinator_epoch,
std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
CoordinatorAdmission::default(),
database_url,
CoordinatorQuotaConfiguration::default(),
)
}
pub fn new_with_admin_token_and_database_url(
coordinator_epoch: u64,
admin_token: impl Into<String>,
database_url: Option<&str>,
) -> Result<Self, CoordinatorServiceError> {
Self::new_with_admin_token_database_url_and_quota(
coordinator_epoch,
admin_token,
database_url,
CoordinatorQuotaConfiguration::default(),
)
}
pub fn new_with_admin_token_database_url_and_quota(
coordinator_epoch: u64,
admin_token: impl Into<String>,
database_url: Option<&str>,
quota_configuration: CoordinatorQuotaConfiguration,
) -> Result<Self, CoordinatorServiceError> {
Self::try_new_with_optional_admin_token_admission_and_database_url(
coordinator_epoch,
Some(admin_token.into()),
CoordinatorAdmission::default(),
database_url,
quota_configuration,
)
}
fn try_new_with_optional_admin_token_admission_and_database_url(
coordinator_epoch: u64,
admin_token: Option<String>,
admission: CoordinatorAdmission,
database_url: Option<&str>,
quota_configuration: CoordinatorQuotaConfiguration,
) -> Result<Self, CoordinatorServiceError> {
let mut store = RuntimeDurableStore::from_database_url(database_url)
.map_err(CoordinatorServiceError::Durable)?;
let coordinator = Coordinator::try_boot(&mut store, coordinator_epoch)
.map_err(CoordinatorServiceError::Durable)?;
let admin_token_digest = admin_token
.filter(|token| !token.trim().is_empty())
.map(Digest::sha256);
Ok(Self {
coordinator,
store,
node_descriptors: BTreeMap::new(),
node_last_seen_epoch_seconds: BTreeMap::new(),
node_stale_after_seconds: std::env::var("CLUSTERFLUX_NODE_STALE_AFTER_SECONDS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_NODE_STALE_AFTER_SECONDS),
debug_freeze_timeout: std::time::Duration::from_secs(5),
enrollment_grants: BTreeMap::new(),
task_events: VecDeque::new(),
process_scope_history: VecDeque::new(),
debug_audit_events: VecDeque::new(),
debug_epochs: BTreeMap::new(),
debug_epoch_runtime: BTreeMap::new(),
debug_breakpoints: BTreeMap::new(),
debug_commands: BTreeMap::new(),
task_assignments: BTreeMap::new(),
task_restart_checkpoints: BTreeMap::new(),
task_restart_checkpoint_order: VecDeque::new(),
task_attempts: BTreeMap::new(),
restart_launches: BTreeSet::new(),
main_runtime: main_runtime::CoordinatorMainRuntime::default(),
pending_task_launches: VecDeque::new(),
task_placements: BTreeMap::new(),
active_tasks: BTreeSet::new(),
task_cancellations: BTreeSet::new(),
task_aborts: BTreeSet::new(),
process_cancellations: BTreeSet::new(),
process_aborts: BTreeSet::new(),
agent_replay_nonces: BTreeMap::new(),
node_replay_nonces: BTreeMap::new(),
panel_snapshots: BTreeMap::new(),
stopped_panels: BTreeSet::new(),
panel_event_limits: BTreeMap::new(),
artifact_registry: ArtifactRegistry::default(),
artifact_reverse_transfers: BTreeMap::new(),
artifact_transfer_by_token: BTreeMap::new(),
artifact_relay: relay::ArtifactRelayLedger::default(),
transport: NativeQuicTransport,
quota: quota::CoordinatorQuota::new(quota_configuration),
admission,
#[cfg(test)]
server_time_override: None,
admin_token_digest,
admin_replay_nonces: BTreeMap::new(),
})
}
pub fn durable_store_kind(&self) -> &'static str {
self.store.kind()
}
fn persist_durable_state(&mut self) -> Result<(), CoordinatorServiceError> {
self.coordinator
.try_persist(&mut self.store)
.map_err(CoordinatorServiceError::Durable)
}
fn current_epoch_seconds(&self) -> Result<u64, CoordinatorServiceError> {
#[cfg(test)]
if let Some(now) = self.server_time_override {
return Ok(now);
}
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.map_err(|err| CoordinatorServiceError::Protocol(format!("system clock error: {err}")))
}
fn handle_quota_status(
&self,
tenant: String,
project: String,
actor_user: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let durable_project = self.coordinator.project(&project).ok_or_else(|| {
CoordinatorError::Unauthorized("quota status requires an existing project".to_owned())
})?;
if durable_project.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"quota status project is outside the tenant scope".to_owned(),
)
.into());
}
let now_epoch_seconds = self.current_epoch_seconds()?;
let status = self
.quota
.project_status(&tenant, &project, now_epoch_seconds);
Ok(CoordinatorResponse::QuotaStatus {
tenant,
project,
actor,
policy_label: status.policy_label,
limits: status.limits,
window_seconds: status.window_seconds,
usage: status.usage,
window_started_epoch_seconds: status.window_started_epoch_seconds,
})
}
#[cfg(test)]
fn set_server_time(&mut self, now_epoch_seconds: u64) {
self.server_time_override = Some(now_epoch_seconds);
}
pub fn issue_cli_session(
&mut self,
tenant: TenantId,
project: ProjectId,
user: UserId,
session_secret: &str,
expires_at_epoch_seconds: Option<u64>,
) -> Result<crate::CliSessionRecord, CoordinatorServiceError> {
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"CLI session project belongs to a different tenant".to_owned(),
)
.into());
}
} else {
self.coordinator
.upsert_project(tenant.clone(), project.clone(), "Session project");
}
self.coordinator
.grant_project_debug(tenant.clone(), project.clone(), user.clone());
let record = self.coordinator.issue_cli_session(
tenant,
project,
user,
session_secret,
expires_at_epoch_seconds,
);
self.persist_durable_state()?;
Ok(record)
}
pub fn revoke_cli_session(
&mut self,
session_secret: &str,
) -> Result<crate::CliSessionRecord, CoordinatorServiceError> {
let record = self.coordinator.revoke_cli_session(session_secret)?;
self.persist_durable_state()?;
Ok(record)
}
}
#[cfg(test)]
mod tests;

147
crates/clusterflux-coordinator/src/service/admin.rs Normal file
View file

@ -0,0 +1,147 @@
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
admin_request_proof_from_token_digest, CredentialKind, Digest, TenantId, UserId,
};
use crate::CoordinatorError;
use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
const ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS: u64 = 300;
impl CoordinatorService {
pub(super) fn handle_admin_status(
&mut self,
tenant: String,
actor_user: String,
admin_proof: Digest,
admin_nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
self.verify_admin_request(
"admin_status",
&tenant,
&actor_user,
&tenant,
&admin_proof,
&admin_nonce,
issued_at_epoch_seconds,
)?;
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
Ok(CoordinatorResponse::AdminStatus {
suspended: self.coordinator.tenant_suspended(&tenant),
tenant,
actor,
safe_default: "read_only".to_owned(),
})
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_suspend_tenant(
&mut self,
tenant: String,
actor_user: String,
target_tenant: String,
admin_proof: Digest,
admin_nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
self.verify_admin_request(
"suspend_tenant",
&tenant,
&actor_user,
&target_tenant,
&admin_proof,
&admin_nonce,
issued_at_epoch_seconds,
)?;
let actor_tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let target_tenant = TenantId::new(target_tenant);
self.coordinator.upsert_tenant(actor_tenant.clone());
self.coordinator.upsert_user(
actor_tenant,
actor.clone(),
CredentialKind::CliDeviceSession,
);
let policy = self
.coordinator
.suspend_tenant(target_tenant.clone(), actor.clone());
self.persist_durable_state()?;
Ok(CoordinatorResponse::TenantSuspended {
tenant: target_tenant,
actor,
policy,
})
}
#[allow(clippy::too_many_arguments)]
fn verify_admin_request(
&mut self,
operation: &str,
tenant: &str,
actor_user: &str,
target_tenant: &str,
admin_proof: &Digest,
admin_nonce: &str,
issued_at_epoch_seconds: u64,
) -> Result<(), CoordinatorServiceError> {
let expected = self.admin_token_digest.as_ref().ok_or_else(|| {
CoordinatorError::Unauthorized(
"self-hosted admin credential is not configured".to_owned(),
)
})?;
if admin_nonce.trim().is_empty() || admin_nonce.len() > 256 {
return Err(CoordinatorError::Unauthorized(
"admin request nonce is missing or invalid".to_owned(),
)
.into());
}
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or_default();
if now.abs_diff(issued_at_epoch_seconds) > ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS {
return Err(CoordinatorError::Unauthorized(
"admin request timestamp is outside the allowed 300-second window".to_owned(),
)
.into());
}
let expected_proof = admin_request_proof_from_token_digest(
expected,
operation,
tenant,
actor_user,
target_tenant,
admin_nonce,
issued_at_epoch_seconds,
);
if admin_proof != &expected_proof {
return Err(CoordinatorError::Unauthorized(
"admin request proof is invalid".to_owned(),
)
.into());
}
self.admin_replay_nonces.retain(|_, issued_at| {
now <= issued_at.saturating_add(ADMIN_REQUEST_MAX_CLOCK_SKEW_SECONDS)
});
if self.admin_replay_nonces.contains_key(admin_nonce) {
return Err(CoordinatorError::Unauthorized(
"admin request nonce was already used".to_owned(),
)
.into());
}
if self.admin_replay_nonces.len() >= super::MAX_REPLAY_NONCES_PER_AUTHORITY {
return Err(CoordinatorError::Unauthorized(
"admin request replay window is full; retry after the bounded signature window advances"
.to_owned(),
)
.into());
}
self.admin_replay_nonces
.insert(admin_nonce.to_owned(), issued_at_epoch_seconds);
Ok(())
}
}

769
crates/clusterflux-coordinator/src/service/artifacts.rs Normal file
View file

@ -0,0 +1,769 @@
use std::io::{Read, Seek, SeekFrom, Write};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
generate_opaque_token, Actor, ArtifactId, AuthContext, DataPlaneObject, DataPlaneScope, Digest,
DownloadPolicy, NodeEndpoint, NodeId, ProjectId, RendezvousRequest, ResourceLimits,
ResourceMeter, StorageLocation, TenantId, UserId,
};
use sha2::{Digest as _, Sha256};
use crate::CoordinatorError;
use super::relay::RelayFinishReason;
use super::{
bounded_ttl, ArtifactTransferAssignment, CoordinatorResponse, CoordinatorService,
CoordinatorServiceError,
};
pub(super) const MAX_ARTIFACT_REVERSE_CHUNK_BYTES: u64 = 256 * 1024;
#[derive(Debug)]
pub(super) struct ArtifactReverseTransfer {
transfer_id: String,
token_digest: Digest,
tenant: TenantId,
project: ProjectId,
source_node: NodeId,
artifact: ArtifactId,
expected_digest: Digest,
expected_size_bytes: u64,
expires_at_epoch_seconds: u64,
spool: tempfile::NamedTempFile,
received_bytes: u64,
content_hasher: Sha256,
delivered_offset: u64,
error: Option<String>,
}
impl CoordinatorService {
pub(super) fn handle_create_artifact_download_link(
&mut self,
tenant: String,
project: String,
actor_user: String,
artifact: String,
max_bytes: u64,
ttl_seconds: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = user_context(tenant, project, actor_user);
let artifact = ArtifactId::new(artifact);
let policy = DownloadPolicy { max_bytes };
let action = self
.artifact_registry
.download_action(&context, &artifact, &policy)?;
self.ensure_download_source_connectivity(&action.source)?;
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay.expire(now_epoch_seconds);
self.quota.can_charge_download(
&context.tenant,
&context.project,
downloadable_size,
now_epoch_seconds,
)?;
let token_nonce = generate_opaque_token("artifact_download")
.map_err(CoordinatorServiceError::Protocol)?;
let ttl_seconds = bounded_ttl(
ttl_seconds,
self.admission.max_artifact_download_ttl_seconds,
);
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let account = match &context.actor {
Actor::User(user) => user.clone(),
Actor::Agent(_) | Actor::Node(_) | Actor::Task(_) => {
return Err(CoordinatorServiceError::Protocol(
"artifact relay download requires a user account".to_owned(),
))
}
};
self.artifact_relay
.reserve(
token_nonce.clone(),
context.tenant.clone(),
context.project.clone(),
account,
downloadable_size,
MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
expires_at_epoch_seconds,
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let link = match self.artifact_registry.create_download_link(
&context,
&artifact,
&policy,
&token_nonce,
now_epoch_seconds,
ttl_seconds,
) {
Ok(link) => link,
Err(error) => {
self.artifact_relay
.finish(&token_nonce, RelayFinishReason::Cancelled);
return Err(error.into());
}
};
self.artifact_relay
.rekey(&token_nonce, link.scoped_token_digest.as_str().to_owned())
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
Ok(CoordinatorResponse::ArtifactDownloadLink { link })
}
pub(super) fn handle_open_artifact_download_stream(
&mut self,
tenant: String,
project: String,
actor_user: String,
artifact: String,
max_bytes: u64,
token_digest: Digest,
chunk_bytes: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = user_context(tenant, project, actor_user);
let artifact = ArtifactId::new(artifact);
let policy = DownloadPolicy { max_bytes };
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay.expire(now_epoch_seconds);
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let downloadable_size = self
.artifact_registry
.downloadable_size(&context, &artifact, &policy)?;
let validation_limits = ResourceLimits::unlimited();
let mut validation_meter = ResourceMeter::default();
let mut stream = self.artifact_registry.open_download_stream(
clusterflux_core::DownloadStreamRequest {
context: &context,
artifact: &artifact,
policy: &policy,
presented_token_digest: &token_digest,
now_epoch_seconds,
limits: &validation_limits,
},
&mut validation_meter,
)?;
self.ensure_download_source_connectivity(&stream.link.source)?;
self.expire_artifact_reverse_transfers(now_epoch_seconds);
if let Some(transfer_id) = self.artifact_transfer_by_token.get(&token_digest).cloned() {
if let Some(message) = self
.artifact_reverse_transfers
.get(&transfer_id)
.and_then(|transfer| transfer.error.clone())
{
self.artifact_reverse_transfers.remove(&transfer_id);
self.artifact_transfer_by_token.remove(&token_digest);
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Failed);
return Err(CoordinatorServiceError::Protocol(format!(
"retaining node could not stream artifact: {message}"
)));
}
let (content_offset, content, end, complete) = {
let transfer = self
.artifact_reverse_transfers
.get_mut(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"artifact reverse transfer index is inconsistent".to_owned(),
)
})?;
if transfer.tenant != context.tenant
|| transfer.project != context.project
|| transfer.artifact != artifact
|| transfer.token_digest != token_digest
{
return Err(clusterflux_core::DownloadError::InvalidToken.into());
}
if transfer.received_bytes != transfer.expected_size_bytes {
self.artifact_relay
.charge_egress(
token_digest.as_str(),
self.artifact_relay.framing_overhead_bytes(),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
return Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link,
streamed_bytes: 0,
charged_download_bytes: self.quota.used_download_bytes(
&context.tenant,
&context.project,
now_epoch_seconds,
),
content_bytes_available: false,
content_offset: None,
content_eof: false,
content_base64: None,
content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
});
}
if chunk_bytes == 0 && downloadable_size != 0 {
return Err(CoordinatorServiceError::Protocol(
"artifact download chunk_bytes must be greater than zero".to_owned(),
));
}
let requested = chunk_bytes
.min(downloadable_size)
.min(MAX_ARTIFACT_REVERSE_CHUNK_BYTES);
let start = transfer.delivered_offset;
let end = start.saturating_add(requested).min(transfer.received_bytes);
let length = usize::try_from(end.saturating_sub(start)).map_err(|_| {
CoordinatorServiceError::Protocol(
"artifact download chunk length does not fit memory bounds".to_owned(),
)
})?;
let mut content = vec![0_u8; length];
let mut spool = transfer.spool.reopen().map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"open bounded artifact transfer spool: {error}"
))
})?;
spool.seek(SeekFrom::Start(start)).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"seek bounded artifact transfer spool: {error}"
))
})?;
spool.read_exact(&mut content).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"read bounded artifact transfer spool: {error}"
))
})?;
(start, content, end, end == transfer.received_bytes)
};
let streamed_bytes = content.len() as u64;
self.artifact_registry.stream_download_chunk(
&mut stream,
&validation_limits,
&mut validation_meter,
streamed_bytes,
)?;
let charged_download_bytes = self.quota.charge_download(
&context.tenant,
&context.project,
streamed_bytes,
now_epoch_seconds,
)?;
let content_base64 = BASE64_STANDARD.encode(content);
self.artifact_relay
.charge_egress(
token_digest.as_str(),
(content_base64.len() as u64)
.saturating_add(self.artifact_relay.framing_overhead_bytes()),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
if let Some(transfer) = self.artifact_reverse_transfers.get_mut(&transfer_id) {
transfer.delivered_offset = end;
}
if complete {
self.artifact_reverse_transfers.remove(&transfer_id);
self.artifact_transfer_by_token.remove(&token_digest);
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Completed);
}
return Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link,
streamed_bytes,
charged_download_bytes,
content_bytes_available: true,
content_offset: Some(content_offset),
content_eof: complete,
content_base64: Some(content_base64),
content_source: Some("retaining_node_reverse_stream".to_owned()),
});
}
let StorageLocation::RetainedNode(source_node) = &stream.link.source else {
return Err(clusterflux_core::DownloadError::Unavailable.into());
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?;
let spool = tempfile::Builder::new()
.prefix("clusterflux-artifact-transfer-")
.tempfile()
.map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"create bounded artifact transfer spool: {error}"
))
})?;
let transfer = ArtifactReverseTransfer {
transfer_id: transfer_id.clone(),
token_digest: token_digest.clone(),
tenant: context.tenant.clone(),
project: context.project.clone(),
source_node: source_node.clone(),
artifact,
expected_digest: metadata.digest.clone(),
expected_size_bytes: metadata.size,
expires_at_epoch_seconds: stream.link.expires_at_epoch_seconds,
spool,
received_bytes: 0,
content_hasher: Sha256::new(),
delivered_offset: 0,
error: None,
};
self.artifact_reverse_transfers
.insert(transfer_id.clone(), transfer);
self.artifact_transfer_by_token
.insert(token_digest, transfer_id);
self.artifact_relay
.charge_egress(
stream.link.scoped_token_digest.as_str(),
self.artifact_relay.framing_overhead_bytes(),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link,
streamed_bytes: 0,
charged_download_bytes: self.quota.used_download_bytes(
&context.tenant,
&context.project,
now_epoch_seconds,
),
content_bytes_available: false,
content_offset: None,
content_eof: false,
content_base64: None,
content_source: Some("retaining_node_reverse_stream_pending".to_owned()),
})
}
pub(super) fn handle_revoke_artifact_download_link(
&mut self,
tenant: String,
project: String,
actor_user: String,
artifact: String,
token_digest: Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = user_context(tenant, project, actor_user);
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let link = self.artifact_registry.revoke_download_link(
&context,
&ArtifactId::new(artifact),
&token_digest,
)?;
if let Some(transfer_id) = self.artifact_transfer_by_token.remove(&token_digest) {
self.artifact_reverse_transfers.remove(&transfer_id);
}
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Cancelled);
Ok(CoordinatorResponse::ArtifactDownloadLinkRevoked { link })
}
pub(super) fn handle_export_artifact_to_node(
&mut self,
tenant: String,
project: String,
actor_user: String,
artifact: String,
receiver_node: String,
direct_connectivity: bool,
failure_reason: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = user_context(tenant, project, actor_user);
let artifact = ArtifactId::new(artifact);
let receiver_node = NodeId::new(receiver_node);
let action = self.artifact_registry.download_action(
&context,
&artifact,
&DownloadPolicy {
max_bytes: u64::MAX,
},
)?;
let StorageLocation::RetainedNode(source_node) = action.source else {
return Err(clusterflux_core::DownloadError::Unavailable.into());
};
let metadata = self
.artifact_registry
.metadata(&artifact)
.ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination =
self.export_endpoint(&receiver_node, &context.tenant, &context.project)?;
let plan = self.transport.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: DataPlaneScope {
tenant: context.tenant.clone(),
project: context.project.clone(),
process: metadata.process.clone(),
object: DataPlaneObject::Artifact(artifact.clone()),
authorization_subject: format!(
"artifact-export:{}-to-{}",
source_node, receiver_node
),
},
source,
destination,
},
direct_connectivity,
failure_reason,
)?;
Ok(CoordinatorResponse::ArtifactExportPlan {
plan,
source_node,
receiver_node,
artifact_size_bytes: metadata.size,
})
}
pub(super) fn handle_poll_artifact_transfer(
&mut self,
tenant: String,
project: String,
node: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
let transfer = self.artifact_reverse_transfers.values().find(|transfer| {
transfer.tenant == tenant
&& transfer.project == project
&& transfer.source_node == node
&& transfer.error.is_none()
&& transfer.received_bytes < transfer.expected_size_bytes
});
Ok(CoordinatorResponse::ArtifactTransferAssignment {
transfer: transfer.map(|transfer| ArtifactTransferAssignment {
transfer_id: transfer.transfer_id.clone(),
artifact: transfer.artifact.clone(),
expected_digest: transfer.expected_digest.clone(),
expected_size_bytes: transfer.expected_size_bytes,
offset: transfer.received_bytes,
max_chunk_bytes: MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
}),
})
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_upload_artifact_transfer_chunk(
&mut self,
tenant: String,
project: String,
node: String,
transfer_id: String,
artifact: String,
offset: u64,
content_base64: String,
chunk_digest: Digest,
eof: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
let token_digest = {
let transfer = self
.artifact_reverse_transfers
.get(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"unknown artifact reverse transfer".to_owned(),
)
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer is outside the signed node scope".to_owned(),
)
.into());
}
transfer.token_digest.clone()
};
let now_epoch_seconds = self.current_epoch_seconds()?;
let ingress_wire_bytes = (content_base64.len() as u64)
.saturating_add(self.artifact_relay.framing_overhead_bytes());
self.artifact_relay
.charge_ingress(token_digest.as_str(), ingress_wire_bytes, now_epoch_seconds)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let transfer = self
.artifact_reverse_transfers
.get_mut(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer is outside the signed node scope".to_owned(),
)
.into());
}
if transfer.received_bytes != offset {
return Err(CoordinatorServiceError::Protocol(format!(
"artifact reverse transfer expected offset {}, received {offset}",
transfer.received_bytes
)));
}
let content = BASE64_STANDARD.decode(content_base64).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"artifact reverse transfer chunk is not valid base64: {error}"
))
})?;
if content.len() as u64 > MAX_ARTIFACT_REVERSE_CHUNK_BYTES {
return Err(CoordinatorServiceError::Protocol(
"artifact reverse transfer chunk exceeds the control-plane limit".to_owned(),
));
}
if Digest::sha256(&content) != chunk_digest {
return Err(CoordinatorServiceError::Protocol(
"artifact reverse transfer chunk digest mismatch".to_owned(),
));
}
let next_offset = offset.saturating_add(content.len() as u64);
if next_offset > transfer.expected_size_bytes {
return Err(CoordinatorServiceError::Protocol(
"artifact reverse transfer exceeds retained metadata size".to_owned(),
));
}
let complete = next_offset == transfer.expected_size_bytes;
if eof != complete {
return Err(CoordinatorServiceError::Protocol(
"artifact reverse transfer EOF does not match retained metadata size".to_owned(),
));
}
transfer
.spool
.as_file_mut()
.seek(SeekFrom::Start(offset))
.and_then(|_| transfer.spool.as_file_mut().write_all(&content))
.map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"write bounded artifact transfer spool: {error}"
))
})?;
transfer.content_hasher.update(&content);
transfer.received_bytes = next_offset;
if complete {
transfer.spool.as_file().sync_all().map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"sync bounded artifact transfer spool: {error}"
))
})?;
let digest_hex = format!("{:x}", transfer.content_hasher.clone().finalize());
let digest =
Digest::from_sha256_hex(&digest_hex).map_err(CoordinatorServiceError::Protocol)?;
if digest != transfer.expected_digest {
transfer.spool.as_file_mut().set_len(0).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"reset invalid artifact transfer spool: {error}"
))
})?;
transfer.received_bytes = 0;
transfer.content_hasher = Sha256::new();
return Err(CoordinatorServiceError::Protocol(
"artifact reverse transfer content digest mismatch".to_owned(),
));
}
}
Ok(CoordinatorResponse::ArtifactTransferChunkAccepted {
transfer_id,
next_offset,
complete,
})
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_fail_artifact_transfer(
&mut self,
tenant: String,
project: String,
node: String,
transfer_id: String,
artifact: String,
message: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
let token_digest = {
let transfer = self
.artifact_reverse_transfers
.get(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"unknown artifact reverse transfer".to_owned(),
)
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer failure is outside the signed node scope".to_owned(),
)
.into());
}
transfer.token_digest.clone()
};
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay
.charge_ingress(
token_digest.as_str(),
(message.len() as u64).saturating_add(self.artifact_relay.framing_overhead_bytes()),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let transfer = self
.artifact_reverse_transfers
.get_mut(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol("unknown artifact reverse transfer".to_owned())
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer failure is outside the signed node scope".to_owned(),
)
.into());
}
let message = message.trim();
transfer.error = Some(if message.is_empty() {
"retained artifact bytes are unavailable".to_owned()
} else {
message.chars().take(1024).collect()
});
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Failed);
Ok(CoordinatorResponse::ArtifactTransferFailed { transfer_id })
}
fn authorize_artifact_transfer_node(
&self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer node is outside its enrolled tenant/project scope"
.to_owned(),
)
.into());
}
Ok(())
}
fn expire_artifact_reverse_transfers(&mut self, now_epoch_seconds: u64) {
let expired = self
.artifact_reverse_transfers
.iter()
.filter(|(_, transfer)| transfer.expires_at_epoch_seconds < now_epoch_seconds)
.map(|(id, transfer)| (id.clone(), transfer.token_digest.clone()))
.collect::<Vec<_>>();
for (id, token) in expired {
self.artifact_reverse_transfers.remove(&id);
self.artifact_transfer_by_token.remove(&token);
self.artifact_relay
.finish(token.as_str(), RelayFinishReason::Expired);
}
self.artifact_relay.expire(now_epoch_seconds);
}
fn export_endpoint(
&self,
node: &NodeId,
tenant: &TenantId,
project: &ProjectId,
) -> Result<NodeEndpoint, CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.ok_or(CoordinatorError::UnknownNode)?;
if &identity.tenant != tenant || &identity.project != project {
return Err(CoordinatorError::Unauthorized(
"artifact export node is outside the tenant/project scope".to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity"
))
})?;
if descriptor.tenant != *tenant || descriptor.project != *project {
return Err(CoordinatorError::Unauthorized(
"artifact export node descriptor is outside the tenant/project scope".to_owned(),
)
.into());
}
if !self.node_is_live(node) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export"
))
.into(),
);
}
if !descriptor.direct_connectivity {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"direct connectivity unavailable to node {node} for artifact export"
))
.into(),
);
}
Ok(NodeEndpoint {
node: node.clone(),
advertised_addr: format!("{node}.mesh.invalid:4433"),
public_key_fingerprint: Digest::sha256(&identity.public_key),
})
}
fn ensure_download_source_connectivity(
&self,
source: &StorageLocation,
) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else {
return Ok(());
};
let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download"
))
})?;
if !self.node_is_live(node) {
return Err(
clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} is offline for artifact download"
)),
);
}
Ok(())
}
}
fn user_context(tenant: String, project: String, actor_user: String) -> AuthContext {
AuthContext {
tenant: TenantId::new(tenant),
project: ProjectId::new(project),
actor: Actor::User(UserId::new(actor_user)),
}
}

139
crates/clusterflux-coordinator/src/service/authenticated.rs Normal file
View file

@ -0,0 +1,139 @@
use clusterflux_core::{AuthContext, UserId};
use super::{
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
CoordinatorServiceError,
};
impl CoordinatorService {
pub(super) fn handle_authenticated_agent_key_request(
&mut self,
context: &AuthContext,
actor: &UserId,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request = match request {
AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { agent, public_key } => {
CoordinatorRequest::RegisterAgentPublicKey {
tenant: context.tenant.as_str().to_owned(),
project: context.project.as_str().to_owned(),
user: actor.as_str().to_owned(),
agent,
public_key,
}
}
AuthenticatedCoordinatorRequest::ListAgentPublicKeys => {
CoordinatorRequest::ListAgentPublicKeys {
tenant: context.tenant.as_str().to_owned(),
project: context.project.as_str().to_owned(),
user: actor.as_str().to_owned(),
}
}
AuthenticatedCoordinatorRequest::RotateAgentPublicKey { agent, public_key } => {
CoordinatorRequest::RotateAgentPublicKey {
tenant: context.tenant.as_str().to_owned(),
project: context.project.as_str().to_owned(),
user: actor.as_str().to_owned(),
agent,
public_key,
}
}
AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { agent } => {
CoordinatorRequest::RevokeAgentPublicKey {
tenant: context.tenant.as_str().to_owned(),
project: context.project.as_str().to_owned(),
user: actor.as_str().to_owned(),
agent,
}
}
_ => unreachable!("caller filters authenticated agent key operations"),
};
self.handle_request(request)
}
pub(super) fn handle_authenticated_launch_task(
&mut self,
context: &AuthContext,
actor: &UserId,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let AuthenticatedCoordinatorRequest::LaunchTask {
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
} = request
else {
unreachable!("caller filters authenticated task launches");
};
self.handle_launch_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
Some(actor.as_str().to_owned()),
None,
None,
None,
None,
*task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
)
}
pub(super) fn handle_authenticated_schedule_task(
&mut self,
context: &AuthContext,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let AuthenticatedCoordinatorRequest::ScheduleTask {
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
} = request
else {
unreachable!("caller filters authenticated task scheduling");
};
self.handle_schedule_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
)
}
pub(super) fn handle_authenticated_artifact_export(
&mut self,
context: &AuthContext,
actor: &UserId,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let AuthenticatedCoordinatorRequest::ExportArtifactToNode {
artifact,
receiver_node,
direct_connectivity,
failure_reason,
} = request
else {
unreachable!("caller filters authenticated artifact exports");
};
self.handle_export_artifact_to_node(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
receiver_node,
direct_connectivity,
failure_reason,
)
}
}

204
crates/clusterflux-coordinator/src/service/authorization.rs Normal file
View file

@ -0,0 +1,204 @@
use clusterflux_core::{Actor, AuthContext, UserId};
use crate::CoordinatorError;
use super::{AuthenticatedCoordinatorRequest, CoordinatorServiceError};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum PublicUserOperation {
AuthStatus,
RevokeCliSession,
CreateProject,
SelectProject,
ListProjects,
RegisterAgentPublicKey,
ListAgentPublicKeys,
RotateAgentPublicKey,
RevokeAgentPublicKey,
CreateNodeEnrollmentGrant,
ListNodeDescriptors,
RevokeNodeCredential,
StartProcess,
ScheduleTask,
LaunchTask,
CancelProcess,
AbortProcess,
ListProcesses,
QuotaStatus,
RestartTask,
ResolveTaskFailure,
DebugAttach,
SetDebugBreakpoints,
InspectDebugBreakpoints,
CreateDebugEpoch,
ResumeDebugEpoch,
InspectDebugEpoch,
ListTaskEvents,
ListTaskSnapshots,
JoinTask,
CreateArtifactDownloadLink,
OpenArtifactDownloadStream,
RevokeArtifactDownloadLink,
ExportArtifactToNode,
}
impl PublicUserOperation {
pub(super) fn as_str(self) -> &'static str {
match self {
Self::AuthStatus => "auth_status",
Self::RevokeCliSession => "revoke_cli_session",
Self::CreateProject => "create_project",
Self::SelectProject => "select_project",
Self::ListProjects => "list_projects",
Self::RegisterAgentPublicKey => "register_agent_public_key",
Self::ListAgentPublicKeys => "list_agent_public_keys",
Self::RotateAgentPublicKey => "rotate_agent_public_key",
Self::RevokeAgentPublicKey => "revoke_agent_public_key",
Self::CreateNodeEnrollmentGrant => "create_node_enrollment_grant",
Self::ListNodeDescriptors => "list_node_descriptors",
Self::RevokeNodeCredential => "revoke_node_credential",
Self::StartProcess => "start_process",
Self::ScheduleTask => "schedule_task",
Self::LaunchTask => "launch_task",
Self::CancelProcess => "cancel_process",
Self::AbortProcess => "abort_process",
Self::ListProcesses => "list_processes",
Self::QuotaStatus => "quota_status",
Self::RestartTask => "restart_task",
Self::ResolveTaskFailure => "resolve_task_failure",
Self::DebugAttach => "debug_attach",
Self::SetDebugBreakpoints => "set_debug_breakpoints",
Self::InspectDebugBreakpoints => "inspect_debug_breakpoints",
Self::CreateDebugEpoch => "create_debug_epoch",
Self::ResumeDebugEpoch => "resume_debug_epoch",
Self::InspectDebugEpoch => "inspect_debug_epoch",
Self::ListTaskEvents => "list_task_events",
Self::ListTaskSnapshots => "list_task_snapshots",
Self::JoinTask => "join_task",
Self::CreateArtifactDownloadLink => "create_artifact_download_link",
Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
Self::RevokeArtifactDownloadLink => "revoke_artifact_download_link",
Self::ExportArtifactToNode => "export_artifact_to_node",
}
}
}
impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
fn from(request: &AuthenticatedCoordinatorRequest) -> Self {
match request {
AuthenticatedCoordinatorRequest::AuthStatus => Self::AuthStatus,
AuthenticatedCoordinatorRequest::RevokeCliSession => Self::RevokeCliSession,
AuthenticatedCoordinatorRequest::CreateProject { .. } => Self::CreateProject,
AuthenticatedCoordinatorRequest::SelectProject { .. } => Self::SelectProject,
AuthenticatedCoordinatorRequest::ListProjects => Self::ListProjects,
AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. } => {
Self::RegisterAgentPublicKey
}
AuthenticatedCoordinatorRequest::ListAgentPublicKeys => Self::ListAgentPublicKeys,
AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. } => {
Self::RotateAgentPublicKey
}
AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. } => {
Self::RevokeAgentPublicKey
}
AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { .. } => {
Self::CreateNodeEnrollmentGrant
}
AuthenticatedCoordinatorRequest::ListNodeDescriptors => Self::ListNodeDescriptors,
AuthenticatedCoordinatorRequest::RevokeNodeCredential { .. } => {
Self::RevokeNodeCredential
}
AuthenticatedCoordinatorRequest::StartProcess { .. } => Self::StartProcess,
AuthenticatedCoordinatorRequest::ScheduleTask { .. } => Self::ScheduleTask,
AuthenticatedCoordinatorRequest::LaunchTask { .. } => Self::LaunchTask,
AuthenticatedCoordinatorRequest::CancelProcess { .. } => Self::CancelProcess,
AuthenticatedCoordinatorRequest::AbortProcess { .. } => Self::AbortProcess,
AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
AuthenticatedCoordinatorRequest::DebugAttach { .. } => Self::DebugAttach,
AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } => {
Self::SetDebugBreakpoints
}
AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. } => {
Self::InspectDebugBreakpoints
}
AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. } => Self::CreateDebugEpoch,
AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. } => Self::ResumeDebugEpoch,
AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
Self::CreateArtifactDownloadLink
}
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream { .. } => {
Self::OpenArtifactDownloadStream
}
AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink { .. } => {
Self::RevokeArtifactDownloadLink
}
AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
Self::ExportArtifactToNode
}
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(super) struct AuthorizedPublicUser {
pub(super) actor: UserId,
pub(super) operation: PublicUserOperation,
}
pub(super) fn authorize_authenticated_user_operation(
context: &AuthContext,
request: &AuthenticatedCoordinatorRequest,
) -> Result<AuthorizedPublicUser, CoordinatorServiceError> {
let operation = PublicUserOperation::from(request);
let actor = match &context.actor {
Actor::User(user) => user.clone(),
_ => {
return Err(CoordinatorError::Unauthorized(format!(
"authenticated {} request requires a user CLI session",
operation.as_str()
))
.into());
}
};
Ok(AuthorizedPublicUser { actor, operation })
}
#[cfg(test)]
mod tests {
use clusterflux_core::{AgentId, ProjectId, TenantId};
use super::*;
#[test]
fn authenticated_public_authorization_requires_user_context_and_names_operation() {
let request = AuthenticatedCoordinatorRequest::DebugAttach {
process: "vp".to_owned(),
};
let agent_context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::Agent(AgentId::from("agent-ci")),
};
let denied = authorize_authenticated_user_operation(&agent_context, &request).unwrap_err();
assert!(denied
.to_string()
.contains("authenticated debug_attach request requires a user CLI session"));
let user_context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::User(UserId::from("user")),
};
let authorized = authorize_authenticated_user_operation(&user_context, &request).unwrap();
assert_eq!(authorized.actor, UserId::from("user"));
assert_eq!(authorized.operation, PublicUserOperation::DebugAttach);
}
}

1073
crates/clusterflux-coordinator/src/service/debug.rs Normal file
View file

File diff suppressed because it is too large Load diff

92
crates/clusterflux-coordinator/src/service/debug/validation.rs Normal file
View file

@ -0,0 +1,92 @@
use std::collections::BTreeSet;
use crate::CoordinatorServiceError;
use super::{DebugAcknowledgementState, DebugEpochRuntime};
pub(super) fn runtime_all_in_state(
runtime: &DebugEpochRuntime,
state: DebugAcknowledgementState,
) -> bool {
!runtime.expected.is_empty()
&& runtime.expected.iter().all(|key| {
runtime
.acknowledgements
.get(key)
.is_some_and(|ack| ack.state == state)
})
}
pub(super) fn validate_probe_symbols(
probe_symbols: Vec<String>,
) -> Result<Vec<String>, CoordinatorServiceError> {
if probe_symbols.len() > 128 {
return Err(CoordinatorServiceError::Protocol(
"debug breakpoint request exceeds 128 probe symbols".to_owned(),
));
}
let mut unique = BTreeSet::new();
for symbol in probe_symbols {
if symbol.trim().is_empty()
|| symbol.len() > 256
|| !symbol
.chars()
.all(|character| character.is_ascii_alphanumeric() || "._:-/".contains(character))
{
return Err(CoordinatorServiceError::Protocol(
"debug breakpoint probe symbol is invalid".to_owned(),
));
}
unique.insert(symbol);
}
Ok(unique.into_iter().collect())
}
pub(super) fn validate_debug_snapshot(
stack_frames: &[String],
local_values: &[(String, String)],
task_args: &[(String, String)],
handles: &[(String, String)],
command_status: Option<&str>,
recent_output: &[String],
message: Option<&str>,
) -> Result<(), CoordinatorServiceError> {
const MAX_ITEMS: usize = 128;
const MAX_TEXT_BYTES: usize = 16 * 1024;
if [
stack_frames.len(),
local_values.len(),
task_args.len(),
handles.len(),
recent_output.len(),
]
.into_iter()
.any(|count| count > MAX_ITEMS)
{
return Err(CoordinatorServiceError::Protocol(
"debug participant snapshot exceeds the 128-item field limit".to_owned(),
));
}
let text_bytes = stack_frames.iter().map(String::len).sum::<usize>()
+ local_values
.iter()
.map(|(name, value)| name.len() + value.len())
.sum::<usize>()
+ task_args
.iter()
.map(|(name, value)| name.len() + value.len())
.sum::<usize>()
+ handles
.iter()
.map(|(name, value)| name.len() + value.len())
.sum::<usize>()
+ recent_output.iter().map(String::len).sum::<usize>()
+ command_status.map(str::len).unwrap_or(0)
+ message.map(str::len).unwrap_or(0);
if text_bytes > MAX_TEXT_BYTES {
return Err(CoordinatorServiceError::Protocol(format!(
"debug participant snapshot is {text_bytes} bytes; maximum is {MAX_TEXT_BYTES}"
)));
}
Ok(())
}

575
crates/clusterflux-coordinator/src/service/debug_requests.rs Normal file
View file

@ -0,0 +1,575 @@
use std::collections::BTreeSet;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
Actor, Capability, CredentialKind, Digest, EnvironmentResource, ProcessId, ProjectId,
RestartDecision, RestartPolicy, RestartRequest, TaskDispatch, TaskInstanceId, TenantId, UserId,
WasmExportAbi,
};
use crate::{CoordinatorError, CoordinatorServiceError};
use super::keys::task_restart_key;
use super::protocol::{TaskAttemptState, TaskFailureResolution};
use super::{
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
TaskReplacementBundle, TaskTerminalState, WorkflowActor,
};
impl CoordinatorService {
pub(super) fn handle_debug_request(
&mut self,
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
match request {
CoordinatorRequest::DebugAttach {
tenant,
project,
actor_user,
process,
} => self.handle_debug_attach(tenant, project, actor_user, process),
CoordinatorRequest::SetDebugBreakpoints {
tenant,
project,
actor_user,
process,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant,
project,
actor_user,
process,
probe_symbols,
),
CoordinatorRequest::InspectDebugBreakpoints {
tenant,
project,
actor_user,
process,
} => self.handle_inspect_debug_breakpoints(tenant, project, actor_user, process),
CoordinatorRequest::CreateDebugEpoch {
tenant,
project,
actor_user,
process,
stopped_task,
reason,
} => self.handle_create_debug_epoch(
tenant,
project,
actor_user,
process,
stopped_task,
reason,
),
CoordinatorRequest::ResumeDebugEpoch {
tenant,
project,
actor_user,
process,
epoch,
} => self.handle_resume_debug_epoch(tenant, project, actor_user, process, epoch),
CoordinatorRequest::InspectDebugEpoch {
tenant,
project,
actor_user,
process,
epoch,
} => self.handle_inspect_debug_epoch(tenant, project, actor_user, process, epoch),
_ => unreachable!("handle_debug_request only accepts debug coordinator requests"),
}
}
pub(super) fn handle_authenticated_debug_request(
&mut self,
tenant: &TenantId,
project: &ProjectId,
actor: &UserId,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
match request {
AuthenticatedCoordinatorRequest::DebugAttach { process } => self.handle_debug_attach(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::SetDebugBreakpoints {
process,
probe_symbols,
} => self.handle_set_debug_breakpoints(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
probe_symbols,
),
AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { process } => self
.handle_inspect_debug_breakpoints(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::CreateDebugEpoch {
process,
stopped_task,
reason,
} => self.handle_create_debug_epoch(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
stopped_task,
reason,
),
AuthenticatedCoordinatorRequest::ResumeDebugEpoch { process, epoch } => self
.handle_resume_debug_epoch(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
epoch,
),
AuthenticatedCoordinatorRequest::InspectDebugEpoch { process, epoch } => self
.handle_inspect_debug_epoch(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
epoch,
),
_ => unreachable!(
"handle_authenticated_debug_request only accepts debug coordinator requests"
),
}
}
pub(super) fn handle_restart_task(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
replacement_bundle: Option<TaskReplacementBundle>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let process = ProcessId::new(process);
let task = TaskInstanceId::new(task);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: project.clone(),
actor: Actor::User(actor.clone()),
};
let authorization = self.coordinator.authorize_debug_attach(&context, &process);
if !authorization.allowed {
let _ = self.record_debug_audit_event(
tenant,
project,
process,
Some(task),
actor,
"restart_task",
false,
authorization.reason.clone(),
)?;
return Err(CoordinatorError::Unauthorized(format!(
"task restart denied: {}",
authorization.reason
))
.into());
}
let active_key = self
.active_tasks
.iter()
.find(|(task_tenant, task_project, task_process, _, task_id)| {
task_tenant == &tenant
&& task_project == &project
&& task_process == &process
&& task_id == &task
})
.cloned();
let process_key = super::keys::process_control_key(&tenant, &project, &process);
let active_main = self
.main_runtime
.controls
.get(&process_key)
.is_some_and(|control| {
control.task_instance == task
&& matches!(control.state.as_str(), "running" | "stopping")
});
let active_task = active_key.is_some() || active_main;
let completed_event_observed = self.task_events.iter().any(|event| {
event.tenant == tenant
&& event.project == project
&& event.process == process
&& event.task == task
});
let checkpoint_key = task_restart_key(&tenant, &project, &process, &task);
let checkpoint = self.task_restart_checkpoints.get(&checkpoint_key).cloned();
let mut accepted = false;
let mut restarted_task_instance = None;
let mut restarted_attempt_id = None;
let mut clean_boundary_available = false;
let mut requires_whole_process_restart = true;
let message = if active_main {
"selected coordinator main is still active; restart the whole virtual process to rerun its capless entry boundary".to_owned()
} else if active_task {
"selected task is still active; wait for its terminal event or abort the whole virtual process before restarting from its clean entry boundary".to_owned()
} else if !completed_event_observed {
"selected task is not known in the active process; restart the whole virtual process or inspect task list".to_owned()
} else if let Some(checkpoint) = checkpoint {
let vfs_available =
checkpoint
.checkpoint
.vfs_manifest
.objects
.iter()
.all(|(path, object)| {
let artifact = super::keys::artifact_id_from_path(path);
self.artifact_registry
.metadata(&artifact)
.is_some_and(|metadata| {
metadata.tenant == tenant
&& metadata.project == project
&& metadata.digest == object.digest
&& metadata.size == object.size
&& !metadata.retaining_nodes.is_empty()
})
});
if !vfs_available {
"selected task checkpoint references VFS artifacts that are no longer retained; restart the whole virtual process".to_owned()
} else {
let replacement = replacement_bundle
.as_ref()
.map(|bundle| {
validate_task_replacement(
bundle,
&checkpoint.assignment.task_spec.task_definition,
checkpoint.assignment.task_spec.environment_id.as_deref(),
)
})
.transpose()?;
let request = RestartRequest {
task: task.clone(),
entrypoint: replacement.as_ref().map_or_else(
|| checkpoint.checkpoint.boundary.task_entrypoint.clone(),
|replacement| replacement.export.clone(),
),
serialized_args: checkpoint.checkpoint.boundary.serialized_args.clone(),
environment_digest: replacement.as_ref().map_or_else(
|| checkpoint.checkpoint.boundary.environment_digest.clone(),
|replacement| replacement.environment_digest.clone(),
),
task_abi: replacement.as_ref().map_or_else(
|| checkpoint.checkpoint.boundary.task_abi.clone(),
|replacement| replacement.restart_compatibility.clone(),
),
source_edited: replacement.is_some(),
};
match RestartPolicy.decide(&checkpoint.checkpoint, &request) {
RestartDecision::RestartTask { from_vfs_epoch, .. } => {
clean_boundary_available = true;
let mut assignment = checkpoint.assignment;
assignment.task = task.clone();
assignment.task_spec.task_instance = task.clone();
let next_attempt = self
.task_attempts
.get(&checkpoint_key)
.map_or(1, |attempts| attempts.len() + 1);
assignment.artifact_path = format!(
"/vfs/artifacts/{}-attempt-{next_attempt}-result.json",
task.as_str()
);
if let Some(replacement) = replacement {
assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
export: Some(replacement.export),
abi: WasmExportAbi::TaskV1,
};
assignment.task_spec.environment = replacement
.environment
.map(|environment| environment.requirements);
assignment.task_spec.environment_digest =
Some(replacement.environment_digest);
assignment.task_spec.required_capabilities =
replacement.required_capabilities;
assignment.task_spec.bundle_digest =
Some(replacement.bundle_digest.clone());
assignment.wasm_module_base64 = replacement.wasm_module_base64;
}
let restart_task_spec = assignment.task_spec.clone();
let restart_artifact_path = assignment.artifact_path.clone();
self.restart_launches.insert(checkpoint_key.clone());
let launch = self.handle_launch_task_with_actor(
tenant.clone(),
project.clone(),
WorkflowActor {
kind: "user".to_owned(),
user: Some(actor.clone()),
agent: None,
credential_kind: CredentialKind::CliDeviceSession,
public_key_fingerprint: None,
authenticated_without_browser: false,
scopes: vec!["process:restart-task".to_owned()],
},
assignment.task_spec,
true,
assignment.artifact_path,
assignment.wasm_module_base64,
);
self.restart_launches.remove(&checkpoint_key);
let launch = launch?;
let queued = matches!(launch, CoordinatorResponse::TaskQueued { .. });
accepted = matches!(
&launch,
CoordinatorResponse::TaskLaunched { .. }
| CoordinatorResponse::TaskQueued { .. }
);
if accepted {
restarted_task_instance = Some(task.clone());
restarted_attempt_id = self
.task_attempts
.get(&checkpoint_key)
.and_then(|attempts| attempts.last())
.map(|attempt| attempt.attempt_id.clone());
if restarted_attempt_id.is_none() {
restarted_attempt_id = Some(self.begin_task_attempt(
&restart_task_spec,
None,
Some(&restart_artifact_path),
queued,
)?);
}
}
requires_whole_process_restart = !accepted;
format!(
"selected logical task {task} restarted as a new attempt from clean VFS entry boundary epoch {from_vfs_epoch}; unflushed task changes were discarded"
)
}
RestartDecision::RestartWholeVirtualProcess { message } => message,
}
}
} else {
"selected task has terminal metadata but no captured clean VFS entry boundary; restart the whole virtual process".to_owned()
};
let audit_event = self.record_debug_audit_event(
tenant,
project,
process.clone(),
Some(task.clone()),
actor.clone(),
"restart_task",
true,
&message,
)?;
Ok(CoordinatorResponse::TaskRestart {
process,
task,
restarted_task_instance,
restarted_attempt_id,
actor,
accepted,
clean_boundary_available,
active_task,
completed_event_observed,
requires_whole_process_restart,
message,
charged_debug_read_bytes: audit_event.charged_debug_read_bytes,
used_debug_read_bytes: audit_event.used_debug_read_bytes,
audit_event,
})
}
pub(super) fn handle_resolve_task_failure(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
resolution: TaskFailureResolution,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let process = ProcessId::new(process);
let task = TaskInstanceId::new(task);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: project.clone(),
actor: Actor::User(actor),
};
let authorization = self.coordinator.authorize_debug_attach(&context, &process);
if !authorization.allowed {
return Err(CoordinatorError::Unauthorized(format!(
"task failure resolution denied: {}",
authorization.reason
))
.into());
}
let key = task_restart_key(&tenant, &project, &process, &task);
let attempt = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
.filter(|attempt| attempt.state == TaskAttemptState::FailedAwaitingAction)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"task is not failed awaiting operator action".to_owned(),
)
})?;
attempt.state = match resolution {
TaskFailureResolution::AcceptFailure => TaskAttemptState::Failed,
TaskFailureResolution::Cancel => TaskAttemptState::Cancelled,
};
attempt.command_state = Some(
match resolution {
TaskFailureResolution::AcceptFailure => "failure_accepted",
TaskFailureResolution::Cancel => "cancelled",
}
.to_owned(),
);
let attempt_id = attempt.attempt_id.clone();
let mut event = self
.task_events
.iter()
.rev()
.find(|event| {
event.tenant == tenant
&& event.project == project
&& event.process == process
&& event.task == task
&& event.attempt_id.as_deref() == Some(attempt_id.as_str())
})
.cloned()
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"failed attempt terminal event is unavailable".to_owned(),
)
})?;
if resolution == TaskFailureResolution::Cancel {
event.terminal_state = TaskTerminalState::Cancelled;
event.stderr_tail = "operator cancelled task after failure".to_owned();
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,
attempt_id,
resolution,
})
}
}
struct ValidatedTaskReplacement {
bundle_digest: Digest,
wasm_module_base64: String,
export: String,
restart_compatibility: Digest,
environment: Option<EnvironmentResource>,
environment_digest: Digest,
required_capabilities: BTreeSet<Capability>,
}
fn validate_task_replacement(
replacement: &TaskReplacementBundle,
task_definition: &clusterflux_core::TaskDefinitionId,
environment_id: Option<&str>,
) -> Result<ValidatedTaskReplacement, CoordinatorServiceError> {
let module = BASE64_STANDARD
.decode(&replacement.wasm_module_base64)
.map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"replacement task bundle is not valid base64: {error}"
))
})?;
let actual_digest = Digest::sha256(&module);
if actual_digest != replacement.bundle_digest {
return Err(CoordinatorServiceError::Protocol(format!(
"replacement task bundle digest mismatch: expected {}, actual {actual_digest}",
replacement.bundle_digest
)));
}
let descriptors = super::main_runtime::task_descriptors(&module)?;
let descriptor = descriptors.get(task_definition.as_str()).ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"replacement bundle has no task definition `{task_definition}`"
))
})?;
if descriptor
.get("abi_version")
.and_then(serde_json::Value::as_u64)
!= Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
{
return Err(CoordinatorServiceError::Protocol(format!(
"replacement task `{task_definition}` uses an unsupported task ABI"
)));
}
let export = descriptor
.get("export")
.and_then(serde_json::Value::as_str)
.filter(|export| !export.is_empty())
.ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"replacement task `{task_definition}` omitted its export"
))
})?
.to_owned();
let restart_compatibility = serde_json::from_value(
descriptor
.get("restart_compatibility_hash")
.cloned()
.ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"replacement task `{task_definition}` omitted restart compatibility metadata"
))
})?,
)?;
let mut required_capabilities = descriptor
.get("required_capabilities")
.and_then(serde_json::Value::as_array)
.into_iter()
.flatten()
.map(|capability| {
super::main_runtime::capability_from_descriptor(capability.as_str().ok_or_else(
|| {
CoordinatorServiceError::Protocol(
"replacement task capability is not a string".to_owned(),
)
},
)?)
.map_err(CoordinatorServiceError::Protocol)
})
.collect::<Result<BTreeSet<_>, _>>()?;
let environment = environment_id
.map(|environment_id| {
super::main_runtime::bundle_environments(&module)?
.remove(environment_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"replacement bundle has no environment `{environment_id}`"
))
})
})
.transpose()?;
if let Some(environment) = &environment {
required_capabilities.extend(environment.requirements.capabilities.iter().cloned());
}
let environment_digest = environment.as_ref().map_or_else(
|| Digest::sha256("clusterflux.environment.unconstrained.v1"),
|environment| environment.digest.clone(),
);
Ok(ValidatedTaskReplacement {
bundle_digest: replacement.bundle_digest.clone(),
wasm_module_base64: replacement.wasm_module_base64.clone(),
export,
restart_compatibility,
environment,
environment_digest,
required_capabilities,
})
}

47
crates/clusterflux-coordinator/src/service/durable_runtime.rs Normal file
View file

@ -0,0 +1,47 @@
use crate::{
DurableState, DurableStore, FallibleDurableStore, InMemoryDurableStore, PostgresDurableStore,
};
pub(super) enum RuntimeDurableStore {
InMemory(InMemoryDurableStore),
Postgres(PostgresDurableStore),
}
impl RuntimeDurableStore {
pub(super) fn from_database_url(database_url: Option<&str>) -> Result<Self, String> {
match database_url.map(str::trim).filter(|url| !url.is_empty()) {
Some(url) => PostgresDurableStore::connect(url)
.map(Self::Postgres)
.map_err(|error| error.to_string()),
None => Ok(Self::InMemory(InMemoryDurableStore::default())),
}
}
pub(super) fn kind(&self) -> &'static str {
match self {
Self::InMemory(_) => "in_memory",
Self::Postgres(_) => "postgres",
}
}
}
impl FallibleDurableStore for RuntimeDurableStore {
type Error = String;
fn load_state(&mut self) -> Result<DurableState, Self::Error> {
match self {
Self::InMemory(store) => Ok(store.load()),
Self::Postgres(store) => store.load_state().map_err(|error| error.to_string()),
}
}
fn save_state(&mut self, state: &DurableState) -> Result<(), Self::Error> {
match self {
Self::InMemory(store) => {
store.save(state.clone());
Ok(())
}
Self::Postgres(store) => store.save_state(state).map_err(|error| error.to_string()),
}
}
}

73
crates/clusterflux-coordinator/src/service/keys.rs Normal file
View file

@ -0,0 +1,73 @@
use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);
pub(super) type TaskAssignmentKey = (TenantId, ProjectId, NodeId);
pub(super) type PanelStopKey = (TenantId, ProjectId, ProcessId);
pub(super) type EnrollmentGrantKey = (TenantId, ProjectId, String);
pub(super) type ProcessControlKey = (TenantId, ProjectId, ProcessId);
pub(super) fn task_control_key(
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
node: &NodeId,
task: &TaskInstanceId,
) -> TaskControlKey {
(
tenant.clone(),
project.clone(),
process.clone(),
node.clone(),
task.clone(),
)
}
pub(super) fn task_restart_key(
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
) -> TaskRestartKey {
(
tenant.clone(),
project.clone(),
process.clone(),
task.clone(),
)
}
pub(super) fn process_control_key(
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> ProcessControlKey {
(tenant.clone(), project.clone(), process.clone())
}
pub(super) fn panel_stop_key(
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> PanelStopKey {
(tenant.clone(), project.clone(), process.clone())
}
pub(super) fn enrollment_grant_key(
tenant: &TenantId,
project: &ProjectId,
grant: &str,
) -> EnrollmentGrantKey {
(tenant.clone(), project.clone(), grant.to_owned())
}
pub(super) fn artifact_id_from_path(path: &VfsPath) -> ArtifactId {
let value = path
.as_str()
.strip_prefix("/vfs/artifacts/")
.unwrap_or(path.as_str())
.replace('/', ":");
ArtifactId::new(value)
}

661
crates/clusterflux-coordinator/src/service/logs.rs Normal file
View file

@ -0,0 +1,661 @@
use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
};
use crate::CoordinatorError;
use super::keys::{process_control_key, task_control_key, task_restart_key};
use super::protocol::TaskAttemptState;
use super::{
artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
};
impl CoordinatorService {
pub(super) fn handle_report_task_log(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
stdout_bytes: u64,
stderr_bytes: u64,
stdout_tail: String,
stderr_tail: String,
stdout_truncated: bool,
stderr_truncated: bool,
backpressured: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
validate_task_log_tail("stdout_tail", &stdout_tail)?;
validate_task_log_tail("stderr_tail", &stderr_tail)?;
let reported_bytes = checked_reported_log_bytes(stdout_bytes, stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota
.can_charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
self.quota
.charge_log_bytes(&tenant, &project, reported_bytes, now_epoch_seconds)?;
Ok(CoordinatorResponse::TaskLogRecorded {
process,
task,
stdout_bytes,
stderr_bytes,
stdout_tail: if stdout_truncated {
format!("{stdout_tail}\n... truncated")
} else {
stdout_tail
},
stderr_tail: if stderr_truncated {
format!("{stderr_tail}\n... truncated")
} else {
stderr_tail
},
backpressured,
})
}
pub(super) fn handle_report_vfs_metadata(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
artifact_path: Option<String>,
artifact_digest: Option<Digest>,
artifact_size_bytes: Option<u64>,
large_bytes_uploaded: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let artifact_path = artifact_path
.map(VfsPath::new)
.transpose()
.map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if let (Some(path), Some(digest)) = (&artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
tenant,
project,
process: process.clone(),
producer_task: task.clone(),
retaining_node: node,
digest,
size: artifact_size_bytes.unwrap_or_default(),
})?;
}
Ok(CoordinatorResponse::VfsMetadataRecorded {
process,
task,
artifact_path,
large_bytes_uploaded,
})
}
pub(super) fn handle_task_completed(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
terminal_state: Option<TaskTerminalState>,
status_code: Option<i32>,
stdout_bytes: u64,
stderr_bytes: u64,
stdout_tail: String,
stderr_tail: String,
stdout_truncated: bool,
stderr_truncated: bool,
artifact_path: Option<String>,
artifact_digest: Option<Digest>,
artifact_size_bytes: Option<u64>,
result: Option<TaskBoundaryValue>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
validate_task_log_tail("stdout_tail", &stdout_tail)?;
validate_task_log_tail("stderr_tail", &stderr_tail)?;
let artifact_path = artifact_path
.map(VfsPath::new)
.transpose()
.map_err(|err| CoordinatorServiceError::InvalidArtifactPath(err.to_string()))?;
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
let checkpoint = self
.task_restart_checkpoints
.get(&super::keys::task_restart_key(
&tenant, &project, &process, &task,
))
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"signed node task completion does not name a coordinator-issued task instance"
.to_owned(),
)
})?;
if checkpoint.assignment.node != node {
return Err(CoordinatorError::Unauthorized(
"signed node task completion came from a node other than the assigned node"
.to_owned(),
)
.into());
}
let mut event = TaskCompletionEvent {
tenant,
project,
process,
node,
executor: super::TaskExecutor::Node,
task_definition: checkpoint.assignment.task_spec.task_definition.clone(),
task,
attempt_id: None,
placement: None,
terminal_state: terminal_state
.unwrap_or_else(|| TaskTerminalState::from_status_code(status_code)),
status_code,
stdout_bytes,
stderr_bytes,
stdout_tail,
stderr_tail,
stdout_truncated,
stderr_truncated,
artifact_path,
artifact_digest: artifact_digest.clone(),
artifact_size_bytes,
result,
};
let reported_bytes = checked_reported_log_bytes(event.stdout_bytes, event.stderr_bytes)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota.can_charge_log_bytes(
&event.tenant,
&event.project,
reported_bytes,
now_epoch_seconds,
)?;
self.quota.charge_log_bytes(
&event.tenant,
&event.project,
reported_bytes,
now_epoch_seconds,
)?;
let task_key = task_control_key(
&event.tenant,
&event.project,
&event.process,
&event.node,
&event.task,
);
let process_key = process_control_key(&event.tenant, &event.project, &event.process);
let process_was_aborted = self.process_aborts.contains(&process_key);
event.placement = self.task_placements.remove(&task_key);
if let (Some(path), Some(digest)) = (&event.artifact_path, artifact_digest) {
self.flush_artifact_metadata(ArtifactFlush {
id: artifact_id_from_path(path),
tenant: event.tenant.clone(),
project: event.project.clone(),
process: event.process.clone(),
producer_task: event.task.clone(),
retaining_node: event.node.clone(),
digest,
size: artifact_size_bytes.unwrap_or(stdout_bytes),
})?;
}
self.task_cancellations.remove(&task_key);
self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key);
let no_active_tasks =
!self
.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &event.tenant
&& task_project == &event.project
&& task_process == &event.process
});
if no_active_tasks {
self.process_aborts.remove(&process_key);
if self.process_cancellations.remove(&process_key)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
}
if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key(
&event.tenant,
&event.project,
&event.process,
&event.task,
);
self.task_restart_checkpoints.remove(&checkpoint_key);
self.task_restart_checkpoint_order
.retain(|retained| retained != &checkpoint_key);
}
let awaiting_operator = self.finish_task_attempt(&mut event);
self.record_task_completion_event(event.clone());
if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
Ok(CoordinatorResponse::TaskRecorded {
process: event.process,
task: event.task,
events_recorded: self.task_events.len(),
})
}
pub(super) fn handle_list_task_events(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: Option<String>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = process.map(ProcessId::new);
if let Some(process) = &process {
self.authorize_task_event_process_scope(&tenant, &project, process)?;
}
let events = self
.task_events
.iter()
.filter(|event| {
event.tenant == tenant
&& event.project == project
&& process
.as_ref()
.is_none_or(|process| event.process == *process)
})
.cloned()
.collect();
Ok(CoordinatorResponse::TaskEvents { events })
}
pub(super) fn handle_list_task_snapshots(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = ProcessId::new(process);
self.authorize_task_event_process_scope(&tenant, &project, &process)?;
let snapshots = self
.task_attempts
.iter()
.filter(
|((attempt_tenant, attempt_project, attempt_process, _), _)| {
attempt_tenant == &tenant
&& attempt_project == &project
&& attempt_process == &process
},
)
.flat_map(|(_, attempts)| attempts.iter().cloned())
.collect();
Ok(CoordinatorResponse::TaskSnapshots { snapshots })
}
fn authorize_task_event_process_scope(
&self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorServiceError> {
let active_in_scope = self
.coordinator
.active_process(tenant, project, process)
.is_some();
let historical_in_scope = self.task_events.iter().any(|event| {
event.tenant == *tenant && event.project == *project && event.process == *process
}) || self.process_scope_history.iter().any(
|(historical_tenant, historical_project, historical_process)| {
historical_tenant == tenant
&& historical_project == project
&& historical_process == process
},
);
let process_exists_outside_scope = self
.coordinator
.active_process_exists_outside_scope(tenant, project, process)
|| self.task_events.iter().any(|event| {
event.process == *process && (event.tenant != *tenant || event.project != *project)
})
|| self.process_scope_history.iter().any(
|(historical_tenant, historical_project, historical_process)| {
historical_process == process
&& (historical_tenant != tenant || historical_project != project)
},
);
if !active_in_scope && !historical_in_scope && process_exists_outside_scope {
return Err(CoordinatorError::Unauthorized(
"task event access is outside the virtual process tenant/project scope".to_owned(),
)
.into());
}
Ok(())
}
pub(super) fn handle_join_task(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = ProcessId::new(process);
let task = TaskInstanceId::new(task);
Ok(CoordinatorResponse::TaskJoined {
join: self.task_join_result(tenant, project, process, task),
})
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_join_child_task(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
parent_task: String,
task: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let parent_task = TaskInstanceId::new(parent_task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
if !self.active_tasks.contains(&super::keys::task_control_key(
&tenant,
&project,
&process,
&node,
&parent_task,
)) {
return Err(CoordinatorError::Unauthorized(
"child task join requires a currently active parent task on the signed node"
.to_owned(),
)
.into());
}
Ok(CoordinatorResponse::TaskJoined {
join: self.task_join_result(tenant, project, process, TaskInstanceId::new(task)),
})
}
pub(super) fn task_join_result(
&self,
tenant: TenantId,
project: ProjectId,
process: ProcessId,
task: TaskInstanceId,
) -> TaskJoinResult {
let attempt_key = task_restart_key(&tenant, &project, &process, &task);
if self
.task_attempts
.get(&attempt_key)
.is_some_and(|attempts| {
attempts
.iter()
.rev()
.find(|attempt| attempt.current)
.is_some_and(|attempt| {
matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
})
{
return TaskJoinResult::pending(
process,
task,
"logical task is still running or awaiting operator action",
);
}
let event = self.task_events.iter().rev().find(|event| {
event.tenant == tenant
&& event.project == project
&& event.process == process
&& event.task == task
});
if let Some(event) = event {
TaskJoinResult::from_remote_completion(
event.process.clone(),
event.task.clone(),
event.node.clone(),
join_state_for_terminal(&event.terminal_state),
event.result.clone(),
event.status_code,
join_message_for_event(event),
)
} else {
let known = self.task_is_known_or_active(&tenant, &project, &process, &task);
TaskJoinResult::pending(
process,
task,
if known {
"waiting for signed node task_completed event before join returns"
} else {
"no signed node completion event has been observed for this task"
},
)
}
}
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
let process_scope = (
event.tenant.clone(),
event.project.clone(),
event.process.clone(),
);
self.process_scope_history
.retain(|retained| retained != &process_scope);
while self.process_scope_history.len() >= super::MAX_TASK_EVENTS_TOTAL {
self.process_scope_history.pop_front();
}
self.process_scope_history.push_back(process_scope);
while self
.task_events
.iter()
.filter(|retained| {
retained.tenant == event.tenant
&& retained.project == event.project
&& retained.process == event.process
})
.count()
>= super::MAX_TASK_EVENTS_PER_PROCESS
{
let Some(index) = self.task_events.iter().position(|retained| {
retained.tenant == event.tenant
&& retained.project == event.project
&& retained.process == event.process
}) else {
break;
};
self.task_events.remove(index);
}
while self.task_events.len() >= super::MAX_TASK_EVENTS_TOTAL {
self.task_events.pop_front();
}
self.task_events.push_back(event);
}
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
let Some(attempt) = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
else {
return false;
};
event.attempt_id = Some(attempt.attempt_id.clone());
attempt.status_code = event.status_code;
attempt.artifact_path = event.artifact_path.clone();
attempt.artifact_digest = event.artifact_digest.clone();
attempt.artifact_size_bytes = event.artifact_size_bytes;
attempt.error = (!event.stderr_tail.trim().is_empty()).then(|| event.stderr_tail.clone());
let awaiting_operator = event.terminal_state == TaskTerminalState::Failed
&& attempt.failure_policy == clusterflux_core::TaskFailurePolicy::AwaitOperator;
attempt.state = if awaiting_operator {
TaskAttemptState::FailedAwaitingAction
} else {
match event.terminal_state {
TaskTerminalState::Completed => TaskAttemptState::Completed,
TaskTerminalState::Failed => TaskAttemptState::Failed,
TaskTerminalState::Cancelled => TaskAttemptState::Cancelled,
}
};
attempt.command_state = Some(if awaiting_operator {
"failed_awaiting_action".to_owned()
} else {
format!("{:?}", event.terminal_state).to_ascii_lowercase()
});
awaiting_operator
}
fn flush_artifact_metadata(
&mut self,
flush: ArtifactFlush,
) -> Result<(), CoordinatorServiceError> {
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_registry
.expire_download_links(now_epoch_seconds);
let pinned = self
.task_restart_checkpoints
.values()
.flat_map(|checkpoint| checkpoint.assignment.task_spec.required_artifacts.iter())
.chain(
self.pending_task_launches
.iter()
.flat_map(|pending| pending.task_spec.required_artifacts.iter()),
)
.cloned()
.collect::<std::collections::BTreeSet<ArtifactId>>();
self.artifact_registry
.flush_metadata_bounded(flush, &pinned)
.map(|_| ())
.map_err(CoordinatorServiceError::Protocol)
}
fn task_is_known_or_active(
&self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task: &TaskInstanceId,
) -> bool {
self.active_tasks
.iter()
.any(|(task_tenant, task_project, task_process, _, task_id)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& task_id == task
})
|| self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
&& &pending.task == task
})
|| self.task_assignments.values().any(|assignments| {
assignments.iter().any(|assignment| {
&assignment.tenant == tenant
&& &assignment.project == project
&& &assignment.process == process
&& &assignment.task == task
})
})
}
}
fn checked_reported_log_bytes(
stdout_bytes: u64,
stderr_bytes: u64,
) -> Result<u64, CoordinatorServiceError> {
stdout_bytes.checked_add(stderr_bytes).ok_or_else(|| {
CoordinatorServiceError::Protocol(
"reported task log byte counts exceed the supported range".to_owned(),
)
})
}
fn validate_task_log_tail(kind: &str, value: &str) -> Result<(), CoordinatorServiceError> {
if value.len() > MAX_TASK_LOG_TAIL_BYTES {
return Err(CoordinatorServiceError::InvalidTaskLogTail(format!(
"{kind} is {} bytes; max is {MAX_TASK_LOG_TAIL_BYTES}",
value.len()
)));
}
Ok(())
}
fn bounded_log_tail(mut value: String, truncated: &mut bool) -> String {
if value.len() <= MAX_TASK_LOG_TAIL_BYTES {
return value;
}
let mut boundary = MAX_TASK_LOG_TAIL_BYTES;
while !value.is_char_boundary(boundary) {
boundary -= 1;
}
value.truncate(boundary);
*truncated = true;
value
}
fn join_state_for_terminal(terminal: &TaskTerminalState) -> TaskJoinState {
match terminal {
TaskTerminalState::Completed => TaskJoinState::Completed,
TaskTerminalState::Failed => TaskJoinState::Failed,
TaskTerminalState::Cancelled => TaskJoinState::Cancelled,
}
}
fn join_message_for_event(event: &TaskCompletionEvent) -> String {
match event.terminal_state {
TaskTerminalState::Completed => {
"joined result from signed node task_completed event".to_owned()
}
TaskTerminalState::Failed => {
let stderr = event.stderr_tail.trim();
if stderr.is_empty() {
"remote task failed".to_owned()
} else {
format!("remote task failed: {stderr}")
}
}
TaskTerminalState::Cancelled => "remote task was cancelled".to_owned(),
}
}

1039
crates/clusterflux-coordinator/src/service/main_runtime.rs Normal file
View file

File diff suppressed because it is too large Load diff

446
crates/clusterflux-coordinator/src/service/nodes.rs Normal file
View file

@ -0,0 +1,446 @@
use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
generate_opaque_token, verify_node_request_signature, Actor, ArtifactId, CredentialKind,
Digest, NodeCapabilities, NodeDescriptor, NodeId, NodeSignedRequest, ProjectId,
SourceProviderKind, TenantId, UserId,
};
use crate::CoordinatorError;
use super::{
bounded_ttl, enrollment_grant_key, CoordinatorResponse, CoordinatorService,
CoordinatorServiceError,
};
impl CoordinatorService {
pub fn set_node_stale_after_seconds(&mut self, seconds: u64) {
self.node_stale_after_seconds = seconds.max(1);
}
pub(super) fn liveness_now_epoch_seconds(&self) -> u64 {
#[cfg(test)]
if let Some(now) = self.server_time_override {
return now;
}
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
})
}
pub(super) fn live_node_descriptors(&self) -> Vec<NodeDescriptor> {
self.node_descriptors
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor
})
.collect()
}
pub(super) fn handle_attach_node(
&mut self,
tenant: String,
project: String,
node: String,
public_key: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
self.coordinator.ensure_tenant_active(&tenant)?;
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator.upsert_user(
tenant.clone(),
UserId::from("local-user"),
CredentialKind::CliDeviceSession,
);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), "local");
self.coordinator.upsert_source_provider_config(
tenant.clone(),
project.clone(),
SourceProviderKind::Filesystem,
Digest::sha256("local-filesystem"),
);
self.coordinator.enroll_node(
tenant.clone(),
project.clone(),
node.clone(),
public_key,
"node:attach",
);
self.persist_durable_state()?;
Ok(CoordinatorResponse::NodeAttached {
node,
tenant,
project,
})
}
pub(super) fn handle_create_node_enrollment_grant(
&mut self,
tenant: String,
project: String,
actor_user: String,
ttl_seconds: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
self.coordinator.ensure_tenant_active(&tenant)?;
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator
.upsert_user(tenant.clone(), actor, CredentialKind::CliDeviceSession);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), "local");
self.coordinator.upsert_source_provider_config(
tenant.clone(),
project.clone(),
SourceProviderKind::Filesystem,
Digest::sha256("local-filesystem"),
);
let now_epoch_seconds = self.current_epoch_seconds()?;
self.enrollment_grants.retain(|_, grant| {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
if self
.enrollment_grants
.values()
.filter(|grant| grant.tenant == tenant && grant.project == project)
.count()
>= super::MAX_ENROLLMENT_GRANTS_PER_PROJECT
{
return Err(CoordinatorServiceError::Protocol(
"node enrollment grant limit reached for this project; consume a grant or wait for one to expire"
.to_owned(),
));
}
let grant =
generate_opaque_token("node_grant").map_err(CoordinatorServiceError::Protocol)?;
let ttl_seconds = bounded_ttl(ttl_seconds, self.admission.max_node_enrollment_ttl_seconds);
let scope = "node:attach".to_owned();
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let enrollment = self.coordinator.create_node_enrollment_grant(
tenant.clone(),
project.clone(),
grant.clone(),
scope.clone(),
expires_at_epoch_seconds,
);
self.enrollment_grants
.insert(enrollment_grant_key(&tenant, &project, &grant), enrollment);
self.persist_durable_state()?;
Ok(CoordinatorResponse::NodeEnrollmentGrantCreated {
tenant,
project,
grant,
scope,
expires_at_epoch_seconds,
})
}
pub(super) fn handle_exchange_node_enrollment_grant(
&mut self,
tenant: String,
project: String,
node: String,
public_key: String,
enrollment_grant: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let now_epoch_seconds = self.current_epoch_seconds()?;
self.enrollment_grants.retain(|_, grant| {
!grant.consumed && grant.expires_at_epoch_seconds >= now_epoch_seconds
});
self.coordinator.ensure_tenant_active(&tenant)?;
let grant_key = enrollment_grant_key(&tenant, &project, &enrollment_grant);
let grant =
self.enrollment_grants
.get_mut(&grant_key)
.ok_or(CoordinatorError::Enrollment(
clusterflux_core::EnrollmentError::Expired,
))?;
let credential = self.coordinator.exchange_node_enrollment_grant(
grant,
node.clone(),
&public_key,
"node:attach",
now_epoch_seconds,
)?;
self.enrollment_grants.remove(&grant_key);
self.persist_durable_state()?;
Ok(CoordinatorResponse::NodeEnrollmentExchanged {
node,
tenant,
project,
credential,
})
}
pub(super) fn handle_node_heartbeat(
&mut self,
node: String,
node_signature: Option<NodeSignedRequest>,
payload_digest: &Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let node = NodeId::new(node);
self.authenticate_node_request(&node, node_signature, "node_heartbeat", payload_digest)?;
Ok(CoordinatorResponse::NodeHeartbeat {
node,
epoch: self.coordinator.coordinator_epoch(),
})
}
pub(super) fn handle_report_node_capabilities(
&mut self,
tenant: String,
project: String,
node: String,
capabilities: NodeCapabilities,
cached_environment_digests: Vec<Digest>,
dependency_cache_digests: Vec<Digest>,
source_snapshots: Vec<Digest>,
artifact_locations: Vec<String>,
direct_connectivity: bool,
_online: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"node capability report is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
capabilities.validate_public_report()?;
for (kind, count) in [
("cached environments", cached_environment_digests.len()),
("dependency caches", dependency_cache_digests.len()),
("source snapshots", source_snapshots.len()),
("artifact locations", artifact_locations.len()),
] {
if count > super::MAX_NODE_REPORTED_OBJECTS_PER_KIND {
return Err(CoordinatorServiceError::Protocol(format!(
"node capability report contains {count} {kind}; limit is {}",
super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
)));
}
}
if cached_environment_digests
.iter()
.chain(&dependency_cache_digests)
.chain(&source_snapshots)
.any(|digest| !digest.is_valid_sha256())
{
return Err(CoordinatorServiceError::Protocol(
"node capability report contains an invalid digest".to_owned(),
));
}
if artifact_locations.iter().any(|artifact| {
artifact.trim().is_empty()
|| artifact.len() > 256
|| artifact
.chars()
.any(|character| matches!(character, '/' | '\\' | '\0'))
}) {
return Err(CoordinatorServiceError::Protocol(
"node capability report contains an invalid artifact id".to_owned(),
));
}
let artifact_locations = artifact_locations
.into_iter()
.map(ArtifactId::new)
.collect::<BTreeSet<_>>();
self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations);
let online = self.node_is_live(&node);
self.node_descriptors.insert(
node.clone(),
NodeDescriptor {
id: node.clone(),
tenant,
project,
capabilities,
cached_environments: cached_environment_digests.into_iter().collect(),
dependency_caches: dependency_cache_digests.into_iter().collect(),
source_snapshots: source_snapshots.into_iter().collect(),
artifact_locations,
direct_connectivity,
online,
},
);
Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
node,
node_descriptors: self.node_descriptors.len(),
})
}
pub(super) fn handle_list_node_descriptors(
&mut self,
tenant: String,
project: String,
actor_user: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let descriptors = self
.live_node_descriptors()
.into_iter()
.filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
.collect();
Ok(CoordinatorResponse::NodeDescriptors { descriptors, actor })
}
pub(super) fn handle_revoke_node_credential(
&mut self,
tenant: String,
project: String,
actor_user: String,
node: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let node = NodeId::new(node);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: project.clone(),
actor: Actor::User(actor.clone()),
};
self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node);
let queued_assignments_removed = self
.task_assignments
.remove(&(tenant.clone(), project.clone(), node.clone()))
.map_or(0, |assignments| assignments.len());
self.active_tasks
.retain(|(task_tenant, task_project, _, task_node, _)| {
task_tenant != &tenant || task_project != &project || task_node != &node
});
self.task_cancellations
.retain(|(task_tenant, task_project, _, task_node, _)| {
task_tenant != &tenant || task_project != &project || task_node != &node
});
self.task_aborts
.retain(|(task_tenant, task_project, _, task_node, _)| {
task_tenant != &tenant || task_project != &project || task_node != &node
});
self.task_placements
.retain(|(task_tenant, task_project, _, task_node, _), _| {
task_tenant != &tenant || task_project != &project || task_node != &node
});
self.persist_durable_state()?;
Ok(CoordinatorResponse::NodeCredentialRevoked {
node,
tenant,
project,
actor,
descriptor_removed,
queued_assignments_removed,
})
}
pub(super) fn authenticate_node_request(
&mut self,
node: &NodeId,
node_signature: Option<NodeSignedRequest>,
request_kind: &str,
payload_digest: &Digest,
) -> Result<(), CoordinatorServiceError> {
let identity = self
.coordinator
.node_identity(node)
.ok_or(CoordinatorError::UnknownNode)?;
let signature = node_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
"node request requires a signed proof of enrolled private-key possession"
.to_owned(),
)
})?;
if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
return Err(CoordinatorError::Unauthorized(
"node signed request nonce is missing or invalid".to_owned(),
)
.into());
}
let now_epoch_seconds = unix_timestamp_seconds();
if signature
.issued_at_epoch_seconds
.abs_diff(now_epoch_seconds)
> super::NODE_SIGNATURE_WINDOW_SECONDS
{
return Err(CoordinatorError::Unauthorized(
"node signed request is expired or outside the allowed clock skew".to_owned(),
)
.into());
}
let replay_key = (node.clone(), signature.nonce.clone());
self.node_replay_nonces.retain(|_, accepted_at| {
now_epoch_seconds <= accepted_at.saturating_add(super::NODE_SIGNATURE_WINDOW_SECONDS)
});
if self.node_replay_nonces.contains_key(&replay_key) {
return Err(CoordinatorError::Unauthorized(
"node signed request nonce has already been used".to_owned(),
)
.into());
}
verify_node_request_signature(
&identity.public_key,
node,
request_kind,
payload_digest,
&signature,
)
.map_err(CoordinatorError::Unauthorized)?;
if self
.node_replay_nonces
.keys()
.filter(|(retained_node, _)| retained_node == node)
.count()
>= super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
{
return Err(CoordinatorError::Unauthorized(
"node signed request replay window is full; retry after the bounded signature window advances"
.to_owned(),
)
.into());
}
self.node_replay_nonces
.insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
descriptor.online = true;
}
Ok(())
}
}
fn unix_timestamp_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}

307
crates/clusterflux-coordinator/src/service/panels.rs Normal file
View file

@ -0,0 +1,307 @@
use clusterflux_core::{
Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
ProcessId, ProjectId, RateLimit, TenantId, UserId,
};
use crate::CoordinatorError;
use super::artifact_id_from_path;
use super::keys::panel_stop_key;
use super::{CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
impl CoordinatorService {
pub(super) fn clear_operator_panel_state(
&mut self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) {
let stop_key = panel_stop_key(tenant, project, process);
self.panel_snapshots.remove(&stop_key);
self.stopped_panels.remove(&stop_key);
self.panel_event_limits
.retain(|(event_tenant, event_project, event_process, _), _| {
event_tenant != tenant || event_project != project || event_process != process
});
}
pub(super) fn handle_render_operator_panel(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
max_download_bytes: u64,
stopped: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let stop_key = panel_stop_key(&tenant, &project, &process);
let panel = if stopped {
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
self.stopped_panels.insert(stop_key);
let stop_key = panel_stop_key(&tenant, &project, &process);
let panel = match self.panel_snapshots.get(&stop_key).cloned() {
Some(panel) => stopped_panel_snapshot(panel),
None => self.render_operator_panel(
tenant.clone(),
project.clone(),
process.clone(),
UserId::new(actor_user),
max_download_bytes,
true,
)?,
};
self.panel_snapshots.insert(stop_key, panel.clone());
panel
} else {
self.stopped_panels.remove(&stop_key);
let panel = self.render_operator_panel(
tenant.clone(),
project.clone(),
process.clone(),
UserId::new(actor_user),
max_download_bytes,
false,
)?;
self.panel_snapshots.insert(stop_key, panel.clone());
panel
};
Ok(CoordinatorResponse::OperatorPanel { panel })
}
pub(super) fn handle_submit_panel_event(
&mut self,
tenant: String,
project: String,
process: String,
widget_id: String,
kind: PanelEventKind,
max_events: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let stop_key = panel_stop_key(&tenant, &project, &process);
let stopped = self.stopped_panels.contains(&stop_key);
let panel = if stopped {
match self.panel_snapshots.get(&stop_key).cloned() {
Some(panel) => stopped_panel_snapshot(panel),
None => {
let panel = self.render_operator_panel(
tenant.clone(),
project.clone(),
process.clone(),
UserId::from("panel-user"),
self.quota.download_limit(),
true,
)?;
self.panel_snapshots.insert(stop_key.clone(), panel.clone());
panel
}
}
} else {
let panel = self.render_operator_panel(
tenant.clone(),
project.clone(),
process.clone(),
UserId::from("panel-user"),
self.quota.download_limit(),
false,
)?;
self.panel_snapshots.insert(stop_key, panel.clone());
panel
};
let event = PanelEvent {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
widget_id: widget_id.clone(),
kind,
};
let limit_key = (tenant, project, process, widget_id);
let mut limit = self
.panel_event_limits
.get(&limit_key)
.cloned()
.unwrap_or(RateLimit {
max_events,
used_events: 0,
});
limit.max_events = max_events;
panel.accept_event(&event, &mut limit)?;
self.panel_event_limits.insert(limit_key, limit.clone());
Ok(CoordinatorResponse::PanelEventAccepted {
used_events: limit.used_events,
max_events: limit.max_events,
})
}
fn render_operator_panel(
&self,
tenant: TenantId,
project: ProjectId,
process: ProcessId,
actor_user: UserId,
max_download_bytes: u64,
stopped: bool,
) -> Result<PanelState, CoordinatorServiceError> {
self.ensure_operator_panel_scope(&tenant, &project, &process)?;
let events = self
.task_events
.iter()
.filter(|event| {
event.tenant == tenant && event.project == project && event.process == process
})
.collect::<Vec<_>>();
let completed = events
.iter()
.filter(|event| event.status_code == Some(0))
.count() as u64;
let total = events.len().max(1) as u64;
let stdout_bytes = events.iter().map(|event| event.stdout_bytes).sum::<u64>();
let stderr_bytes = events.iter().map(|event| event.stderr_bytes).sum::<u64>();
let last_task = events.last().map(|event| event.task.clone());
let mut panel = PanelState::new(tenant.clone(), project.clone(), process.clone());
if stopped {
panel.freeze_program_ui_events();
}
panel.add_widget(PanelWidget {
id: "process-status".to_owned(),
label: "Process Status".to_owned(),
kind: PanelWidgetKind::Text {
value: if stopped {
"stopped".to_owned()
} else {
"running".to_owned()
},
},
})?;
panel.add_widget(PanelWidget {
id: "task-progress".to_owned(),
label: "Tasks".to_owned(),
kind: PanelWidgetKind::Progress {
current: completed,
total,
},
})?;
panel.add_widget(PanelWidget {
id: "task-summary".to_owned(),
label: "Task Summary".to_owned(),
kind: PanelWidgetKind::Text {
value: if events.is_empty() {
"no task events recorded".to_owned()
} else {
events
.iter()
.map(|event| {
format!(
"{} [{}]:{:?}:{}",
event.task_definition, event.task, event.status_code, event.node
)
})
.collect::<Vec<_>>()
.join(", ")
},
},
})?;
panel.add_widget(PanelWidget {
id: "recent-logs".to_owned(),
label: "Recent Logs".to_owned(),
kind: PanelWidgetKind::Text {
value: format!("stdout={stdout_bytes} stderr={stderr_bytes}"),
},
})?;
panel.add_widget(PanelWidget {
id: "debug-process".to_owned(),
label: "Debug Process".to_owned(),
kind: PanelWidgetKind::Button {
action: "debug-process".to_owned(),
},
})?;
panel.add_widget(PanelWidget {
id: "cancel-process".to_owned(),
label: "Cancel Process".to_owned(),
kind: PanelWidgetKind::Button {
action: "cancel-process".to_owned(),
},
})?;
if last_task.is_some() {
panel.add_widget(PanelWidget {
id: "restart-selected-task".to_owned(),
label: "Restart Selected Task".to_owned(),
kind: PanelWidgetKind::Button {
action: "restart-task".to_owned(),
},
})?;
}
let mut actions = vec![
clusterflux_core::ControlPlaneAction::DebugProcess,
clusterflux_core::ControlPlaneAction::CancelProcess,
];
if let Some(task) = last_task.clone() {
actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
}
panel.set_control_plane_actions(actions);
if let Some(path) = events
.iter()
.rev()
.find_map(|event| event.artifact_path.as_ref())
{
let artifact = artifact_id_from_path(path);
let context = clusterflux_core::AuthContext {
tenant,
project,
actor: Actor::User(actor_user),
};
panel.add_download_widget_from_action(
"download-artifact",
"Download Artifact",
self.artifact_registry.download_action(
&context,
&artifact,
&DownloadPolicy {
max_bytes: max_download_bytes,
},
),
)?;
}
Ok(panel)
}
fn ensure_operator_panel_scope(
&self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
) -> Result<(), CoordinatorServiceError> {
let active = self
.coordinator
.active_process(tenant, project, process)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"operator panel requires an active virtual process".to_owned(),
)
})?;
debug_assert_eq!(active.tenant, *tenant);
debug_assert_eq!(active.project, *project);
Ok(())
}
}
fn stopped_panel_snapshot(mut panel: PanelState) -> PanelState {
panel.freeze_program_ui_events();
if let Some(status) = panel.widgets.get_mut("process-status") {
status.kind = PanelWidgetKind::Text {
value: "stopped".to_owned(),
};
}
panel
}

698
crates/clusterflux-coordinator/src/service/process_launch.rs Normal file
View file

@ -0,0 +1,698 @@
use std::collections::BTreeMap;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskBoundaryValue, TaskCheckpoint,
TaskDispatch, TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath,
WasmTaskInvocation,
};
use crate::CoordinatorError;
use super::keys::{process_control_key, task_control_key, task_restart_key};
use super::{
CoordinatorResponse, CoordinatorService, CoordinatorServiceError, TaskAssignment, WorkflowActor,
};
use super::processes::*;
use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
impl CoordinatorService {
pub(super) fn capture_task_restart_checkpoint(
&mut self,
assignment: &TaskAssignment,
) -> Result<(), CoordinatorServiceError> {
let task_spec = &assignment.task_spec;
let environment_digest = task_spec.environment_digest.clone().unwrap_or_else(|| {
task_spec.environment.as_ref().map_or_else(
|| Digest::sha256("clusterflux.environment.unconstrained.v1"),
|environment| {
Digest::sha256(
serde_json::to_vec(environment)
.expect("serializable environment requirements"),
)
},
)
});
let task_entrypoint = match &task_spec.dispatch {
clusterflux_core::TaskDispatch::CoordinatorNodeWasm { export, .. } => export
.clone()
.or_else(|| assignment_task_descriptor(assignment)?.get("export")?.as_str().map(str::to_owned))
.ok_or_else(|| {
CoordinatorServiceError::Protocol(format!(
"cannot capture restart checkpoint for task `{}`: bundle descriptor omitted its Wasm export",
task_spec.task_definition
))
})?,
};
let mut objects = BTreeMap::new();
let mut missing_required_artifact = false;
for artifact in &task_spec.required_artifacts {
let Some(metadata) = self.artifact_registry.metadata(artifact) else {
missing_required_artifact = true;
continue;
};
if metadata.tenant != assignment.tenant
|| metadata.project != assignment.project
|| metadata.retaining_nodes.is_empty()
{
missing_required_artifact = true;
continue;
}
let path = VfsPath::new(format!("/vfs/artifacts/{artifact}"))
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
objects.insert(
path.clone(),
VfsObject {
path,
digest: metadata.digest.clone(),
size: metadata.size,
producer: metadata.producer_task.clone(),
node: metadata.producer_node.clone(),
},
);
}
let checkpoint = TaskCheckpoint {
task: assignment.task.clone(),
boundary: CheckpointBoundary {
task_entrypoint,
serialized_args: Digest::sha256(serde_json::to_vec(&task_spec.args)?),
environment_digest,
vfs_epoch: task_spec.vfs_epoch,
task_abi: assignment_task_compatibility(assignment)
.unwrap_or(Digest::sha256(serde_json::to_vec(&task_spec.dispatch)?)),
},
vfs_manifest: VfsManifest {
epoch: task_spec.vfs_epoch,
producer: assignment.task.clone(),
node: assignment.node.clone(),
objects,
large_bytes_uploaded: false,
},
depends_on_live_stack: false,
depends_on_live_socket: false,
depends_on_ephemeral_artifact_durability: missing_required_artifact,
};
let key = task_restart_key(
&assignment.tenant,
&assignment.project,
&assignment.process,
&assignment.task,
);
self.task_restart_checkpoint_order
.retain(|retained| retained != &key);
self.task_restart_checkpoints.insert(
key.clone(),
TaskRestartCheckpoint {
checkpoint,
assignment: assignment.clone(),
},
);
self.task_restart_checkpoint_order.push_back(key);
while self
.task_restart_checkpoint_order
.iter()
.filter(|(tenant, project, process, _)| {
tenant == &assignment.tenant
&& project == &assignment.project
&& process == &assignment.process
})
.count()
> super::MAX_RESTART_CHECKPOINTS_PER_PROCESS
{
let Some(index) = self.task_restart_checkpoint_order.iter().position(
|(tenant, project, process, _)| {
tenant == &assignment.tenant
&& project == &assignment.project
&& process == &assignment.process
},
) else {
break;
};
if let Some(expired) = self.task_restart_checkpoint_order.remove(index) {
self.task_restart_checkpoints.remove(&expired);
}
}
while self.task_restart_checkpoint_order.len() > super::MAX_RESTART_CHECKPOINTS_TOTAL {
if let Some(expired) = self.task_restart_checkpoint_order.pop_front() {
self.task_restart_checkpoints.remove(&expired);
}
}
Ok(())
}
pub(super) fn handle_schedule_task(
&mut self,
tenant: String,
project: String,
environment: Option<clusterflux_core::EnvironmentRequirements>,
environment_digest: Option<Digest>,
required_capabilities: Vec<clusterflux_core::Capability>,
dependency_cache: Option<Digest>,
source_snapshot: Option<Digest>,
required_artifacts: Vec<String>,
prefer_node: Option<String>,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let now_epoch_seconds = self.current_epoch_seconds()?;
let request = PlacementRequest {
tenant: tenant.clone(),
project: project.clone(),
environment,
environment_digest,
environment_cache_required: false,
required_capabilities: required_capabilities.into_iter().collect(),
dependency_cache,
source_snapshot,
required_artifacts: required_artifacts
.into_iter()
.map(ArtifactId::new)
.collect(),
quota_available: self
.quota
.can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
.is_ok(),
policy_allowed: self.admission.workflow_placement_allowed,
prefer_node: prefer_node.map(NodeId::new),
};
let nodes = self.live_node_descriptors();
let placement = DefaultScheduler.place(&nodes, &request)?;
Ok(CoordinatorResponse::TaskPlacement { placement })
}
pub(super) fn handle_launch_task(
&mut self,
tenant: String,
project: String,
actor_user: Option<String>,
actor_agent: Option<String>,
agent_public_key_fingerprint: Option<Digest>,
agent_signature: Option<AgentSignedRequest>,
request_payload_digest: Option<&Digest>,
task_spec: TaskSpec,
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
if matches!(
&task_spec.dispatch,
TaskDispatch::CoordinatorNodeWasm {
abi: clusterflux_core::WasmExportAbi::EntrypointV1,
..
}
) {
return self.handle_launch_coordinator_main(
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
request_payload_digest,
task_spec,
wasm_module_base64,
);
}
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = task_spec.process.clone();
let task = task_spec.task_instance.clone();
let actor = self.workflow_actor(
&tenant,
&project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
request_payload_digest,
"launch_task",
&process,
Some(&task),
)?;
self.handle_launch_task_with_actor(
tenant,
project,
actor,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
)
}
#[allow(clippy::too_many_arguments)]
pub(super) fn handle_launch_child_task(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
parent_task: String,
task_spec: TaskSpec,
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let parent_task = TaskInstanceId::new(parent_task);
if task_spec.process != process {
return Err(CoordinatorError::Unauthorized(
"child task must remain in its parent virtual process".to_owned(),
)
.into());
}
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
let parent_key = task_control_key(&tenant, &project, &process, &node, &parent_task);
if !self.active_tasks.contains(&parent_key) {
return Err(CoordinatorError::Unauthorized(
"child task launch requires a currently active parent task on the signed node"
.to_owned(),
)
.into());
}
let actor = WorkflowActor {
kind: "task".to_owned(),
user: None,
agent: None,
credential_kind: CredentialKind::TaskCredential,
public_key_fingerprint: None,
authenticated_without_browser: true,
scopes: vec!["process:spawn-child".to_owned()],
};
self.handle_launch_task_with_actor(
tenant,
project,
actor,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
)
}
pub(super) fn handle_launch_task_with_actor(
&mut self,
tenant: TenantId,
project: ProjectId,
actor: WorkflowActor,
task_spec: TaskSpec,
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
if task_spec.tenant != tenant || task_spec.project != project {
return Err(CoordinatorError::Unauthorized(
"task specification is outside the authenticated tenant/project scope".to_owned(),
)
.into());
}
if !task_spec.product_mode_uses_remote_dispatch() {
return Err(CoordinatorError::Unauthorized(
"task specification must use the Wasm coordinator/node dispatch ABI".to_owned(),
)
.into());
}
if task_spec
.environment_id
.as_deref()
.is_some_and(|environment| environment.trim().is_empty() || environment.len() > 128)
{
return Err(CoordinatorError::Unauthorized(
"task specification environment id is invalid".to_owned(),
)
.into());
}
let process = task_spec.process.clone();
let task = task_spec.task_instance.clone();
let active = self
.coordinator
.active_process(&tenant, &project, &process)
.cloned()
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"task launch requires an active coordinator-side virtual process".to_owned(),
)
})?;
debug_assert_eq!(active.tenant, tenant);
debug_assert_eq!(active.project, project);
if self
.process_cancellations
.contains(&process_control_key(&tenant, &project, &process))
{
return Err(CoordinatorError::Unauthorized(
"task launch is blocked because the virtual process is cancelling".to_owned(),
)
.into());
}
if self.task_instance_exists(&tenant, &project, &process, &task) {
return Err(CoordinatorServiceError::Protocol(format!(
"task instance {task} already exists in virtual process {process}; every spawn must use a fresh task-instance id"
)));
}
let in_flight = self
.active_tasks
.iter()
.filter(|(task_tenant, task_project, task_process, _, _)| {
task_tenant == &tenant && task_project == &project && task_process == &process
})
.count()
+ self
.pending_task_launches
.iter()
.filter(|pending| {
pending.tenant == tenant
&& pending.project == project
&& pending.process == process
})
.count();
if in_flight >= super::MAX_IN_FLIGHT_TASKS_PER_PROCESS {
return Err(CoordinatorServiceError::Protocol(format!(
"virtual process task limit of {} reached; join or cancel existing work before spawning more",
super::MAX_IN_FLIGHT_TASKS_PER_PROCESS
)));
}
if task_spec.vfs_epoch != active.coordinator_epoch {
return Err(CoordinatorError::Unauthorized(format!(
"task specification VFS epoch {} does not match active process epoch {}",
task_spec.vfs_epoch, active.coordinator_epoch
))
.into());
}
let bundle_digest = task_spec.bundle_digest.as_ref().ok_or_else(|| {
CoordinatorError::Unauthorized(
"Wasm task specification is missing its bundle digest".to_owned(),
)
})?;
if !bundle_digest.is_valid_sha256() {
return Err(CoordinatorError::Unauthorized(
"Wasm task specification has an invalid bundle digest".to_owned(),
)
.into());
}
let module = BASE64_STANDARD
.decode(&wasm_module_base64)
.map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"Wasm task module is not valid base64: {error}"
))
})?;
let actual_digest = Digest::sha256(&module);
if &actual_digest != bundle_digest {
return Err(CoordinatorError::Unauthorized(format!(
"Wasm task module digest does not match bundle digest: expected {bundle_digest}, actual {actual_digest}"
))
.into());
}
WasmTaskInvocation::new(
task_spec.task_definition.clone(),
task.clone(),
task_spec.args.clone(),
)
.validate()
.map_err(CoordinatorServiceError::Protocol)?;
for artifact in &task_spec.required_artifacts {
let metadata = self.artifact_registry.metadata(artifact).ok_or_else(|| {
CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is unavailable or has expired"
))
})?;
if metadata.tenant != tenant || metadata.project != project {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} is outside the task tenant/project scope"
))
.into());
}
if metadata.retaining_nodes.is_empty() {
return Err(CoordinatorError::Unauthorized(format!(
"required artifact {artifact} has no retaining node"
))
.into());
}
}
VfsPath::new(&artifact_path)
.map_err(|error| CoordinatorServiceError::InvalidArtifactPath(error.to_string()))?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota
.can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
let request = PlacementRequest {
tenant: tenant.clone(),
project: project.clone(),
environment: task_spec.environment.clone(),
environment_digest: task_spec.environment_digest.clone(),
environment_cache_required: task_spec.environment_id.is_some()
&& task_spec.environment.is_none(),
required_capabilities: task_spec.required_capabilities.clone(),
dependency_cache: task_spec.dependency_cache.clone(),
source_snapshot: task_spec.source_snapshot.clone(),
required_artifacts: task_spec.required_artifacts.iter().cloned().collect(),
quota_available: self
.quota
.can_charge_workflow_spawn(&tenant, &project, now_epoch_seconds)
.is_ok(),
policy_allowed: self.admission.workflow_placement_allowed,
prefer_node: None,
};
let nodes = self.live_node_descriptors();
let placement = match DefaultScheduler.place(&nodes, &request) {
Ok(placement) => placement,
Err(err) if wait_for_node => {
let reason = if err.message.is_empty() {
"waiting for any capable node".to_owned()
} else {
err.message
};
let charged_spawns =
self.quota
.charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
self.begin_task_attempt(&task_spec, None, Some(&artifact_path), true)?;
self.pending_task_launches.push_back(PendingTaskLaunch {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task: task.clone(),
request,
epoch: active.coordinator_epoch,
artifact_path,
task_spec,
wasm_module_base64,
});
return Ok(CoordinatorResponse::TaskQueued {
process,
task,
actor,
reason,
charged_spawns,
queued_tasks: self.pending_task_launches.len(),
});
}
Err(err) => return Err(err.into()),
};
let charged_spawns =
self.quota
.charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
let assignment = TaskAssignment {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task: task.clone(),
node: placement.node.clone(),
epoch: active.coordinator_epoch,
artifact_path,
task_spec,
wasm_module_base64,
};
self.begin_task_attempt(
&assignment.task_spec,
Some(assignment.node.clone()),
Some(&assignment.artifact_path),
false,
)?;
self.capture_task_restart_checkpoint(&assignment)?;
let task_key = task_control_key(&tenant, &project, &process, &placement.node, &task);
self.task_placements
.insert(task_key.clone(), placement.clone());
self.active_tasks.insert(task_key);
self.task_assignments
.entry((tenant, project, placement.node.clone()))
.or_default()
.push_back(assignment.clone());
Ok(CoordinatorResponse::TaskLaunched {
process,
task,
actor,
placement,
assignment: Box::new(assignment),
charged_spawns,
})
}
fn task_instance_exists(
&self,
tenant: &TenantId,
project: &ProjectId,
process: &ProcessId,
task_instance: &clusterflux_core::TaskInstanceId,
) -> bool {
self.active_tasks.iter().any(
|(task_tenant, task_project, task_process, _, existing_instance)| {
task_tenant == tenant
&& task_project == project
&& task_process == process
&& existing_instance == task_instance
},
) || self.pending_task_launches.iter().any(|pending| {
&pending.tenant == tenant
&& &pending.project == project
&& &pending.process == process
&& &pending.task == task_instance
}) || (!self.restart_launches.contains(&task_restart_key(
tenant,
project,
process,
task_instance,
)) && self.task_events.iter().any(|event| {
&event.tenant == tenant
&& &event.project == project
&& &event.process == process
&& &event.task == task_instance
}))
}
pub(super) fn begin_task_attempt(
&mut self,
task_spec: &TaskSpec,
node: Option<NodeId>,
artifact_path: Option<&str>,
queued: bool,
) -> Result<String, CoordinatorServiceError> {
let key = task_restart_key(
&task_spec.tenant,
&task_spec.project,
&task_spec.process,
&task_spec.task_instance,
);
while !self.task_attempts.contains_key(&key)
&& self.task_attempts.len() >= super::MAX_TASK_ATTEMPT_HISTORIES
{
let removable = self.task_attempts.iter().find_map(|(candidate, attempts)| {
attempts
.iter()
.all(|attempt| {
!matches!(
&attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
.then(|| candidate.clone())
});
let Some(removable) = removable else {
return Err(CoordinatorServiceError::Protocol(
"task attempt history capacity is exhausted by active attempts".to_owned(),
));
};
self.task_attempts.remove(&removable);
}
let attempts = self.task_attempts.entry(key).or_default();
for attempt in attempts.iter_mut() {
attempt.current = false;
}
let attempt_id = clusterflux_core::generate_opaque_token("ta")
.map_err(CoordinatorServiceError::Protocol)?;
let attempt_number = u32::try_from(attempts.len() + 1).unwrap_or(u32::MAX);
let mut argument_summary = task_spec
.args
.iter()
.map(|argument| {
let mut value = serde_json::to_string(argument)
.unwrap_or_else(|_| "<invalid canonical argument>".to_owned());
value.truncate(value.len().min(1024));
value
})
.collect::<Vec<_>>();
argument_summary.truncate(64);
let mut handle_summary = task_spec
.required_artifacts
.iter()
.map(|artifact| format!("artifact:{artifact}"))
.collect::<Vec<_>>();
for argument in &task_spec.args {
if let TaskBoundaryValue::Structured(boundary) = argument {
handle_summary.extend(boundary.handles.iter().map(|handle| format!("{handle:?}")));
}
}
handle_summary.truncate(256);
attempts.push(TaskAttemptSnapshot {
process: task_spec.process.clone(),
task: task_spec.task_instance.clone(),
attempt_id: attempt_id.clone(),
attempt_number,
task_definition: task_spec.task_definition.clone(),
display_name: task_spec.task_definition.as_str().replace(['_', '-'], " "),
state: if queued {
TaskAttemptState::Queued
} else {
TaskAttemptState::Running
},
current: true,
node,
environment_id: task_spec.environment_id.clone(),
environment_digest: task_spec.environment_digest.clone(),
argument_summary,
handle_summary,
command_state: Some(if queued { "queued" } else { "running" }.to_owned()),
vfs_checkpoint: format!("vfs-epoch:{}", task_spec.vfs_epoch),
probe_symbol: None,
source_path: None,
source_line: None,
restart_compatible: true,
failure_policy: task_spec.failure_policy,
artifact_path: artifact_path.and_then(|path| VfsPath::new(path).ok()),
artifact_digest: None,
artifact_size_bytes: None,
status_code: None,
error: None,
});
if attempts.len() > 128 {
attempts.remove(0);
}
Ok(attempt_id)
}
pub(super) fn assign_task_attempt(&mut self, task_spec: &TaskSpec, node: NodeId) {
let key = task_restart_key(
&task_spec.tenant,
&task_spec.project,
&task_spec.process,
&task_spec.task_instance,
);
if let Some(attempt) = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
{
attempt.node = Some(node);
attempt.state = TaskAttemptState::Running;
attempt.command_state = Some("running".to_owned());
}
}
}
fn assignment_task_compatibility(assignment: &TaskAssignment) -> Option<Digest> {
let descriptor = assignment_task_descriptor(assignment)?;
serde_json::from_value(descriptor.get("restart_compatibility_hash")?.clone()).ok()
}
fn assignment_task_descriptor(assignment: &TaskAssignment) -> Option<serde_json::Value> {
let module = BASE64_STANDARD
.decode(&assignment.wasm_module_base64)
.ok()?;
let mut descriptors = super::main_runtime::task_descriptors(&module).ok()?;
descriptors.remove(assignment.task_spec.task_definition.as_str())
}

809
crates/clusterflux-coordinator/src/service/processes.rs Normal file
View file

@ -0,0 +1,809 @@
use std::collections::{BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
AgentId, AgentSignedRequest, CredentialKind, DefaultScheduler, Digest, NodeId,
PlacementRequest, ProcessId, ProjectId, RendezvousRequest, Scheduler, SourcePreparation,
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
};
use crate::CoordinatorError;
use super::keys::{process_control_key, task_control_key};
use super::{
CoordinatorResponse, CoordinatorService, CoordinatorServiceError, SourcePreparationDisposition,
SourcePreparationStatus, TaskAssignment, TaskCancellationTarget, VirtualProcessStatus,
WorkflowActor,
};
#[derive(Clone, Debug)]
pub(super) struct PendingTaskLaunch {
pub(super) tenant: TenantId,
pub(super) project: ProjectId,
pub(super) process: ProcessId,
pub(super) task: TaskInstanceId,
pub(super) request: PlacementRequest,
pub(super) epoch: u64,
pub(super) artifact_path: String,
pub(super) task_spec: TaskSpec,
pub(super) wasm_module_base64: String,
}
#[derive(Clone, Debug)]
pub(super) struct TaskRestartCheckpoint {
pub(super) checkpoint: TaskCheckpoint,
pub(super) assignment: TaskAssignment,
}
impl CoordinatorService {
pub(super) fn handle_poll_task_assignment(
&mut self,
tenant: String,
project: String,
node: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"task assignment poll is outside the enrolled tenant/project scope".to_owned(),
)
.into());
}
let assignment_key = (tenant.clone(), project.clone(), node.clone());
let assignment = self
.task_assignments
.get_mut(&assignment_key)
.and_then(VecDeque::pop_front);
if assignment.is_some() {
return Ok(CoordinatorResponse::TaskAssignment {
assignment: assignment.map(Box::new),
});
}
let assignment = self.assign_pending_task_to_node(&tenant, &project, &node)?;
Ok(CoordinatorResponse::TaskAssignment {
assignment: assignment.map(Box::new),
})
}
fn assign_pending_task_to_node(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
) -> Result<Option<TaskAssignment>, CoordinatorServiceError> {
let Some(descriptor) = self.node_descriptors.get(node).cloned() else {
return Ok(None);
};
let mut remaining = VecDeque::new();
let mut selected = None;
while let Some(pending) = self.pending_task_launches.pop_front() {
if selected.is_some() {
remaining.push_back(pending);
continue;
}
if &pending.tenant != tenant || &pending.project != project {
remaining.push_back(pending);
continue;
}
if self.process_cancellations.contains(&process_control_key(
&pending.tenant,
&pending.project,
&pending.process,
)) {
continue;
}
let Some(active) = self.coordinator.active_process(
&pending.tenant,
&pending.project,
&pending.process,
) else {
continue;
};
if active.tenant != pending.tenant
|| active.project != pending.project
|| active.coordinator_epoch != pending.epoch
{
continue;
}
let Ok(placement) =
DefaultScheduler.place(std::slice::from_ref(&descriptor), &pending.request)
else {
remaining.push_back(pending);
continue;
};
let assignment = TaskAssignment {
tenant: pending.tenant.clone(),
project: pending.project.clone(),
process: pending.process.clone(),
task: pending.task.clone(),
node: placement.node.clone(),
epoch: pending.epoch,
artifact_path: pending.artifact_path,
task_spec: pending.task_spec,
wasm_module_base64: pending.wasm_module_base64,
};
self.assign_task_attempt(&assignment.task_spec, assignment.node.clone());
self.capture_task_restart_checkpoint(&assignment)?;
let task_key = task_control_key(
&pending.tenant,
&pending.project,
&pending.process,
&placement.node,
&pending.task,
);
self.task_placements.insert(task_key.clone(), placement);
self.active_tasks.insert(task_key);
selected = Some(assignment);
}
self.pending_task_launches = remaining;
Ok(selected)
}
pub(super) fn handle_request_rendezvous(
&mut self,
scope: clusterflux_core::DataPlaneScope,
source: clusterflux_core::NodeEndpoint,
destination: clusterflux_core::NodeEndpoint,
direct_connectivity: bool,
failure_reason: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let now_epoch_seconds = self.current_epoch_seconds()?;
let charged_rendezvous_attempts = self.quota.charge_rendezvous_attempt(
&scope.tenant,
&scope.project,
now_epoch_seconds,
)?;
let plan = self.transport.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope,
source,
destination,
},
direct_connectivity,
failure_reason,
)?;
Ok(CoordinatorResponse::RendezvousPlan {
plan,
charged_rendezvous_attempts,
})
}
pub(super) fn handle_request_source_preparation(
&mut self,
tenant: String,
project: String,
provider: clusterflux_core::SourceProviderKind,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let preparation = SourcePreparation::node_task(tenant.clone(), project.clone(), provider);
let request = PlacementRequest {
tenant,
project,
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: preparation.required_capabilities.clone(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: Default::default(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let nodes = self.live_node_descriptors();
let disposition = match DefaultScheduler.place(&nodes, &request) {
Ok(placement) => SourcePreparationDisposition::Assigned {
node: placement.node,
},
Err(err) => SourcePreparationDisposition::Pending {
reason: if err.message.is_empty() {
"waiting for any capable node to prepare source".to_owned()
} else {
err.message
},
},
};
Ok(CoordinatorResponse::SourcePreparation {
status: SourcePreparationStatus {
preparation,
disposition,
},
})
}
pub(super) fn handle_complete_source_preparation(
&mut self,
tenant: String,
project: String,
node: String,
provider: clusterflux_core::SourceProviderKind,
source_snapshot: Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let node = NodeId::new(node);
let identity = self
.coordinator
.node_identity(&node)
.ok_or(CoordinatorError::UnknownNode)?;
if identity.tenant != tenant || identity.project != project {
return Err(CoordinatorError::Unauthorized(
"source preparation completion is outside the enrolled tenant/project scope"
.to_owned(),
)
.into());
}
let descriptor = self.node_descriptors.get_mut(&node).ok_or_else(|| {
CoordinatorError::Unauthorized(
"source preparation completion requires a node capability report".to_owned(),
)
})?;
if !descriptor.source_snapshots.contains(&source_snapshot)
&& descriptor.source_snapshots.len() >= super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
{
return Err(CoordinatorServiceError::Protocol(format!(
"node source snapshot retention limit of {} reached; refresh the node capability report",
super::MAX_NODE_REPORTED_OBJECTS_PER_KIND
)));
}
descriptor.source_snapshots.insert(source_snapshot.clone());
Ok(CoordinatorResponse::SourcePreparationCompleted {
node,
provider,
source_snapshot,
})
}
pub(super) fn handle_start_process(
&mut self,
tenant: String,
project: String,
actor_user: Option<String>,
actor_agent: Option<String>,
agent_public_key_fingerprint: Option<Digest>,
agent_signature: Option<AgentSignedRequest>,
request_payload_digest: Option<&Digest>,
process: String,
restart: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
self.coordinator.ensure_tenant_active(&tenant)?;
let actor = self.workflow_actor(
&tenant,
&project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
request_payload_digest,
"start_process",
&process,
None,
)?;
let replacing_existing = if let Some(active) = self
.coordinator
.active_process_for_project(&tenant, &project)
{
if active.id != process || !restart {
return Err(CoordinatorError::Unauthorized(format!(
"project already has active virtual process {}; attach to or restart it, request cooperative cancellation, abort it, or use another Coordinator Project",
active.id
))
.into());
}
true
} else {
false
};
if replacing_existing {
self.main_runtime.interrupt_process(
&tenant,
&project,
&process,
"virtual process incarnation replaced",
);
self.main_runtime
.controls
.remove(&process_control_key(&tenant, &project, &process));
}
let now_epoch_seconds = self.current_epoch_seconds()?;
let charged_spawns =
self.quota
.charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
self.process_cancellations
.remove(&process_control_key(&tenant, &project, &process));
self.process_aborts
.remove(&process_control_key(&tenant, &project, &process));
self.clear_debug_state_for_process(&tenant, &project, &process);
self.clear_operator_panel_state(&tenant, &project, &process);
self.task_cancellations
.retain(|(task_tenant, task_project, task_process, _, _)| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.task_aborts
.retain(|(task_tenant, task_project, task_process, _, _)| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.active_tasks
.retain(|(task_tenant, task_project, task_process, _, _)| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.task_placements
.retain(|(task_tenant, task_project, task_process, _, _), _| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != tenant
|| assignment.project != project
|| assignment.process != process
});
!assignments.is_empty()
});
self.pending_task_launches.retain(|pending| {
pending.tenant != tenant || pending.project != project || pending.process != process
});
self.task_restart_checkpoints.retain(
|(checkpoint_tenant, checkpoint_project, checkpoint_process, _), _| {
checkpoint_tenant != &tenant
|| checkpoint_project != &project
|| checkpoint_process != &process
},
);
self.task_restart_checkpoint_order.retain(
|(checkpoint_tenant, checkpoint_project, checkpoint_process, _)| {
checkpoint_tenant != &tenant
|| checkpoint_project != &project
|| checkpoint_process != &process
},
);
self.task_events.retain(|event| {
event.tenant != tenant || event.project != project || event.process != process
});
self.task_attempts
.retain(|(attempt_tenant, attempt_project, attempt_process, _), _| {
attempt_tenant != &tenant
|| attempt_project != &project
|| attempt_process != &process
});
self.restart_launches
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
attempt_tenant != &tenant
|| attempt_project != &project
|| attempt_process != &process
});
self.debug_audit_events.retain(|event| {
event.tenant != tenant || event.project != project || event.process != process
});
self.coordinator
.start_process(tenant, project, process.clone());
Ok(CoordinatorResponse::ProcessStarted {
process,
epoch: self.coordinator.coordinator_epoch(),
actor,
charged_spawns,
})
}
pub(super) fn handle_reconnect_node(
&mut self,
node: String,
process: String,
epoch: u64,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let node = NodeId::new(node);
let process = ProcessId::new(process);
self.coordinator
.reconnect_node(&node, Some((&process, epoch)))?;
Ok(CoordinatorResponse::NodeReconnected { node, process })
}
pub(super) fn handle_cancel_task(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.coordinator
.authorize_node_for_process(&node, &tenant, &project, &process)?;
let active = self
.coordinator
.active_process(&tenant, &project, &process)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"task cancellation requires an active virtual process".to_owned(),
)
})?;
if !active.connected_nodes.contains(&node) {
return Err(CoordinatorError::Unauthorized(
"task cancellation target node is not connected to the virtual process".to_owned(),
)
.into());
}
self.task_cancellations
.insert(task_control_key(&tenant, &project, &process, &node, &task));
Ok(CoordinatorResponse::TaskCancellationRequested {
process,
task,
node,
})
}
pub(super) fn handle_cancel_process(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let _actor_user = actor_user;
let active = self
.coordinator
.active_process(&tenant, &project, &process)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"process cancellation requires an active virtual process".to_owned(),
)
})?;
debug_assert_eq!(active.tenant, tenant);
debug_assert_eq!(active.project, project);
self.process_cancellations
.insert(process_control_key(&tenant, &project, &process));
self.main_runtime.interrupt_process(
&tenant,
&project,
&process,
"virtual process cancellation requested",
);
self.clear_debug_state_for_process(&tenant, &project, &process);
self.pending_task_launches.retain(|pending| {
pending.tenant != tenant || pending.project != project || pending.process != process
});
let mut cancelled_tasks = Vec::new();
let mut affected_nodes = BTreeSet::new();
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
if task_tenant == &tenant && task_project == &project && task_process == &process {
self.task_cancellations
.insert(task_control_key(&tenant, &project, &process, node, task));
affected_nodes.insert(node.clone());
cancelled_tasks.push(TaskCancellationTarget {
process: process.clone(),
task: task.clone(),
node: node.clone(),
});
}
}
let process_key = process_control_key(&tenant, &project, &process);
if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
self.coordinator
.abort_process(&tenant, &project, &process)?;
self.clear_operator_panel_state(&tenant, &project, &process);
}
Ok(CoordinatorResponse::ProcessCancellationRequested {
process,
cancelled_tasks,
affected_nodes: affected_nodes.into_iter().collect(),
})
}
pub(super) fn handle_abort_process(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let _actor_user = actor_user;
let active = self
.coordinator
.active_process(&tenant, &project, &process)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"process abort requires an active virtual process".to_owned(),
)
})?;
debug_assert_eq!(active.tenant, tenant);
debug_assert_eq!(active.project, project);
let process_key = process_control_key(&tenant, &project, &process);
self.process_cancellations.remove(&process_key);
self.task_cancellations
.retain(|(task_tenant, task_project, task_process, _, _)| {
task_tenant != &tenant || task_project != &project || task_process != &process
});
self.process_aborts.insert(process_key);
self.main_runtime
.interrupt_process(&tenant, &project, &process, "virtual process aborted");
self.main_runtime
.controls
.remove(&process_control_key(&tenant, &project, &process));
self.clear_debug_state_for_process(&tenant, &project, &process);
self.clear_operator_panel_state(&tenant, &project, &process);
self.pending_task_launches.retain(|pending| {
pending.tenant != tenant || pending.project != project || pending.process != process
});
self.task_assignments.retain(|_, assignments| {
assignments.retain(|assignment| {
assignment.tenant != tenant
|| assignment.project != project
|| assignment.process != process
});
!assignments.is_empty()
});
let mut aborted_tasks = Vec::new();
let mut affected_nodes = BTreeSet::new();
for (task_tenant, task_project, task_process, node, task) in self.active_tasks.iter() {
if task_tenant == &tenant && task_project == &project && task_process == &process {
self.task_aborts
.insert(task_control_key(&tenant, &project, &process, node, task));
affected_nodes.insert(node.clone());
aborted_tasks.push(TaskCancellationTarget {
process: process.clone(),
task: task.clone(),
node: node.clone(),
});
}
}
self.coordinator
.abort_process(&tenant, &project, &process)?;
let active_restart_tasks = aborted_tasks
.iter()
.map(|target| target.task.clone())
.collect::<BTreeSet<_>>();
self.task_restart_checkpoints.retain(
|(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task), _| {
checkpoint_tenant != &tenant
|| checkpoint_project != &project
|| checkpoint_process != &process
|| active_restart_tasks.contains(checkpoint_task)
},
);
self.task_restart_checkpoint_order.retain(
|(checkpoint_tenant, checkpoint_project, checkpoint_process, checkpoint_task)| {
checkpoint_tenant != &tenant
|| checkpoint_project != &project
|| checkpoint_process != &process
|| active_restart_tasks.contains(checkpoint_task)
},
);
if aborted_tasks.is_empty() {
self.process_aborts
.remove(&process_control_key(&tenant, &project, &process));
}
Ok(CoordinatorResponse::ProcessAborted {
process,
aborted_tasks,
affected_nodes: affected_nodes.into_iter().collect(),
})
}
pub(super) fn handle_list_processes(
&mut self,
tenant: String,
project: String,
actor_user: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let processes = self
.coordinator
.active_processes_for_project(&tenant, &project)
.into_iter()
.map(|active| {
let process_key = process_control_key(&active.tenant, &active.project, &active.id);
let main = self.main_runtime.controls.get(&process_key);
let state = if self.process_cancellations.contains(&process_key) {
"cancelling"
} else {
main.map_or("running", |main| main.state.as_str())
};
let main_wait_state = main.and_then(|main| {
if main.state != "running" {
return None;
}
if self.pending_task_launches.iter().any(|pending| {
pending.tenant == active.tenant
&& pending.project == active.project
&& pending.process == active.id
}) {
Some("waiting_for_node".to_owned())
} else if self.main_runtime.is_waiting_for_task(
&active.tenant,
&active.project,
&active.id,
) {
Some("waiting_for_task".to_owned())
} else {
Some("executing".to_owned())
}
});
VirtualProcessStatus {
process: active.id,
state: state.to_owned(),
main_task_definition: main.map(|main| main.task_definition.clone()),
main_task_instance: main.map(|main| main.task_instance.clone()),
main_state: main.map(|main| main.state.clone()),
main_wait_state,
main_debug_epoch: main.and_then(|main| main.debug.requested_epoch()),
connected_nodes: active.connected_nodes.into_iter().collect(),
coordinator_epoch: active.coordinator_epoch,
}
})
.collect();
Ok(CoordinatorResponse::ProcessStatuses { processes, actor })
}
pub(super) fn handle_poll_task_control(
&mut self,
tenant: String,
project: String,
process: String,
node: String,
task: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let process = ProcessId::new(process);
let node = NodeId::new(node);
let task = TaskInstanceId::new(task);
self.authorize_node_for_process_or_termination(&node, &tenant, &project, &process)?;
let cancel_requested = self
.task_cancellations
.contains(&task_control_key(&tenant, &project, &process, &node, &task))
|| self
.process_cancellations
.contains(&process_control_key(&tenant, &project, &process));
let abort_requested = self
.task_aborts
.contains(&task_control_key(&tenant, &project, &process, &node, &task))
|| self
.process_aborts
.contains(&process_control_key(&tenant, &project, &process));
Ok(CoordinatorResponse::TaskControl {
process,
task,
cancel_requested,
abort_requested,
})
}
pub(super) fn workflow_actor(
&mut self,
tenant: &TenantId,
project: &ProjectId,
actor_user: Option<String>,
actor_agent: Option<String>,
agent_public_key_fingerprint: Option<Digest>,
agent_signature: Option<AgentSignedRequest>,
request_payload_digest: Option<&Digest>,
request_kind: &str,
process: &ProcessId,
task: Option<&TaskInstanceId>,
) -> Result<WorkflowActor, CoordinatorServiceError> {
if let Some(agent) = actor_agent {
let agent = AgentId::new(agent);
let signature = agent_signature.ok_or_else(|| {
CoordinatorError::Unauthorized(
"agent workflow dispatch requires a signed request proving private-key possession"
.to_owned(),
)
})?;
let request_payload_digest = request_payload_digest.ok_or_else(|| {
CoordinatorError::Unauthorized(
"agent workflow dispatch requires a canonical signed request payload"
.to_owned(),
)
})?;
if signature.nonce.trim().is_empty() || signature.nonce.len() > 256 {
return Err(CoordinatorError::Unauthorized(
"agent signed request nonce is missing or invalid".to_owned(),
)
.into());
}
let now_epoch_seconds = unix_timestamp_seconds();
let replay_key = (
tenant.clone(),
project.clone(),
agent.clone(),
signature.nonce.clone(),
);
const AGENT_SIGNATURE_WINDOW_SECONDS: u64 = 300;
self.agent_replay_nonces.retain(|_, issued_at| {
now_epoch_seconds <= issued_at.saturating_add(AGENT_SIGNATURE_WINDOW_SECONDS)
});
if self.agent_replay_nonces.contains_key(&replay_key) {
return Err(CoordinatorError::Unauthorized(
"agent signed request nonce has already been used".to_owned(),
)
.into());
}
let canonical_scope = clusterflux_core::AgentWorkflowRequestScope::new(
tenant.clone(),
project.clone(),
request_kind,
process.clone(),
task.cloned(),
)
.map_err(CoordinatorError::Unauthorized)?;
let record = self.coordinator.authorize_agent_project_run(
canonical_scope.for_agent(&agent),
agent_public_key_fingerprint.as_ref(),
request_payload_digest,
&signature,
now_epoch_seconds,
)?;
if self
.agent_replay_nonces
.keys()
.filter(|(retained_tenant, retained_project, retained_agent, _)| {
retained_tenant == tenant
&& retained_project == project
&& retained_agent == &agent
})
.count()
>= super::MAX_REPLAY_NONCES_PER_AUTHORITY
{
return Err(CoordinatorError::Unauthorized(
"agent signed request replay window is full; retry after the bounded signature window advances"
.to_owned(),
)
.into());
}
self.agent_replay_nonces
.insert(replay_key, signature.issued_at_epoch_seconds);
return Ok(WorkflowActor {
kind: "agent".to_owned(),
user: Some(record.user),
agent: Some(agent),
credential_kind: CredentialKind::PublicKey,
public_key_fingerprint: Some(record.public_key_fingerprint),
authenticated_without_browser: true,
scopes: record.scopes,
});
}
let actor = UserId::new(actor_user.unwrap_or_else(|| "user".to_owned()));
Ok(WorkflowActor {
kind: "user".to_owned(),
user: Some(actor),
agent: None,
credential_kind: CredentialKind::BrowserSession,
public_key_fingerprint: None,
authenticated_without_browser: false,
scopes: vec!["project:read".to_owned(), "project:run".to_owned()],
})
}
}
fn unix_timestamp_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}

673
crates/clusterflux-coordinator/src/service/protocol.rs Normal file
View file

@ -0,0 +1,673 @@
use std::collections::BTreeMap;
use clusterflux_core::{
AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind,
DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements,
LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest,
PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation,
SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId,
UserId, VfsPath,
};
use serde::{Deserialize, Serialize};
mod responses;
pub use responses::*;
use crate::{AgentPublicKeyRecord, ProjectRecord};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TaskReplacementBundle {
pub bundle_digest: Digest,
pub wasm_module_base64: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskFailureResolution {
AcceptFailure,
Cancel,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum CoordinatorRequest {
Ping,
Authenticated {
session_secret: String,
request: AuthenticatedCoordinatorRequest,
},
AuthStatus {
tenant: String,
project: String,
actor_user: String,
},
AdminStatus {
tenant: String,
actor_user: String,
admin_proof: Digest,
admin_nonce: String,
issued_at_epoch_seconds: u64,
},
SuspendTenant {
tenant: String,
actor_user: String,
target_tenant: String,
admin_proof: Digest,
admin_nonce: String,
issued_at_epoch_seconds: u64,
},
CreateProject {
tenant: String,
actor_user: String,
project: String,
name: String,
},
SelectProject {
tenant: String,
actor_user: String,
project: String,
},
ListProjects {
tenant: String,
actor_user: String,
},
RegisterAgentPublicKey {
tenant: String,
project: String,
user: String,
agent: String,
public_key: String,
},
ListAgentPublicKeys {
tenant: String,
project: String,
user: String,
},
RotateAgentPublicKey {
tenant: String,
project: String,
user: String,
agent: String,
public_key: String,
},
RevokeAgentPublicKey {
tenant: String,
project: String,
user: String,
agent: String,
},
AttachNode {
tenant: String,
project: String,
node: String,
public_key: String,
},
CreateNodeEnrollmentGrant {
tenant: String,
project: String,
actor_user: String,
#[serde(default = "default_node_enrollment_ttl_seconds")]
ttl_seconds: u64,
},
ExchangeNodeEnrollmentGrant {
tenant: String,
project: String,
node: String,
public_key: String,
enrollment_grant: String,
},
NodeHeartbeat {
node: String,
#[serde(default)]
node_signature: Option<NodeSignedRequest>,
},
SignedNode {
node: String,
node_signature: NodeSignedRequest,
request: Box<CoordinatorRequest>,
},
ReportNodeCapabilities {
tenant: String,
project: String,
node: String,
capabilities: NodeCapabilities,
cached_environment_digests: Vec<Digest>,
#[serde(default)]
dependency_cache_digests: Vec<Digest>,
source_snapshots: Vec<Digest>,
artifact_locations: Vec<String>,
direct_connectivity: bool,
online: bool,
},
ListNodeDescriptors {
tenant: String,
project: String,
actor_user: String,
},
RevokeNodeCredential {
tenant: String,
project: String,
actor_user: String,
node: String,
},
ScheduleTask {
tenant: String,
project: String,
environment: Option<EnvironmentRequirements>,
environment_digest: Option<Digest>,
required_capabilities: Vec<Capability>,
#[serde(default)]
dependency_cache: Option<Digest>,
source_snapshot: Option<Digest>,
required_artifacts: Vec<String>,
prefer_node: Option<String>,
},
LaunchTask {
tenant: String,
project: String,
#[serde(default)]
actor_user: Option<String>,
#[serde(default)]
actor_agent: Option<String>,
#[serde(default)]
agent_public_key_fingerprint: Option<Digest>,
#[serde(default)]
agent_signature: Option<AgentSignedRequest>,
task_spec: TaskSpec,
#[serde(default)]
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
},
LaunchChildTask {
tenant: String,
project: String,
process: String,
node: String,
parent_task: String,
task_spec: TaskSpec,
#[serde(default)]
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
},
JoinChildTask {
tenant: String,
project: String,
process: String,
node: String,
parent_task: String,
task: String,
},
PollTaskAssignment {
tenant: String,
project: String,
node: String,
},
PollArtifactTransfer {
tenant: String,
project: String,
node: String,
},
UploadArtifactTransferChunk {
tenant: String,
project: String,
node: String,
transfer_id: String,
artifact: String,
offset: u64,
content_base64: String,
chunk_digest: Digest,
eof: bool,
},
FailArtifactTransfer {
tenant: String,
project: String,
node: String,
transfer_id: String,
artifact: String,
message: String,
},
RequestRendezvous {
scope: DataPlaneScope,
source: NodeEndpoint,
destination: NodeEndpoint,
direct_connectivity: bool,
failure_reason: String,
},
RequestSourcePreparation {
tenant: String,
project: String,
provider: SourceProviderKind,
},
CompleteSourcePreparation {
tenant: String,
project: String,
node: String,
provider: SourceProviderKind,
source_snapshot: Digest,
},
StartProcess {
tenant: String,
project: String,
#[serde(default)]
actor_user: Option<String>,
#[serde(default)]
actor_agent: Option<String>,
#[serde(default)]
agent_public_key_fingerprint: Option<Digest>,
#[serde(default)]
agent_signature: Option<AgentSignedRequest>,
process: String,
#[serde(default)]
restart: bool,
},
ReconnectNode {
node: String,
process: String,
epoch: u64,
},
CancelTask {
tenant: String,
project: String,
process: String,
node: String,
task: String,
},
CancelProcess {
tenant: String,
project: String,
actor_user: String,
process: String,
},
AbortProcess {
tenant: String,
project: String,
actor_user: String,
process: String,
},
ListProcesses {
tenant: String,
project: String,
actor_user: String,
},
QuotaStatus {
tenant: String,
project: String,
actor_user: String,
},
PollTaskControl {
tenant: String,
project: String,
process: String,
node: String,
task: String,
},
RestartTask {
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
#[serde(default)]
replacement_bundle: Option<TaskReplacementBundle>,
},
ResolveTaskFailure {
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
resolution: TaskFailureResolution,
},
DebugAttach {
tenant: String,
project: String,
actor_user: String,
process: String,
},
SetDebugBreakpoints {
tenant: String,
project: String,
actor_user: String,
process: String,
probe_symbols: Vec<String>,
},
InspectDebugBreakpoints {
tenant: String,
project: String,
actor_user: String,
process: String,
},
CreateDebugEpoch {
tenant: String,
project: String,
actor_user: String,
process: String,
stopped_task: String,
reason: String,
},
ResumeDebugEpoch {
tenant: String,
project: String,
actor_user: String,
process: String,
epoch: u64,
},
InspectDebugEpoch {
tenant: String,
project: String,
actor_user: String,
process: String,
epoch: u64,
},
PollDebugCommand {
tenant: String,
project: String,
process: String,
node: String,
task: String,
},
ReportDebugState {
tenant: String,
project: String,
process: String,
node: String,
task: String,
epoch: u64,
state: DebugAcknowledgementState,
#[serde(default)]
stack_frames: Vec<String>,
#[serde(default)]
local_values: Vec<(String, String)>,
#[serde(default)]
task_args: Vec<(String, String)>,
#[serde(default)]
handles: Vec<(String, String)>,
#[serde(default)]
command_status: Option<String>,
#[serde(default)]
recent_output: Vec<String>,
#[serde(default)]
message: Option<String>,
},
ReportDebugProbeHit {
tenant: String,
project: String,
process: String,
node: String,
task: String,
probe_symbol: String,
},
ReportTaskLog {
tenant: String,
project: String,
process: String,
node: String,
task: String,
stdout_bytes: u64,
stderr_bytes: u64,
#[serde(default)]
stdout_tail: String,
#[serde(default)]
stderr_tail: String,
stdout_truncated: bool,
stderr_truncated: bool,
backpressured: bool,
},
ReportVfsMetadata {
tenant: String,
project: String,
process: String,
node: String,
task: String,
artifact_path: Option<String>,
artifact_digest: Option<Digest>,
artifact_size_bytes: Option<u64>,
large_bytes_uploaded: bool,
},
TaskCompleted {
tenant: String,
project: String,
process: String,
node: String,
task: String,
#[serde(default)]
terminal_state: Option<TaskTerminalState>,
status_code: Option<i32>,
stdout_bytes: u64,
stderr_bytes: u64,
#[serde(default)]
stdout_tail: String,
#[serde(default)]
stderr_tail: String,
#[serde(default)]
stdout_truncated: bool,
#[serde(default)]
stderr_truncated: bool,
artifact_path: Option<String>,
artifact_digest: Option<Digest>,
artifact_size_bytes: Option<u64>,
#[serde(default)]
result: Option<TaskBoundaryValue>,
},
ListTaskEvents {
tenant: String,
project: String,
actor_user: String,
#[serde(default)]
process: Option<String>,
},
ListTaskSnapshots {
tenant: String,
project: String,
actor_user: String,
process: String,
},
JoinTask {
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
},
RenderOperatorPanel {
tenant: String,
project: String,
process: String,
actor_user: String,
max_download_bytes: u64,
stopped: bool,
},
SubmitPanelEvent {
tenant: String,
project: String,
process: String,
widget_id: String,
kind: PanelEventKind,
max_events: u64,
},
CreateArtifactDownloadLink {
tenant: String,
project: String,
actor_user: String,
artifact: String,
max_bytes: u64,
#[serde(default = "default_download_ttl_seconds")]
ttl_seconds: u64,
},
OpenArtifactDownloadStream {
tenant: String,
project: String,
actor_user: String,
artifact: String,
max_bytes: u64,
token_digest: Digest,
chunk_bytes: u64,
},
RevokeArtifactDownloadLink {
tenant: String,
project: String,
actor_user: String,
artifact: String,
token_digest: Digest,
},
ExportArtifactToNode {
tenant: String,
project: String,
actor_user: String,
artifact: String,
receiver_node: String,
direct_connectivity: bool,
failure_reason: String,
},
}
impl CoordinatorRequest {
pub fn operation(&self) -> Result<String, String> {
serde_json::to_value(self)
.map_err(|err| format!("failed to encode coordinator request operation: {err}"))
.map(|value| clusterflux_core::coordinator_payload_operation(&value))
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum AuthenticatedCoordinatorRequest {
AuthStatus,
RevokeCliSession,
CreateProject {
project: String,
name: String,
},
SelectProject {
project: String,
},
ListProjects,
RegisterAgentPublicKey {
agent: String,
public_key: String,
},
ListAgentPublicKeys,
RotateAgentPublicKey {
agent: String,
public_key: String,
},
RevokeAgentPublicKey {
agent: String,
},
CreateNodeEnrollmentGrant {
#[serde(default = "default_node_enrollment_ttl_seconds")]
ttl_seconds: u64,
},
ListNodeDescriptors,
RevokeNodeCredential {
node: String,
},
StartProcess {
process: String,
#[serde(default)]
restart: bool,
},
ScheduleTask {
environment: Option<EnvironmentRequirements>,
environment_digest: Option<Digest>,
required_capabilities: Vec<Capability>,
#[serde(default)]
dependency_cache: Option<Digest>,
source_snapshot: Option<Digest>,
required_artifacts: Vec<String>,
prefer_node: Option<String>,
},
LaunchTask {
task_spec: Box<TaskSpec>,
#[serde(default)]
wait_for_node: bool,
artifact_path: String,
wasm_module_base64: String,
},
CancelProcess {
process: String,
},
AbortProcess {
process: String,
},
ListProcesses,
QuotaStatus,
RestartTask {
process: String,
task: String,
#[serde(default)]
replacement_bundle: Option<TaskReplacementBundle>,
},
ResolveTaskFailure {
process: String,
task: String,
resolution: TaskFailureResolution,
},
DebugAttach {
process: String,
},
SetDebugBreakpoints {
process: String,
probe_symbols: Vec<String>,
},
InspectDebugBreakpoints {
process: String,
},
CreateDebugEpoch {
process: String,
stopped_task: String,
reason: String,
},
ResumeDebugEpoch {
process: String,
epoch: u64,
},
InspectDebugEpoch {
process: String,
epoch: u64,
},
ListTaskEvents {
#[serde(default)]
process: Option<String>,
},
ListTaskSnapshots {
process: String,
},
JoinTask {
process: String,
task: String,
},
CreateArtifactDownloadLink {
artifact: String,
max_bytes: u64,
#[serde(default = "default_download_ttl_seconds")]
ttl_seconds: u64,
},
OpenArtifactDownloadStream {
artifact: String,
max_bytes: u64,
token_digest: Digest,
chunk_bytes: u64,
},
RevokeArtifactDownloadLink {
artifact: String,
token_digest: Digest,
},
ExportArtifactToNode {
artifact: String,
receiver_node: String,
direct_connectivity: bool,
failure_reason: String,
},
}
fn default_download_ttl_seconds() -> u64 {
900
}
fn default_node_enrollment_ttl_seconds() -> u64 {
900
}

545
crates/clusterflux-coordinator/src/service/protocol/responses.rs Normal file
View file

@ -0,0 +1,545 @@
use super::*;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskTerminalState {
Completed,
Failed,
Cancelled,
}
impl TaskTerminalState {
pub(crate) fn from_status_code(status_code: Option<i32>) -> Self {
match status_code {
Some(0) => Self::Completed,
_ => Self::Failed,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskExecutor {
CoordinatorMain,
Node,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCompletionEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub node: NodeId,
pub executor: TaskExecutor,
pub task_definition: clusterflux_core::TaskDefinitionId,
pub task: TaskInstanceId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub attempt_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placement: Option<Placement>,
pub terminal_state: TaskTerminalState,
pub status_code: Option<i32>,
pub stdout_bytes: u64,
pub stderr_bytes: u64,
pub stdout_tail: String,
pub stderr_tail: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub artifact_path: Option<VfsPath>,
pub artifact_digest: Option<Digest>,
pub artifact_size_bytes: Option<u64>,
pub result: Option<TaskBoundaryValue>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskAttemptState {
Queued,
Running,
FailedAwaitingAction,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskAttemptSnapshot {
pub process: ProcessId,
pub task: TaskInstanceId,
pub attempt_id: String,
pub attempt_number: u32,
pub task_definition: clusterflux_core::TaskDefinitionId,
pub display_name: String,
pub state: TaskAttemptState,
pub current: bool,
pub node: Option<NodeId>,
pub environment_id: Option<String>,
pub environment_digest: Option<Digest>,
pub argument_summary: Vec<String>,
pub handle_summary: Vec<String>,
pub command_state: Option<String>,
pub vfs_checkpoint: String,
pub probe_symbol: Option<String>,
pub source_path: Option<String>,
pub source_line: Option<u32>,
pub restart_compatible: bool,
pub failure_policy: clusterflux_core::TaskFailurePolicy,
pub artifact_path: Option<VfsPath>,
pub artifact_digest: Option<Digest>,
pub artifact_size_bytes: Option<u64>,
pub status_code: Option<i32>,
pub error: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugAuditEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub task: Option<TaskInstanceId>,
pub actor: UserId,
pub operation: String,
pub allowed: bool,
pub reason: String,
pub charged_debug_read_bytes: u64,
pub used_debug_read_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct WorkflowActor {
pub kind: String,
pub user: Option<UserId>,
pub agent: Option<AgentId>,
pub credential_kind: CredentialKind,
pub public_key_fingerprint: Option<Digest>,
pub authenticated_without_browser: bool,
pub scopes: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskAssignment {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub task: TaskInstanceId,
pub node: NodeId,
pub epoch: u64,
pub artifact_path: String,
pub task_spec: TaskSpec,
pub wasm_module_base64: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactTransferAssignment {
pub transfer_id: String,
pub artifact: ArtifactId,
pub expected_digest: Digest,
pub expected_size_bytes: u64,
pub offset: u64,
pub max_chunk_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCancellationTarget {
pub process: ProcessId,
pub task: TaskInstanceId,
pub node: NodeId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DebugAcknowledgementState {
Frozen,
Running,
Failed,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugParticipantAcknowledgement {
pub node: NodeId,
pub task_definition: clusterflux_core::TaskDefinitionId,
pub task: TaskInstanceId,
pub epoch: u64,
pub state: DebugAcknowledgementState,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
pub message: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VirtualProcessStatus {
pub process: ProcessId,
pub state: String,
pub main_task_definition: Option<clusterflux_core::TaskDefinitionId>,
pub main_task_instance: Option<TaskInstanceId>,
pub main_state: Option<String>,
pub main_wait_state: Option<String>,
pub main_debug_epoch: Option<u64>,
pub connected_nodes: Vec<NodeId>,
pub coordinator_epoch: u64,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SourcePreparationDisposition {
Pending { reason: String },
Assigned { node: NodeId },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourcePreparationStatus {
pub preparation: SourcePreparation,
pub disposition: SourcePreparationDisposition,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CoordinatorResponse {
Pong {
epoch: u64,
},
AuthStatus {
tenant: TenantId,
project: ProjectId,
actor: UserId,
authenticated: bool,
account_status: String,
suspended: bool,
disabled: bool,
deleted: bool,
manual_review: bool,
sanitized_reason: Option<String>,
next_actions: Vec<String>,
private_moderation_details_exposed: bool,
signup_failure_details_exposed: bool,
},
AdminStatus {
tenant: TenantId,
actor: UserId,
suspended: bool,
safe_default: String,
},
TenantSuspended {
tenant: TenantId,
actor: UserId,
policy: crate::ServicePolicyRecord,
},
ProjectCreated {
project: ProjectRecord,
actor: UserId,
},
ProjectSelected {
project: ProjectRecord,
actor: UserId,
},
Projects {
projects: Vec<ProjectRecord>,
actor: UserId,
},
CliSessionRevoked {
tenant: TenantId,
project: ProjectId,
actor: UserId,
},
AgentPublicKey {
record: AgentPublicKeyRecord,
actor: UserId,
},
AgentPublicKeys {
records: Vec<AgentPublicKeyRecord>,
actor: UserId,
},
NodeAttached {
node: NodeId,
tenant: TenantId,
project: ProjectId,
},
NodeEnrollmentGrantCreated {
tenant: TenantId,
project: ProjectId,
grant: String,
scope: String,
expires_at_epoch_seconds: u64,
},
NodeEnrollmentExchanged {
node: NodeId,
tenant: TenantId,
project: ProjectId,
credential: clusterflux_core::NodeCredential,
},
NodeHeartbeat {
node: NodeId,
epoch: u64,
},
NodeCapabilitiesRecorded {
node: NodeId,
node_descriptors: usize,
},
NodeDescriptors {
descriptors: Vec<NodeDescriptor>,
actor: UserId,
},
NodeCredentialRevoked {
node: NodeId,
tenant: TenantId,
project: ProjectId,
actor: UserId,
descriptor_removed: bool,
queued_assignments_removed: usize,
},
TaskPlacement {
placement: Placement,
},
TaskLaunched {
process: ProcessId,
task: TaskInstanceId,
actor: WorkflowActor,
placement: Placement,
assignment: Box<TaskAssignment>,
charged_spawns: u64,
},
MainLaunched {
process: ProcessId,
task_definition: clusterflux_core::TaskDefinitionId,
task_instance: TaskInstanceId,
actor: WorkflowActor,
state: String,
},
TaskQueued {
process: ProcessId,
task: TaskInstanceId,
actor: WorkflowActor,
reason: String,
charged_spawns: u64,
queued_tasks: usize,
},
TaskAssignment {
assignment: Option<Box<TaskAssignment>>,
},
ArtifactTransferAssignment {
transfer: Option<ArtifactTransferAssignment>,
},
ArtifactTransferChunkAccepted {
transfer_id: String,
next_offset: u64,
complete: bool,
},
ArtifactTransferFailed {
transfer_id: String,
},
RendezvousPlan {
plan: DirectBulkTransferPlan,
charged_rendezvous_attempts: u64,
},
SourcePreparation {
status: SourcePreparationStatus,
},
SourcePreparationCompleted {
node: NodeId,
provider: SourceProviderKind,
source_snapshot: Digest,
},
ProcessStarted {
process: ProcessId,
epoch: u64,
actor: WorkflowActor,
charged_spawns: u64,
},
NodeReconnected {
node: NodeId,
process: ProcessId,
},
TaskCancellationRequested {
process: ProcessId,
task: TaskInstanceId,
node: NodeId,
},
ProcessCancellationRequested {
process: ProcessId,
cancelled_tasks: Vec<TaskCancellationTarget>,
affected_nodes: Vec<NodeId>,
},
ProcessAborted {
process: ProcessId,
aborted_tasks: Vec<TaskCancellationTarget>,
affected_nodes: Vec<NodeId>,
},
ProcessStatuses {
processes: Vec<VirtualProcessStatus>,
actor: UserId,
},
QuotaStatus {
tenant: TenantId,
project: ProjectId,
actor: UserId,
#[serde(default, skip_serializing_if = "Option::is_none")]
policy_label: Option<String>,
limits: ResourceLimits,
window_seconds: BTreeMap<LimitKind, u64>,
usage: BTreeMap<LimitKind, u64>,
window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
},
TaskControl {
process: ProcessId,
task: TaskInstanceId,
cancel_requested: bool,
abort_requested: bool,
},
TaskRestart {
process: ProcessId,
task: TaskInstanceId,
restarted_task_instance: Option<clusterflux_core::TaskInstanceId>,
restarted_attempt_id: Option<String>,
actor: UserId,
accepted: bool,
clean_boundary_available: bool,
active_task: bool,
completed_event_observed: bool,
requires_whole_process_restart: bool,
message: String,
audit_event: DebugAuditEvent,
charged_debug_read_bytes: u64,
used_debug_read_bytes: u64,
},
DebugCommand {
process: ProcessId,
task: TaskInstanceId,
epoch: Option<u64>,
command: Option<String>,
},
DebugStateRecorded {
process: ProcessId,
node: NodeId,
task: TaskInstanceId,
epoch: u64,
state: DebugAcknowledgementState,
},
DebugAttach {
process: ProcessId,
actor: UserId,
authorization: Authorization,
audit_event: DebugAuditEvent,
charged_debug_read_bytes: u64,
used_debug_read_bytes: u64,
},
DebugBreakpoints {
process: ProcessId,
actor: UserId,
probe_symbols: Vec<String>,
hit_epoch: Option<u64>,
hit_task: Option<TaskInstanceId>,
hit_probe_symbol: Option<String>,
audit_event: DebugAuditEvent,
charged_debug_read_bytes: u64,
used_debug_read_bytes: u64,
},
DebugProbeHit {
process: ProcessId,
node: NodeId,
task: TaskInstanceId,
probe_symbol: String,
breakpoint_matched: bool,
debug_epoch: Option<u64>,
},
DebugEpoch {
process: ProcessId,
actor: UserId,
epoch: u64,
command: String,
affected_tasks: Vec<TaskCancellationTarget>,
all_stop_requested: bool,
audit_event: DebugAuditEvent,
charged_debug_read_bytes: u64,
used_debug_read_bytes: u64,
},
DebugEpochStatus {
process: ProcessId,
actor: UserId,
epoch: u64,
command: String,
expected_tasks: Vec<TaskCancellationTarget>,
acknowledgements: Vec<DebugParticipantAcknowledgement>,
fully_frozen: bool,
partially_frozen: bool,
fully_resumed: bool,
failed: bool,
failure_messages: Vec<String>,
audit_event: DebugAuditEvent,
charged_debug_read_bytes: u64,
used_debug_read_bytes: u64,
},
TaskLogRecorded {
process: ProcessId,
task: TaskInstanceId,
stdout_bytes: u64,
stderr_bytes: u64,
stdout_tail: String,
stderr_tail: String,
backpressured: bool,
},
VfsMetadataRecorded {
process: ProcessId,
task: TaskInstanceId,
artifact_path: Option<VfsPath>,
large_bytes_uploaded: bool,
},
TaskRecorded {
process: ProcessId,
task: TaskInstanceId,
events_recorded: usize,
},
TaskEvents {
events: Vec<TaskCompletionEvent>,
},
TaskSnapshots {
snapshots: Vec<TaskAttemptSnapshot>,
},
TaskFailureResolved {
process: ProcessId,
task: TaskInstanceId,
attempt_id: String,
resolution: TaskFailureResolution,
},
TaskJoined {
join: TaskJoinResult,
},
OperatorPanel {
panel: PanelState,
},
PanelEventAccepted {
used_events: u64,
max_events: u64,
},
ArtifactDownloadLink {
link: DownloadLink,
},
ArtifactDownloadLinkRevoked {
link: DownloadLink,
},
ArtifactDownloadStream {
link: DownloadLink,
streamed_bytes: u64,
charged_download_bytes: u64,
content_bytes_available: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_offset: Option<u64>,
#[serde(default)]
content_eof: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_base64: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
content_source: Option<String>,
},
ArtifactExportPlan {
plan: DirectBulkTransferPlan,
source_node: NodeId,
receiver_node: NodeId,
artifact_size_bytes: u64,
},
Error {
message: String,
},
}

421
crates/clusterflux-coordinator/src/service/quota.rs Normal file
View file

@ -0,0 +1,421 @@
use std::collections::BTreeMap;
use clusterflux_core::{LimitError, LimitKind, ProjectId, ResourceLimits, ResourceMeter, TenantId};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorQuotaConfiguration {
pub limits: ResourceLimits,
pub window_seconds: BTreeMap<LimitKind, u64>,
pub policy_label: Option<String>,
}
pub(super) struct CoordinatorQuotaStatus {
pub(super) policy_label: Option<String>,
pub(super) limits: ResourceLimits,
pub(super) window_seconds: BTreeMap<LimitKind, u64>,
pub(super) usage: BTreeMap<LimitKind, u64>,
pub(super) window_started_epoch_seconds: BTreeMap<LimitKind, u64>,
}
impl CoordinatorQuotaConfiguration {
pub fn new(
limits: ResourceLimits,
window_seconds: impl IntoIterator<Item = (LimitKind, u64)>,
) -> Result<Self, String> {
let window_seconds = window_seconds.into_iter().collect::<BTreeMap<_, _>>();
if window_seconds.values().any(|seconds| *seconds == 0) {
return Err("quota windows must be at least one second".to_owned());
}
Ok(Self {
limits,
window_seconds,
policy_label: None,
})
}
pub fn with_policy_label(mut self, label: impl Into<String>) -> Self {
let label = label.into();
self.policy_label = (!label.trim().is_empty()).then_some(label);
self
}
pub fn unlimited() -> Self {
Self {
limits: ResourceLimits::unlimited(),
window_seconds: LimitKind::ALL
.into_iter()
.map(|kind| (kind, u64::MAX))
.collect(),
policy_label: None,
}
}
pub fn window_seconds(&self, kind: LimitKind) -> u64 {
self.window_seconds
.get(&kind)
.copied()
.unwrap_or(u64::MAX)
.max(1)
}
}
impl Default for CoordinatorQuotaConfiguration {
fn default() -> Self {
Self::unlimited()
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct ProjectQuotaScope {
tenant: TenantId,
project: ProjectId,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct MeterKey {
scope: ProjectQuotaScope,
kind: LimitKind,
window: u64,
}
#[derive(Clone, Debug)]
pub(super) struct CoordinatorQuota {
configuration: CoordinatorQuotaConfiguration,
meters: BTreeMap<MeterKey, ResourceMeter>,
}
impl Default for CoordinatorQuota {
fn default() -> Self {
Self::new(CoordinatorQuotaConfiguration::default())
}
}
impl CoordinatorQuota {
pub(super) fn new(configuration: CoordinatorQuotaConfiguration) -> Self {
Self {
configuration,
meters: BTreeMap::new(),
}
}
fn key(
&self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
now_epoch_seconds: u64,
) -> MeterKey {
MeterKey {
scope: ProjectQuotaScope {
tenant: tenant.clone(),
project: project.clone(),
},
kind,
window: now_epoch_seconds / self.configuration.window_seconds(kind),
}
}
fn meter(
&self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
now_epoch_seconds: u64,
) -> Option<&ResourceMeter> {
self.meters
.get(&self.key(tenant, project, kind, now_epoch_seconds))
}
fn meter_mut(
&mut self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
now_epoch_seconds: u64,
) -> &mut ResourceMeter {
let key = self.key(tenant, project, kind, now_epoch_seconds);
self.meters.retain(|existing, _| {
existing.scope != key.scope || existing.kind != kind || existing.window == key.window
});
self.meters.entry(key).or_default()
}
fn can_charge(
&self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
amount: u64,
now_epoch_seconds: u64,
) -> Result<(), LimitError> {
self.meter(tenant, project, kind, now_epoch_seconds)
.cloned()
.unwrap_or_default()
.can_charge(&self.configuration.limits, kind, amount)
}
fn charge(
&mut self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
amount: u64,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
let limits = self.configuration.limits.clone();
let meter = self.meter_mut(tenant, project, kind, now_epoch_seconds);
meter.charge(&limits, kind, amount)?;
Ok(meter.used(&kind))
}
fn used(
&self,
tenant: &TenantId,
project: &ProjectId,
kind: LimitKind,
now_epoch_seconds: u64,
) -> u64 {
self.meter(tenant, project, kind, now_epoch_seconds)
.map_or(0, |meter| meter.used(&kind))
}
pub(super) fn can_charge_workflow_spawn(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> Result<(), LimitError> {
self.can_charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
}
pub(super) fn charge_api_call(
&mut self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(tenant, project, LimitKind::ApiCall, 1, now_epoch_seconds)
}
pub(super) fn can_charge_log_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), LimitError> {
self.can_charge(
tenant,
project,
LimitKind::LogBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn charge_log_bytes(
&mut self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(
tenant,
project,
LimitKind::LogBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn charge_workflow_spawn(
&mut self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(tenant, project, LimitKind::Spawn, 1, now_epoch_seconds)
}
#[cfg(test)]
pub(super) fn used_workflow_spawns(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> u64 {
self.used(tenant, project, LimitKind::Spawn, now_epoch_seconds)
}
#[cfg(test)]
pub(super) fn used_api_calls(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> u64 {
self.used(tenant, project, LimitKind::ApiCall, now_epoch_seconds)
}
#[cfg(test)]
pub(super) fn used_log_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> u64 {
self.used(tenant, project, LimitKind::LogBytes, now_epoch_seconds)
}
#[cfg(test)]
pub(super) fn active_meter_count(&self) -> usize {
self.meters.len()
}
pub(super) fn charge_rendezvous_attempt(
&mut self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(
tenant,
project,
LimitKind::RendezvousAttempt,
1,
now_epoch_seconds,
)
}
pub(super) fn can_charge_download(
&self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), LimitError> {
self.can_charge(
tenant,
project,
LimitKind::ArtifactDownloadBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn charge_download(
&mut self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(
tenant,
project,
LimitKind::ArtifactDownloadBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn download_limit(&self) -> u64 {
self.configuration
.limits
.limit(&LimitKind::ArtifactDownloadBytes)
}
pub(super) fn used_download_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> u64 {
self.used(
tenant,
project,
LimitKind::ArtifactDownloadBytes,
now_epoch_seconds,
)
}
pub(super) fn charge_debug_read(
&mut self,
tenant: &TenantId,
project: &ProjectId,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<u64, LimitError> {
self.charge(
tenant,
project,
LimitKind::DebugReadBytes,
bytes,
now_epoch_seconds,
)
}
pub(super) fn used_debug_read_bytes(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> u64 {
self.used(
tenant,
project,
LimitKind::DebugReadBytes,
now_epoch_seconds,
)
}
pub(super) fn project_status(
&self,
tenant: &TenantId,
project: &ProjectId,
now_epoch_seconds: u64,
) -> CoordinatorQuotaStatus {
let mut usage = BTreeMap::new();
let mut window_starts = BTreeMap::new();
for kind in LimitKind::ALL {
let seconds = self.configuration.window_seconds(kind);
usage.insert(kind, self.used(tenant, project, kind, now_epoch_seconds));
window_starts.insert(kind, (now_epoch_seconds / seconds).saturating_mul(seconds));
}
CoordinatorQuotaStatus {
policy_label: self.configuration.policy_label.clone(),
limits: self.configuration.limits.clone(),
window_seconds: self.configuration.window_seconds.clone(),
usage,
window_started_epoch_seconds: window_starts,
}
}
#[cfg(test)]
pub(super) fn set_workflow_limits(&mut self, limits: ResourceLimits) {
self.configuration
.limits
.limits
.insert(LimitKind::Spawn, limits.limit(&LimitKind::Spawn));
self.meters.clear();
}
#[cfg(test)]
pub(super) fn set_download_limits(&mut self, limits: ResourceLimits) {
self.configuration.limits.limits.insert(
LimitKind::ArtifactDownloadBytes,
limits.limit(&LimitKind::ArtifactDownloadBytes),
);
self.meters.clear();
}
#[cfg(test)]
pub(super) fn set_rendezvous_limits(&mut self, limits: ResourceLimits) {
self.configuration.limits.limits.insert(
LimitKind::RendezvousAttempt,
limits.limit(&LimitKind::RendezvousAttempt),
);
self.meters.clear();
}
}

486
crates/clusterflux-coordinator/src/service/relay.rs Normal file
View file

@ -0,0 +1,486 @@
use std::collections::BTreeMap;
use clusterflux_core::{ProjectId, TenantId, UserId};
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactRelayConfiguration {
pub enabled: bool,
pub max_artifact_bytes: u64,
pub max_active_per_project: usize,
pub max_active_per_tenant: usize,
pub max_active_per_account: usize,
pub max_active_global: usize,
pub max_period_bytes_per_project: u64,
pub max_period_bytes_per_tenant: u64,
pub max_period_bytes_per_account: u64,
pub max_period_bytes_global: u64,
pub period_seconds: u64,
pub framing_overhead_bytes: u64,
pub max_tracked_scopes: usize,
}
impl ArtifactRelayConfiguration {
pub fn unlimited() -> Self {
Self {
enabled: true,
max_artifact_bytes: u64::MAX,
max_active_per_project: usize::MAX,
max_active_per_tenant: usize::MAX,
max_active_per_account: usize::MAX,
max_active_global: usize::MAX,
max_period_bytes_per_project: u64::MAX,
max_period_bytes_per_tenant: u64::MAX,
max_period_bytes_per_account: u64::MAX,
max_period_bytes_global: u64::MAX,
period_seconds: u64::MAX,
framing_overhead_bytes: 512,
max_tracked_scopes: usize::MAX,
}
}
pub fn validate(&self) -> Result<(), ArtifactRelayError> {
if self.period_seconds == 0
|| self.max_active_per_project == 0
|| self.max_active_per_tenant == 0
|| self.max_active_per_account == 0
|| self.max_active_global == 0
|| self.max_tracked_scopes == 0
{
return Err(ArtifactRelayError::InvalidConfiguration);
}
Ok(())
}
}
impl Default for ArtifactRelayConfiguration {
fn default() -> Self {
Self::unlimited()
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ArtifactRelayUsage {
pub active_transfers: usize,
pub ingress_bytes: u64,
pub egress_bytes: u64,
pub abandoned_or_failed_bytes: u64,
pub reserved_bytes: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum RelayFinishReason {
Completed,
Failed,
Cancelled,
Expired,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ArtifactRelayError {
#[error("artifact relay is disabled by hosted policy")]
Disabled,
#[error("artifact exceeds the configured relay size limit")]
ArtifactTooLarge,
#[error("artifact relay concurrency limit reached for {0}")]
Concurrency(&'static str),
#[error("artifact relay period byte budget exhausted for {0}")]
PeriodBudget(&'static str),
#[error("artifact relay retained scope state limit reached")]
StateLimit,
#[error("artifact relay reservation is missing or expired")]
MissingReservation,
#[error("artifact relay configuration contains a zero bound")]
InvalidConfiguration,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct RelayScope {
tenant: TenantId,
project: ProjectId,
account: UserId,
}
#[derive(Clone, Debug)]
struct RelayReservation {
scope: RelayScope,
remaining_reserved_bytes: u64,
ingress_bytes: u64,
egress_bytes: u64,
expires_at_epoch_seconds: u64,
}
#[derive(Clone, Debug)]
pub(super) struct ArtifactRelayLedger {
configuration: ArtifactRelayConfiguration,
reservations: BTreeMap<String, RelayReservation>,
period: u64,
project_used: BTreeMap<(TenantId, ProjectId), u64>,
tenant_used: BTreeMap<TenantId, u64>,
account_used: BTreeMap<(TenantId, UserId), u64>,
global_used: u64,
ingress_used: u64,
egress_used: u64,
abandoned_or_failed_used: u64,
}
impl Default for ArtifactRelayLedger {
fn default() -> Self {
Self::new(ArtifactRelayConfiguration::default())
}
}
impl ArtifactRelayLedger {
pub(super) fn new(configuration: ArtifactRelayConfiguration) -> Self {
Self {
configuration,
reservations: BTreeMap::new(),
period: 0,
project_used: BTreeMap::new(),
tenant_used: BTreeMap::new(),
account_used: BTreeMap::new(),
global_used: 0,
ingress_used: 0,
egress_used: 0,
abandoned_or_failed_used: 0,
}
}
pub(super) fn configure(
&mut self,
configuration: ArtifactRelayConfiguration,
) -> Result<(), ArtifactRelayError> {
configuration.validate()?;
if !self.reservations.is_empty() {
return Err(ArtifactRelayError::Concurrency("configuration change"));
}
*self = Self::new(configuration);
Ok(())
}
fn prepare_period(&mut self, now_epoch_seconds: u64) {
let period = now_epoch_seconds / self.configuration.period_seconds.max(1);
if self.period != period {
self.period = period;
self.project_used.clear();
self.tenant_used.clear();
self.account_used.clear();
self.global_used = 0;
self.ingress_used = 0;
self.egress_used = 0;
self.abandoned_or_failed_used = 0;
}
}
pub(super) fn estimated_wire_bytes(&self, artifact_bytes: u64, chunk_bytes: u64) -> u64 {
let encoded = artifact_bytes
.saturating_add(2)
.saturating_div(3)
.saturating_mul(4);
let chunks = artifact_bytes
.saturating_add(chunk_bytes.saturating_sub(1))
.saturating_div(chunk_bytes.max(1))
.max(1);
encoded.saturating_mul(2).saturating_add(
chunks
.saturating_mul(2)
.saturating_mul(self.configuration.framing_overhead_bytes),
)
}
pub(super) fn framing_overhead_bytes(&self) -> u64 {
self.configuration.framing_overhead_bytes
}
pub(super) fn reserve(
&mut self,
key: String,
tenant: TenantId,
project: ProjectId,
account: UserId,
artifact_bytes: u64,
chunk_bytes: u64,
expires_at_epoch_seconds: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.expire(now_epoch_seconds);
self.prepare_period(now_epoch_seconds);
if !self.configuration.enabled {
return Err(ArtifactRelayError::Disabled);
}
if artifact_bytes > self.configuration.max_artifact_bytes {
return Err(ArtifactRelayError::ArtifactTooLarge);
}
let scope = RelayScope {
tenant,
project,
account,
};
self.check_concurrency(&scope)?;
self.check_tracked_scope_capacity(&scope)?;
let reserved = self.estimated_wire_bytes(artifact_bytes, chunk_bytes);
self.check_budget(&scope, reserved)?;
self.reservations.insert(
key,
RelayReservation {
scope,
remaining_reserved_bytes: reserved,
ingress_bytes: 0,
egress_bytes: 0,
expires_at_epoch_seconds,
},
);
Ok(())
}
pub(super) fn rekey(&mut self, from: &str, to: String) -> Result<(), ArtifactRelayError> {
let reservation = self
.reservations
.remove(from)
.ok_or(ArtifactRelayError::MissingReservation)?;
self.reservations.insert(to, reservation);
Ok(())
}
fn check_concurrency(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
if self.reservations.len() >= self.configuration.max_active_global {
return Err(ArtifactRelayError::Concurrency("global"));
}
let project = self
.reservations
.values()
.filter(|reservation| {
reservation.scope.tenant == scope.tenant
&& reservation.scope.project == scope.project
})
.count();
if project >= self.configuration.max_active_per_project {
return Err(ArtifactRelayError::Concurrency("project"));
}
let tenant = self
.reservations
.values()
.filter(|reservation| reservation.scope.tenant == scope.tenant)
.count();
if tenant >= self.configuration.max_active_per_tenant {
return Err(ArtifactRelayError::Concurrency("tenant"));
}
let account = self
.reservations
.values()
.filter(|reservation| {
reservation.scope.tenant == scope.tenant
&& reservation.scope.account == scope.account
})
.count();
if account >= self.configuration.max_active_per_account {
return Err(ArtifactRelayError::Concurrency("account"));
}
Ok(())
}
fn check_tracked_scope_capacity(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
let project_key = (scope.tenant.clone(), scope.project.clone());
let account_key = (scope.tenant.clone(), scope.account.clone());
let new_scopes = usize::from(!self.project_used.contains_key(&project_key))
+ usize::from(!self.tenant_used.contains_key(&scope.tenant))
+ usize::from(!self.account_used.contains_key(&account_key));
let tracked = self.project_used.len() + self.tenant_used.len() + self.account_used.len();
if tracked.saturating_add(new_scopes) > self.configuration.max_tracked_scopes {
return Err(ArtifactRelayError::StateLimit);
}
Ok(())
}
fn reserved_for(&self, scope: &RelayScope) -> (u64, u64, u64, u64) {
let mut project = 0_u64;
let mut tenant = 0_u64;
let mut account = 0_u64;
let mut global = 0_u64;
for reservation in self.reservations.values() {
let bytes = reservation.remaining_reserved_bytes;
global = global.saturating_add(bytes);
if reservation.scope.tenant == scope.tenant {
tenant = tenant.saturating_add(bytes);
if reservation.scope.project == scope.project {
project = project.saturating_add(bytes);
}
if reservation.scope.account == scope.account {
account = account.saturating_add(bytes);
}
}
}
(project, tenant, account, global)
}
fn check_budget(&self, scope: &RelayScope, additional: u64) -> Result<(), ArtifactRelayError> {
let (project_reserved, tenant_reserved, account_reserved, global_reserved) =
self.reserved_for(scope);
let project_used = self
.project_used
.get(&(scope.tenant.clone(), scope.project.clone()))
.copied()
.unwrap_or(0);
let tenant_used = self.tenant_used.get(&scope.tenant).copied().unwrap_or(0);
let account_used = self
.account_used
.get(&(scope.tenant.clone(), scope.account.clone()))
.copied()
.unwrap_or(0);
for (used, reserved, limit, label) in [
(
project_used,
project_reserved,
self.configuration.max_period_bytes_per_project,
"project",
),
(
tenant_used,
tenant_reserved,
self.configuration.max_period_bytes_per_tenant,
"tenant",
),
(
account_used,
account_reserved,
self.configuration.max_period_bytes_per_account,
"account",
),
(
self.global_used,
global_reserved,
self.configuration.max_period_bytes_global,
"global",
),
] {
if used.saturating_add(reserved).saturating_add(additional) > limit {
return Err(ArtifactRelayError::PeriodBudget(label));
}
}
Ok(())
}
fn charge(
&mut self,
key: &str,
bytes: u64,
ingress: bool,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.prepare_period(now_epoch_seconds);
let reservation = self
.reservations
.get(key)
.cloned()
.ok_or(ArtifactRelayError::MissingReservation)?;
let additional = bytes.saturating_sub(reservation.remaining_reserved_bytes);
if additional > 0 {
self.check_budget(&reservation.scope, additional)?;
}
let reservation = self
.reservations
.get_mut(key)
.ok_or(ArtifactRelayError::MissingReservation)?;
reservation.remaining_reserved_bytes =
reservation.remaining_reserved_bytes.saturating_sub(bytes);
if ingress {
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
self.ingress_used = self.ingress_used.saturating_add(bytes);
} else {
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
self.egress_used = self.egress_used.saturating_add(bytes);
}
let project_key = (
reservation.scope.tenant.clone(),
reservation.scope.project.clone(),
);
let account_key = (
reservation.scope.tenant.clone(),
reservation.scope.account.clone(),
);
*self.project_used.entry(project_key).or_default() = self
.project_used
.get(&(
reservation.scope.tenant.clone(),
reservation.scope.project.clone(),
))
.copied()
.unwrap_or(0)
.saturating_add(bytes);
*self
.tenant_used
.entry(reservation.scope.tenant.clone())
.or_default() = self
.tenant_used
.get(&reservation.scope.tenant)
.copied()
.unwrap_or(0)
.saturating_add(bytes);
*self.account_used.entry(account_key).or_default() = self
.account_used
.get(&(
reservation.scope.tenant.clone(),
reservation.scope.account.clone(),
))
.copied()
.unwrap_or(0)
.saturating_add(bytes);
self.global_used = self.global_used.saturating_add(bytes);
Ok(())
}
pub(super) fn charge_ingress(
&mut self,
key: &str,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.charge(key, bytes, true, now_epoch_seconds)
}
pub(super) fn charge_egress(
&mut self,
key: &str,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.charge(key, bytes, false, now_epoch_seconds)
}
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
if let Some(reservation) = self.reservations.remove(key) {
if reason != RelayFinishReason::Completed {
self.abandoned_or_failed_used = self
.abandoned_or_failed_used
.saturating_add(reservation.ingress_bytes)
.saturating_add(reservation.egress_bytes);
}
}
}
pub(super) fn expire(&mut self, now_epoch_seconds: u64) {
let expired = self
.reservations
.iter()
.filter(|(_, reservation)| reservation.expires_at_epoch_seconds < now_epoch_seconds)
.map(|(key, _)| key.clone())
.collect::<Vec<_>>();
for key in expired {
self.finish(&key, RelayFinishReason::Expired);
}
}
pub(super) fn usage(&self) -> ArtifactRelayUsage {
ArtifactRelayUsage {
active_transfers: self.reservations.len(),
ingress_bytes: self.ingress_used,
egress_bytes: self.egress_used,
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
reserved_bytes: self
.reservations
.values()
.map(|reservation| reservation.remaining_reserved_bytes)
.fold(0_u64, u64::saturating_add),
}
}
}

814
crates/clusterflux-coordinator/src/service/routing.rs Normal file
View file

@ -0,0 +1,814 @@
use super::*;
impl CoordinatorService {
pub fn handle_request(
&mut self,
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
self.pump_main_runtime_commands();
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize coordinator request for authentication: {error}"
))
})?;
let request_payload_digest =
clusterflux_core::signed_request_payload_digest(&request_payload);
match request {
CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong {
epoch: self.coordinator.coordinator_epoch(),
}),
CoordinatorRequest::Authenticated {
session_secret,
request,
} => self.handle_authenticated_request(session_secret, request),
CoordinatorRequest::AuthStatus {
tenant,
project,
actor_user,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let account_state = self.coordinator.account_policy_state(&tenant);
Ok(CoordinatorResponse::AuthStatus {
tenant,
project,
actor,
authenticated: true,
account_status: account_state.account_status,
suspended: account_state.suspended,
disabled: account_state.disabled,
deleted: account_state.deleted,
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
CoordinatorRequest::AdminStatus {
tenant,
actor_user,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
} => self.handle_admin_status(
tenant,
actor_user,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
),
CoordinatorRequest::SuspendTenant {
tenant,
actor_user,
target_tenant,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
} => self.handle_suspend_tenant(
tenant,
actor_user,
target_tenant,
admin_proof,
admin_nonce,
issued_at_epoch_seconds,
),
CoordinatorRequest::CreateProject {
tenant,
actor_user,
project,
name,
} => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let project = ProjectId::new(project);
self.coordinator.ensure_tenant_active(&tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the signed-in tenant scope".to_owned(),
)
.into());
}
}
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator.upsert_user(
tenant.clone(),
actor.clone(),
CredentialKind::BrowserSession,
);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), name);
self.coordinator.grant_project_debug(
tenant.clone(),
project.clone(),
actor.clone(),
);
self.persist_durable_state()?;
let project = self
.coordinator
.project(&project)
.expect("project was just created")
.clone();
Ok(CoordinatorResponse::ProjectCreated { project, actor })
}
CoordinatorRequest::SelectProject {
tenant,
actor_user,
project,
} => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let project_id = ProjectId::new(project);
let project = self
.coordinator
.project(&project_id)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"project is not visible to the signed-in user".to_owned(),
)
})?
.clone();
if project.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project is outside the signed-in tenant scope".to_owned(),
)
.into());
}
self.coordinator
.upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
self.persist_durable_state()?;
Ok(CoordinatorResponse::ProjectSelected { project, actor })
}
CoordinatorRequest::ListProjects { tenant, actor_user } => {
let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: ProjectId::from("__project_listing__"),
actor: Actor::User(actor.clone()),
};
self.coordinator
.upsert_user(tenant, actor.clone(), CredentialKind::BrowserSession);
self.persist_durable_state()?;
Ok(CoordinatorResponse::Projects {
projects: self.coordinator.list_projects(&context),
actor,
})
}
CoordinatorRequest::RegisterAgentPublicKey {
tenant,
project,
user,
agent,
public_key,
}
| CoordinatorRequest::RotateAgentPublicKey {
tenant,
project,
user,
agent,
public_key,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let agent = AgentId::new(agent);
self.coordinator.ensure_tenant_active(&tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the signed-in tenant scope".to_owned(),
)
.into());
}
}
self.coordinator.upsert_tenant(tenant.clone());
self.coordinator.upsert_user(
tenant.clone(),
actor.clone(),
CredentialKind::CliDeviceSession,
);
self.coordinator
.upsert_project(tenant.clone(), project.clone(), "local");
let record = self.coordinator.register_agent_public_key(
tenant,
project,
actor.clone(),
agent,
public_key,
);
self.persist_durable_state()?;
Ok(CoordinatorResponse::AgentPublicKey { record, actor })
}
CoordinatorRequest::ListAgentPublicKeys {
tenant,
project,
user,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let context = clusterflux_core::AuthContext {
tenant,
project,
actor: Actor::User(actor.clone()),
};
Ok(CoordinatorResponse::AgentPublicKeys {
records: self.coordinator.list_agent_public_keys(&context),
actor,
})
}
CoordinatorRequest::RevokeAgentPublicKey {
tenant,
project,
user,
agent,
} => {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(user);
let agent = AgentId::new(agent);
let context = clusterflux_core::AuthContext {
tenant,
project,
actor: Actor::User(actor.clone()),
};
let record = self.coordinator.revoke_agent_public_key(&context, &agent)?;
self.persist_durable_state()?;
Ok(CoordinatorResponse::AgentPublicKey { record, actor })
}
CoordinatorRequest::AttachNode {
tenant,
project,
node,
public_key,
} => self.handle_attach_node(tenant, project, node, public_key),
CoordinatorRequest::CreateNodeEnrollmentGrant {
tenant,
project,
actor_user,
ttl_seconds,
} => self.handle_create_node_enrollment_grant(tenant, project, actor_user, ttl_seconds),
CoordinatorRequest::ExchangeNodeEnrollmentGrant {
tenant,
project,
node,
public_key,
enrollment_grant,
} => self.handle_exchange_node_enrollment_grant(
tenant,
project,
node,
public_key,
enrollment_grant,
),
CoordinatorRequest::NodeHeartbeat {
node,
node_signature,
} => self.handle_node_heartbeat(node, node_signature, &request_payload_digest),
CoordinatorRequest::SignedNode {
node,
node_signature,
request,
} => self.handle_signed_node_request(node, node_signature, *request),
CoordinatorRequest::ReportNodeCapabilities { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::ListNodeDescriptors {
tenant,
project,
actor_user,
} => self.handle_list_node_descriptors(tenant, project, actor_user),
CoordinatorRequest::RevokeNodeCredential {
tenant,
project,
actor_user,
node,
} => self.handle_revoke_node_credential(tenant, project, actor_user, node),
CoordinatorRequest::ScheduleTask {
tenant,
project,
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
} => self.handle_schedule_task(
tenant,
project,
environment,
environment_digest,
required_capabilities,
dependency_cache,
source_snapshot,
required_artifacts,
prefer_node,
),
CoordinatorRequest::LaunchTask {
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
} => self.handle_launch_task(
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
Some(&request_payload_digest),
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
),
CoordinatorRequest::LaunchChildTask { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::JoinChildTask { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::PollTaskAssignment { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::RequestRendezvous {
scope,
source,
destination,
direct_connectivity,
failure_reason,
} => self.handle_request_rendezvous(
scope,
source,
destination,
direct_connectivity,
failure_reason,
),
CoordinatorRequest::RequestSourcePreparation {
tenant,
project,
provider,
} => self.handle_request_source_preparation(tenant, project, provider),
CoordinatorRequest::CompleteSourcePreparation { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::StartProcess {
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
process,
restart,
} => self.handle_start_process(
tenant,
project,
actor_user,
actor_agent,
agent_public_key_fingerprint,
agent_signature,
Some(&request_payload_digest),
process,
restart,
),
CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::CancelTask {
tenant,
project,
process,
node,
task,
} => self.handle_cancel_task(tenant, project, process, node, task),
CoordinatorRequest::CancelProcess {
tenant,
project,
actor_user,
process,
} => self.handle_cancel_process(tenant, project, actor_user, process),
CoordinatorRequest::AbortProcess {
tenant,
project,
actor_user,
process,
} => self.handle_abort_process(tenant, project, actor_user, process),
CoordinatorRequest::ListProcesses {
tenant,
project,
actor_user,
} => self.handle_list_processes(tenant, project, actor_user),
CoordinatorRequest::QuotaStatus {
tenant,
project,
actor_user,
} => self.handle_quota_status(tenant, project, actor_user),
CoordinatorRequest::PollTaskControl { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::PollArtifactTransfer { .. }
| CoordinatorRequest::UploadArtifactTransferChunk { .. }
| CoordinatorRequest::FailArtifactTransfer { .. } => {
self.reject_unsigned_node_request()
}
CoordinatorRequest::RestartTask {
tenant,
project,
actor_user,
process,
task,
replacement_bundle,
} => self.handle_restart_task(
tenant,
project,
actor_user,
process,
task,
replacement_bundle,
),
CoordinatorRequest::ResolveTaskFailure {
tenant,
project,
actor_user,
process,
task,
resolution,
} => self.handle_resolve_task_failure(
tenant, project, actor_user, process, task, resolution,
),
request @ (CoordinatorRequest::DebugAttach { .. }
| CoordinatorRequest::SetDebugBreakpoints { .. }
| CoordinatorRequest::InspectDebugBreakpoints { .. }
| CoordinatorRequest::CreateDebugEpoch { .. }
| CoordinatorRequest::ResumeDebugEpoch { .. }
| CoordinatorRequest::InspectDebugEpoch { .. }) => self.handle_debug_request(request),
CoordinatorRequest::PollDebugCommand { .. }
| CoordinatorRequest::ReportDebugState { .. }
| CoordinatorRequest::ReportDebugProbeHit { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportTaskLog { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ReportVfsMetadata { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::TaskCompleted { .. } => self.reject_unsigned_node_request(),
CoordinatorRequest::ListTaskEvents {
tenant,
project,
actor_user,
process,
} => self.handle_list_task_events(tenant, project, actor_user, process),
CoordinatorRequest::ListTaskSnapshots {
tenant,
project,
actor_user,
process,
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
CoordinatorRequest::JoinTask {
tenant,
project,
actor_user,
process,
task,
} => self.handle_join_task(tenant, project, actor_user, process, task),
CoordinatorRequest::RenderOperatorPanel {
tenant,
project,
process,
actor_user,
max_download_bytes,
stopped,
} => self.handle_render_operator_panel(
tenant,
project,
actor_user,
process,
max_download_bytes,
stopped,
),
CoordinatorRequest::SubmitPanelEvent {
tenant,
project,
process,
widget_id,
kind,
max_events,
} => self
.handle_submit_panel_event(tenant, project, process, widget_id, kind, max_events),
CoordinatorRequest::CreateArtifactDownloadLink {
tenant,
project,
actor_user,
artifact,
max_bytes,
ttl_seconds,
} => self.handle_create_artifact_download_link(
tenant,
project,
actor_user,
artifact,
max_bytes,
ttl_seconds,
),
CoordinatorRequest::OpenArtifactDownloadStream {
tenant,
project,
actor_user,
artifact,
max_bytes,
token_digest,
chunk_bytes,
} => self.handle_open_artifact_download_stream(
tenant,
project,
actor_user,
artifact,
max_bytes,
token_digest,
chunk_bytes,
),
CoordinatorRequest::RevokeArtifactDownloadLink {
tenant,
project,
actor_user,
artifact,
token_digest,
} => self.handle_revoke_artifact_download_link(
tenant,
project,
actor_user,
artifact,
token_digest,
),
CoordinatorRequest::ExportArtifactToNode {
tenant,
project,
actor_user,
artifact,
receiver_node,
direct_connectivity,
failure_reason,
} => self.handle_export_artifact_to_node(
tenant,
project,
actor_user,
artifact,
receiver_node,
direct_connectivity,
failure_reason,
),
}
}
fn handle_authenticated_request(
&mut self,
session_secret: String,
request: AuthenticatedCoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let context = self.coordinator.authenticate_cli_session(&session_secret)?;
let authorized = authorize_authenticated_user_operation(&context, &request)?;
let now_epoch_seconds = self.current_epoch_seconds()?;
self.quota
.charge_api_call(&context.tenant, &context.project, now_epoch_seconds)?;
let _authorized_operation = authorized.operation;
let actor = authorized.actor;
match request {
AuthenticatedCoordinatorRequest::AuthStatus => {
let account_state = self.coordinator.account_policy_state(&context.tenant);
Ok(CoordinatorResponse::AuthStatus {
tenant: context.tenant,
project: context.project,
actor,
authenticated: true,
account_status: account_state.account_status,
suspended: account_state.suspended,
disabled: account_state.disabled,
deleted: account_state.deleted,
manual_review: account_state.manual_review,
sanitized_reason: account_state.sanitized_reason,
next_actions: account_state.next_actions,
private_moderation_details_exposed: false,
signup_failure_details_exposed: false,
})
}
AuthenticatedCoordinatorRequest::RevokeCliSession => {
self.coordinator.revoke_cli_session(&session_secret)?;
self.persist_durable_state()?;
Ok(CoordinatorResponse::CliSessionRevoked {
tenant: context.tenant,
project: context.project,
actor,
})
}
AuthenticatedCoordinatorRequest::CreateProject { project, name } => {
let project = ProjectId::new(project);
self.coordinator.ensure_tenant_active(&context.tenant)?;
if let Some(existing) = self.coordinator.project(&project) {
if existing.tenant != context.tenant {
return Err(CoordinatorError::Unauthorized(
"project id is outside the authenticated tenant scope".to_owned(),
)
.into());
}
}
self.coordinator
.upsert_project(context.tenant.clone(), project.clone(), name);
self.coordinator.grant_project_debug(
context.tenant.clone(),
project.clone(),
actor.clone(),
);
self.persist_durable_state()?;
let project = self
.coordinator
.project(&project)
.expect("project was just created")
.clone();
Ok(CoordinatorResponse::ProjectCreated { project, actor })
}
AuthenticatedCoordinatorRequest::SelectProject { project } => {
let project_id = ProjectId::new(project);
let project = self
.coordinator
.project(&project_id)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"project is not visible to the authenticated user".to_owned(),
)
})?
.clone();
if project.tenant != context.tenant {
return Err(CoordinatorError::Unauthorized(
"project is outside the authenticated tenant scope".to_owned(),
)
.into());
}
Ok(CoordinatorResponse::ProjectSelected { project, actor })
}
AuthenticatedCoordinatorRequest::ListProjects => Ok(CoordinatorResponse::Projects {
projects: self.coordinator.list_projects(&context),
actor,
}),
request @ (AuthenticatedCoordinatorRequest::RegisterAgentPublicKey { .. }
| AuthenticatedCoordinatorRequest::ListAgentPublicKeys
| AuthenticatedCoordinatorRequest::RotateAgentPublicKey { .. }
| AuthenticatedCoordinatorRequest::RevokeAgentPublicKey { .. }) => {
self.handle_authenticated_agent_key_request(&context, &actor, request)
}
AuthenticatedCoordinatorRequest::CreateNodeEnrollmentGrant { ttl_seconds } => self
.handle_create_node_enrollment_grant(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
ttl_seconds,
),
AuthenticatedCoordinatorRequest::ListNodeDescriptors => self
.handle_list_node_descriptors(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::RevokeNodeCredential { node } => self
.handle_revoke_node_credential(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
node,
),
AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
.handle_start_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
Some(actor.as_str().to_owned()),
None,
None,
None,
None,
process,
restart,
),
request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
self.handle_authenticated_schedule_task(&context, request)
}
request @ AuthenticatedCoordinatorRequest::LaunchTask { .. } => {
self.handle_authenticated_launch_task(&context, &actor, request)
}
AuthenticatedCoordinatorRequest::CancelProcess { process } => self
.handle_cancel_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::QuotaStatus => self.handle_quota_status(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
),
AuthenticatedCoordinatorRequest::RestartTask {
process,
task,
replacement_bundle,
} => self.handle_restart_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
replacement_bundle,
),
AuthenticatedCoordinatorRequest::ResolveTaskFailure {
process,
task,
resolution,
} => self.handle_resolve_task_failure(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
resolution,
),
request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. }
| AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. }
| AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. }
| AuthenticatedCoordinatorRequest::CreateDebugEpoch { .. }
| AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. }
| AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. }) => self
.handle_authenticated_debug_request(
&context.tenant,
&context.project,
&actor,
request,
),
AuthenticatedCoordinatorRequest::ListTaskEvents { process } => self
.handle_list_task_events(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => self
.handle_list_task_snapshots(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
),
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink {
artifact,
max_bytes,
ttl_seconds,
} => self.handle_create_artifact_download_link(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
max_bytes,
ttl_seconds,
),
AuthenticatedCoordinatorRequest::OpenArtifactDownloadStream {
artifact,
max_bytes,
token_digest,
chunk_bytes,
} => self.handle_open_artifact_download_stream(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
max_bytes,
token_digest,
chunk_bytes,
),
AuthenticatedCoordinatorRequest::RevokeArtifactDownloadLink {
artifact,
token_digest,
} => self.handle_revoke_artifact_download_link(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
artifact,
token_digest,
),
request @ AuthenticatedCoordinatorRequest::ExportArtifactToNode { .. } => {
self.handle_authenticated_artifact_export(&context, &actor, request)
}
}
}
}

365
crates/clusterflux-coordinator/src/service/signed_nodes.rs Normal file
View file

@ -0,0 +1,365 @@
use clusterflux_core::{NodeId, NodeSignedRequest};
use crate::CoordinatorError;
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
impl CoordinatorService {
pub(super) fn handle_signed_node_request(
&mut self,
signed_node: String,
node_signature: NodeSignedRequest,
request: CoordinatorRequest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let request_kind = signed_node_request_kind(&request)?;
let request_node = signed_node_request_node(&request)?;
let request_payload = serde_json::to_value(&request).map_err(|error| {
CoordinatorServiceError::Protocol(format!(
"failed to canonicalize signed node request: {error}"
))
})?;
let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
let signed_node = NodeId::new(signed_node);
if request_node != signed_node {
return Err(CoordinatorError::Unauthorized(
"signed node request node does not match the wrapped request node".to_owned(),
)
.into());
}
self.authenticate_node_request(
&signed_node,
Some(node_signature),
request_kind,
&payload_digest,
)?;
match request {
CoordinatorRequest::ReportNodeCapabilities {
tenant,
project,
node,
capabilities,
cached_environment_digests,
dependency_cache_digests,
source_snapshots,
artifact_locations,
direct_connectivity,
online,
} => self.handle_report_node_capabilities(
tenant,
project,
node,
capabilities,
cached_environment_digests,
dependency_cache_digests,
source_snapshots,
artifact_locations,
direct_connectivity,
online,
),
CoordinatorRequest::PollTaskAssignment {
tenant,
project,
node,
} => self.handle_poll_task_assignment(tenant, project, node),
CoordinatorRequest::PollArtifactTransfer {
tenant,
project,
node,
} => self.handle_poll_artifact_transfer(tenant, project, node),
CoordinatorRequest::UploadArtifactTransferChunk {
tenant,
project,
node,
transfer_id,
artifact,
offset,
content_base64,
chunk_digest,
eof,
} => self.handle_upload_artifact_transfer_chunk(
tenant,
project,
node,
transfer_id,
artifact,
offset,
content_base64,
chunk_digest,
eof,
),
CoordinatorRequest::FailArtifactTransfer {
tenant,
project,
node,
transfer_id,
artifact,
message,
} => self.handle_fail_artifact_transfer(
tenant,
project,
node,
transfer_id,
artifact,
message,
),
CoordinatorRequest::LaunchChildTask {
tenant,
project,
process,
node,
parent_task,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
} => self.handle_launch_child_task(
tenant,
project,
process,
node,
parent_task,
task_spec,
wait_for_node,
artifact_path,
wasm_module_base64,
),
CoordinatorRequest::JoinChildTask {
tenant,
project,
process,
node,
parent_task,
task,
} => self.handle_join_child_task(tenant, project, process, node, parent_task, task),
CoordinatorRequest::CompleteSourcePreparation {
tenant,
project,
node,
provider,
source_snapshot,
} => self.handle_complete_source_preparation(
tenant,
project,
node,
provider,
source_snapshot,
),
CoordinatorRequest::ReconnectNode {
node,
process,
epoch,
} => self.handle_reconnect_node(node, process, epoch),
CoordinatorRequest::PollTaskControl {
tenant,
project,
process,
node,
task,
} => self.handle_poll_task_control(tenant, project, process, node, task),
CoordinatorRequest::PollDebugCommand {
tenant,
project,
process,
node,
task,
} => self.handle_poll_debug_command(tenant, project, process, node, task),
CoordinatorRequest::ReportDebugState {
tenant,
project,
process,
node,
task,
epoch,
state,
stack_frames,
local_values,
task_args,
handles,
command_status,
recent_output,
message,
} => self.handle_report_debug_state(
tenant,
project,
process,
node,
task,
epoch,
state,
stack_frames,
local_values,
task_args,
handles,
command_status,
recent_output,
message,
),
CoordinatorRequest::ReportDebugProbeHit {
tenant,
project,
process,
node,
task,
probe_symbol,
} => self.handle_report_debug_probe_hit(
tenant,
project,
process,
node,
task,
probe_symbol,
),
CoordinatorRequest::ReportTaskLog {
tenant,
project,
process,
node,
task,
stdout_bytes,
stderr_bytes,
stdout_tail,
stderr_tail,
stdout_truncated,
stderr_truncated,
backpressured,
} => self.handle_report_task_log(
tenant,
project,
process,
node,
task,
stdout_bytes,
stderr_bytes,
stdout_tail,
stderr_tail,
stdout_truncated,
stderr_truncated,
backpressured,
),
CoordinatorRequest::ReportVfsMetadata {
tenant,
project,
process,
node,
task,
artifact_path,
artifact_digest,
artifact_size_bytes,
large_bytes_uploaded,
} => self.handle_report_vfs_metadata(
tenant,
project,
process,
node,
task,
artifact_path,
artifact_digest,
artifact_size_bytes,
large_bytes_uploaded,
),
CoordinatorRequest::TaskCompleted {
tenant,
project,
process,
node,
task,
terminal_state,
status_code,
stdout_bytes,
stderr_bytes,
stdout_tail,
stderr_tail,
stdout_truncated,
stderr_truncated,
artifact_path,
artifact_digest,
artifact_size_bytes,
result,
} => self.handle_task_completed(
tenant,
project,
process,
node,
task,
terminal_state,
status_code,
stdout_bytes,
stderr_bytes,
stdout_tail,
stderr_tail,
stdout_truncated,
stderr_truncated,
artifact_path,
artifact_digest,
artifact_size_bytes,
result,
),
_ => self.reject_unsigned_node_request(),
}
}
pub(super) fn reject_unsigned_node_request(
&self,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
Err(CoordinatorError::Unauthorized(
"node-originated request requires signed_node envelope proof".to_owned(),
)
.into())
}
}
fn signed_node_request_kind(
request: &CoordinatorRequest,
) -> Result<&'static str, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { .. } => Ok("report_node_capabilities"),
CoordinatorRequest::PollTaskAssignment { .. } => Ok("poll_task_assignment"),
CoordinatorRequest::PollArtifactTransfer { .. } => Ok("poll_artifact_transfer"),
CoordinatorRequest::UploadArtifactTransferChunk { .. } => {
Ok("upload_artifact_transfer_chunk")
}
CoordinatorRequest::FailArtifactTransfer { .. } => Ok("fail_artifact_transfer"),
CoordinatorRequest::LaunchChildTask { .. } => Ok("launch_child_task"),
CoordinatorRequest::JoinChildTask { .. } => Ok("join_child_task"),
CoordinatorRequest::CompleteSourcePreparation { .. } => Ok("complete_source_preparation"),
CoordinatorRequest::ReconnectNode { .. } => Ok("reconnect_node"),
CoordinatorRequest::PollTaskControl { .. } => Ok("poll_task_control"),
CoordinatorRequest::PollDebugCommand { .. } => Ok("poll_debug_command"),
CoordinatorRequest::ReportDebugState { .. } => Ok("report_debug_state"),
CoordinatorRequest::ReportDebugProbeHit { .. } => Ok("report_debug_probe_hit"),
CoordinatorRequest::ReportTaskLog { .. } => Ok("report_task_log"),
CoordinatorRequest::ReportVfsMetadata { .. } => Ok("report_vfs_metadata"),
CoordinatorRequest::TaskCompleted { .. } => Ok("task_completed"),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}
fn signed_node_request_node(
request: &CoordinatorRequest,
) -> Result<NodeId, CoordinatorServiceError> {
match request {
CoordinatorRequest::ReportNodeCapabilities { node, .. }
| CoordinatorRequest::PollTaskAssignment { node, .. }
| CoordinatorRequest::PollArtifactTransfer { node, .. }
| CoordinatorRequest::UploadArtifactTransferChunk { node, .. }
| CoordinatorRequest::FailArtifactTransfer { node, .. }
| CoordinatorRequest::LaunchChildTask { node, .. }
| CoordinatorRequest::JoinChildTask { node, .. }
| CoordinatorRequest::CompleteSourcePreparation { node, .. }
| CoordinatorRequest::ReconnectNode { node, .. }
| CoordinatorRequest::PollTaskControl { node, .. }
| CoordinatorRequest::PollDebugCommand { node, .. }
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => Ok(NodeId::new(node.clone())),
_ => Err(CoordinatorError::Unauthorized(
"signed_node envelope only accepts node-originated coordinator requests".to_owned(),
)
.into()),
}
}

200
crates/clusterflux-coordinator/src/service/tcp.rs Normal file
View file

@ -0,0 +1,200 @@
use std::io::{BufRead, BufReader, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::sync::{Arc, Mutex};
use super::{CoordinatorRequest, CoordinatorResponse, CoordinatorService, CoordinatorServiceError};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ClientAuthorityMode {
Strict,
LocalTrustedLoopback,
}
impl CoordinatorService {
pub fn serve_tcp(self, listener: TcpListener) -> Result<(), CoordinatorServiceError> {
if !listener.local_addr()?.ip().is_loopback() {
return Err(CoordinatorServiceError::Protocol(
"the native coordinator transport is plaintext and restricted to loopback; expose a remote coordinator only through a secure transport"
.to_owned(),
));
}
self.serve_tcp_with_authority(listener, ClientAuthorityMode::Strict)
}
pub fn serve_tcp_local_trusted(
self,
listener: TcpListener,
) -> Result<(), CoordinatorServiceError> {
if !listener.local_addr()?.ip().is_loopback() {
return Err(CoordinatorServiceError::Protocol(
"local trusted request mode is restricted to a loopback listener".to_owned(),
));
}
self.serve_tcp_with_authority(listener, ClientAuthorityMode::LocalTrustedLoopback)
}
fn serve_tcp_with_authority(
self,
listener: TcpListener,
authority_mode: ClientAuthorityMode,
) -> Result<(), CoordinatorServiceError> {
let shared = Arc::new(Mutex::new(self));
for stream in listener.incoming() {
let stream = stream?;
let service = Arc::clone(&shared);
std::thread::spawn(move || {
if let Err(err) = handle_shared_stream(service, stream, authority_mode) {
eprintln!("coordinator stream failed: {err}");
}
});
}
Ok(())
}
pub fn handle_stream(&mut self, stream: TcpStream) -> Result<(), CoordinatorServiceError> {
self.handle_stream_with_authority(stream, ClientAuthorityMode::Strict)
}
#[cfg(test)]
pub(super) fn handle_stream_local_trusted(
&mut self,
stream: TcpStream,
) -> Result<(), CoordinatorServiceError> {
self.handle_stream_with_authority(stream, ClientAuthorityMode::LocalTrustedLoopback)
}
fn handle_stream_with_authority(
&mut self,
stream: TcpStream,
authority_mode: ClientAuthorityMode,
) -> Result<(), CoordinatorServiceError> {
let mut reader = BufReader::new(stream.try_clone()?);
let mut writer = stream;
loop {
let mut line = String::new();
if reader.read_line(&mut line)? == 0 {
return Ok(());
}
if line.trim().is_empty() {
continue;
}
let response = match decode_wire_request(&line) {
Ok(request) => match authorize_client_request(&request, authority_mode)
.and_then(|()| self.handle_request(request))
{
Ok(response) => response,
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
};
serde_json::to_writer(&mut writer, &response)?;
writer.write_all(b"\n")?;
writer.flush()?;
}
}
}
fn handle_shared_stream(
service: Arc<Mutex<CoordinatorService>>,
stream: TcpStream,
authority_mode: ClientAuthorityMode,
) -> Result<(), CoordinatorServiceError> {
let mut reader = BufReader::new(stream.try_clone()?);
let mut writer = stream;
loop {
let mut line = String::new();
if reader.read_line(&mut line)? == 0 {
return Ok(());
}
if line.trim().is_empty() {
continue;
}
let response = match decode_wire_request(&line) {
Ok(request) => match authorize_client_request(&request, authority_mode) {
Ok(()) => match service.lock() {
Ok(mut service) => match service.handle_request(request) {
Ok(response) => response,
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(_) => CoordinatorResponse::Error {
message: "coordinator service lock poisoned".to_owned(),
},
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
},
Err(err) => CoordinatorResponse::Error {
message: err.to_string(),
},
};
serde_json::to_writer(&mut writer, &response)?;
writer.write_all(b"\n")?;
writer.flush()?;
}
}
pub fn bind_listener(addr: &str) -> Result<(TcpListener, SocketAddr), CoordinatorServiceError> {
let listener = TcpListener::bind(addr)?;
let addr = listener.local_addr()?;
Ok((listener, addr))
}
fn decode_wire_request(line: &str) -> Result<CoordinatorRequest, CoordinatorServiceError> {
serde_json::from_str::<super::CoordinatorWireRequest>(line)?
.into_request()
.map_err(CoordinatorServiceError::Protocol)
}
fn authorize_client_request(
request: &CoordinatorRequest,
authority_mode: ClientAuthorityMode,
) -> Result<(), CoordinatorServiceError> {
if authority_mode == ClientAuthorityMode::LocalTrustedLoopback {
return Ok(());
}
match request {
CoordinatorRequest::Ping
| CoordinatorRequest::Authenticated { .. }
| CoordinatorRequest::ExchangeNodeEnrollmentGrant { .. }
| CoordinatorRequest::SignedNode { .. }
| CoordinatorRequest::NodeHeartbeat {
node_signature: Some(_),
..
}
| CoordinatorRequest::StartProcess {
actor_agent: Some(_),
agent_signature: Some(_),
..
}
| CoordinatorRequest::LaunchTask {
actor_agent: Some(_),
agent_signature: Some(_),
..
}
| CoordinatorRequest::AdminStatus { .. }
| CoordinatorRequest::SuspendTenant { .. } => Ok(()),
_ => Err(CoordinatorServiceError::Protocol(
"strict Core Client authority requires an authenticated CLI session, signed Agent, signed Node, enrollment grant exchange, or admin credential; request-body identity fields are not authority"
.to_owned(),
)),
}
}
#[cfg(test)]
mod transport_boundary_tests {
use super::*;
#[test]
fn native_plaintext_service_refuses_non_loopback_listener() {
let (listener, _) = bind_listener("0.0.0.0:0").unwrap();
let error = CoordinatorService::new(1).serve_tcp(listener).unwrap_err();
assert!(error.to_string().contains("restricted to loopback"));
}
}

6404
crates/clusterflux-coordinator/src/service/tests.rs Normal file
View file

File diff suppressed because it is too large Load diff

59
crates/clusterflux-coordinator/src/service/wire_protocol.rs Normal file
View file

@ -0,0 +1,59 @@
use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::CoordinatorRequest;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum CoordinatorWireRequest {
Envelope(CoordinatorRequestEnvelope),
}
impl CoordinatorWireRequest {
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
match self {
Self::Envelope(envelope) => envelope.into_request(),
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct CoordinatorRequestEnvelope {
#[serde(rename = "type")]
pub envelope_type: String,
pub protocol_version: u64,
pub request_id: String,
pub operation: String,
#[serde(default)]
pub authentication: Option<Value>,
pub payload: CoordinatorRequest,
}
impl CoordinatorRequestEnvelope {
pub fn into_request(self) -> Result<CoordinatorRequest, String> {
if self.envelope_type != COORDINATOR_WIRE_REQUEST_TYPE {
return Err(format!(
"unsupported coordinator wire request type {}; expected {}",
self.envelope_type, COORDINATOR_WIRE_REQUEST_TYPE
));
}
if self.protocol_version != COORDINATOR_PROTOCOL_VERSION {
return Err(format!(
"unsupported coordinator protocol version {}; expected {}",
self.protocol_version, COORDINATOR_PROTOCOL_VERSION
));
}
if self.request_id.trim().is_empty() {
return Err("coordinator wire request_id must be non-empty".to_owned());
}
let payload_operation = self.payload.operation()?;
if self.operation != payload_operation {
return Err(format!(
"coordinator wire operation {} does not match payload operation {}",
self.operation, payload_operation
));
}
Ok(self.payload)
}
}

168
crates/clusterflux-coordinator/src/sessions.rs Normal file
View file

@ -0,0 +1,168 @@
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{Actor, AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId};
use crate::{CliSessionRecord, Coordinator, CoordinatorError, CredentialRecord};
impl Coordinator {
pub fn issue_cli_session(
&mut self,
tenant: TenantId,
project: ProjectId,
user: UserId,
session_secret: &str,
expires_at_epoch_seconds: Option<u64>,
) -> CliSessionRecord {
self.upsert_tenant(tenant.clone());
self.upsert_user(
tenant.clone(),
user.clone(),
CredentialKind::CliDeviceSession,
);
let session_digest = Digest::sha256(session_secret);
let record = CliSessionRecord {
session_digest: session_digest.clone(),
tenant: tenant.clone(),
project: project.clone(),
user: user.clone(),
credential_kind: CredentialKind::CliDeviceSession,
expires_at_epoch_seconds,
revoked: false,
};
self.durable
.cli_sessions
.insert(session_digest.clone(), record.clone());
let credential_subject = format!("cli-session:{}", session_digest.as_str());
self.durable.credentials.insert(
credential_subject.clone(),
CredentialRecord {
subject: credential_subject,
tenant,
project: Some(project),
kind: CredentialKind::CliDeviceSession,
public_key_fingerprint: None,
},
);
record
}
pub fn authenticate_cli_session(
&self,
session_secret: &str,
) -> Result<AuthContext, CoordinatorError> {
self.authenticate_cli_session_at(session_secret, unix_timestamp_seconds())
}
pub fn authenticate_cli_session_at(
&self,
session_secret: &str,
now_epoch_seconds: u64,
) -> Result<AuthContext, CoordinatorError> {
if session_secret.trim().is_empty() {
return Err(CoordinatorError::Unauthorized(
"CLI session credential is missing".to_owned(),
));
}
let session_digest = Digest::sha256(session_secret);
let record = self
.durable
.cli_sessions
.get(&session_digest)
.ok_or_else(|| {
CoordinatorError::Unauthorized(
"CLI session credential is not recognized".to_owned(),
)
})?;
if record.revoked {
return Err(CoordinatorError::Unauthorized(
"CLI session credential has been revoked".to_owned(),
));
}
if record
.expires_at_epoch_seconds
.is_some_and(|expires_at| expires_at <= now_epoch_seconds)
{
return Err(CoordinatorError::Unauthorized(
"CLI session credential has expired; run clusterflux login --browser again"
.to_owned(),
));
}
if record.credential_kind != CredentialKind::CliDeviceSession {
return Err(CoordinatorError::Unauthorized(
"credential is not a CLI session".to_owned(),
));
}
self.ensure_tenant_active(&record.tenant)?;
Ok(AuthContext {
tenant: record.tenant.clone(),
project: record.project.clone(),
actor: Actor::User(record.user.clone()),
})
}
pub fn revoke_cli_session(
&mut self,
session_secret: &str,
) -> Result<CliSessionRecord, CoordinatorError> {
self.authenticate_cli_session(session_secret)?;
let session_digest = Digest::sha256(session_secret);
let record = self
.durable
.cli_sessions
.get_mut(&session_digest)
.expect("authenticated session must exist");
record.revoked = true;
Ok(record.clone())
}
}
fn unix_timestamp_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use crate::InMemoryDurableStore;
use super::*;
#[test]
fn repeated_logins_for_one_user_create_distinct_session_credentials() {
let store = InMemoryDurableStore::default();
let mut coordinator = Coordinator::boot(&store, 1);
let tenant = TenantId::from("tenant");
let user = UserId::from("user");
coordinator.issue_cli_session(
tenant.clone(),
ProjectId::from("project-one"),
user.clone(),
"session-one",
None,
);
coordinator.issue_cli_session(
tenant,
ProjectId::from("project-two"),
user,
"session-two",
None,
);
let subjects = coordinator
.durable
.credentials
.values()
.map(|credential| credential.subject.clone())
.collect::<BTreeSet<_>>();
assert_eq!(coordinator.durable.cli_sessions.len(), 2);
assert_eq!(subjects.len(), 2);
assert!(subjects
.iter()
.all(|subject| subject.starts_with("cli-session:sha256:")));
}
}

22
crates/clusterflux-core/Cargo.toml Normal file
View file

@ -0,0 +1,22 @@
[package]
name = "clusterflux-core"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
ed25519-dalek.workspace = true
serde.workspace = true
serde_json.workspace = true
hex.workspace = true
sha2.workspace = true
syn.workspace = true
thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
getrandom.workspace = true
[dev-dependencies]
tempfile.workspace = true

1178
crates/clusterflux-core/src/artifact.rs Normal file
View file

File diff suppressed because it is too large Load diff

829
crates/clusterflux-core/src/auth.rs Normal file
View file

@ -0,0 +1,829 @@
#[cfg(not(target_arch = "wasm32"))]
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64::{engine::general_purpose::STANDARD, Engine as _};
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
AgentId, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId,
};
pub fn admin_request_proof(
admin_token: &str,
operation: &str,
tenant: &str,
actor_user: &str,
target_tenant: &str,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Digest {
admin_request_proof_from_token_digest(
&Digest::sha256(admin_token),
operation,
tenant,
actor_user,
target_tenant,
nonce,
issued_at_epoch_seconds,
)
}
pub fn admin_request_proof_from_token_digest(
admin_token_digest: &Digest,
operation: &str,
tenant: &str,
actor_user: &str,
target_tenant: &str,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Digest {
let issued_at_epoch_seconds = issued_at_epoch_seconds.to_string();
Digest::from_parts([
b"clusterflux-admin-request-proof:v1".as_slice(),
admin_token_digest.as_str().as_bytes(),
operation.as_bytes(),
tenant.as_bytes(),
actor_user.as_bytes(),
target_tenant.as_bytes(),
nonce.as_bytes(),
issued_at_epoch_seconds.as_bytes(),
])
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Actor {
User(UserId),
Agent(AgentId),
Node(NodeId),
Task(TaskInstanceId),
}
impl Actor {
pub fn kind(&self) -> IdentityKind {
match self {
Self::User(_) => IdentityKind::User,
Self::Agent(_) => IdentityKind::Agent,
Self::Node(_) => IdentityKind::Node,
Self::Task(_) => IdentityKind::Task,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum IdentityKind {
User,
Agent,
Node,
Project,
Task,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum CredentialKind {
BrowserSession,
CliDeviceSession,
PublicKey,
NodeCredential,
TaskCredential,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AuthContext {
pub tenant: TenantId,
pub project: ProjectId,
pub actor: Actor,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Action {
CreateProject,
AttachNode,
CreateNodeEnrollmentGrant,
ExchangeNodeEnrollmentGrant,
LoginBrowser,
LoginCli,
EnrollAgent,
List,
Inspect,
Mutate,
ClaimTask,
DebugAttach,
DebugRead,
DownloadArtifact,
PublishArtifact,
RunNativeCommand,
RunContainer,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowserLoginFlow {
pub authorization_url: String,
pub callback_path: String,
pub state: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PublicKeyIdentity {
pub subject: Actor,
pub public_key: String,
pub fingerprint: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgentSignedRequest {
pub nonce: String,
pub issued_at_epoch_seconds: u64,
pub signature: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeSignedRequest {
pub nonce: String,
pub issued_at_epoch_seconds: u64,
pub signature: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnrollmentGrant {
pub tenant: TenantId,
pub project: ProjectId,
pub grant_id: String,
pub scope: String,
pub expires_at_epoch_seconds: u64,
pub consumed: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeCredential {
pub node: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub public_key_fingerprint: Digest,
pub scope: String,
pub capability_policy_digest: Digest,
pub credential_kind: CredentialKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum EnrollmentError {
Expired,
AlreadyConsumed,
WrongScope,
}
impl EnrollmentGrant {
pub fn exchange_for_node_identity(
&mut self,
node: NodeId,
public_key: &str,
requested_scope: &str,
now_epoch_seconds: u64,
) -> Result<NodeCredential, EnrollmentError> {
if self.consumed {
return Err(EnrollmentError::AlreadyConsumed);
}
if now_epoch_seconds > self.expires_at_epoch_seconds {
return Err(EnrollmentError::Expired);
}
if requested_scope != self.scope {
return Err(EnrollmentError::WrongScope);
}
self.consumed = true;
let capability_policy_digest =
node_capability_policy_digest(&self.tenant, &self.project, &self.scope);
Ok(NodeCredential {
node,
tenant: self.tenant.clone(),
project: self.project.clone(),
public_key_fingerprint: Digest::sha256(public_key),
scope: self.scope.clone(),
capability_policy_digest,
credential_kind: CredentialKind::NodeCredential,
})
}
}
pub fn node_capability_policy_digest(
tenant: &TenantId,
project: &ProjectId,
scope: &str,
) -> Digest {
Digest::from_parts([
b"node-capability-policy:v1".as_slice(),
tenant.as_str().as_bytes(),
project.as_str().as_bytes(),
scope.as_bytes(),
])
}
pub fn agent_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "agent private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
Ok(format!(
"ed25519:{}",
STANDARD.encode(signing_key.verifying_key().to_bytes())
))
}
pub fn node_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
agent_ed25519_public_key_from_private_key(private_key)
}
pub fn derive_ed25519_private_key_from_seed(seed: &str) -> String {
let digest = Digest::sha256(seed);
let hex = digest.as_str().trim_start_matches("sha256:");
let bytes = hex::decode(hex).expect("sha256 digest hex should decode");
format!("ed25519:{}", STANDARD.encode(bytes))
}
/// Generates a new Ed25519 private key from the operating system CSPRNG.
///
/// Seed-derived keys remain available for deterministic test fixtures, but
/// must not be used for persisted user, Agent, or Node credentials.
#[cfg(not(target_arch = "wasm32"))]
pub fn generate_ed25519_private_key() -> Result<String, String> {
let mut bytes = [0_u8; 32];
getrandom::fill(&mut bytes)
.map_err(|err| format!("operating system random source failed: {err}"))?;
Ok(format!("ed25519:{}", STANDARD.encode(bytes)))
}
/// Generates an opaque, URL-safe 256-bit token suitable for one-time grants,
/// session secrets, and nonces. The label is non-secret domain separation.
#[cfg(not(target_arch = "wasm32"))]
pub fn generate_opaque_token(label: &str) -> Result<String, String> {
let mut bytes = [0_u8; 32];
getrandom::fill(&mut bytes)
.map_err(|err| format!("operating system random source failed: {err}"))?;
Ok(format!("{label}_{}", URL_SAFE_NO_PAD.encode(bytes)))
}
#[derive(Clone, Copy, Debug)]
pub struct AgentWorkflowScope<'a> {
pub tenant: &'a TenantId,
pub project: &'a ProjectId,
pub agent: &'a AgentId,
pub request_kind: &'a str,
pub process: &'a ProcessId,
pub task: Option<&'a TaskInstanceId>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AgentWorkflowRequestScope {
pub tenant: TenantId,
pub project: ProjectId,
pub request_kind: String,
pub process: ProcessId,
pub task: Option<TaskInstanceId>,
}
impl AgentWorkflowRequestScope {
pub fn new(
tenant: TenantId,
project: ProjectId,
request_kind: impl Into<String>,
process: ProcessId,
task: Option<TaskInstanceId>,
) -> Result<Self, String> {
let request_kind = request_kind.into();
match request_kind.as_str() {
"start_process" if task.is_some() => {
return Err("start_process agent scope must not contain a task instance".to_owned())
}
"launch_task" if task.is_none() => {
return Err("launch_task agent scope requires a task instance".to_owned())
}
"start_process" | "launch_task" => {}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
}
Ok(Self {
tenant,
project,
request_kind,
process,
task,
})
}
pub fn for_agent<'a>(&'a self, agent: &'a AgentId) -> AgentWorkflowScope<'a> {
AgentWorkflowScope {
tenant: &self.tenant,
project: &self.project,
agent,
request_kind: &self.request_kind,
process: &self.process,
task: self.task.as_ref(),
}
}
}
pub fn agent_workflow_request_scope_from_payload(
payload: &Value,
) -> Result<AgentWorkflowRequestScope, String> {
let object = payload
.as_object()
.ok_or_else(|| "agent workflow request payload must be an object".to_owned())?;
let request_kind = object
.get("type")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow request type is missing".to_owned())?;
let tenant = object
.get("tenant")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow tenant is missing".to_owned())?;
let project = object
.get("project")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow project is missing".to_owned())?;
let (process, task) = match request_kind {
"start_process" => (
object
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "start_process agent scope is missing process".to_owned())?,
None,
),
"launch_task" => {
let task_spec = object
.get("task_spec")
.and_then(Value::as_object)
.ok_or_else(|| "launch_task agent scope is missing task_spec".to_owned())?;
let process = task_spec
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing process".to_owned())?;
let task = task_spec
.get("task_instance")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
(process, Some(TaskInstanceId::from(task)))
}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
};
AgentWorkflowRequestScope::new(
TenantId::from(tenant),
ProjectId::from(project),
request_kind,
ProcessId::from(process),
task,
)
}
pub fn sign_agent_workflow_request(
private_key: &str,
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<AgentSignedRequest, String> {
let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "agent private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
let message =
agent_workflow_signature_message(scope, payload_digest, &nonce, issued_at_epoch_seconds);
let signature: Signature = signing_key.sign(&message);
Ok(AgentSignedRequest {
nonce,
issued_at_epoch_seconds,
signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
})
}
pub fn verify_agent_workflow_signature(
public_key: &str,
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
signed_request: &AgentSignedRequest,
) -> Result<(), String> {
let public_key = decode_ed25519_key(public_key, 32, "agent public key")?;
let public_key: [u8; 32] = public_key
.try_into()
.map_err(|_| "agent public key must be 32 bytes".to_owned())?;
let verifying_key = VerifyingKey::from_bytes(&public_key)
.map_err(|_| "agent public key is not a valid Ed25519 verifying key".to_owned())?;
let signature = decode_ed25519_key(&signed_request.signature, 64, "agent signature")?;
let signature: [u8; 64] = signature
.try_into()
.map_err(|_| "agent signature must be 64 bytes".to_owned())?;
let signature = Signature::from_bytes(&signature);
let message = agent_workflow_signature_message(
scope,
payload_digest,
&signed_request.nonce,
signed_request.issued_at_epoch_seconds,
);
verifying_key
.verify(&message, &signature)
.map_err(|_| "agent signature does not verify against the registered public key".to_owned())
}
pub fn sign_node_request(
private_key: &str,
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<NodeSignedRequest, String> {
let private_key = decode_ed25519_key(private_key, 32, "node private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "node private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
let message = node_request_signature_message(
node,
request_kind,
payload_digest,
&nonce,
issued_at_epoch_seconds,
);
let signature: Signature = signing_key.sign(&message);
Ok(NodeSignedRequest {
nonce,
issued_at_epoch_seconds,
signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
})
}
pub fn verify_node_request_signature(
public_key: &str,
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
signed_request: &NodeSignedRequest,
) -> Result<(), String> {
let public_key = decode_ed25519_key(public_key, 32, "node public key")?;
let public_key: [u8; 32] = public_key
.try_into()
.map_err(|_| "node public key must be 32 bytes".to_owned())?;
let verifying_key = VerifyingKey::from_bytes(&public_key)
.map_err(|_| "node public key is not a valid Ed25519 verifying key".to_owned())?;
let signature = decode_ed25519_key(&signed_request.signature, 64, "node signature")?;
let signature: [u8; 64] = signature
.try_into()
.map_err(|_| "node signature must be 64 bytes".to_owned())?;
let signature = Signature::from_bytes(&signature);
let message = node_request_signature_message(
node,
request_kind,
payload_digest,
&signed_request.nonce,
signed_request.issued_at_epoch_seconds,
);
verifying_key
.verify(&message, &signature)
.map_err(|_| "node signature does not verify against the enrolled public key".to_owned())
}
fn decode_ed25519_key(value: &str, expected_len: usize, kind: &str) -> Result<Vec<u8>, String> {
let encoded = value
.strip_prefix("ed25519:")
.ok_or_else(|| format!("{kind} must use ed25519:<base64> encoding"))?;
let bytes = STANDARD
.decode(encoded)
.map_err(|_| format!("{kind} is not valid base64"))?;
if bytes.len() != expected_len {
return Err(format!("{kind} must be {expected_len} bytes"));
}
Ok(bytes)
}
fn agent_workflow_signature_message(
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Vec<u8> {
let issued_at = issued_at_epoch_seconds.to_string();
let task = scope.task.map(TaskInstanceId::as_str).unwrap_or("");
let parts = [
"clusterflux-agent-workflow-signature:v2",
scope.tenant.as_str(),
scope.project.as_str(),
scope.agent.as_str(),
scope.request_kind,
scope.process.as_str(),
task,
payload_digest.as_str(),
nonce,
&issued_at,
];
let mut message = Vec::new();
for part in parts {
message.extend_from_slice(part.len().to_string().as_bytes());
message.push(b':');
message.extend_from_slice(part.as_bytes());
message.push(b'\n');
}
message
}
fn node_request_signature_message(
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Vec<u8> {
let issued_at = issued_at_epoch_seconds.to_string();
let parts = [
"clusterflux-node-request-signature:v2",
node.as_str(),
request_kind,
payload_digest.as_str(),
nonce,
&issued_at,
];
let mut message = Vec::new();
for part in parts {
message.extend_from_slice(part.len().to_string().as_bytes());
message.push(b':');
message.extend_from_slice(part.as_bytes());
message.push(b'\n');
}
message
}
/// Computes the stable digest covered by node and agent request signatures.
///
/// The proof field itself is excluded, and explicit JSON nulls are normalized
/// with omitted optional fields because the wire protocol deserializes both to
/// the same request. Every semantically meaningful key and value remains bound
/// by the signature.
pub fn signed_request_payload_digest(value: &Value) -> Digest {
fn canonicalize(value: &Value, top_level: bool) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.filter(|(key, value)| {
!value.is_null()
&& (!top_level
|| !matches!(key.as_str(), "agent_signature" | "node_signature"))
})
.map(|(key, value)| (key.clone(), canonicalize(value, false)))
.collect(),
),
Value::Array(values) => Value::Array(
values
.iter()
.map(|value| canonicalize(value, false))
.collect(),
),
value => value.clone(),
}
}
let canonical = canonicalize(value, true);
let bytes = serde_json::to_vec(&canonical)
.expect("canonical JSON request values are always serializable");
Digest::sha256(bytes)
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Scope {
pub tenant: TenantId,
pub project: ProjectId,
pub process: Option<ProcessId>,
pub task: Option<TaskInstanceId>,
pub node: Option<NodeId>,
pub artifact: Option<ArtifactId>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Authorization {
pub allowed: bool,
pub reason: String,
}
impl Authorization {
pub fn allow(reason: impl Into<String>) -> Self {
Self {
allowed: true,
reason: reason.into(),
}
}
pub fn deny(reason: impl Into<String>) -> Self {
Self {
allowed: false,
reason: reason.into(),
}
}
}
pub fn same_tenant_project(context: &AuthContext, scope: &Scope) -> Authorization {
if context.tenant != scope.tenant {
return Authorization::deny("tenant mismatch");
}
if context.project != scope.project {
return Authorization::deny("project mismatch");
}
Authorization::allow("same tenant and project")
}
pub fn task_credentials_do_not_contain_user_session(
task: &Actor,
credentials: &[CredentialKind],
) -> Authorization {
if !matches!(task, Actor::Task(_)) {
return Authorization::deny("credential check requires task actor");
}
if credentials.iter().any(|credential| {
matches!(
credential,
CredentialKind::BrowserSession | CredentialKind::CliDeviceSession
)
}) {
return Authorization::deny(
"user OAuth/session tokens must not be passed to nodes as task credentials",
);
}
Authorization::allow("task credentials are scoped runtime credentials")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tenant_project_scope_denies_cross_tenant_access() {
let context = AuthContext {
tenant: TenantId::from("tenant-a"),
project: ProjectId::from("project-a"),
actor: Actor::User(UserId::from("user-a")),
};
let scope = Scope {
tenant: TenantId::from("tenant-b"),
project: ProjectId::from("project-a"),
process: None,
task: None,
node: None,
artifact: None,
};
assert!(!same_tenant_project(&context, &scope).allowed);
}
#[test]
fn node_enrollment_exchanges_short_lived_grant_once() {
let mut grant = EnrollmentGrant {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
grant_id: "grant".to_owned(),
scope: "node:attach".to_owned(),
expires_at_epoch_seconds: 100,
consumed: false,
};
let credential = grant
.exchange_for_node_identity(NodeId::from("node"), "public-key", "node:attach", 99)
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert_eq!(credential.tenant, TenantId::from("tenant"));
assert_eq!(credential.project, ProjectId::from("project"));
assert_eq!(credential.node, NodeId::from("node"));
assert_eq!(credential.scope, "node:attach");
assert_eq!(
credential.capability_policy_digest,
node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:attach"
)
);
assert_eq!(
grant.exchange_for_node_identity(
NodeId::from("node2"),
"public-key",
"node:attach",
99
),
Err(EnrollmentError::AlreadyConsumed)
);
}
#[test]
fn node_capability_policy_digest_is_scoped() {
let base = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:attach",
);
let other_project = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("other"),
"node:attach",
);
let other_scope = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:limited",
);
assert!(base.is_valid_sha256());
assert_ne!(base, other_project);
assert_ne!(base, other_scope);
}
#[test]
fn generated_ed25519_private_keys_are_random_and_valid() {
let first = generate_ed25519_private_key().unwrap();
let second = generate_ed25519_private_key().unwrap();
assert_ne!(first, second);
assert!(agent_ed25519_public_key_from_private_key(&first)
.unwrap()
.starts_with("ed25519:"));
assert!(node_ed25519_public_key_from_private_key(&second)
.unwrap()
.starts_with("ed25519:"));
}
#[test]
fn generated_opaque_tokens_are_random_and_url_safe() {
let first = generate_opaque_token("grant").unwrap();
let second = generate_opaque_token("grant").unwrap();
assert_ne!(first, second);
assert!(first.starts_with("grant_"));
assert!(first
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')));
}
#[test]
fn task_credentials_reject_user_session_tokens() {
for credential in [
CredentialKind::BrowserSession,
CredentialKind::CliDeviceSession,
] {
let authz = task_credentials_do_not_contain_user_session(
&Actor::Task(TaskInstanceId::from("task")),
&[CredentialKind::TaskCredential, credential],
);
assert!(!authz.allowed);
assert!(authz.reason.contains("must not be passed"));
}
let scoped = task_credentials_do_not_contain_user_session(
&Actor::Task(TaskInstanceId::from("task")),
&[
CredentialKind::TaskCredential,
CredentialKind::NodeCredential,
],
);
assert!(scoped.allowed);
}
#[test]
fn identities_remain_distinct_for_authorization() {
assert_eq!(Actor::User(UserId::from("user")).kind(), IdentityKind::User);
assert_eq!(
Actor::Agent(AgentId::from("agent")).kind(),
IdentityKind::Agent
);
assert_eq!(Actor::Node(NodeId::from("node")).kind(), IdentityKind::Node);
assert_eq!(
Actor::Task(TaskInstanceId::from("task")).kind(),
IdentityKind::Task
);
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: Some(ProcessId::from("process")),
task: Some(TaskInstanceId::from("task")),
node: Some(NodeId::from("node")),
artifact: Some(ArtifactId::from("artifact")),
};
assert_eq!(scope.process, Some(ProcessId::from("process")));
assert_eq!(scope.artifact, Some(ArtifactId::from("artifact")));
assert_ne!(
CredentialKind::BrowserSession,
CredentialKind::CliDeviceSession
);
assert_ne!(CredentialKind::PublicKey, CredentialKind::NodeCredential);
assert_ne!(
CredentialKind::NodeCredential,
CredentialKind::TaskCredential
);
}
}

459
crates/clusterflux-core/src/bundle.rs Normal file
View file

@ -0,0 +1,459 @@
use serde::{Deserialize, Serialize};
use syn::parse::Parser;
use syn::{Expr, Item, Lit, Meta, Token};
use crate::{Digest, EnvironmentResource, SourceTransferPolicy, TaskDefinitionId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SelectedInput {
pub path: String,
pub digest: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleIdentityInputs {
pub wasm_code: Digest,
pub task_abi: Digest,
pub entrypoints: Vec<String>,
pub default_entrypoint: String,
pub environments: Vec<EnvironmentResource>,
pub source_provider_manifest: Digest,
pub source_transfer_policy: SourceTransferPolicy,
pub selected_inputs: Vec<SelectedInput>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleMetadata {
pub identity: Digest,
pub wasm_code: Digest,
pub task_metadata: BundleTaskMetadata,
pub source_metadata: BundleSourceMetadata,
pub debug_metadata: BundleDebugMetadata,
pub large_input_policy: BundleLargeInputPolicy,
pub restart_compatibility: BundleRestartCompatibility,
pub environments: Vec<EnvironmentResource>,
pub selected_inputs: Vec<SelectedInput>,
pub embeds_full_container_images: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleTaskMetadata {
pub task_abi: Digest,
pub entrypoints: Vec<String>,
pub default_entrypoint: String,
pub boundary: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleSourceMetadata {
pub source_provider_manifest: Digest,
pub transfer_policy: SourceTransferPolicy,
pub selected_inputs: Vec<SelectedInput>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleDebugMetadata {
pub available: bool,
pub source_level_breakpoints: bool,
pub dap_virtual_process: bool,
pub variables_pane_supported: bool,
pub probes: Vec<BundleDebugProbe>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleDebugProbe {
pub id: String,
pub source_path: String,
pub line_start: u32,
pub line_end: u32,
pub function: String,
pub task: TaskDefinitionId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleLargeInputPolicy {
pub selected_inputs_are_content_digests: bool,
pub selected_input_bytes_included: bool,
pub full_repository_bytes_included: bool,
pub silent_task_argument_serialization: bool,
pub supported_handle_types: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleRestartCompatibility {
pub source_edits_can_restart_from_clean_task_boundary: bool,
pub requires_clean_checkpoint_boundary: bool,
pub compares_task_abi: Digest,
pub compares_environment_digests: bool,
pub compares_serialized_args: bool,
pub discards_unflushed_task_local_changes: bool,
pub incompatible_changes_require_whole_process_restart: bool,
}
impl BundleIdentityInputs {
pub fn identity(&self) -> Digest {
let mut parts = vec![
b"bundle:v1".to_vec(),
self.wasm_code.as_str().as_bytes().to_vec(),
self.task_abi.as_str().as_bytes().to_vec(),
self.source_provider_manifest.as_str().as_bytes().to_vec(),
self.default_entrypoint.as_bytes().to_vec(),
format!("{:?}", self.source_transfer_policy).into_bytes(),
];
let mut entrypoints = self.entrypoints.clone();
entrypoints.sort();
for entrypoint in entrypoints {
parts.push(entrypoint.into_bytes());
}
let mut environments = self.environments.clone();
environments.sort_by(|left, right| left.name.cmp(&right.name));
for environment in environments {
parts.push(environment.name.as_bytes().to_vec());
parts.push(format!("{:?}", environment.kind).into_bytes());
parts.push(environment.digest.as_str().as_bytes().to_vec());
}
let mut inputs = self.selected_inputs.clone();
inputs.sort_by(|left, right| left.path.cmp(&right.path));
for input in inputs {
parts.push(input.path.into_bytes());
parts.push(input.digest.as_str().as_bytes().to_vec());
}
Digest::from_parts(parts)
}
pub fn inspectable_metadata(&self) -> BundleMetadata {
let mut entrypoints = self.entrypoints.clone();
entrypoints.sort();
BundleMetadata {
identity: self.identity(),
wasm_code: self.wasm_code.clone(),
task_metadata: BundleTaskMetadata {
task_abi: self.task_abi.clone(),
entrypoints,
default_entrypoint: self.default_entrypoint.clone(),
boundary: "clusterflux_task_exports".to_owned(),
},
source_metadata: BundleSourceMetadata {
source_provider_manifest: self.source_provider_manifest.clone(),
transfer_policy: self.source_transfer_policy.clone(),
selected_inputs: self.selected_inputs.clone(),
},
debug_metadata: BundleDebugMetadata {
available: true,
source_level_breakpoints: true,
dap_virtual_process: true,
variables_pane_supported: true,
probes: Vec::new(),
},
large_input_policy: BundleLargeInputPolicy {
selected_inputs_are_content_digests: true,
selected_input_bytes_included: false,
full_repository_bytes_included: false,
silent_task_argument_serialization: false,
supported_handle_types: vec![
"SourceSnapshot".to_owned(),
"Blob".to_owned(),
"Artifact".to_owned(),
"VFS".to_owned(),
],
},
restart_compatibility: BundleRestartCompatibility {
source_edits_can_restart_from_clean_task_boundary: true,
requires_clean_checkpoint_boundary: true,
compares_task_abi: self.task_abi.clone(),
compares_environment_digests: true,
compares_serialized_args: true,
discards_unflushed_task_local_changes: true,
incompatible_changes_require_whole_process_restart: true,
},
environments: self.environments.clone(),
selected_inputs: self.selected_inputs.clone(),
embeds_full_container_images: false,
}
}
}
pub fn discover_source_debug_probes(
source_path: impl Into<String>,
source: &str,
) -> Vec<BundleDebugProbe> {
let source_path = source_path.into();
let lines = source.lines().collect::<Vec<_>>();
let function_starts = lines
.iter()
.enumerate()
.filter_map(|(index, line)| {
parse_rust_function_name(line).map(|function| (index, function))
})
.collect::<Vec<_>>();
let Ok(file) = syn::parse_file(source) else {
return Vec::new();
};
file.items
.iter()
.filter_map(|item| {
let Item::Fn(function) = item else {
return None;
};
let function_name = function.sig.ident.to_string();
let task = clusterflux_probe_task(function)?;
let (function_index, (line_index, _)) = function_starts
.iter()
.enumerate()
.find(|(_, (_, candidate))| candidate == &function_name)?;
let line_start = (*line_index + 1) as u32;
let next_start = function_starts
.get(function_index + 1)
.map(|(next_index, _)| *next_index)
.unwrap_or(lines.len());
let line_end = next_start.max(*line_index + 1) as u32;
Some(debug_probe(
&source_path,
line_start,
line_end,
&function_name,
task,
))
})
.collect()
}
fn debug_probe(
source_path: &str,
line_start: u32,
line_end: u32,
function: &str,
task: TaskDefinitionId,
) -> BundleDebugProbe {
let id = Digest::from_parts([
b"bundle-debug-probe:v1".as_slice(),
source_path.as_bytes(),
function.as_bytes(),
task.as_str().as_bytes(),
line_start.to_string().as_bytes(),
line_end.to_string().as_bytes(),
])
.as_str()
.to_owned();
BundleDebugProbe {
id,
source_path: source_path.to_owned(),
line_start,
line_end,
function: function.to_owned(),
task,
}
}
fn parse_rust_function_name(line: &str) -> Option<String> {
let start = line.find("fn ")? + 3;
let rest = &line[start..];
let name = rest
.chars()
.take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
.collect::<String>();
(!name.is_empty()).then_some(name)
}
fn clusterflux_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> {
for attribute in &function.attrs {
let mut segments = attribute.path().segments.iter();
let Some(namespace) = segments.next() else {
continue;
};
let Some(kind) = segments.next() else {
continue;
};
if namespace.ident != "clusterflux" || segments.next().is_some() {
continue;
}
let function_name = function.sig.ident.to_string();
let default_name = match kind.ident.to_string().as_str() {
"main" => function_name
.strip_suffix("_main")
.unwrap_or(&function_name)
.to_owned(),
"task" => function_name,
_ => continue,
};
let declared_name = match &attribute.meta {
Meta::List(list) => {
let parser = syn::punctuated::Punctuated::<Meta, Token![,]>::parse_terminated;
parser
.parse2(list.tokens.clone())
.ok()
.and_then(|items| {
items.into_iter().find_map(|item| {
let Meta::NameValue(name_value) = item else {
return None;
};
if !name_value.path.is_ident("name") {
return None;
}
let Expr::Lit(value) = name_value.value else {
return None;
};
let Lit::Str(value) = value.lit else {
return None;
};
Some(value.value())
})
})
.unwrap_or(default_name)
}
_ => default_name,
};
return Some(TaskDefinitionId::new(declared_name));
}
None
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::{EnvironmentKind, EnvironmentRequirements};
use super::*;
fn env(digest: &str) -> EnvironmentResource {
EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: PathBuf::from("envs/linux/Containerfile"),
context_path: PathBuf::from("envs/linux"),
digest: Digest::sha256(digest),
requirements: EnvironmentRequirements::linux_container(),
}
}
#[test]
fn bundle_identity_changes_when_environment_recipe_changes() {
let base = BundleIdentityInputs {
wasm_code: Digest::sha256("wasm"),
task_abi: Digest::sha256("abi"),
entrypoints: vec!["build".to_owned()],
default_entrypoint: "build".to_owned(),
environments: vec![env("recipe-a")],
source_provider_manifest: Digest::sha256("source"),
source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
selected_inputs: vec![],
};
let mut changed = base.clone();
changed.environments = vec![env("recipe-b")];
assert_ne!(base.identity(), changed.identity());
}
#[test]
fn bundle_metadata_is_inspectable_and_does_not_vendor_images_by_default() {
let inputs = BundleIdentityInputs {
wasm_code: Digest::sha256("wasm"),
task_abi: Digest::sha256("abi"),
entrypoints: vec!["build".to_owned(), "test".to_owned()],
default_entrypoint: "build".to_owned(),
environments: vec![env("recipe")],
source_provider_manifest: Digest::sha256("source"),
source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
selected_inputs: vec![SelectedInput {
path: "inputs/config.json".to_owned(),
digest: Digest::sha256("config"),
}],
};
let metadata = inputs.inspectable_metadata();
assert!(metadata.wasm_code.as_str().starts_with("sha256:"));
assert_eq!(metadata.task_metadata.default_entrypoint, "build");
assert!(metadata
.task_metadata
.entrypoints
.contains(&"test".to_owned()));
assert!(metadata.debug_metadata.dap_virtual_process);
assert!(
metadata
.source_metadata
.transfer_policy
.local_source_bytes_remain_node_local
);
assert!(
metadata
.large_input_policy
.selected_inputs_are_content_digests
);
assert!(!metadata.large_input_policy.selected_input_bytes_included);
assert!(!metadata.large_input_policy.full_repository_bytes_included);
assert!(
!metadata
.large_input_policy
.silent_task_argument_serialization
);
assert!(metadata
.large_input_policy
.supported_handle_types
.contains(&"Artifact".to_owned()));
assert!(
metadata
.restart_compatibility
.source_edits_can_restart_from_clean_task_boundary
);
assert!(
metadata
.restart_compatibility
.requires_clean_checkpoint_boundary
);
assert_eq!(
metadata.restart_compatibility.compares_task_abi,
inputs.task_abi
);
assert!(
metadata
.restart_compatibility
.incompatible_changes_require_whole_process_restart
);
assert_eq!(metadata.environments.len(), 1);
assert!(!metadata.embeds_full_container_images);
}
#[test]
fn source_debug_probe_metadata_maps_function_ranges_to_tasks() {
let probes = discover_source_debug_probes(
"src/build.rs",
r#"#[clusterflux::main]
fn build_main() {
let linux = compile_linux();
}
#[clusterflux::task]
fn compile_linux() {
println!("linux");
}
fn helper_without_runtime_probe() {}
#[clusterflux::task(name = "release")]
fn package_release() {
println!("package");
}
"#,
);
assert_eq!(probes.len(), 3);
assert_eq!(probes[0].source_path, "src/build.rs");
assert_eq!(probes[0].function, "build_main");
assert_eq!(probes[0].task, TaskDefinitionId::from("build"));
assert_eq!(probes[0].line_start, 2);
assert_eq!(probes[0].line_end, 6);
assert_eq!(probes[1].function, "compile_linux");
assert_eq!(probes[1].task, TaskDefinitionId::from("compile_linux"));
assert_eq!(probes[2].function, "package_release");
assert_eq!(probes[2].task, TaskDefinitionId::from("release"));
assert!(probes.iter().all(|probe| probe.id.starts_with("sha256:")));
}
}

218
crates/clusterflux-core/src/capability.rs Normal file
View file

@ -0,0 +1,218 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum Capability {
Command,
Containers,
RootlessPodman,
SourceFilesystem,
SourceGit,
HostFilesystem,
Network,
Secrets,
InboundPorts,
ArbitrarySyscalls,
VfsArtifacts,
WindowsCommandDev,
QuicDirect,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum EnvironmentBackend {
Container,
NixFlake,
WindowsCommandDev,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum Os {
Linux,
Windows,
Macos,
Other(String),
}
impl Os {
pub fn current() -> Self {
match std::env::consts::OS {
"linux" => Self::Linux,
"windows" => Self::Windows,
"macos" => Self::Macos,
other => Self::Other(other.to_owned()),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeCapabilities {
pub os: Os,
pub arch: String,
pub capabilities: BTreeSet<Capability>,
pub environment_backends: BTreeSet<EnvironmentBackend>,
pub source_providers: BTreeSet<String>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum CapabilityReportError {
#[error("node architecture `{0}` is invalid")]
InvalidArchitecture(String),
#[error("node OS label `{0}` is invalid")]
InvalidOsLabel(String),
#[error("source provider id `{0}` is invalid")]
InvalidSourceProvider(String),
}
impl NodeCapabilities {
pub fn detect_current() -> Self {
let os = Os::current();
let mut capabilities = BTreeSet::from([
Capability::Command,
Capability::SourceFilesystem,
Capability::VfsArtifacts,
]);
let mut environment_backends = BTreeSet::new();
match os {
Os::Linux => {
if rootless_podman_available() {
capabilities.insert(Capability::Containers);
capabilities.insert(Capability::RootlessPodman);
environment_backends.insert(EnvironmentBackend::Container);
}
}
Os::Windows => {
capabilities.insert(Capability::WindowsCommandDev);
environment_backends.insert(EnvironmentBackend::WindowsCommandDev);
}
Os::Macos | Os::Other(_) => {}
}
Self {
os,
arch: std::env::consts::ARCH.to_owned(),
capabilities,
environment_backends,
source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
}
}
pub fn with_capability(mut self, capability: Capability) -> Self {
self.capabilities.insert(capability);
self
}
pub fn has_all(&self, required: &BTreeSet<Capability>) -> bool {
required
.iter()
.all(|capability| self.capabilities.contains(capability))
}
pub fn validate_public_report(&self) -> Result<(), CapabilityReportError> {
if !valid_capability_label(&self.arch) {
return Err(CapabilityReportError::InvalidArchitecture(
self.arch.clone(),
));
}
if let Os::Other(label) = &self.os {
if !valid_capability_label(label) {
return Err(CapabilityReportError::InvalidOsLabel(label.clone()));
}
}
for provider in &self.source_providers {
if !valid_source_provider_id(provider) {
return Err(CapabilityReportError::InvalidSourceProvider(
provider.clone(),
));
}
}
Ok(())
}
}
#[cfg(not(target_arch = "wasm32"))]
fn rootless_podman_available() -> bool {
const ATTEMPTS: usize = 3;
for attempt in 0..ATTEMPTS {
match std::process::Command::new("podman")
.args(["info", "--format", "{{.Host.Security.Rootless}}"])
.output()
{
Ok(output) if rootless_podman_probe_succeeded(&output) => return true,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return false,
Ok(_) | Err(_) if attempt + 1 < ATTEMPTS => {
std::thread::sleep(std::time::Duration::from_millis(250));
}
Ok(_) | Err(_) => {}
}
}
false
}
#[cfg(not(target_arch = "wasm32"))]
fn rootless_podman_probe_succeeded(output: &std::process::Output) -> bool {
output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == "true"
}
#[cfg(target_arch = "wasm32")]
fn rootless_podman_available() -> bool {
false
}
fn valid_capability_label(label: &str) -> bool {
!label.is_empty()
&& label.len() <= 64
&& label.bytes().all(
|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.'),
)
}
fn valid_source_provider_id(provider: &str) -> bool {
!provider.is_empty()
&& provider.len() <= 64
&& provider
.bytes()
.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
}
#[cfg(test)]
mod tests {
use super::*;
fn capabilities() -> NodeCapabilities {
NodeCapabilities {
os: Os::Linux,
arch: "x86_64".to_owned(),
capabilities: BTreeSet::from([Capability::Command]),
environment_backends: BTreeSet::new(),
source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
}
}
#[test]
fn capability_reports_validate_hostile_strings() {
assert!(capabilities().validate_public_report().is_ok());
let mut invalid_arch = capabilities();
invalid_arch.arch = "x86_64\nmalicious".to_owned();
assert_eq!(
invalid_arch.validate_public_report(),
Err(CapabilityReportError::InvalidArchitecture(
"x86_64\nmalicious".to_owned()
))
);
let mut invalid_provider = capabilities();
invalid_provider
.source_providers
.insert("../checkout".to_owned());
assert_eq!(
invalid_provider.validate_public_report(),
Err(CapabilityReportError::InvalidSourceProvider(
"../checkout".to_owned()
))
);
}
}

284
crates/clusterflux-core/src/checkpoint.rs Normal file
View file

@ -0,0 +1,284 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Digest, TaskInstanceId, VfsManifest};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CheckpointBoundary {
pub task_entrypoint: String,
pub serialized_args: Digest,
pub environment_digest: Digest,
pub vfs_epoch: u64,
pub task_abi: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCheckpoint {
pub task: TaskInstanceId,
pub boundary: CheckpointBoundary,
pub vfs_manifest: VfsManifest,
pub depends_on_live_stack: bool,
pub depends_on_live_socket: bool,
pub depends_on_ephemeral_artifact_durability: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RestartRequest {
pub task: TaskInstanceId,
pub entrypoint: String,
pub serialized_args: Digest,
pub environment_digest: Digest,
pub task_abi: Digest,
pub source_edited: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum RestartDecision {
RestartTask {
task: TaskInstanceId,
from_vfs_epoch: u64,
discard_unflushed_changes: bool,
},
RestartWholeVirtualProcess {
message: String,
},
}
#[derive(Clone, Debug, Default)]
pub struct RestartPolicy;
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum CompatibilityFailure {
#[error("task entrypoint changed")]
Entrypoint,
#[error("serialized task arguments changed")]
Args,
#[error("environment digest changed")]
Environment,
#[error("task ABI changed")]
TaskAbi,
#[error("checkpoint depends on unsupported live stack migration")]
LiveStack,
#[error("checkpoint depends on unsupported live socket checkpointing")]
LiveSocket,
#[error("checkpoint incorrectly treats ephemeral artifacts as durable")]
EphemeralArtifactDurability,
}
impl RestartPolicy {
pub fn decide(&self, checkpoint: &TaskCheckpoint, request: &RestartRequest) -> RestartDecision {
match compatibility_failure(checkpoint, request) {
None => RestartDecision::RestartTask {
task: checkpoint.task.clone(),
from_vfs_epoch: checkpoint.boundary.vfs_epoch,
discard_unflushed_changes: true,
},
Some(failure) => RestartDecision::RestartWholeVirtualProcess {
message: format!(
"cannot restart selected task `{}` from checkpoint: {failure}; restart the whole virtual process",
checkpoint.task
),
},
}
}
}
fn compatibility_failure(
checkpoint: &TaskCheckpoint,
request: &RestartRequest,
) -> Option<CompatibilityFailure> {
if checkpoint.depends_on_live_stack {
return Some(CompatibilityFailure::LiveStack);
}
if checkpoint.depends_on_live_socket {
return Some(CompatibilityFailure::LiveSocket);
}
if checkpoint.depends_on_ephemeral_artifact_durability {
return Some(CompatibilityFailure::EphemeralArtifactDurability);
}
if checkpoint.boundary.task_entrypoint != request.entrypoint {
return Some(CompatibilityFailure::Entrypoint);
}
if checkpoint.boundary.serialized_args != request.serialized_args {
return Some(CompatibilityFailure::Args);
}
if checkpoint.boundary.environment_digest != request.environment_digest {
return Some(CompatibilityFailure::Environment);
}
if checkpoint.boundary.task_abi != request.task_abi {
return Some(CompatibilityFailure::TaskAbi);
}
None
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::{
EnvironmentKind, EnvironmentRequirements, EnvironmentResource, NodeId, VfsOverlay, VfsPath,
};
use super::*;
fn env(digest_input: &str) -> EnvironmentResource {
EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: PathBuf::from("envs/linux/Containerfile"),
context_path: PathBuf::from("envs/linux"),
digest: Digest::sha256(digest_input),
requirements: EnvironmentRequirements::linux_container(),
}
}
fn checkpoint() -> (TaskCheckpoint, EnvironmentResource) {
let environment = env("env");
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node"));
overlay.write(
VfsPath::new("/vfs/artifacts/app").unwrap(),
Digest::sha256("app"),
3,
);
let manifest = overlay.flush();
(
TaskCheckpoint {
task: TaskInstanceId::from("task"),
boundary: CheckpointBoundary {
task_entrypoint: "compile_linux".to_owned(),
serialized_args: Digest::sha256("args"),
environment_digest: environment.digest.clone(),
vfs_epoch: manifest.epoch,
task_abi: Digest::sha256("abi"),
},
vfs_manifest: manifest,
depends_on_live_stack: false,
depends_on_live_socket: false,
depends_on_ephemeral_artifact_durability: false,
},
environment,
)
}
fn restart_request(environment: EnvironmentResource) -> RestartRequest {
RestartRequest {
task: TaskInstanceId::from("task"),
entrypoint: "compile_linux".to_owned(),
serialized_args: Digest::sha256("args"),
environment_digest: environment.digest,
task_abi: Digest::sha256("abi"),
source_edited: true,
}
}
fn assert_whole_process_restart(decision: RestartDecision, expected_reason: &str) {
match decision {
RestartDecision::RestartWholeVirtualProcess { message } => {
assert!(
message.contains(expected_reason),
"restart message `{message}` did not include `{expected_reason}`"
);
assert!(
message.contains("restart the whole virtual process"),
"restart message `{message}` did not direct a whole-process restart"
);
}
RestartDecision::RestartTask { .. } => {
panic!("incompatible checkpoint unexpectedly restarted selected task")
}
}
}
#[test]
fn compatible_restart_uses_task_boundary_and_discards_unflushed_changes() {
let (checkpoint, environment) = checkpoint();
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_eq!(
decision,
RestartDecision::RestartTask {
task: TaskInstanceId::from("task"),
from_vfs_epoch: 1,
discard_unflushed_changes: true
}
);
}
#[test]
fn incompatible_environment_requires_whole_process_restart() {
let (checkpoint, _) = checkpoint();
let request = restart_request(env("changed-env"));
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "environment digest changed");
}
#[test]
fn incompatible_entrypoint_requires_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.entrypoint = "package_linux".to_owned();
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "task entrypoint changed");
}
#[test]
fn incompatible_serialized_args_require_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.serialized_args = Digest::sha256("changed-args");
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "serialized task arguments changed");
}
#[test]
fn incompatible_task_abi_requires_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.task_abi = Digest::sha256("changed-abi");
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "task ABI changed");
}
#[test]
fn restart_never_claims_live_stack_migration() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_live_stack = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "live stack");
}
#[test]
fn restart_never_claims_live_socket_checkpointing() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_live_socket = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "live socket");
}
#[test]
fn restart_never_depends_on_ephemeral_artifact_durability() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_ephemeral_artifact_durability = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "ephemeral artifacts");
}
}

337
crates/clusterflux-core/src/debug.rs Normal file
View file

@ -0,0 +1,337 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{ProcessId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugParticipantKind {
WasmTask,
ControlledNativeCommand,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugRuntimeState {
Running,
Frozen,
Completed,
Failed(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugParticipant {
pub task: TaskInstanceId,
pub name: String,
pub kind: DebugParticipantKind,
pub can_freeze: bool,
pub state: DebugRuntimeState,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugStopReason {
Breakpoint { task: TaskInstanceId, line: u32 },
PauseRequest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ThreadInspection {
pub task: TaskInstanceId,
pub name: String,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpoch {
pub process: ProcessId,
pub epoch: u64,
pub reason: DebugStopReason,
participants: BTreeMap<TaskInstanceId, DebugParticipant>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum DebugEpochError {
#[error("no participant could freeze; `{task}` rejected the freeze request")]
CannotFreeze { task: TaskInstanceId },
#[error("participant `{0}` is not part of this debug epoch")]
UnknownParticipant(TaskInstanceId),
#[error("participant `{0}` did not acknowledge frozen state in this debug epoch")]
ParticipantNotFrozen(TaskInstanceId),
}
impl DebugEpoch {
pub fn all_stop(
process: ProcessId,
epoch: u64,
reason: DebugStopReason,
participants: Vec<DebugParticipant>,
) -> Result<Self, DebugEpochError> {
let first_rejected = participants
.iter()
.find(|participant| {
matches!(participant.state, DebugRuntimeState::Running) && !participant.can_freeze
})
.map(|participant| participant.task.clone());
let participants: BTreeMap<_, _> = participants
.into_iter()
.map(|mut participant| {
if matches!(participant.state, DebugRuntimeState::Running) {
participant.state = if participant.can_freeze {
DebugRuntimeState::Frozen
} else {
DebugRuntimeState::Failed(
"participant did not acknowledge frozen state before the debug deadline"
.to_owned(),
)
};
}
(participant.task.clone(), participant)
})
.collect();
if !participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
{
return Err(DebugEpochError::CannotFreeze {
task: first_rejected.unwrap_or_else(|| TaskInstanceId::from("debug-epoch")),
});
}
Ok(Self {
process,
epoch,
reason,
participants,
})
}
pub fn pause(
process: ProcessId,
epoch: u64,
participants: Vec<DebugParticipant>,
) -> Result<Self, DebugEpochError> {
Self::all_stop(process, epoch, DebugStopReason::PauseRequest, participants)
}
pub fn continue_all(&mut self) {
for participant in self.participants.values_mut() {
if participant.state == DebugRuntimeState::Frozen {
participant.state = DebugRuntimeState::Running;
}
}
}
pub fn inspection(&self, task: &TaskInstanceId) -> Result<ThreadInspection, DebugEpochError> {
let participant = self
.participants
.get(task)
.ok_or_else(|| DebugEpochError::UnknownParticipant(task.clone()))?;
if participant.state != DebugRuntimeState::Frozen {
return Err(DebugEpochError::ParticipantNotFrozen(task.clone()));
}
Ok(ThreadInspection {
task: participant.task.clone(),
name: participant.name.clone(),
stack_frames: participant.stack_frames.clone(),
local_values: participant.local_values.clone(),
task_args: participant.task_args.clone(),
handles: participant.handles.clone(),
command_status: participant.command_status.clone(),
recent_output: participant.recent_output.clone(),
})
}
pub fn participant_state(&self, task: &TaskInstanceId) -> Option<&DebugRuntimeState> {
self.participants
.get(task)
.map(|participant| &participant.state)
}
pub fn thread_names(&self) -> Vec<String> {
self.participants
.values()
.map(|participant| participant.name.clone())
.collect()
}
pub fn all_threads_stopped(&self) -> bool {
!self.participants.is_empty()
&& self
.participants
.values()
.all(|participant| participant.state == DebugRuntimeState::Frozen)
}
pub fn partially_frozen(&self) -> bool {
!self.all_threads_stopped()
&& self
.participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn participant(task: &str, kind: DebugParticipantKind, can_freeze: bool) -> DebugParticipant {
DebugParticipant {
task: TaskInstanceId::from(task),
name: task.to_owned(),
kind,
can_freeze,
state: DebugRuntimeState::Running,
stack_frames: vec![format!("{task}::run")],
local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())],
task_args: vec![("target".to_owned(), "linux".to_owned())],
handles: vec![("artifact".to_owned(), "artifact-1".to_owned())],
command_status: Some("running".to_owned()),
recent_output: vec!["building".to_owned()],
}
}
#[test]
fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks() {
let epoch = DebugEpoch::all_stop(
ProcessId::from("process"),
1,
DebugStopReason::Breakpoint {
task: TaskInstanceId::from("compile-linux"),
line: 42,
},
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
true,
),
],
)
.unwrap();
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("main")),
Some(&DebugRuntimeState::Frozen)
);
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("compile-linux")),
Some(&DebugRuntimeState::Frozen)
);
}
#[test]
fn debug_epoch_reports_failure_when_no_participant_can_freeze() {
let error = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
false,
)],
)
.unwrap_err();
assert!(matches!(error, DebugEpochError::CannotFreeze { .. }));
}
#[test]
fn debug_epoch_keeps_frozen_participants_when_another_participant_fails() {
let epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant(
"native",
DebugParticipantKind::ControlledNativeCommand,
false,
),
],
)
.unwrap();
assert!(epoch.partially_frozen());
assert!(!epoch.all_threads_stopped());
assert!(matches!(
epoch.participant_state(&TaskInstanceId::from("native")),
Some(DebugRuntimeState::Failed(_))
));
assert!(matches!(
epoch.inspection(&TaskInstanceId::from("native")),
Err(DebugEpochError::ParticipantNotFrozen(_))
));
}
#[test]
fn continue_resumes_every_frozen_participant() {
let mut epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant("task", DebugParticipantKind::WasmTask, true),
],
)
.unwrap();
epoch.continue_all();
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("main")),
Some(&DebugRuntimeState::Running)
);
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("task")),
Some(&DebugRuntimeState::Running)
);
}
#[test]
fn inspection_exposes_stack_args_handles_command_status_and_output() {
let epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
true,
)],
)
.unwrap();
let inspection = epoch
.inspection(&TaskInstanceId::from("compile-linux"))
.unwrap();
assert_eq!(inspection.stack_frames, vec!["compile-linux::run"]);
assert_eq!(
inspection.local_values[0],
("wasm_local_0".to_owned(), "I32(41)".to_owned())
);
assert_eq!(
inspection.task_args[0],
("target".to_owned(), "linux".to_owned())
);
assert_eq!(
inspection.handles[0],
("artifact".to_owned(), "artifact-1".to_owned())
);
assert_eq!(inspection.command_status, Some("running".to_owned()));
assert_eq!(inspection.recent_output, vec!["building"]);
}
}

68
crates/clusterflux-core/src/digest.rs Normal file
View file

@ -0,0 +1,68 @@
use serde::{Deserialize, Serialize};
use sha2::{Digest as ShaDigest, Sha256};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct Digest(String);
impl Digest {
pub fn sha256(bytes: impl AsRef<[u8]>) -> Self {
let mut hasher = Sha256::new();
hasher.update(bytes.as_ref());
Self(format!("sha256:{}", hex::encode(hasher.finalize())))
}
pub fn from_parts(parts: impl IntoIterator<Item = impl AsRef<[u8]>>) -> Self {
let mut hasher = Sha256::new();
for part in parts {
let part = part.as_ref();
hasher.update((part.len() as u64).to_be_bytes());
hasher.update(part);
}
Self(format!("sha256:{}", hex::encode(hasher.finalize())))
}
pub fn from_sha256_hex(hex_digest: impl Into<String>) -> Result<Self, String> {
let digest = Self(format!("sha256:{}", hex_digest.into()));
if digest.is_valid_sha256() {
Ok(digest)
} else {
Err(
"SHA-256 digest must contain exactly 64 lowercase hexadecimal characters"
.to_owned(),
)
}
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn is_valid_sha256(&self) -> bool {
let Some(hex) = self.0.strip_prefix("sha256:") else {
return false;
};
hex.len() == 64
&& hex
.bytes()
.all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'))
}
}
impl std::fmt::Display for Digest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn digest_validates_strict_sha256_syntax() {
assert!(Digest::sha256("bytes").is_valid_sha256());
assert!(!Digest("sha1:abc".to_owned()).is_valid_sha256());
assert!(!Digest("sha256:ABCDEF".to_owned()).is_valid_sha256());
assert!(!Digest("sha256:not-hex".to_owned()).is_valid_sha256());
}
}

288
crates/clusterflux-core/src/environment.rs Normal file
View file

@ -0,0 +1,288 @@
use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Capability, Digest, Os};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum EnvironmentKind {
Containerfile,
Dockerfile,
NixFlake,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentRequirements {
pub os: Option<Os>,
pub arch: Option<String>,
pub capabilities: BTreeSet<Capability>,
}
impl EnvironmentRequirements {
pub fn linux_container() -> Self {
Self {
os: Some(Os::Linux),
arch: None,
capabilities: BTreeSet::from([Capability::Containers, Capability::RootlessPodman]),
}
}
pub fn windows_command_dev() -> Self {
Self {
os: Some(Os::Windows),
arch: None,
capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
}
}
pub fn unconstrained() -> Self {
Self {
os: None,
arch: None,
capabilities: BTreeSet::new(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentResource {
pub name: String,
pub kind: EnvironmentKind,
pub recipe_path: PathBuf,
pub context_path: PathBuf,
pub digest: Digest,
pub requirements: EnvironmentRequirements,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentReference {
pub name: String,
pub byte_offset: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentDiagnostic {
pub reference: EnvironmentReference,
pub message: String,
}
#[derive(Debug, Error)]
pub enum EnvironmentError {
#[error("failed to read environment resources under {path}: {source}")]
Read {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
pub fn discover_environments(
project_root: &Path,
) -> Result<Vec<EnvironmentResource>, EnvironmentError> {
let envs_dir = project_root.join("envs");
if !envs_dir.exists() {
return Ok(Vec::new());
}
let mut resources = Vec::new();
let entries = fs::read_dir(&envs_dir).map_err(|source| EnvironmentError::Read {
path: envs_dir.clone(),
source,
})?;
for entry in entries {
let entry = entry.map_err(|source| EnvironmentError::Read {
path: envs_dir.clone(),
source,
})?;
let path = entry.path();
if !path.is_dir() {
continue;
}
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
if let Some(resource) = discover_one(project_root, name, &path)? {
resources.push(resource);
}
}
resources.sort_by(|left, right| left.name.cmp(&right.name));
Ok(resources)
}
pub fn diagnose_environment_references(
source: &str,
environments: &[EnvironmentResource],
) -> Vec<EnvironmentDiagnostic> {
let known = environments
.iter()
.map(|environment| environment.name.as_str())
.collect::<BTreeSet<_>>();
find_env_macro_references(source)
.into_iter()
.filter(|reference| !known.contains(reference.name.as_str()))
.map(|reference| EnvironmentDiagnostic {
message: format!(
"missing Clusterflux environment `{}`; expected envs/{}/Containerfile or envs/{}/Dockerfile",
reference.name, reference.name, reference.name
),
reference,
})
.collect()
}
fn discover_one(
project_root: &Path,
name: &str,
env_dir: &Path,
) -> Result<Option<EnvironmentResource>, EnvironmentError> {
let candidates = [
("Containerfile", EnvironmentKind::Containerfile),
("Dockerfile", EnvironmentKind::Dockerfile),
("flake.nix", EnvironmentKind::NixFlake),
];
for (file_name, kind) in candidates {
let recipe_path = env_dir.join(file_name);
if !recipe_path.exists() {
continue;
}
let recipe_bytes = fs::read(&recipe_path).map_err(|source| EnvironmentError::Read {
path: recipe_path.clone(),
source,
})?;
let relative_recipe = recipe_path
.strip_prefix(project_root)
.unwrap_or(&recipe_path)
.to_string_lossy();
let digest = Digest::from_parts([
b"environment:v1".as_slice(),
name.as_bytes(),
format!("{kind:?}").as_bytes(),
relative_recipe.as_bytes(),
recipe_bytes.as_slice(),
]);
let requirements = match kind {
EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile
if name.eq_ignore_ascii_case("windows") =>
{
EnvironmentRequirements::windows_command_dev()
}
EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile => {
EnvironmentRequirements::linux_container()
}
EnvironmentKind::NixFlake => EnvironmentRequirements::unconstrained(),
};
return Ok(Some(EnvironmentResource {
name: name.to_owned(),
kind,
recipe_path,
context_path: env_dir.to_path_buf(),
digest,
requirements,
}));
}
Ok(None)
}
fn find_env_macro_references(source: &str) -> Vec<EnvironmentReference> {
let mut references = Vec::new();
let mut cursor = 0;
while let Some(index) = source[cursor..].find("env!(") {
let start = cursor + index;
let mut pos = start + "env!(".len();
while source[pos..].starts_with(char::is_whitespace) {
pos += source[pos..]
.chars()
.next()
.map(char::len_utf8)
.unwrap_or(1);
}
if !source[pos..].starts_with('"') {
cursor = pos;
continue;
}
pos += 1;
let name_start = pos;
while pos < source.len() && !source[pos..].starts_with('"') {
pos += source[pos..]
.chars()
.next()
.map(char::len_utf8)
.unwrap_or(1);
}
if pos < source.len() {
references.push(EnvironmentReference {
name: source[name_start..pos].to_owned(),
byte_offset: start,
});
}
cursor = pos.saturating_add(1);
}
references
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn discovers_containerfile_environments_by_logical_name() {
let temp = tempfile::tempdir().unwrap();
let linux = temp.path().join("envs/linux");
fs::create_dir_all(&linux).unwrap();
fs::write(linux.join("Containerfile"), "FROM alpine\n").unwrap();
let envs = discover_environments(temp.path()).unwrap();
assert_eq!(envs.len(), 1);
assert_eq!(envs[0].name, "linux");
assert_eq!(envs[0].kind, EnvironmentKind::Containerfile);
assert!(!envs[0].digest.as_str().is_empty());
}
#[test]
fn missing_env_macro_reference_reports_clear_diagnostic() {
let source = r#"fn main() { let _ = env!("windows"); }"#;
let diagnostics = diagnose_environment_references(source, &[]);
assert_eq!(diagnostics.len(), 1);
assert!(diagnostics[0]
.message
.contains("envs/windows/Containerfile"));
}
#[test]
fn windows_environment_name_uses_windows_development_requirements() {
let temp = tempfile::tempdir().unwrap();
let windows = temp.path().join("envs/windows");
fs::create_dir_all(&windows).unwrap();
fs::write(
windows.join("Dockerfile"),
"# user-attached windows dev contract\n",
)
.unwrap();
let envs = discover_environments(temp.path()).unwrap();
assert_eq!(envs[0].name, "windows");
assert_eq!(envs[0].requirements.os, Some(Os::Windows));
assert!(envs[0]
.requirements
.capabilities
.contains(&Capability::WindowsCommandDev));
}
}

1002
crates/clusterflux-core/src/execution.rs Normal file
View file

File diff suppressed because it is too large Load diff

45
crates/clusterflux-core/src/ids.rs Normal file
View file

@ -0,0 +1,45 @@
use serde::{Deserialize, Serialize};
macro_rules! id_type {
($name:ident) => {
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct $name(String);
impl $name {
pub fn new(value: impl Into<String>) -> Self {
let value = value.into();
assert!(
!value.trim().is_empty(),
concat!(stringify!($name), " cannot be empty")
);
Self(value)
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl From<&str> for $name {
fn from(value: &str) -> Self {
Self::new(value)
}
}
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
};
}
id_type!(AgentId);
id_type!(ArtifactId);
id_type!(NodeId);
id_type!(ProcessId);
id_type!(ProjectId);
id_type!(TaskDefinitionId);
id_type!(TaskInstanceId);
id_type!(TenantId);
id_type!(UserId);

102
crates/clusterflux-core/src/lib.rs Normal file
View file

@ -0,0 +1,102 @@
pub mod artifact;
pub mod auth;
pub mod bundle;
pub mod capability;
pub mod checkpoint;
pub mod debug;
pub mod digest;
pub mod environment;
pub mod execution;
pub mod ids;
pub mod limits;
pub mod operator_panel;
pub mod policy;
pub mod project;
pub mod scheduler;
pub mod source;
pub mod transport;
pub mod vfs;
pub mod wire;
pub use artifact::{
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
DownloadStreamRequest, RetentionPolicy, StorageLocation,
};
pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest,
agent_ed25519_public_key_from_private_key, agent_workflow_request_scope_from_payload,
derive_ed25519_private_key_from_seed, node_capability_policy_digest,
node_ed25519_public_key_from_private_key, sign_agent_workflow_request, sign_node_request,
signed_request_payload_digest, verify_agent_workflow_signature, verify_node_request_signature,
Action, Actor, AgentSignedRequest, AgentWorkflowRequestScope, AgentWorkflowScope, AuthContext,
Authorization, BrowserLoginFlow, CredentialKind, EnrollmentError, EnrollmentGrant,
IdentityKind, NodeCredential, NodeSignedRequest, PublicKeyIdentity, Scope,
};
#[cfg(not(target_arch = "wasm32"))]
pub use auth::{generate_ed25519_private_key, generate_opaque_token};
pub use bundle::{
discover_source_debug_probes, BundleDebugMetadata, BundleDebugProbe, BundleIdentityInputs,
BundleLargeInputPolicy, BundleMetadata, BundleRestartCompatibility, BundleSourceMetadata,
BundleTaskMetadata, SelectedInput,
};
pub use capability::{Capability, CapabilityReportError, EnvironmentBackend, NodeCapabilities, Os};
pub use checkpoint::{
CheckpointBoundary, CompatibilityFailure, RestartDecision, RestartPolicy, RestartRequest,
TaskCheckpoint,
};
pub use debug::{
DebugEpoch, DebugEpochError, DebugParticipant, DebugParticipantKind, DebugRuntimeState,
DebugStopReason, ThreadInspection,
};
pub use digest::Digest;
pub use environment::{
diagnose_environment_references, discover_environments, EnvironmentDiagnostic, EnvironmentKind,
EnvironmentReference, EnvironmentRequirements, EnvironmentResource,
};
pub use execution::{
CommandBackendKind, CommandInvocation, CommandNetworkPolicy, CommandPlan, GuestRuntimeKind,
NativeCommandPolicy, StructuredTaskBoundary, TaskBoundaryHandle, TaskBoundaryValue,
TaskDispatch, TaskFailurePolicy, TaskJoinResult, TaskJoinState, TaskSpec, WasmExportAbi,
WasmHostCommandRequest, WasmHostCommandResult, WasmHostDebugProbeRequest,
WasmHostDebugProbeResult, WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult,
WasmHostTaskControlRequest, WasmHostTaskControlResult, WasmHostTaskHandle,
WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation,
WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
};
pub use ids::{
AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
UserId,
};
pub use limits::{
LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,
ResourceMeter, TaskArgumentBudget, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
};
pub use operator_panel::{
ControlPlaneAction, PanelError, PanelEvent, PanelEventKind, PanelState, PanelWidget,
PanelWidgetKind, RateLimit,
};
pub use policy::{
CapabilityPolicy, Decision, LocalTrustedPolicy, PolicyReason, ResourceRequest, ServicePolicy,
};
pub use project::{Entrypoint, ProjectModel, ProjectModelError};
pub use scheduler::{
DefaultScheduler, NodeDescriptor, Placement, PlacementError, PlacementRequest, Scheduler,
};
pub use source::{
SourceManifestError, SourcePreparation, SourceProviderKind, SourceProviderManifest,
SourceProviderModule, SourceTransferMode, SourceTransferPolicy,
};
pub use transport::{
BulkTransferDecision, DataPlaneObject, DataPlaneScope, DirectBulkTransferPlan,
NativeQuicTransport, NodeEndpoint, RendezvousRequest, Transport, TransportError, TransportKind,
};
pub use vfs::{
ReuseDecision, SyncPolicy, VfsError, VfsManifest, VfsObject, VfsOverlay, VfsPath,
VfsSyncDecision,
};
pub use wire::{
coordinator_authentication_metadata, coordinator_payload_operation, coordinator_wire_request,
COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE,
};

273
crates/clusterflux-core/src/limits.rs Normal file
View file

@ -0,0 +1,273 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::TaskInstanceId;
/// Fastest supported interval for a node's signed artifact/assignment polling loop.
/// The coordinator's bounded replay window is sized against this protocol limit.
pub const MIN_SIGNED_NODE_POLL_INTERVAL_MS: u64 = 20;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum LimitKind {
ApiCall,
Spawn,
LogBytes,
MetadataBytes,
DebugReadBytes,
UiEvent,
RendezvousAttempt,
ArtifactDownloadBytes,
HostedFuel,
HostedMemoryBytes,
HostedWallClockMs,
HostedStateBytes,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceLimits {
pub limits: BTreeMap<LimitKind, u64>,
}
impl ResourceLimits {
pub fn new(limits: impl IntoIterator<Item = (LimitKind, u64)>) -> Self {
Self {
limits: limits.into_iter().collect(),
}
}
pub fn unlimited() -> Self {
Self::new(LimitKind::ALL.into_iter().map(|kind| (kind, u64::MAX)))
}
pub fn limit(&self, kind: &LimitKind) -> u64 {
*self.limits.get(kind).unwrap_or(&0)
}
}
impl Default for ResourceLimits {
fn default() -> Self {
Self::unlimited()
}
}
impl LimitKind {
pub const ALL: [Self; 12] = [
Self::ApiCall,
Self::Spawn,
Self::LogBytes,
Self::MetadataBytes,
Self::DebugReadBytes,
Self::UiEvent,
Self::RendezvousAttempt,
Self::ArtifactDownloadBytes,
Self::HostedFuel,
Self::HostedMemoryBytes,
Self::HostedWallClockMs,
Self::HostedStateBytes,
];
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceMeter {
used: BTreeMap<LimitKind, u64>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum LimitError {
#[error(
"resource limit exceeded for {kind:?}: requested {requested}, used {used}, limit {limit}"
)]
Exceeded {
kind: LimitKind,
requested: u64,
used: u64,
limit: u64,
},
#[error("task argument is too large: {size} bytes exceeds {limit} bytes")]
LargeTaskArgument { size: u64, limit: u64 },
}
impl ResourceMeter {
pub fn can_charge(
&self,
limits: &ResourceLimits,
kind: LimitKind,
amount: u64,
) -> Result<(), LimitError> {
let used = self.used.get(&kind).copied().unwrap_or(0);
let limit = limits.limit(&kind);
if used.saturating_add(amount) > limit {
return Err(LimitError::Exceeded {
kind,
requested: amount,
used,
limit,
});
}
Ok(())
}
pub fn charge(
&mut self,
limits: &ResourceLimits,
kind: LimitKind,
amount: u64,
) -> Result<(), LimitError> {
self.can_charge(limits, kind, amount)?;
let used = self.used.get(&kind).copied().unwrap_or(0);
self.used.insert(kind, used + amount);
Ok(())
}
pub fn used(&self, kind: &LimitKind) -> u64 {
self.used.get(kind).copied().unwrap_or(0)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogRecord {
pub task: TaskInstanceId,
pub bytes: Vec<u8>,
pub truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogBuffer {
max_bytes: usize,
used_bytes: usize,
records: Vec<LogRecord>,
backpressured: bool,
}
impl LogBuffer {
pub fn new(max_bytes: usize) -> Self {
Self {
max_bytes,
used_bytes: 0,
records: Vec::new(),
backpressured: false,
}
}
pub fn push(&mut self, task: TaskInstanceId, bytes: impl AsRef<[u8]>) {
let bytes = bytes.as_ref();
let remaining = self.max_bytes.saturating_sub(self.used_bytes);
let truncated = bytes.len() > remaining;
let stored = bytes[..bytes.len().min(remaining)].to_vec();
self.used_bytes += stored.len();
if truncated {
self.backpressured = true;
}
self.records.push(LogRecord {
task,
bytes: stored,
truncated,
});
}
pub fn records(&self) -> &[LogRecord] {
&self.records
}
pub fn backpressured(&self) -> bool {
self.backpressured
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum LargeArgumentPolicy {
Allow,
Warn,
Reject,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskArgumentBudget {
pub max_inline_bytes: u64,
pub policy: LargeArgumentPolicy,
}
impl TaskArgumentBudget {
pub fn validate(&self, size: u64) -> Result<Option<String>, LimitError> {
if size <= self.max_inline_bytes {
return Ok(None);
}
match self.policy {
LargeArgumentPolicy::Allow => Ok(None),
LargeArgumentPolicy::Warn => Ok(Some(format!(
"task argument is {size} bytes; prefer SourceSnapshot, Blob, Artifact, or VFS handles"
))),
LargeArgumentPolicy::Reject => Err(LimitError::LargeTaskArgument {
size,
limit: self.max_inline_bytes,
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resource_meter_rejects_usage_before_work_starts() {
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::Spawn, 1)]),
};
let mut meter = ResourceMeter::default();
meter.charge(&limits, LimitKind::Spawn, 1).unwrap();
let error = meter.charge(&limits, LimitKind::Spawn, 1).unwrap_err();
assert!(matches!(error, LimitError::Exceeded { .. }));
}
#[test]
fn resource_meter_can_check_limits_without_consuming() {
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 4)]),
};
let mut meter = ResourceMeter::default();
meter
.can_charge(&limits, LimitKind::ArtifactDownloadBytes, 4)
.unwrap();
assert_eq!(meter.used(&LimitKind::ArtifactDownloadBytes), 0);
meter
.charge(&limits, LimitKind::ArtifactDownloadBytes, 3)
.unwrap();
assert!(matches!(
meter.can_charge(&limits, LimitKind::ArtifactDownloadBytes, 2),
Err(LimitError::Exceeded { .. })
));
}
#[test]
fn log_buffer_caps_backpressures_and_keeps_task_association() {
let mut logs = LogBuffer::new(4);
logs.push(TaskInstanceId::from("task-a"), b"abcdef");
assert!(logs.backpressured());
assert_eq!(logs.records()[0].task, TaskInstanceId::from("task-a"));
assert_eq!(logs.records()[0].bytes, b"abcd");
assert!(logs.records()[0].truncated);
}
#[test]
fn large_task_arguments_are_rejected_or_warned() {
let reject = TaskArgumentBudget {
max_inline_bytes: 4,
policy: LargeArgumentPolicy::Reject,
};
assert!(reject.validate(5).is_err());
let warn = TaskArgumentBudget {
max_inline_bytes: 4,
policy: LargeArgumentPolicy::Warn,
};
assert!(warn.validate(5).unwrap().unwrap().contains("Artifact"));
}
}

377
crates/clusterflux-core/src/operator_panel.rs Normal file
View file

@ -0,0 +1,377 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
ArtifactId, DownloadAction, DownloadError, ProcessId, ProjectId, TaskInstanceId, TenantId,
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PanelWidgetKind {
Text {
value: String,
},
Progress {
current: u64,
total: u64,
},
Button {
action: String,
},
Toggle {
value: bool,
},
Select {
options: Vec<String>,
selected: String,
},
ArtifactDownload {
artifact: ArtifactId,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelWidget {
pub id: String,
pub label: String,
pub kind: PanelWidgetKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ControlPlaneAction {
RestartTask(TaskInstanceId),
CancelProcess,
DebugProcess,
DownloadArtifact(ArtifactId),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelState {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub widgets: BTreeMap<String, PanelWidget>,
pub program_ui_events_enabled: bool,
pub control_plane_actions: Vec<ControlPlaneAction>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PanelEventKind {
ButtonClicked,
ToggleChanged(bool),
SelectChanged(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub widget_id: String,
pub kind: PanelEventKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RateLimit {
pub max_events: u64,
pub used_events: u64,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum PanelError {
#[error("custom HTML or JavaScript is not supported in operator panels")]
CustomContentDenied,
#[error(
"operator panel widget `{0}` is not allowed to collect secrets or OAuth-like credentials"
)]
CredentialCollectionDenied(String),
#[error("panel event scope does not match tenant/project/process")]
ScopeMismatch,
#[error("program UI events are disabled while debug process is stopped")]
ProgramEventsDisabled,
#[error("panel event rate limit exceeded")]
RateLimited,
#[error("unknown panel widget `{0}`")]
UnknownWidget(String),
#[error("artifact download action is unavailable: {0}")]
DownloadUnavailable(String),
}
impl PanelState {
pub fn new(tenant: TenantId, project: ProjectId, process: ProcessId) -> Self {
Self {
tenant,
project,
process,
widgets: BTreeMap::new(),
program_ui_events_enabled: true,
control_plane_actions: Vec::new(),
}
}
pub fn add_widget(&mut self, widget: PanelWidget) -> Result<(), PanelError> {
validate_widget(&widget)?;
self.widgets.insert(widget.id.clone(), widget);
Ok(())
}
pub fn add_download_widget_from_action(
&mut self,
widget_id: impl Into<String>,
label: impl Into<String>,
action: Result<DownloadAction, DownloadError>,
) -> Result<(), PanelError> {
let action = action.map_err(|err| PanelError::DownloadUnavailable(err.to_string()))?;
let artifact = action.artifact;
self.add_widget(PanelWidget {
id: widget_id.into(),
label: label.into(),
kind: PanelWidgetKind::ArtifactDownload {
artifact: artifact.clone(),
},
})?;
self.control_plane_actions
.push(ControlPlaneAction::DownloadArtifact(artifact));
Ok(())
}
pub fn reject_custom_content(_html_or_js: &str) -> Result<(), PanelError> {
Err(PanelError::CustomContentDenied)
}
pub fn freeze_program_ui_events(&mut self) {
self.program_ui_events_enabled = false;
}
pub fn set_control_plane_actions(&mut self, actions: Vec<ControlPlaneAction>) {
self.control_plane_actions = actions;
}
pub fn accept_event(
&self,
event: &PanelEvent,
limit: &mut RateLimit,
) -> Result<(), PanelError> {
if !self.program_ui_events_enabled {
return Err(PanelError::ProgramEventsDisabled);
}
if self.tenant != event.tenant
|| self.project != event.project
|| self.process != event.process
{
return Err(PanelError::ScopeMismatch);
}
if !self.widgets.contains_key(&event.widget_id) {
return Err(PanelError::UnknownWidget(event.widget_id.clone()));
}
if limit.used_events >= limit.max_events {
return Err(PanelError::RateLimited);
}
limit.used_events += 1;
Ok(())
}
pub fn control_plane_actions_available(&self) -> &[ControlPlaneAction] {
&self.control_plane_actions
}
}
fn validate_widget(widget: &PanelWidget) -> Result<(), PanelError> {
let mut checked_text = vec![widget.id.as_str(), widget.label.as_str()];
match &widget.kind {
PanelWidgetKind::Button { action } => checked_text.push(action),
PanelWidgetKind::Select { options, selected } => {
checked_text.push(selected);
checked_text.extend(options.iter().map(String::as_str));
}
PanelWidgetKind::Text { .. }
| PanelWidgetKind::Progress { .. }
| PanelWidgetKind::Toggle { .. }
| PanelWidgetKind::ArtifactDownload { .. } => {}
}
let combined = checked_text.join(" ").to_ascii_lowercase();
if combined.contains("password")
|| combined.contains("token")
|| combined.contains("oauth")
|| combined.contains("secret")
{
return Err(PanelError::CredentialCollectionDenied(widget.id.clone()));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn panel() -> PanelState {
PanelState::new(
TenantId::from("tenant"),
ProjectId::from("project"),
ProcessId::from("process"),
)
}
#[test]
fn panel_uses_typed_widgets_and_rejects_custom_content() {
let mut panel = panel();
panel
.add_widget(PanelWidget {
id: "progress".to_owned(),
label: "Build".to_owned(),
kind: PanelWidgetKind::Progress {
current: 1,
total: 2,
},
})
.unwrap();
assert!(PanelState::reject_custom_content("<script>alert(1)</script>").is_err());
assert!(panel.widgets.contains_key("progress"));
}
#[test]
fn panel_rejects_password_or_oauth_collection_widgets() {
let mut panel = panel();
let error = panel
.add_widget(PanelWidget {
id: "oauth_token".to_owned(),
label: "OAuth Token".to_owned(),
kind: PanelWidgetKind::Text {
value: String::new(),
},
})
.unwrap_err();
assert!(matches!(error, PanelError::CredentialCollectionDenied(_)));
}
#[test]
fn panel_rejects_credential_collection_in_interactive_fields() {
let mut panel = panel();
let button_error = panel
.add_widget(PanelWidget {
id: "continue".to_owned(),
label: "Continue".to_owned(),
kind: PanelWidgetKind::Button {
action: "collect-secret".to_owned(),
},
})
.unwrap_err();
assert!(matches!(
button_error,
PanelError::CredentialCollectionDenied(_)
));
let select_error = panel
.add_widget(PanelWidget {
id: "auth-mode".to_owned(),
label: "Auth Mode".to_owned(),
kind: PanelWidgetKind::Select {
options: vec!["password".to_owned(), "public key".to_owned()],
selected: "public key".to_owned(),
},
})
.unwrap_err();
assert!(matches!(
select_error,
PanelError::CredentialCollectionDenied(_)
));
}
#[test]
fn panel_events_are_scoped_and_rate_limited() {
let mut panel = panel();
panel
.add_widget(PanelWidget {
id: "restart".to_owned(),
label: "Restart".to_owned(),
kind: PanelWidgetKind::Button {
action: "restart".to_owned(),
},
})
.unwrap();
let event = PanelEvent {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
widget_id: "restart".to_owned(),
kind: PanelEventKind::ButtonClicked,
};
let mut limit = RateLimit {
max_events: 1,
used_events: 0,
};
panel.accept_event(&event, &mut limit).unwrap();
assert_eq!(
panel.accept_event(&event, &mut limit),
Err(PanelError::RateLimited)
);
}
#[test]
fn stopped_debug_process_keeps_control_plane_actions_available() {
let mut panel = panel();
panel.freeze_program_ui_events();
panel.set_control_plane_actions(vec![
ControlPlaneAction::RestartTask(TaskInstanceId::from("task")),
ControlPlaneAction::DownloadArtifact(ArtifactId::from("artifact")),
]);
let event = PanelEvent {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
widget_id: "missing".to_owned(),
kind: PanelEventKind::ButtonClicked,
};
let mut limit = RateLimit {
max_events: 1,
used_events: 0,
};
assert_eq!(
panel.accept_event(&event, &mut limit),
Err(PanelError::ProgramEventsDisabled)
);
assert_eq!(panel.control_plane_actions_available().len(), 2);
}
#[test]
fn download_widget_is_only_created_from_available_action() {
let mut panel = panel();
let action = Ok(DownloadAction {
artifact: ArtifactId::from("artifact"),
source: crate::StorageLocation::RetainedNode(crate::NodeId::from("node")),
scoped_token_subject: "tenant/project/process/artifact".to_owned(),
});
panel
.add_download_widget_from_action("download-artifact", "Download", action)
.unwrap();
assert!(matches!(
panel.widgets["download-artifact"].kind,
PanelWidgetKind::ArtifactDownload { .. }
));
assert!(matches!(
panel.control_plane_actions_available()[0],
ControlPlaneAction::DownloadArtifact(_)
));
let before = panel.widgets.len();
let error = panel
.add_download_widget_from_action(
"missing-download",
"Download",
Err(DownloadError::Unavailable),
)
.unwrap_err();
assert_eq!(panel.widgets.len(), before);
assert!(matches!(error, PanelError::DownloadUnavailable(_)));
}
}

134
crates/clusterflux-core/src/policy.rs Normal file
View file

@ -0,0 +1,134 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use crate::{Action, AuthContext, Capability, Scope};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PolicyReason {
Allowed,
MissingCapability(Capability),
HostedNativeComputeDenied,
HostedContainerDenied,
QuotaExceeded(String),
Unauthorized(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Decision {
pub allowed: bool,
pub reason: PolicyReason,
}
impl Decision {
pub fn allow() -> Self {
Self {
allowed: true,
reason: PolicyReason::Allowed,
}
}
pub fn deny(reason: PolicyReason) -> Self {
Self {
allowed: false,
reason,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceRequest {
pub action: Action,
pub required_capabilities: BTreeSet<Capability>,
pub hosted_control_plane: bool,
}
pub trait CapabilityPolicy {
fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision;
}
pub trait ServicePolicy: CapabilityPolicy + Send + Sync {}
impl<T> ServicePolicy for T where T: CapabilityPolicy + Send + Sync {}
#[derive(Clone, Debug, Default)]
pub struct LocalTrustedPolicy;
impl CapabilityPolicy for LocalTrustedPolicy {
fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision {
let authz = crate::auth::same_tenant_project(context, scope);
if !authz.allowed {
return Decision::deny(PolicyReason::Unauthorized(authz.reason));
}
if request.hosted_control_plane && request.action == Action::RunNativeCommand {
return Decision::deny(PolicyReason::HostedNativeComputeDenied);
}
if request.hosted_control_plane && request.action == Action::RunContainer {
return Decision::deny(PolicyReason::HostedContainerDenied);
}
Decision::allow()
}
}
#[cfg(test)]
mod tests {
use crate::{Actor, ProjectId, TenantId, UserId};
use super::*;
#[test]
fn public_policy_interface_denies_hosted_native_compute() {
let policy = LocalTrustedPolicy;
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::User(UserId::from("user")),
};
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: None,
task: None,
node: None,
artifact: None,
};
let request = ResourceRequest {
action: Action::RunNativeCommand,
required_capabilities: BTreeSet::new(),
hosted_control_plane: true,
};
let decision = policy.decide(&context, &scope, &request);
assert!(!decision.allowed);
assert_eq!(decision.reason, PolicyReason::HostedNativeComputeDenied);
}
#[test]
fn local_trusted_policy_allows_owner_controlled_native_capability_request() {
let policy = LocalTrustedPolicy;
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::User(UserId::from("owner")),
};
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: None,
task: None,
node: None,
artifact: None,
};
let request = ResourceRequest {
action: Action::RunNativeCommand,
required_capabilities: BTreeSet::from([Capability::Command]),
hosted_control_plane: false,
};
let decision = policy.decide(&context, &scope, &request);
assert!(decision.allowed);
assert_eq!(decision.reason, PolicyReason::Allowed);
}
}

303
crates/clusterflux-core/src/project.rs Normal file
View file

@ -0,0 +1,303 @@
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use syn::{punctuated::Punctuated, Expr, Item, Lit, Meta, Token};
use thiserror::Error;
use crate::{discover_environments, environment::EnvironmentError, EnvironmentResource};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Entrypoint {
pub name: String,
pub function: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProjectModel {
pub root: PathBuf,
pub environments: Vec<EnvironmentResource>,
pub entrypoints: BTreeMap<String, Entrypoint>,
pub default_entrypoint: String,
pub required_config_file: Option<PathBuf>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ProjectModelError {
#[error("environment discovery failed: {0}")]
Environment(String),
#[error("Clusterflux entrypoint discovery failed: {0}")]
EntrypointDiscovery(String),
#[error(
"no Clusterflux entrypoint is declared; add `#[clusterflux::main]` to a function under src/"
)]
NoEntrypoints,
#[error("unknown Clusterflux entrypoint `{name}`; available entrypoints: {available:?}")]
UnknownEntrypoint {
name: String,
available: Vec<String>,
},
}
impl ProjectModel {
pub fn discover_without_config(root: &Path) -> Result<Self, ProjectModelError> {
let environments = discover_environments(root).map_err(|err| {
ProjectModelError::Environment(match err {
EnvironmentError::Read { path, source } => {
format!("failed to read {}: {source}", path.display())
}
})
})?;
let entrypoints = discover_entrypoints(root)?;
let default_entrypoint = if entrypoints.contains_key("build") {
"build".to_owned()
} else {
entrypoints.keys().next().cloned().unwrap_or_default()
};
Ok(Self {
root: root.to_path_buf(),
environments,
entrypoints,
default_entrypoint,
required_config_file: None,
})
}
pub fn select_entrypoint(&self, name: Option<&str>) -> Result<&Entrypoint, ProjectModelError> {
if self.entrypoints.is_empty() {
return Err(ProjectModelError::NoEntrypoints);
}
let name = name.unwrap_or(&self.default_entrypoint);
self.entrypoints
.get(name)
.ok_or_else(|| ProjectModelError::UnknownEntrypoint {
name: name.to_owned(),
available: self.entrypoints.keys().cloned().collect(),
})
}
}
fn discover_entrypoints(root: &Path) -> Result<BTreeMap<String, Entrypoint>, ProjectModelError> {
let source_root = root.join("src");
if !source_root.is_dir() {
return Ok(BTreeMap::new());
}
let mut source_files = Vec::new();
collect_rust_sources(&source_root, &mut source_files)?;
source_files.sort();
let mut entrypoints = BTreeMap::new();
for path in source_files {
let source = fs::read_to_string(&path).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to read {}: {error}",
path.display()
))
})?;
let syntax = syn::parse_file(&source).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to parse {}: {error}",
path.display()
))
})?;
collect_entrypoint_items(&syntax.items, &path, &mut entrypoints)?;
}
Ok(entrypoints)
}
fn collect_rust_sources(
directory: &Path,
files: &mut Vec<PathBuf>,
) -> Result<(), ProjectModelError> {
let entries = fs::read_dir(directory).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to read {}: {error}",
directory.display()
))
})?;
for entry in entries {
let entry = entry.map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to inspect {}: {error}",
directory.display()
))
})?;
let path = entry.path();
let file_type = entry.file_type().map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to inspect {}: {error}",
path.display()
))
})?;
if file_type.is_dir() {
collect_rust_sources(&path, files)?;
} else if file_type.is_file() && path.extension().is_some_and(|ext| ext == "rs") {
files.push(path);
}
}
Ok(())
}
fn collect_entrypoint_items(
items: &[Item],
path: &Path,
entrypoints: &mut BTreeMap<String, Entrypoint>,
) -> Result<(), ProjectModelError> {
for item in items {
match item {
Item::Fn(function) => {
let Some(attribute) = function.attrs.iter().find(|attribute| {
let segments = attribute
.path()
.segments
.iter()
.map(|segment| segment.ident.to_string())
.collect::<Vec<_>>();
segments.as_slice() == ["clusterflux", "main"]
}) else {
continue;
};
let function_name = function.sig.ident.to_string();
let default_name = function_name
.strip_suffix("_main")
.unwrap_or(&function_name);
let name = entrypoint_name(attribute, default_name);
let entrypoint = Entrypoint {
name: name.clone(),
function: function_name,
};
if let Some(existing) = entrypoints.insert(name.clone(), entrypoint) {
return Err(ProjectModelError::EntrypointDiscovery(format!(
"duplicate entrypoint `{name}` in {}; it was already declared by `{}`",
path.display(),
existing.function
)));
}
}
Item::Mod(module) => {
if let Some((_, nested)) = &module.content {
collect_entrypoint_items(nested, path, entrypoints)?;
}
}
_ => {}
}
}
Ok(())
}
fn entrypoint_name(attribute: &syn::Attribute, default: &str) -> String {
let Meta::List(_) = &attribute.meta else {
return default.to_owned();
};
let Ok(arguments) = attribute.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)
else {
return default.to_owned();
};
arguments
.into_iter()
.find_map(|meta| {
let Meta::NameValue(name_value) = meta else {
return None;
};
if !name_value.path.is_ident("name") {
return None;
}
let Expr::Lit(expression) = name_value.value else {
return None;
};
let Lit::Str(value) = expression.lit else {
return None;
};
Some(value.value())
})
.unwrap_or_else(|| default.to_owned())
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn project_works_without_hand_written_configuration_file() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("envs/linux")).unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(
temp.path().join("envs/linux/Containerfile"),
"FROM alpine\n",
)
.unwrap();
fs::write(
temp.path().join("src/main.rs"),
"#[clusterflux::main]\npub fn build_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert_eq!(model.required_config_file, None);
assert_eq!(model.environments[0].name, "linux");
assert_eq!(model.select_entrypoint(None).unwrap().name, "build");
}
#[test]
fn project_can_define_multiple_default_entrypoints() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src/nested")).unwrap();
fs::write(
temp.path().join("src/lib.rs"),
"#[clusterflux::main(name = \"check\")]\npub fn test_main() {}\n",
)
.unwrap();
fs::write(
temp.path().join("src/nested/release.rs"),
"#[clusterflux::main]\npub fn release_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert_eq!(
model.select_entrypoint(Some("check")).unwrap().function,
"test_main"
);
assert_eq!(
model.select_entrypoint(Some("release")).unwrap().function,
"release_main"
);
}
#[test]
fn unknown_entrypoint_lists_available_choices() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(
temp.path().join("src/main.rs"),
"#[clusterflux::main]\npub fn build_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
let error = model.select_entrypoint(Some("deploy")).unwrap_err();
assert!(matches!(error, ProjectModelError::UnknownEntrypoint { .. }));
}
#[test]
fn project_without_declared_entrypoint_does_not_invent_product_surfaces() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(temp.path().join("src/main.rs"), "fn main() {}\n").unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert!(model.entrypoints.is_empty());
assert_eq!(model.default_entrypoint, "");
assert_eq!(
model.select_entrypoint(None).unwrap_err(),
ProjectModelError::NoEntrypoints
);
}
}

472
crates/clusterflux-core/src/scheduler.rs Normal file
View file

@ -0,0 +1,472 @@
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
ArtifactId, Capability, Digest, EnvironmentRequirements, NodeCapabilities, NodeId, ProjectId,
TenantId,
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeDescriptor {
pub id: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub capabilities: NodeCapabilities,
pub cached_environments: BTreeSet<Digest>,
pub dependency_caches: BTreeSet<Digest>,
pub source_snapshots: BTreeSet<Digest>,
pub artifact_locations: BTreeSet<ArtifactId>,
pub direct_connectivity: bool,
pub online: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PlacementRequest {
pub tenant: TenantId,
pub project: ProjectId,
pub environment: Option<EnvironmentRequirements>,
pub environment_digest: Option<Digest>,
#[serde(default)]
pub environment_cache_required: bool,
pub required_capabilities: BTreeSet<Capability>,
pub dependency_cache: Option<Digest>,
pub source_snapshot: Option<Digest>,
pub required_artifacts: BTreeSet<ArtifactId>,
pub quota_available: bool,
pub policy_allowed: bool,
pub prefer_node: Option<NodeId>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Placement {
pub node: NodeId,
pub score: i64,
pub reasons: Vec<String>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
#[error("no capable node for placement: {message}")]
pub struct PlacementError {
pub message: String,
}
pub trait Scheduler {
fn place(
&self,
nodes: &[NodeDescriptor],
request: &PlacementRequest,
) -> Result<Placement, PlacementError>;
}
#[derive(Clone, Debug, Default)]
pub struct DefaultScheduler;
impl Scheduler for DefaultScheduler {
fn place(
&self,
nodes: &[NodeDescriptor],
request: &PlacementRequest,
) -> Result<Placement, PlacementError> {
let mut scored = Vec::new();
let mut rejection_counts = BTreeMap::<String, usize>::new();
for node in nodes {
match compatibility(node, request) {
Ok(mut placement) => {
locality_score(node, request, &mut placement);
scored.push(placement);
}
Err(reasons) => {
for reason in reasons {
*rejection_counts.entry(reason).or_default() += 1;
}
}
}
}
scored
.into_iter()
.max_by_key(|placement| placement.score)
.ok_or_else(|| PlacementError {
message: rejection_counts
.into_iter()
.map(|(reason, count)| format!("{reason} ({count} node(s))"))
.collect::<Vec<_>>()
.join("; "),
})
}
}
fn compatibility(
node: &NodeDescriptor,
request: &PlacementRequest,
) -> Result<Placement, Vec<String>> {
let mut reasons = Vec::new();
if !node.online {
reasons.push("node offline".to_owned());
}
if node.tenant != request.tenant {
reasons.push("tenant mismatch".to_owned());
}
if node.project != request.project {
reasons.push("project mismatch".to_owned());
}
if !request.quota_available {
reasons.push("quota unavailable for placement".to_owned());
}
if !request.policy_allowed {
reasons.push("policy denied placement".to_owned());
}
for capability in &request.required_capabilities {
if !node.capabilities.capabilities.contains(capability) {
reasons.push(format!("missing capability {capability:?}"));
}
}
if let Some(environment) = &request.environment {
if let Some(required_os) = &environment.os {
if &node.capabilities.os != required_os {
reasons.push(format!("environment requires os {required_os:?}"));
}
}
if let Some(required_arch) = &environment.arch {
if &node.capabilities.arch != required_arch {
reasons.push(format!("environment requires arch {required_arch}"));
}
}
for capability in &environment.capabilities {
if !node.capabilities.capabilities.contains(capability) {
reasons.push(format!("environment requires capability {capability:?}"));
}
}
}
if request.environment_cache_required {
match request.environment_digest.as_ref() {
Some(digest) if !node.cached_environments.contains(digest) => {
reasons.push(format!(
"required named environment cache {digest} is unavailable"
));
}
None => reasons.push("required named environment cache digest is missing".to_owned()),
Some(_) => {}
}
}
let source_transfer_required = request
.source_snapshot
.as_ref()
.is_some_and(|digest| !node.source_snapshots.contains(digest));
if source_transfer_required && !node.direct_connectivity {
reasons.push("source snapshot unavailable and direct connectivity unavailable".to_owned());
}
let missing_artifacts = request
.required_artifacts
.iter()
.filter(|artifact| !node.artifact_locations.contains(*artifact))
.count();
if missing_artifacts > 0 && !node.direct_connectivity {
reasons.push(format!(
"{missing_artifacts} required artifact(s) unavailable and direct connectivity unavailable"
));
}
if reasons.is_empty() {
Ok(Placement {
node: node.id.clone(),
score: 0,
reasons: Vec::new(),
})
} else {
Err(reasons)
}
}
fn locality_score(node: &NodeDescriptor, request: &PlacementRequest, placement: &mut Placement) {
if request.prefer_node.as_ref() == Some(&node.id) {
placement.score += 100;
placement.reasons.push("preferred node".to_owned());
}
if request
.environment_digest
.as_ref()
.is_some_and(|digest| node.cached_environments.contains(digest))
{
placement.score += 50;
placement.reasons.push("warm environment cache".to_owned());
}
if request
.source_snapshot
.as_ref()
.is_some_and(|digest| node.source_snapshots.contains(digest))
{
placement.score += 40;
placement
.reasons
.push("source snapshot already local".to_owned());
}
if request
.dependency_cache
.as_ref()
.is_some_and(|digest| node.dependency_caches.contains(digest))
{
placement.score += 30;
placement.reasons.push("warm dependency cache".to_owned());
}
let artifact_hits = request
.required_artifacts
.iter()
.filter(|artifact| node.artifact_locations.contains(*artifact))
.count() as i64;
if artifact_hits > 0 {
placement.score += 10 * artifact_hits;
placement.reasons.push(format!(
"{artifact_hits} required artifact(s) already local"
));
}
}
#[cfg(test)]
mod tests {
use crate::{EnvironmentBackend, Os};
use super::*;
fn node(id: &str, cached_source: bool) -> NodeDescriptor {
let source = Digest::sha256("source");
NodeDescriptor {
id: NodeId::from(id),
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
capabilities: NodeCapabilities {
os: Os::Linux,
arch: "x86_64".to_owned(),
capabilities: BTreeSet::from([
Capability::Command,
Capability::Containers,
Capability::RootlessPodman,
]),
environment_backends: BTreeSet::from([EnvironmentBackend::Container]),
source_providers: BTreeSet::from(["filesystem".to_owned()]),
},
cached_environments: BTreeSet::from([Digest::sha256("env")]),
dependency_caches: if cached_source {
BTreeSet::from([Digest::sha256("deps")])
} else {
BTreeSet::new()
},
source_snapshots: if cached_source {
BTreeSet::from([source])
} else {
BTreeSet::new()
},
artifact_locations: BTreeSet::new(),
direct_connectivity: true,
online: true,
}
}
#[test]
fn scheduler_prefers_warm_source_and_environment() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::linux_container()),
environment_digest: Some(Digest::sha256("env")),
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: Some(Digest::sha256("deps")),
source_snapshot: Some(Digest::sha256("source")),
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let placement = DefaultScheduler
.place(&[node("cold", false), node("warm", true)], &request)
.unwrap();
assert_eq!(placement.node, NodeId::from("warm"));
assert!(placement
.reasons
.iter()
.any(|reason| reason.contains("source")));
assert!(placement
.reasons
.iter()
.any(|reason| reason.contains("dependency")));
}
#[test]
fn scheduler_requires_requested_named_environment_cache() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: Some(Digest::sha256("missing-environment")),
environment_cache_required: true,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("uncached", false)], &request)
.unwrap_err();
assert!(error.message.contains("named environment cache"));
}
#[test]
fn scheduler_failure_names_missing_constraint() {
let mut request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
request.required_capabilities.insert(Capability::Command);
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("WindowsCommandDev"));
}
#[test]
fn scheduler_failure_names_environment_constraint() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::windows_command_dev()),
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("environment requires os Windows"));
assert!(error
.message
.contains("environment requires capability WindowsCommandDev"));
}
#[test]
fn scheduler_requires_direct_connectivity_when_transfer_is_needed() {
let mut disconnected = node("disconnected", false);
disconnected.direct_connectivity = false;
let mut local = node("local", true);
local.direct_connectivity = false;
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: Some(Digest::sha256("source")),
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let placement = DefaultScheduler
.place(&[disconnected, local], &request)
.unwrap();
assert_eq!(placement.node, NodeId::from("local"));
let mut disconnected = node("disconnected", false);
disconnected.direct_connectivity = false;
let error = DefaultScheduler
.place(&[disconnected], &request)
.unwrap_err();
assert!(error
.message
.contains("source snapshot unavailable and direct connectivity unavailable"));
}
#[test]
fn scheduler_failure_names_required_artifact_transfer_constraint() {
let mut disconnected = node("disconnected", true);
disconnected.direct_connectivity = false;
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::from([ArtifactId::from("cache")]),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[disconnected], &request)
.unwrap_err();
assert!(error
.message
.contains("1 required artifact(s) unavailable and direct connectivity unavailable"));
}
#[test]
fn scheduler_failure_names_quota_and_policy_constraints() {
let mut request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: false,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("quota unavailable for placement"));
request.quota_available = true;
request.policy_allowed = false;
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("policy denied placement"));
}
}

324
crates/clusterflux-core/src/source.rs Normal file
View file

@ -0,0 +1,324 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Capability, Digest, ProjectId, TenantId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum SourceProviderKind {
Filesystem,
Git,
Custom(String),
}
impl SourceProviderKind {
pub fn provider_id(&self) -> &str {
match self {
SourceProviderKind::Filesystem => "filesystem",
SourceProviderKind::Git => "git",
SourceProviderKind::Custom(provider) => provider,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum SourceTransferMode {
RequiredContent,
ExplicitSnapshotChunks,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceTransferPolicy {
pub local_source_bytes_remain_node_local: bool,
pub coordinator_receives_source_bytes_by_default: bool,
pub default_full_repo_tarball: bool,
pub allowed_remote_transfer: BTreeSet<SourceTransferMode>,
}
impl SourceTransferPolicy {
pub fn local_first_snapshot_chunks() -> Self {
Self {
local_source_bytes_remain_node_local: true,
coordinator_receives_source_bytes_by_default: false,
default_full_repo_tarball: false,
allowed_remote_transfer: BTreeSet::from([
SourceTransferMode::RequiredContent,
SourceTransferMode::ExplicitSnapshotChunks,
]),
}
}
}
impl Default for SourceTransferPolicy {
fn default() -> Self {
Self::local_first_snapshot_chunks()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceProviderManifest {
pub kind: SourceProviderKind,
pub digest: Digest,
pub description: String,
#[serde(default)]
pub coordinator_requires_checkout_access: bool,
#[serde(default)]
pub transfer_policy: SourceTransferPolicy,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum SourceManifestError {
#[error("source provider manifest digest is not a valid sha256 digest: {0}")]
InvalidDigest(String),
#[error("custom source provider id `{0}` is invalid")]
InvalidProviderId(String),
#[error("source provider manifest description must be non-empty")]
EmptyDescription,
#[error("source provider manifest description is too long")]
DescriptionTooLong,
#[error("source provider manifest description contains control characters")]
DescriptionControlCharacter,
#[error("source provider manifest would require coordinator checkout access")]
CoordinatorCheckoutAccess,
#[error("source provider manifest would send source bytes to the coordinator by default")]
CoordinatorReceivesSourceBytes,
#[error("source provider manifest would default to a full-repo tarball")]
DefaultFullRepoTarball,
#[error("source provider manifest has no allowed remote transfer mode")]
MissingRemoteTransferMode,
}
pub trait SourceProviderModule {
fn kind(&self) -> SourceProviderKind;
fn manifest(&self) -> SourceProviderManifest;
}
impl SourceProviderManifest {
pub fn local_first(kind: SourceProviderKind, description: impl Into<String>) -> Self {
let transfer_policy = SourceTransferPolicy::local_first_snapshot_chunks();
let digest = Self::digest_for(&kind, false, &transfer_policy);
Self {
kind,
digest,
description: description.into(),
coordinator_requires_checkout_access: false,
transfer_policy,
}
}
pub fn validate_public_mvp(&self) -> Result<(), SourceManifestError> {
self.validate_shape()?;
if self.coordinator_requires_checkout_access {
return Err(SourceManifestError::CoordinatorCheckoutAccess);
}
if self
.transfer_policy
.coordinator_receives_source_bytes_by_default
{
return Err(SourceManifestError::CoordinatorReceivesSourceBytes);
}
if self.transfer_policy.default_full_repo_tarball {
return Err(SourceManifestError::DefaultFullRepoTarball);
}
if self.transfer_policy.allowed_remote_transfer.is_empty() {
return Err(SourceManifestError::MissingRemoteTransferMode);
}
Ok(())
}
fn validate_shape(&self) -> Result<(), SourceManifestError> {
if !self.digest.is_valid_sha256() {
return Err(SourceManifestError::InvalidDigest(
self.digest.as_str().to_owned(),
));
}
if let SourceProviderKind::Custom(provider) = &self.kind {
if !valid_provider_id(provider) {
return Err(SourceManifestError::InvalidProviderId(provider.clone()));
}
}
if self.description.trim().is_empty() {
return Err(SourceManifestError::EmptyDescription);
}
if self.description.len() > 256 {
return Err(SourceManifestError::DescriptionTooLong);
}
if self.description.chars().any(char::is_control) {
return Err(SourceManifestError::DescriptionControlCharacter);
}
Ok(())
}
fn digest_for(
kind: &SourceProviderKind,
coordinator_requires_checkout_access: bool,
transfer_policy: &SourceTransferPolicy,
) -> Digest {
let mut modes = transfer_policy
.allowed_remote_transfer
.iter()
.map(|mode| format!("{mode:?}"))
.collect::<Vec<_>>();
modes.sort();
let mut parts = vec![
b"source-provider-manifest:v2".to_vec(),
kind.provider_id().as_bytes().to_vec(),
coordinator_requires_checkout_access
.to_string()
.into_bytes(),
transfer_policy
.local_source_bytes_remain_node_local
.to_string()
.into_bytes(),
transfer_policy
.coordinator_receives_source_bytes_by_default
.to_string()
.into_bytes(),
transfer_policy
.default_full_repo_tarball
.to_string()
.into_bytes(),
];
parts.extend(modes.into_iter().map(String::into_bytes));
Digest::from_parts(parts)
}
}
fn valid_provider_id(provider: &str) -> bool {
!provider.is_empty()
&& provider.len() <= 64
&& provider
.bytes()
.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourcePreparation {
pub tenant: TenantId,
pub project: ProjectId,
pub provider: SourceProviderKind,
pub required_capabilities: BTreeSet<Capability>,
pub coordinator_requires_checkout_access: bool,
}
impl SourcePreparation {
pub fn node_task(tenant: TenantId, project: ProjectId, provider: SourceProviderKind) -> Self {
let capability = match provider {
SourceProviderKind::Filesystem => Capability::SourceFilesystem,
SourceProviderKind::Git => Capability::SourceGit,
SourceProviderKind::Custom(_) => Capability::SourceFilesystem,
};
Self {
tenant,
project,
provider,
required_capabilities: BTreeSet::from([capability]),
coordinator_requires_checkout_access: false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn source_preparation_can_be_scheduled_as_node_task() {
let prep = SourcePreparation::node_task(
TenantId::from("tenant"),
ProjectId::from("project"),
SourceProviderKind::Git,
);
assert!(!prep.coordinator_requires_checkout_access);
assert!(prep.required_capabilities.contains(&Capability::SourceGit));
}
#[test]
fn local_first_source_manifest_rejects_bulk_coordinator_paths() {
let manifest = SourceProviderManifest::local_first(
SourceProviderKind::Git,
"node-side Git snapshot provider",
);
assert!(manifest.validate_public_mvp().is_ok());
assert!(!manifest.coordinator_requires_checkout_access);
assert!(
!manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default
);
assert!(!manifest.transfer_policy.default_full_repo_tarball);
assert!(manifest
.transfer_policy
.allowed_remote_transfer
.contains(&SourceTransferMode::ExplicitSnapshotChunks));
}
#[test]
fn source_manifest_validation_treats_manifest_as_hostile_input() {
let mut manifest = SourceProviderManifest::local_first(
SourceProviderKind::Custom("gitlab-lfs".to_owned()),
"custom provider",
);
assert!(manifest.validate_public_mvp().is_ok());
manifest.kind = SourceProviderKind::Custom("../checkout".to_owned());
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::InvalidProviderId(
"../checkout".to_owned()
))
);
manifest.kind = SourceProviderKind::Git;
manifest.digest = Digest::sha256("valid");
manifest.coordinator_requires_checkout_access = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::CoordinatorCheckoutAccess)
);
manifest.coordinator_requires_checkout_access = false;
manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::CoordinatorReceivesSourceBytes)
);
manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default = false;
manifest.transfer_policy.default_full_repo_tarball = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::DefaultFullRepoTarball)
);
}
#[test]
fn source_manifest_rejects_malformed_digest_from_json() {
let mut manifest = SourceProviderManifest::local_first(
SourceProviderKind::Filesystem,
"filesystem provider",
);
let value = serde_json::to_value(&manifest).unwrap();
let mut object = value.as_object().unwrap().clone();
object.insert(
"digest".to_owned(),
serde_json::Value::String("sha256:not-a-real-digest".to_owned()),
);
manifest = serde_json::from_value(serde_json::Value::Object(object)).unwrap();
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::InvalidDigest(
"sha256:not-a-real-digest".to_owned()
))
);
}
}

243
crates/clusterflux-core/src/transport.rs Normal file
View file

@ -0,0 +1,243 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{ArtifactId, Digest, NodeId, ProcessId, ProjectId, TenantId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum TransportKind {
NativeQuic,
}
pub trait Transport {
fn kind(&self) -> TransportKind;
fn authenticated_direct_connections(&self) -> bool;
}
#[derive(Clone, Debug, Default)]
pub struct NativeQuicTransport;
impl Transport for NativeQuicTransport {
fn kind(&self) -> TransportKind {
TransportKind::NativeQuic
}
fn authenticated_direct_connections(&self) -> bool {
true
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DataPlaneScope {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub object: DataPlaneObject,
pub authorization_subject: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DataPlaneObject {
Artifact(ArtifactId),
Blob(Digest),
SourceSnapshot(Digest),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeEndpoint {
pub node: NodeId,
pub advertised_addr: String,
pub public_key_fingerprint: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RendezvousRequest {
pub scope: DataPlaneScope,
pub source: NodeEndpoint,
pub destination: NodeEndpoint,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DirectBulkTransferPlan {
pub transport: TransportKind,
pub scope: DataPlaneScope,
pub source: NodeEndpoint,
pub destination: NodeEndpoint,
pub authorization_digest: Digest,
pub coordinator_assisted_rendezvous: bool,
pub coordinator_bulk_relay_allowed: bool,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum TransportError {
#[error(
"direct node-to-node connectivity is unavailable for scoped data-plane transfer: {reason}; coordinator bulk relay is disabled"
)]
DirectConnectivityUnavailable { reason: String },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum BulkTransferDecision {
DirectAuthenticated { scope: DataPlaneScope },
FailClear { message: String },
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
#[error("bulk relay through coordinator is not allowed by default")]
pub struct BulkRelayDenied;
impl NativeQuicTransport {
pub fn plan_authenticated_direct_bulk_transfer(
&self,
request: RendezvousRequest,
direct_connectivity: bool,
failure_reason: impl Into<String>,
) -> Result<DirectBulkTransferPlan, TransportError> {
if !direct_connectivity {
return Err(TransportError::DirectConnectivityUnavailable {
reason: failure_reason.into(),
});
}
let authorization_digest = data_plane_authorization_digest(&request);
Ok(DirectBulkTransferPlan {
transport: self.kind(),
scope: request.scope,
source: request.source,
destination: request.destination,
authorization_digest,
coordinator_assisted_rendezvous: true,
coordinator_bulk_relay_allowed: false,
})
}
}
pub fn direct_bulk_transfer_or_error(
scope: DataPlaneScope,
direct_connectivity: bool,
) -> BulkTransferDecision {
if direct_connectivity {
BulkTransferDecision::DirectAuthenticated { scope }
} else {
BulkTransferDecision::FailClear {
message:
"direct node-to-node connectivity is unavailable; coordinator bulk relay is disabled"
.to_owned(),
}
}
}
fn data_plane_authorization_digest(request: &RendezvousRequest) -> Digest {
let object = match &request.scope.object {
DataPlaneObject::Artifact(artifact) => format!("artifact:{artifact}"),
DataPlaneObject::Blob(digest) => format!("blob:{}", digest.as_str()),
DataPlaneObject::SourceSnapshot(digest) => format!("source:{}", digest.as_str()),
};
Digest::from_parts([
b"dataplane-auth:v1".as_slice(),
request.scope.tenant.as_str().as_bytes(),
request.scope.project.as_str().as_bytes(),
request.scope.process.as_str().as_bytes(),
object.as_bytes(),
request.scope.authorization_subject.as_bytes(),
request.source.node.as_str().as_bytes(),
request.source.public_key_fingerprint.as_str().as_bytes(),
request.destination.node.as_str().as_bytes(),
request
.destination
.public_key_fingerprint
.as_str()
.as_bytes(),
])
}
#[cfg(test)]
mod tests {
use super::*;
fn endpoint(name: &str) -> NodeEndpoint {
NodeEndpoint {
node: NodeId::from(name),
advertised_addr: format!("{name}.mesh.invalid:4433"),
public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
}
}
fn scope(project: &str) -> DataPlaneScope {
DataPlaneScope {
tenant: TenantId::from("tenant"),
project: ProjectId::from(project),
process: ProcessId::from("process"),
object: DataPlaneObject::Artifact(ArtifactId::from("artifact")),
authorization_subject: "node-a-to-node-b".to_owned(),
}
}
#[test]
fn failed_direct_transfer_does_not_silently_relay() {
let decision = direct_bulk_transfer_or_error(scope("project"), false);
assert!(matches!(decision, BulkTransferDecision::FailClear { .. }));
}
#[test]
fn native_quic_rendezvous_plan_is_scoped_and_disallows_coordinator_bulk_relay() {
let transport = NativeQuicTransport;
let request = RendezvousRequest {
scope: scope("project"),
source: endpoint("node-a"),
destination: endpoint("node-b"),
};
let plan = transport
.plan_authenticated_direct_bulk_transfer(request.clone(), true, "")
.unwrap();
let changed_scope_plan = transport
.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope("other-project"),
..request
},
true,
"",
)
.unwrap();
assert_eq!(plan.transport, TransportKind::NativeQuic);
assert_eq!(plan.scope.tenant, TenantId::from("tenant"));
assert_eq!(plan.scope.project, ProjectId::from("project"));
assert_eq!(plan.scope.process, ProcessId::from("process"));
assert_eq!(
plan.scope.object,
DataPlaneObject::Artifact(ArtifactId::from("artifact"))
);
assert_eq!(plan.source.node, NodeId::from("node-a"));
assert_eq!(plan.destination.node, NodeId::from("node-b"));
assert!(plan.coordinator_assisted_rendezvous);
assert!(!plan.coordinator_bulk_relay_allowed);
assert_ne!(
plan.authorization_digest,
changed_scope_plan.authorization_digest
);
}
#[test]
fn failed_direct_rendezvous_reports_clear_error_instead_of_relaying() {
let error = NativeQuicTransport
.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope("project"),
source: endpoint("node-a"),
destination: endpoint("node-b"),
},
false,
"nat traversal failed",
)
.unwrap_err();
assert!(error.to_string().contains("nat traversal failed"));
assert!(error
.to_string()
.contains("coordinator bulk relay is disabled"));
}
}

241
crates/clusterflux-core/src/vfs.rs Normal file
View file

@ -0,0 +1,241 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Digest, NodeId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct VfsPath(String);
impl VfsPath {
pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
let path = path.into();
if !path.starts_with("/vfs/") {
return Err(VfsError::InvalidPath(path));
}
Ok(Self(path))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsObject {
pub path: VfsPath,
pub digest: Digest,
pub size: u64,
pub producer: TaskInstanceId,
pub node: NodeId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsManifest {
pub epoch: u64,
pub producer: TaskInstanceId,
pub node: NodeId,
pub objects: BTreeMap<VfsPath, VfsObject>,
pub large_bytes_uploaded: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SyncPolicy {
MetadataOnly,
ExplicitNode(NodeId),
ExplicitStore(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum VfsSyncDecision {
NoBytesMoved,
MoveBytesToNode(NodeId),
MoveBytesToStore(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReuseDecision {
SameNodeZeroCopy,
NeedsTransfer { from: NodeId, to: NodeId },
Unavailable,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum VfsError {
#[error("VFS path must start with /vfs/: {0}")]
InvalidPath(String),
#[error("path is not visible in the published VFS manifest: {0}")]
NotVisible(String),
}
#[derive(Clone, Debug)]
pub struct VfsOverlay {
task: TaskInstanceId,
node: NodeId,
epoch: u64,
pending: BTreeMap<VfsPath, VfsObject>,
published: BTreeMap<VfsPath, VfsObject>,
}
impl VfsOverlay {
pub fn new(task: TaskInstanceId, node: NodeId) -> Self {
Self {
task,
node,
epoch: 0,
pending: BTreeMap::new(),
published: BTreeMap::new(),
}
}
pub fn write(&mut self, path: VfsPath, digest: Digest, size: u64) -> VfsObject {
let object = VfsObject {
path: path.clone(),
digest,
size,
producer: self.task.clone(),
node: self.node.clone(),
};
self.pending.insert(path, object.clone());
object
}
pub fn flush(&mut self) -> VfsManifest {
self.epoch += 1;
self.published.append(&mut self.pending);
VfsManifest {
epoch: self.epoch,
producer: self.task.clone(),
node: self.node.clone(),
objects: self.published.clone(),
large_bytes_uploaded: false,
}
}
pub fn sync(&self, policy: SyncPolicy) -> VfsSyncDecision {
match policy {
SyncPolicy::MetadataOnly => VfsSyncDecision::NoBytesMoved,
SyncPolicy::ExplicitNode(node) => VfsSyncDecision::MoveBytesToNode(node),
SyncPolicy::ExplicitStore(store) => VfsSyncDecision::MoveBytesToStore(store),
}
}
pub fn read_published<'a>(
manifest: &'a VfsManifest,
path: &VfsPath,
) -> Result<&'a VfsObject, VfsError> {
manifest
.objects
.get(path)
.ok_or_else(|| VfsError::NotVisible(path.as_str().to_owned()))
}
pub fn reuse_for_consumer(
manifest: &VfsManifest,
path: &VfsPath,
consumer_node: &NodeId,
) -> ReuseDecision {
let Some(object) = manifest.objects.get(path) else {
return ReuseDecision::Unavailable;
};
if &object.node == consumer_node {
ReuseDecision::SameNodeZeroCopy
} else {
ReuseDecision::NeedsTransfer {
from: object.node.clone(),
to: consumer_node.clone(),
}
}
}
pub fn discard_unflushed(&mut self) {
self.pending.clear();
}
pub fn pending_len(&self) -> usize {
self.pending.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn path() -> VfsPath {
VfsPath::new("/vfs/artifacts/app").unwrap()
}
#[test]
fn flush_publishes_manifest_without_large_byte_upload() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let manifest = overlay.flush();
assert_eq!(manifest.epoch, 1);
assert!(!manifest.large_bytes_uploaded);
assert!(manifest.objects.contains_key(&path()));
}
#[test]
fn downstream_task_can_read_after_flush_but_not_before() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let empty = VfsManifest {
epoch: 0,
producer: TaskInstanceId::from("task"),
node: NodeId::from("node-a"),
objects: BTreeMap::new(),
large_bytes_uploaded: false,
};
assert!(VfsOverlay::read_published(&empty, &path()).is_err());
let manifest = overlay.flush();
assert!(VfsOverlay::read_published(&manifest, &path()).is_ok());
}
#[test]
fn sync_is_explicit_and_policy_driven() {
let overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
assert_eq!(
overlay.sync(SyncPolicy::MetadataOnly),
VfsSyncDecision::NoBytesMoved
);
assert_eq!(
overlay.sync(SyncPolicy::ExplicitStore("s3://bucket/app".to_owned())),
VfsSyncDecision::MoveBytesToStore("s3://bucket/app".to_owned())
);
}
#[test]
fn same_node_reuse_avoids_transfer() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let manifest = overlay.flush();
assert_eq!(
VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-a")),
ReuseDecision::SameNodeZeroCopy
);
assert_eq!(
VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-b")),
ReuseDecision::NeedsTransfer {
from: NodeId::from("node-a"),
to: NodeId::from("node-b")
}
);
}
#[test]
fn unflushed_task_local_changes_can_be_discarded() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
overlay.discard_unflushed();
assert_eq!(overlay.pending_len(), 0);
}
}

114
crates/clusterflux-core/src/wire.rs Normal file
View file

@ -0,0 +1,114 @@
use serde_json::{json, Value};
pub const COORDINATOR_PROTOCOL_VERSION: u64 = 1;
pub const COORDINATOR_WIRE_REQUEST_TYPE: &str = "coordinator_request";
pub fn coordinator_wire_request(request_id: impl Into<String>, payload: Value) -> Value {
let operation = coordinator_payload_operation(&payload);
let authentication = coordinator_authentication_metadata(&payload);
json!({
"type": COORDINATOR_WIRE_REQUEST_TYPE,
"protocol_version": COORDINATOR_PROTOCOL_VERSION,
"request_id": request_id.into(),
"operation": operation,
"authentication": authentication,
"payload": payload,
})
}
pub fn coordinator_payload_operation(payload: &Value) -> String {
payload
.get("type")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_owned()
}
pub fn coordinator_authentication_metadata(payload: &Value) -> Value {
let operation = coordinator_payload_operation(payload);
match operation.as_str() {
"authenticated" => json!({
"kind": "cli_session",
"session": true,
"request_operation": payload
.get("request")
.map(coordinator_payload_operation)
.unwrap_or_else(|| "unknown".to_owned()),
}),
"signed_node" => json!({
"kind": "node_signature",
"node": payload.get("node").and_then(Value::as_str),
}),
"node_heartbeat" if payload.get("node_signature").is_some() => json!({
"kind": "node_signature",
"node": payload.get("node").and_then(Value::as_str),
}),
"start_process" | "launch_task" if payload.get("agent_signature").is_some() => json!({
"kind": "agent_signature",
"agent": payload.get("actor_agent").and_then(Value::as_str),
"fingerprint": payload.get("agent_public_key_fingerprint").and_then(Value::as_str),
}),
"admin_status" | "suspend_tenant" if payload.get("admin_proof").is_some() => json!({
"kind": "admin_proof",
"actor": payload.get("actor_user").and_then(Value::as_str),
"nonce": payload.get("admin_nonce").and_then(Value::as_str),
"issued_at_epoch_seconds": payload.get("issued_at_epoch_seconds").and_then(Value::as_u64),
}),
"exchange_node_enrollment_grant" => json!({
"kind": "node_enrollment_grant",
"node": payload.get("node").and_then(Value::as_str),
}),
_ => json!({
"kind": "none",
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn coordinator_wire_request_wraps_payload_without_exposing_secret_metadata() {
let envelope = coordinator_wire_request(
"cli-1",
json!({
"type": "authenticated",
"session_secret": "secret-value",
"request": { "type": "list_projects" },
}),
);
assert_eq!(envelope["type"], COORDINATOR_WIRE_REQUEST_TYPE);
assert_eq!(envelope["protocol_version"], COORDINATOR_PROTOCOL_VERSION);
assert_eq!(envelope["request_id"], "cli-1");
assert_eq!(envelope["operation"], "authenticated");
assert_eq!(envelope["authentication"]["kind"], "cli_session");
assert_eq!(
envelope["authentication"]["request_operation"],
"list_projects"
);
assert_eq!(envelope["authentication"].get("session_secret"), None);
assert_eq!(envelope["payload"]["session_secret"], "secret-value");
}
#[test]
fn coordinator_wire_request_describes_signature_metadata() {
let envelope = coordinator_wire_request(
"node-1",
json!({
"type": "signed_node",
"node": "node-a",
"node_signature": {
"nonce": "nonce",
"issued_at_epoch_seconds": 1,
"signature": "ed25519:sig"
},
"request": { "type": "poll_task_assignment", "node": "node-a" },
}),
);
assert_eq!(envelope["authentication"]["kind"], "node_signature");
assert_eq!(envelope["authentication"]["node"], "node-a");
}
}

21
crates/clusterflux-dap/Cargo.toml Normal file
View file

@ -0,0 +1,21 @@
[package]
name = "clusterflux-dap"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[[bin]]
name = "clusterflux-debug-dap"
path = "src/main.rs"
[dependencies]
anyhow.workspace = true
base64.workspace = true
clusterflux-core = { path = "../clusterflux-core" }
clusterflux-control = { path = "../clusterflux-control" }
clusterflux-node = { path = "../clusterflux-node" }
serde_json.workspace = true
[dev-dependencies]
tempfile.workspace = true

854
crates/clusterflux-dap/src/adapter.rs Normal file
View file

@ -0,0 +1,854 @@
use std::io::{self, BufReader};
use anyhow::Result;
use clusterflux_core::DebugRuntimeState;
use serde_json::{json, Value};
use crate::breakpoints::{
freeze_all, freeze_all_at_line, next_breakpoint_after, position_confirmed_breakpoint_stop,
request_thread, request_thread_id, resolve_breakpoints_for_source,
restart_requires_whole_process, simulated_freeze_failure_thread, step_thread,
stopped_thread_for_breakpoint,
};
use crate::dap_protocol::{initialize_capabilities, read_message, DapWriter};
use crate::demo_backend::{
is_explicit_demo_backend, start_simulated_backend, LINUX_THREAD, MAIN_THREAD,
};
use crate::runtime_client::{
attach_services_runtime, create_debug_epoch, relaunch_services_main_runtime, restart_task,
resume_debug_epoch, run_live_services_runtime, run_local_services_runtime,
wait_for_debug_epoch_frozen, wait_for_debug_epoch_resumed, wait_for_services_runtime_outcome,
RuntimeContinuationOutcome,
};
use crate::variables::variables_response;
use crate::virtual_model::{AdapterState, DapSessionMode, RuntimeBackend};
pub(crate) fn run_adapter() -> Result<()> {
let mut writer = DapWriter::new();
let mut reader = BufReader::new(io::stdin());
let mut state = AdapterState::default();
let mut _local_runtime_session = None;
while let Some(request) = read_message(&mut reader)? {
let command = request
.get("command")
.and_then(Value::as_str)
.unwrap_or_default();
match command {
"initialize" => writer.response(&request, true, initialize_capabilities())?,
"launch" | "attach" => {
let args = request
.get("arguments")
.cloned()
.unwrap_or_else(|| json!({}));
state.entry = args
.get("entry")
.and_then(Value::as_str)
.unwrap_or("build")
.to_owned();
let project = args
.get("project")
.and_then(Value::as_str)
.unwrap_or(".")
.to_owned();
let runtime_backend = match runtime_backend_from_launch_arg(
args.get("runtimeBackend").and_then(Value::as_str),
) {
Ok(runtime_backend) => runtime_backend,
Err(message) => {
writer.error_response(&request, message)?;
continue;
}
};
let coordinator_endpoint = args
.get("coordinatorEndpoint")
.and_then(Value::as_str)
.map(str::to_owned);
let source_path = args
.get("sourcePath")
.and_then(Value::as_str)
.map(str::to_owned);
if let Err(err) = state.configure_launch(
project,
state.entry.clone(),
runtime_backend,
coordinator_endpoint,
source_path,
) {
writer.error_response(&request, err.to_string())?;
continue;
}
state.restart_existing = args
.get("restartExisting")
.and_then(Value::as_bool)
.unwrap_or(false);
if let Some(process_id) = args.get("processId").and_then(Value::as_str) {
state.process = clusterflux_core::ProcessId::new(process_id);
}
if command == "attach" {
state.session_mode = DapSessionMode::Attach;
state.command_status =
format!("attached to existing virtual process {}", state.process);
}
writer.response(&request, true, json!({}))?;
writer.event("initialized", json!({}))?;
}
"setBreakpoints" => {
let requested_source_path = request
.get("arguments")
.and_then(|value| value.get("source"))
.and_then(|value| value.get("path"))
.and_then(Value::as_str);
let requested_lines = request
.get("arguments")
.and_then(|value| value.get("breakpoints"))
.and_then(Value::as_array)
.map(|items| {
items
.iter()
.filter_map(|item| item.get("line").and_then(Value::as_i64))
.collect::<Vec<_>>()
})
.unwrap_or_default();
let response = resolve_breakpoints_for_source(
&mut state,
requested_source_path,
requested_lines,
)
.iter()
.map(|breakpoint| breakpoint.to_dap())
.collect::<Vec<_>>();
writer.response(&request, true, json!({ "breakpoints": response }))?;
}
"setExceptionBreakpoints" => {
writer.response(&request, true, json!({ "breakpoints": [] }))?;
}
"configurationDone" => {
if state.session_mode == DapSessionMode::Attach {
match state.runtime_backend {
RuntimeBackend::Simulated => {
state.command_status =
format!("attached to existing virtual process {}", state.process);
}
RuntimeBackend::LocalServices | RuntimeBackend::LiveServices => {
match attach_services_runtime(&state) {
Ok(record) => state.apply_attach_record(record),
Err(err) => {
writer.error_response(
&request,
format!("services runtime attach failed: {err:#}"),
)?;
continue;
}
}
}
}
} else {
match state.runtime_backend {
RuntimeBackend::LocalServices => match run_local_services_runtime(&state) {
Ok((record, session)) => {
state.apply_runtime_record(record);
_local_runtime_session = Some(session);
}
Err(err) => {
writer.error_response(
&request,
format!("local services runtime launch failed: {err:#}"),
)?;
continue;
}
},
RuntimeBackend::LiveServices => match run_live_services_runtime(&state) {
Ok(record) => {
state.apply_runtime_record(record);
}
Err(err) => {
writer.error_response(
&request,
format!("live services runtime launch failed: {err:#}"),
)?;
continue;
}
},
RuntimeBackend::Simulated => {
start_simulated_backend(&mut state);
}
}
}
writer.response(&request, true, json!({}))?;
let session_message = if state.session_mode == DapSessionMode::Attach {
format!(
"Attached to Clusterflux virtual process {} with {:?} runtime\n",
state.process, state.runtime_backend
)
} else {
format!(
"Clusterflux bundle refreshed for entry `{}`; starting virtual process {} with {:?} runtime\n",
state.entry, state.process, state.runtime_backend
)
};
writer.output("console", session_message)?;
if !state.breakpoints.is_empty() {
let stopped_thread = stopped_thread_for_breakpoint(&state);
if matches!(
state.runtime_backend,
RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
) && state.coordinator_debug_epoch.is_some()
{
position_confirmed_breakpoint_stop(&mut state, stopped_thread);
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Clusterflux Debug Epoch all-stop confirmed by every active participant",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
continue;
}
match freeze_all(&mut state, stopped_thread, None) {
Ok(()) => {
if let Err(err) = record_coordinator_debug_epoch(
&mut state,
stopped_thread,
"breakpoint",
) {
let message = format!("coordinator debug epoch failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
continue;
}
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Clusterflux Debug Epoch all-stop",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Err(failure) => {
let message = failure.message();
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
}
}
} else {
writer.event("terminated", json!({}))?;
}
}
"threads" => {
let threads = state
.threads
.values()
.map(|thread| {
json!({
"id": thread.id,
"name": format!("{}/{}", state.process, thread.name),
})
})
.collect::<Vec<_>>();
writer.response(&request, true, json!({ "threads": threads }))?;
}
"stackTrace" => {
let thread = request_thread(&request, &state);
let source_path = crate::source::stack_source_path(&state);
let stack_frames = if state.runtime_backend != RuntimeBackend::Simulated
&& thread.runtime_stack_frames.is_empty()
{
Vec::new()
} else {
let frame_name = thread
.runtime_stack_frames
.first()
.cloned()
.unwrap_or_else(|| format!("{}::run", thread.name));
vec![json!({
"id": thread.frame_id,
"name": frame_name,
"line": thread.line,
"column": 1,
"source": {
"name": crate::source::source_name(&source_path),
"path": source_path,
"presentationHint": "normal"
}
})]
};
let total_frames = stack_frames.len();
writer.response(
&request,
true,
json!({
"stackFrames": stack_frames,
"totalFrames": total_frames
}),
)?;
}
"source" => match crate::source::source_response(&state, &request) {
Ok(body) => writer.response(&request, true, body)?,
Err(err) => writer.error_response(&request, err.to_string())?,
},
"scopes" => {
let frame_id = request
.get("arguments")
.and_then(|value| value.get("frameId"))
.and_then(Value::as_i64)
.unwrap_or(1000 + crate::demo_backend::MAIN_THREAD);
let thread = state
.threads
.values()
.find(|thread| thread.frame_id == frame_id)
.cloned()
.unwrap_or_else(|| state.threads[&crate::demo_backend::MAIN_THREAD].clone());
writer.response(
&request,
true,
json!({
"scopes": [
{
"name": "Source Locals",
"variablesReference": thread.locals_ref,
"expensive": false,
"presentationHint": "locals"
},
{
"name": "Wasm Frame Locals",
"variablesReference": thread.wasm_locals_ref,
"expensive": false,
"presentationHint": "locals"
},
{
"name": "Task Args and Handles",
"variablesReference": thread.args_ref,
"expensive": false,
"presentationHint": "arguments"
},
{
"name": "Clusterflux Runtime",
"variablesReference": thread.runtime_ref,
"expensive": false
},
{
"name": "Recent Output",
"variablesReference": thread.output_ref,
"expensive": false
}
]
}),
)?;
}
"variables" => {
let reference = request
.get("arguments")
.and_then(|value| value.get("variablesReference"))
.and_then(Value::as_i64)
.unwrap_or(0);
writer.response(&request, true, variables_response(&state, reference))?;
}
"evaluate" => {
let expression = request
.get("arguments")
.and_then(|value| value.get("expression"))
.and_then(Value::as_str)
.unwrap_or_default();
let result = match expression {
"virtual_process_id" => state.process.to_string(),
"debug_epoch" => state.epoch.to_string(),
"command_status" => state.command_status.clone(),
_ => "not available".to_owned(),
};
writer.response(
&request,
true,
json!({ "result": result, "variablesReference": 0 }),
)?;
}
"pause" => {
let stopped_thread = default_thread_id(&state);
match freeze_all(
&mut state,
stopped_thread,
simulated_freeze_failure_thread(&request),
) {
Ok(()) => {
if let Err(err) =
record_coordinator_debug_epoch(&mut state, stopped_thread, "pause")
{
let message = format!("coordinator debug epoch failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
continue;
}
writer.response(&request, true, json!({}))?;
writer.event(
"stopped",
json!({
"reason": "pause",
"description": "Clusterflux Debug Epoch pause",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Err(failure) => {
let message = failure.message();
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
}
}
}
"continue" => {
let thread_id =
request_thread_id(&request).unwrap_or_else(|| default_thread_id(&state));
let next_breakpoint = next_breakpoint_after(&state, thread_id);
let previous_debug_epoch = state.coordinator_debug_epoch.unwrap_or(state.epoch);
if let Err(err) = resume_coordinator_epoch(&mut state) {
let message = format!("coordinator debug epoch resume failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
continue;
}
for thread in state.threads.values_mut() {
if thread.state == DebugRuntimeState::Frozen {
thread.state = DebugRuntimeState::Running;
}
}
writer.response(&request, true, json!({ "allThreadsContinued": true }))?;
writer.event(
"continued",
json!({
"threadId": thread_id,
"allThreadsContinued": true,
}),
)?;
if matches!(
state.runtime_backend,
RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
) {
match wait_for_services_runtime_outcome(&state, previous_debug_epoch) {
Ok(RuntimeContinuationOutcome::Breakpoint(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
position_confirmed_breakpoint_stop(&mut state, stopped_thread);
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Clusterflux Debug Epoch all-stop confirmed by every active participant",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Exception(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
writer.output("stderr", format!("{}\n", state.command_status))?;
writer.event(
"stopped",
json!({
"reason": "exception",
"description": "Task failed and is awaiting operator action",
"threadId": stopped_thread,
"allThreadsStopped": false,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Terminal(record)) => {
state.apply_runtime_record(record);
if state.last_task_failed {
writer.output("stderr", format!("{}\n", state.command_status))?;
}
writer.event("terminated", json!({}))?;
}
Err(err) => {
let message = format!("continued runtime observation failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.event(
"stopped",
json!({
"reason": "exception",
"description": message,
"threadId": thread_id,
"allThreadsStopped": false,
}),
)?;
}
}
continue;
}
if let Some((stopped_thread, line)) = next_breakpoint {
match freeze_all_at_line(&mut state, stopped_thread, line, None) {
Ok(()) => {
if let Err(err) = record_coordinator_debug_epoch(
&mut state,
stopped_thread,
"breakpoint",
) {
let message = format!("coordinator debug epoch failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
continue;
}
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Clusterflux Debug Epoch all-stop",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Err(failure) => {
let message = failure.message();
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
}
}
} else {
writer.event("terminated", json!({}))?;
}
}
"next" | "stepIn" | "stepOut" => {
if state.runtime_backend != RuntimeBackend::Simulated {
writer.error_response(
&request,
"source stepping is not yet available from the executing node; use continue or pause instead of a synthetic step",
)?;
continue;
}
let thread_id = request_thread_id(&request).unwrap_or(LINUX_THREAD);
let description = match command {
"next" => "step over",
"stepIn" => "step in",
"stepOut" => "step out",
_ => "step",
};
step_thread(&mut state, thread_id, description);
writer.response(&request, true, json!({}))?;
writer.event(
"stopped",
json!({
"reason": "step",
"description": format!("Clusterflux Debug Epoch {description}"),
"threadId": thread_id,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
"restart" | "restartFrame" => {
let thread_id = request_thread_id(&request)
.or_else(|| {
request
.get("arguments")
.and_then(|arguments| arguments.get("frameId"))
.and_then(Value::as_i64)
.and_then(|frame_id| {
state
.threads
.values()
.find(|thread| thread.frame_id == frame_id)
.map(|thread| thread.id)
})
})
.unwrap_or_else(|| default_thread_id(&state));
if restart_requires_whole_process(&request) {
let message = format!(
"Incompatible source edit requires whole virtual-process restart for {}",
state.process
);
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
continue;
}
let restarting_failed_task = state
.threads
.get(&thread_id)
.is_some_and(|thread| matches!(thread.state, DebugRuntimeState::Failed(_)));
if state.runtime_backend != RuntimeBackend::Simulated {
if thread_id == MAIN_THREAD && restarting_failed_task {
match relaunch_services_main_runtime(&state) {
Ok(record) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
position_confirmed_breakpoint_stop(&mut state, stopped_thread);
writer.response(&request, true, json!({}))?;
writer.output(
"console",
"Rebuilt the current bundle and restarted the failed coordinator main from its entry boundary\n",
)?;
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Restarted main from the rebuilt bundle; every active participant confirmed all-stop",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Err(err) => {
let message = format!("coordinator main restart failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
}
}
continue;
}
match restart_task_through_coordinator(&mut state, thread_id) {
Ok(message) => {
let previous_debug_epoch =
state.coordinator_debug_epoch.unwrap_or(state.epoch);
if let Some(thread) = state.threads.get_mut(&thread_id) {
thread.state = DebugRuntimeState::Running;
thread.recent_output.push(message.clone());
}
state.last_task_failed = false;
state.command_status = message.clone();
writer.response(&request, true, json!({}))?;
writer.output("console", format!("{message}\n"))?;
if matches!(
state.runtime_backend,
RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
) {
match wait_for_services_runtime_outcome(
&state,
previous_debug_epoch,
) {
Ok(RuntimeContinuationOutcome::Breakpoint(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
position_confirmed_breakpoint_stop(
&mut state,
stopped_thread,
);
writer.event(
"stopped",
json!({
"reason": "breakpoint",
"description": "Restarted task reached a Wasm probe and every active participant confirmed all-stop",
"threadId": stopped_thread,
"allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Exception(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
writer.event(
"stopped",
json!({
"reason": "exception",
"description": "Replacement attempt failed and is awaiting operator action",
"threadId": stopped_thread,
"allThreadsStopped": false,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Terminal(record)) => {
state.apply_runtime_record(record);
writer.event("terminated", json!({}))?;
}
Err(err) => {
writer.event(
"stopped",
json!({
"reason": "exception",
"description": format!("restarted runtime observation failed: {err:#}"),
"threadId": thread_id,
"allThreadsStopped": false,
}),
)?;
}
}
}
}
Err(err) => {
let message = format!("coordinator task restart failed: {err:#}");
writer.output("stderr", format!("{message}\n"))?;
writer.error_response(&request, message)?;
}
}
continue;
}
if let Some(thread) = state.threads.get_mut(&thread_id) {
thread.state = DebugRuntimeState::Running;
thread
.recent_output
.push("task restarted from VFS checkpoint".to_owned());
}
if restarting_failed_task {
state.last_task_failed = false;
}
state.command_status = format!(
"{} task restarted from compatible VFS checkpoint",
if restarting_failed_task {
"failed"
} else {
"selected"
}
);
writer.response(&request, true, json!({}))?;
writer.output(
"console",
format!(
"Restarted {} task from compatible VFS checkpoint on thread {thread_id}\n",
if restarting_failed_task {
"failed"
} else {
"selected"
}
),
)?;
}
"disconnect" | "terminate" => {
writer.response(&request, true, json!({}))?;
writer.event("terminated", json!({}))?;
break;
}
_ => writer.error_response(&request, format!("unsupported DAP command: {command}"))?,
}
}
Ok(())
}
fn restart_task_through_coordinator(state: &mut AdapterState, thread_id: i64) -> Result<String> {
let task = state
.threads
.get(&thread_id)
.or_else(|| state.threads.get(&default_thread_id(state)))
.map(|thread| thread.task.clone())
.ok_or_else(|| anyhow::anyhow!("selected debug thread does not map to a virtual task"))?;
let record = restart_task(state, &task)?;
if !record.accepted {
let mut message = format!(
"coordinator refused task restart for `{task}`: {}",
record.message
);
if record.requires_whole_process_restart {
message.push_str("; whole virtual-process restart required");
}
if record.active_task {
message.push_str("; selected task is still active");
}
if record.completed_event_observed {
message.push_str("; only terminal task-event metadata is available");
}
return Err(anyhow::anyhow!(message));
}
if !record.clean_boundary_available {
return Err(anyhow::anyhow!(
"coordinator accepted task restart for `{task}` without a clean checkpoint boundary"
));
}
let restarted_task = record.restarted_task_instance.ok_or_else(|| {
anyhow::anyhow!("coordinator accepted task restart without its stable task-instance ID")
})?;
if let Some(thread) = state.threads.get_mut(&thread_id) {
thread.task = restarted_task.clone();
thread.attempt_id = record.restarted_attempt_id.clone().ok_or_else(|| {
anyhow::anyhow!("coordinator accepted task restart without a new attempt ID")
})?;
}
state.debug_probes =
crate::breakpoints::load_bundle_debug_probes(&state.project, &state.source_path);
Ok(format!(
"Coordinator restarted logical task `{restarted_task}` as attempt `{}` from the rebuilt bundle and a clean runtime checkpoint boundary",
record.restarted_attempt_id.as_deref().unwrap_or("unknown")
))
}
fn record_coordinator_debug_epoch(
state: &mut AdapterState,
stopped_thread: i64,
reason: &str,
) -> Result<()> {
if state.runtime_backend == RuntimeBackend::Simulated {
return Ok(());
}
let stopped_task = state
.threads
.get(&stopped_thread)
.map(|thread| thread.task.clone())
.unwrap_or_else(|| state.threads[&default_thread_id(state)].task.clone());
let record = create_debug_epoch(state, &stopped_task, reason)?;
let status = wait_for_debug_epoch_frozen(state, record.epoch)?;
state.debug_all_threads_stopped = status.fully_frozen;
state.epoch = record.epoch;
state.coordinator_debug_epoch = Some(record.epoch);
let previous_status = state.command_status.clone();
state.command_status = format!(
"{previous_status}; debug epoch {} {} after {}/{} signed participant freeze acknowledgements",
status.epoch,
status.command,
status.acknowledgements.len(),
status.expected_tasks
);
if let Some(thread) = state.threads.get_mut(&stopped_thread) {
thread.recent_output.push(format!(
"coordinator debug epoch {} reached all-stop after {}/{} signed participant acknowledgements",
status.epoch,
status.acknowledgements.len(),
status.expected_tasks
));
}
Ok(())
}
fn default_thread_id(state: &AdapterState) -> i64 {
if state.runtime_backend == RuntimeBackend::Simulated
&& state.threads.contains_key(&LINUX_THREAD)
{
LINUX_THREAD
} else {
MAIN_THREAD
}
}
fn resume_coordinator_epoch(state: &mut AdapterState) -> Result<()> {
let Some(epoch) = state.coordinator_debug_epoch else {
return Ok(());
};
if state.runtime_backend == RuntimeBackend::Simulated {
return Ok(());
}
let record = resume_debug_epoch(state, epoch)?;
let status = wait_for_debug_epoch_resumed(state, epoch)?;
state.coordinator_debug_epoch = None;
state.command_status = format!(
"debug epoch {} resumed after {}/{} signed participant acknowledgements",
status.epoch,
status.acknowledgements.len(),
status.expected_tasks
);
let status_thread = default_thread_id(state);
if let Some(thread) = state.threads.get_mut(&status_thread) {
thread.recent_output.push(format!(
"coordinator debug epoch {} resumed with {} after {}/{} acknowledgements ({} affected task(s))",
status.epoch,
status.command,
status.acknowledgements.len(),
status.expected_tasks,
record.affected_tasks
));
}
Ok(())
}
pub(crate) fn runtime_backend_from_launch_arg(
value: Option<&str>,
) -> Result<RuntimeBackend, String> {
match value {
None | Some("local-services") => Ok(RuntimeBackend::LocalServices),
Some("live-services") => Ok(RuntimeBackend::LiveServices),
Some(value) if is_explicit_demo_backend(value) => Ok(RuntimeBackend::Simulated),
Some(value) => Err(format!(
"unsupported Clusterflux runtimeBackend `{value}`; use local-services, live-services, or explicit demo"
)),
}
}

340
crates/clusterflux-dap/src/breakpoints.rs Normal file
View file

@ -0,0 +1,340 @@
use std::fs;
use clusterflux_core::{
discover_source_debug_probes, BundleDebugProbe, DebugRuntimeState, TaskInstanceId,
};
use serde_json::{json, Value};
use crate::demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
use crate::virtual_model::{AdapterState, VirtualThread};
pub(crate) fn request_thread<'a>(request: &Value, state: &'a AdapterState) -> &'a VirtualThread {
let thread_id = request_thread_id(request).unwrap_or(MAIN_THREAD);
state
.threads
.get(&thread_id)
.unwrap_or_else(|| &state.threads[&MAIN_THREAD])
}
pub(crate) fn request_thread_id(request: &Value) -> Option<i64> {
request
.get("arguments")
.and_then(|value| value.get("threadId"))
.and_then(Value::as_i64)
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct ResolvedBreakpoint {
pub(crate) id: usize,
pub(crate) line: i64,
pub(crate) verified: bool,
pub(crate) message: String,
}
impl ResolvedBreakpoint {
pub(crate) fn to_dap(&self) -> Value {
json!({
"id": self.id,
"verified": self.verified,
"line": self.line,
"message": self.message,
})
}
}
pub(crate) fn load_bundle_debug_probes(project: &str, source_path: &str) -> Vec<BundleDebugProbe> {
let source = fs::read_to_string(crate::source::resolve_source_path(project, source_path));
source
.map(|source| discover_source_debug_probes(source_path, &source))
.unwrap_or_default()
}
pub(crate) fn resolve_breakpoints(
state: &mut AdapterState,
requested_lines: Vec<i64>,
) -> Vec<ResolvedBreakpoint> {
let resolved = requested_lines
.into_iter()
.enumerate()
.map(|(index, line)| {
let probe = debug_probe_for_line(state, line);
let verified = probe.is_some()
|| (state.debug_probes.is_empty()
&& source_function_name_at_line(state, line).is_some());
let message = match probe {
Some(probe) => format!(
"Mapped to Clusterflux debug probe {} for task {}",
probe.id, probe.task
),
None if verified => "Mapped to Clusterflux virtual source location".to_owned(),
None => "No Clusterflux debug probe metadata covers this source line".to_owned(),
};
ResolvedBreakpoint {
id: index + 1,
line,
verified,
message,
}
})
.collect::<Vec<_>>();
state.breakpoints = resolved
.iter()
.filter(|breakpoint| breakpoint.verified)
.map(|breakpoint| breakpoint.line)
.collect();
resolved
}
pub(crate) fn resolve_breakpoints_for_source(
state: &mut AdapterState,
requested_source_path: Option<&str>,
requested_lines: Vec<i64>,
) -> Vec<ResolvedBreakpoint> {
if requested_source_path.is_none_or(|requested_source_path| {
crate::source::source_paths_match(&state.project, &state.source_path, requested_source_path)
}) {
return resolve_breakpoints(state, requested_lines);
}
requested_lines
.into_iter()
.enumerate()
.map(|(index, line)| ResolvedBreakpoint {
id: index + 1,
line,
verified: false,
message: format!(
"This debug session is configured for `{}`; breakpoints from another source file are not applied",
state.source_path
),
})
.collect()
}
pub(crate) fn restart_requires_whole_process(request: &Value) -> bool {
let Some(arguments) = request.get("arguments") else {
return false;
};
arguments
.get("requiresWholeProcessRestart")
.and_then(Value::as_bool)
.unwrap_or(false)
|| [
"compatibility",
"sourceCompatibility",
"sourceEditCompatibility",
"taskCompatibility",
]
.iter()
.filter_map(|field| arguments.get(field).and_then(Value::as_str))
.any(is_incompatible_restart)
|| arguments
.get("sourceEdit")
.and_then(|value| value.get("compatibility"))
.and_then(Value::as_str)
.is_some_and(is_incompatible_restart)
}
fn is_incompatible_restart(value: &str) -> bool {
value.eq_ignore_ascii_case("incompatible")
|| value.eq_ignore_ascii_case("whole-process")
|| value.eq_ignore_ascii_case("whole_process")
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct FreezeFailure {
pub(crate) thread_id: i64,
pub(crate) task: TaskInstanceId,
}
impl FreezeFailure {
pub(crate) fn message(&self) -> String {
format!(
"debug all-stop failed: participant `{}` on thread {} could not freeze",
self.task, self.thread_id
)
}
}
pub(crate) fn simulated_freeze_failure_thread(request: &Value) -> Option<i64> {
request
.get("arguments")
.and_then(|arguments| arguments.get("simulateFreezeFailure"))
.and_then(Value::as_bool)
.unwrap_or(false)
.then_some(PACKAGE_THREAD)
}
pub(crate) fn freeze_all(
state: &mut AdapterState,
stopped_thread: i64,
forced_failure_thread: Option<i64>,
) -> Result<(), FreezeFailure> {
let line = state
.breakpoints
.iter()
.copied()
.find(|line| thread_for_source_line(state, *line) == stopped_thread)
.or_else(|| state.breakpoints.first().copied())
.unwrap_or_else(|| state.threads[&stopped_thread].line);
freeze_all_at_line(state, stopped_thread, line, forced_failure_thread)
}
pub(crate) fn freeze_all_at_line(
state: &mut AdapterState,
stopped_thread: i64,
line: i64,
forced_failure_thread: Option<i64>,
) -> Result<(), FreezeFailure> {
if let Some(failure) = state.threads.values().find(|thread| {
thread.state == DebugRuntimeState::Running
&& (!thread.freeze_supported || forced_failure_thread == Some(thread.id))
}) {
return Err(FreezeFailure {
thread_id: failure.id,
task: failure.task.clone(),
});
}
state.epoch += 1;
for thread in state.threads.values_mut() {
if thread.state == DebugRuntimeState::Running {
thread.state = DebugRuntimeState::Frozen;
}
}
if let Some(thread) = state.threads.get_mut(&stopped_thread) {
thread.line = line;
thread
.recent_output
.push(format!("debug epoch {} all-stop", state.epoch));
}
Ok(())
}
pub(crate) fn stopped_thread_for_breakpoint(state: &AdapterState) -> i64 {
if let Some(stopped_task) = state.stopped_task.as_ref() {
if let Some(thread) = state
.threads
.values()
.find(|thread| &thread.task == stopped_task)
{
return thread.id;
}
}
state
.breakpoints
.first()
.map(|line| thread_for_source_line(state, *line))
.unwrap_or(MAIN_THREAD)
}
pub(crate) fn position_confirmed_breakpoint_stop(state: &mut AdapterState, stopped_thread: i64) {
let line = state
.breakpoints
.iter()
.copied()
.find(|line| thread_for_source_line(state, *line) == stopped_thread)
.or_else(|| state.breakpoints.first().copied());
if let (Some(line), Some(thread)) = (line, state.threads.get_mut(&stopped_thread)) {
thread.line = line;
thread.recent_output.push(format!(
"debug epoch {} confirmed frozen by all participants",
state.epoch
));
}
}
pub(crate) fn next_breakpoint_after(state: &AdapterState, thread_id: i64) -> Option<(i64, i64)> {
let current_line = state.threads.get(&thread_id)?.line;
state
.breakpoints
.iter()
.copied()
.filter(|line| *line > current_line)
.min()
.map(|line| (thread_for_source_line(state, line), line))
}
fn thread_for_source_line(state: &AdapterState, line: i64) -> i64 {
if let Some(probe) = debug_probe_for_line(state, line) {
return state
.threads
.values()
.find(|thread| thread.task_definition == probe.task)
.map(|thread| thread.id)
.unwrap_or(MAIN_THREAD);
}
if let Some(function_name) = source_function_name_at_line(state, line) {
let lower = function_name.to_ascii_lowercase();
if lower.contains("linux") {
return LINUX_THREAD;
}
if lower.contains("windows") {
return WINDOWS_THREAD;
}
if lower.contains("package") {
return PACKAGE_THREAD;
}
return MAIN_THREAD;
}
state
.threads
.values()
.find(|thread| thread.line == line)
.map(|thread| thread.id)
.unwrap_or(MAIN_THREAD)
}
fn debug_probe_for_line(state: &AdapterState, line: i64) -> Option<&BundleDebugProbe> {
let line = u32::try_from(line).ok()?;
state
.debug_probes
.iter()
.find(|probe| probe.line_start <= line && line <= probe.line_end)
}
pub(crate) fn source_function_name_at_line(state: &AdapterState, line: i64) -> Option<String> {
let source = fs::read_to_string(crate::source::resolve_source_path(
&state.project,
&state.source_path,
))
.ok()?;
let lines = source.lines().collect::<Vec<_>>();
let mut index = usize::try_from(line).ok()?.saturating_sub(1);
if index >= lines.len() {
index = lines.len().saturating_sub(1);
}
lines[..=index]
.iter()
.rev()
.find_map(|line| parse_rust_function_name(line))
}
pub(crate) fn parse_rust_function_name(line: &str) -> Option<String> {
let start = line.find("fn ")? + 3;
let rest = &line[start..];
let name = rest
.chars()
.take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
.collect::<String>();
(!name.is_empty()).then_some(name)
}
pub(crate) fn step_thread(state: &mut AdapterState, thread_id: i64, description: &str) {
state.epoch += 1;
let resolved_thread = if state.threads.contains_key(&thread_id) {
thread_id
} else {
LINUX_THREAD
};
if let Some(thread) = state.threads.get_mut(&resolved_thread) {
thread.state = DebugRuntimeState::Frozen;
thread.line += 1;
thread
.recent_output
.push(format!("debug epoch {} {description}", state.epoch));
}
}

129
crates/clusterflux-dap/src/dap_protocol.rs Normal file
View file

@ -0,0 +1,129 @@
use std::io::{self, BufRead, Write};
use anyhow::{anyhow, Result};
use serde_json::{json, Value};
#[derive(Clone)]
pub(crate) struct DapWriter {
seq: i64,
}
impl DapWriter {
pub(crate) fn new() -> Self {
Self { seq: 1 }
}
pub(crate) fn response(
&mut self,
request: &Value,
success: bool,
body: Value,
) -> io::Result<()> {
let command = request
.get("command")
.and_then(Value::as_str)
.unwrap_or("<unknown>");
let request_seq = request.get("seq").and_then(Value::as_i64).unwrap_or(0);
let seq = self.next_seq();
self.write(json!({
"seq": seq,
"type": "response",
"request_seq": request_seq,
"success": success,
"command": command,
"body": body,
}))
}
pub(crate) fn error_response(
&mut self,
request: &Value,
message: impl Into<String>,
) -> io::Result<()> {
let command = request
.get("command")
.and_then(Value::as_str)
.unwrap_or("<unknown>");
let request_seq = request.get("seq").and_then(Value::as_i64).unwrap_or(0);
let seq = self.next_seq();
self.write(json!({
"seq": seq,
"type": "response",
"request_seq": request_seq,
"success": false,
"command": command,
"message": message.into(),
}))
}
pub(crate) fn event(&mut self, event: &str, body: Value) -> io::Result<()> {
let seq = self.next_seq();
self.write(json!({
"seq": seq,
"type": "event",
"event": event,
"body": body,
}))
}
pub(crate) fn output(&mut self, category: &str, output: impl Into<String>) -> io::Result<()> {
self.event(
"output",
json!({
"category": category,
"output": output.into(),
}),
)
}
fn next_seq(&mut self) -> i64 {
let seq = self.seq;
self.seq += 1;
seq
}
fn write(&mut self, message: Value) -> io::Result<()> {
let payload = serde_json::to_vec(&message)?;
let mut out = io::stdout().lock();
write!(out, "Content-Length: {}\r\n\r\n", payload.len())?;
out.write_all(&payload)?;
out.flush()
}
}
pub(crate) fn initialize_capabilities() -> Value {
json!({
"supportsConfigurationDoneRequest": true,
"supportsTerminateRequest": true,
"supportsRestartRequest": true,
"supportsRestartFrame": true,
"supportsEvaluateForHovers": true,
"supportsStepBack": false,
})
}
pub(crate) fn read_message<R: BufRead>(reader: &mut R) -> Result<Option<Value>> {
let mut content_length = None;
loop {
let mut line = String::new();
let bytes = reader.read_line(&mut line)?;
if bytes == 0 {
return Ok(None);
}
let line = line.trim_end_matches(['\r', '\n']);
if line.is_empty() {
break;
}
if let Some(value) = line.strip_prefix("Content-Length:") {
content_length = Some(value.trim().parse::<usize>()?);
}
}
let length = content_length.ok_or_else(|| anyhow!("DAP message missing Content-Length"))?;
let mut payload = vec![0; length];
reader.read_exact(&mut payload)?;
Ok(Some(serde_json::from_slice(&payload)?))
}

68
crates/clusterflux-dap/src/demo_backend.rs Normal file
View file

@ -0,0 +1,68 @@
use std::collections::BTreeMap;
use clusterflux_core::{DebugRuntimeState, TaskInstanceId};
use crate::virtual_model::{AdapterState, VirtualThread};
pub(crate) const MAIN_THREAD: i64 = 1;
pub(crate) const LINUX_THREAD: i64 = 2;
pub(crate) const WINDOWS_THREAD: i64 = 3;
pub(crate) const PACKAGE_THREAD: i64 = 4;
pub(crate) fn is_explicit_demo_backend(value: &str) -> bool {
value == "simulated" || value == "demo"
}
pub(crate) fn launch_threads(entry: &str) -> BTreeMap<i64, VirtualThread> {
[
thread(MAIN_THREAD, "main", &format!("{entry} virtual process"), 12),
thread(LINUX_THREAD, "compile-linux", "compile linux", 42),
thread(WINDOWS_THREAD, "compile-windows", "compile windows", 52),
thread(PACKAGE_THREAD, "package-release", "package artifacts", 64),
]
.into_iter()
.map(|thread| (thread.id, thread))
.collect()
}
pub(crate) fn start_simulated_backend(state: &mut AdapterState) {
state.command_status = "running under simulated debug adapter state".to_owned();
}
fn thread(id: i64, task: &str, name: &str, line: i64) -> VirtualThread {
VirtualThread {
id,
frame_id: 1000 + id,
locals_ref: 5000 + id,
wasm_locals_ref: 9000 + id,
args_ref: 2000 + id,
runtime_ref: 3000 + id,
output_ref: 4000 + id,
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::from(task),
attempt_id: format!("demo-attempt-{id}"),
task_definition: clusterflux_core::TaskDefinitionId::from(task),
name: name.to_owned(),
line,
state: DebugRuntimeState::Running,
freeze_supported: true,
recent_output: vec![format!("{name}: waiting")],
stdout_bytes: 0,
stderr_bytes: 0,
stdout_tail: String::new(),
stderr_tail: String::new(),
stdout_truncated: false,
stderr_truncated: false,
runtime_stack_frames: Vec::new(),
wasm_local_values: Vec::new(),
runtime_task_args: Vec::new(),
runtime_handles: Vec::new(),
runtime_command_status: None,
runtime_node: Some("demo-node".to_owned()),
runtime_environment: Some("demo".to_owned()),
runtime_vfs_checkpoint: Some("demo-vfs".to_owned()),
restart_compatible: Some(true),
}
}

43
crates/clusterflux-dap/src/main.rs Normal file
View file

@ -0,0 +1,43 @@
use anyhow::Result;
#[cfg(test)]
use std::path::Path;
mod adapter;
mod breakpoints;
mod dap_protocol;
mod demo_backend;
mod runtime_client;
mod source;
mod variables;
mod view_state;
mod virtual_model;
#[cfg(test)]
use adapter::runtime_backend_from_launch_arg;
#[cfg(test)]
use breakpoints::{
freeze_all, resolve_breakpoints, resolve_breakpoints_for_source,
restart_requires_whole_process, stopped_thread_for_breakpoint,
};
#[cfg(test)]
use dap_protocol::{initialize_capabilities, read_message};
#[cfg(test)]
use runtime_client::{client_user_request, parse_task_restart_response};
#[cfg(test)]
use variables::variables_response;
#[cfg(test)]
use virtual_model::{AdapterState, RuntimeLaunchRecord};
#[cfg(test)]
use clusterflux_core::{BundleDebugProbe, DebugRuntimeState, TaskInstanceId};
#[cfg(test)]
use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
#[cfg(test)]
use virtual_model::{process_id, RuntimeBackend};
fn main() -> Result<()> {
adapter::run_adapter()
}
#[cfg(test)]
mod tests;

1172
crates/clusterflux-dap/src/runtime_client.rs Normal file
View file

File diff suppressed because it is too large Load diff

243
crates/clusterflux-dap/src/runtime_client/debug_protocol.rs Normal file
View file

@ -0,0 +1,243 @@
use std::time::{Duration, Instant};
use anyhow::{anyhow, Result};
use clusterflux_core::TaskInstanceId;
use serde_json::{json, Value};
use crate::virtual_model::AdapterState;
use super::{
client_user_request, coordinator_request, DebugEpochRecord, DebugEpochStatusRecord,
TaskRestartRecord,
};
pub(super) fn coordinator_debug_epoch_request(
state: &AdapterState,
payload: Value,
) -> Result<Value> {
let coordinator =
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
coordinator_request(&coordinator, payload)
}
pub(super) fn parse_debug_epoch_response(response: Value) -> Result<DebugEpochRecord> {
let epoch = response
.get("epoch")
.and_then(Value::as_u64)
.ok_or_else(|| anyhow!("coordinator debug epoch response did not include an epoch"))?;
let command = response
.get("command")
.and_then(Value::as_str)
.ok_or_else(|| anyhow!("coordinator debug epoch response did not include a command"))?
.to_owned();
let affected_tasks = response
.get("affected_tasks")
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0);
Ok(DebugEpochRecord {
epoch,
command,
affected_tasks,
})
}
pub(super) fn wait_for_debug_epoch_state(
state: &AdapterState,
epoch: u64,
frozen: bool,
) -> Result<DebugEpochStatusRecord> {
let deadline = Instant::now() + Duration::from_secs(60);
loop {
let response = match coordinator_debug_epoch_request(
state,
client_user_request(
state,
json!({
"type": "inspect_debug_epoch",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
"process": state.process.as_str(),
"epoch": epoch,
}),
),
) {
Ok(response) => response,
Err(error) if !frozen && debug_epoch_was_released(&error, state, epoch) => {
return Ok(DebugEpochStatusRecord {
epoch,
command: "resume".to_owned(),
expected_tasks: 0,
acknowledgements: Vec::new(),
fully_frozen: false,
partially_frozen: false,
fully_resumed: true,
failed: false,
failure_messages: Vec::new(),
});
}
Err(error) => return Err(error),
};
let status = parse_debug_epoch_status(response)?;
if frozen && (status.fully_frozen || status.partially_frozen) {
return Ok(status);
}
if status.failed {
return Err(anyhow!(
"debug epoch {epoch} participant failed: {}",
status.failure_messages.join("; ")
));
}
if !frozen && status.fully_resumed {
return Ok(status);
}
if Instant::now() >= deadline {
return Err(anyhow!(
"debug epoch {epoch} did not receive {}/{} signed participant acknowledgements for {} within 60 seconds",
status.acknowledgements.len(),
status.expected_tasks,
if frozen { "frozen state" } else { "resumed state" }
));
}
std::thread::sleep(Duration::from_millis(100));
}
}
fn debug_epoch_was_released(error: &anyhow::Error, state: &AdapterState, epoch: u64) -> bool {
format!("{error:#}").contains(&format!(
"debug epoch {epoch} is not active for {}",
state.process
))
}
fn parse_debug_epoch_status(response: Value) -> Result<DebugEpochStatusRecord> {
let epoch = response
.get("epoch")
.and_then(Value::as_u64)
.ok_or_else(|| anyhow!("coordinator debug epoch status omitted epoch"))?;
let command = response
.get("command")
.and_then(Value::as_str)
.ok_or_else(|| anyhow!("coordinator debug epoch status omitted command"))?
.to_owned();
let expected_tasks = response
.get("expected_tasks")
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(0);
let acknowledgements = response
.get("acknowledgements")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let failure_messages = response
.get("failure_messages")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::to_owned)
.collect();
Ok(DebugEpochStatusRecord {
epoch,
command,
expected_tasks,
acknowledgements,
fully_frozen: response
.get("fully_frozen")
.and_then(Value::as_bool)
.unwrap_or(false),
partially_frozen: response
.get("partially_frozen")
.and_then(Value::as_bool)
.unwrap_or(false),
fully_resumed: response
.get("fully_resumed")
.and_then(Value::as_bool)
.unwrap_or(false),
failed: response
.get("failed")
.and_then(Value::as_bool)
.unwrap_or(false),
failure_messages,
})
}
pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestartRecord> {
Ok(TaskRestartRecord {
accepted: response
.get("accepted")
.and_then(Value::as_bool)
.ok_or_else(|| anyhow!("coordinator task restart response did not include accepted"))?,
restarted_task_instance: response
.get("restarted_task_instance")
.and_then(Value::as_str)
.map(TaskInstanceId::new),
restarted_attempt_id: response
.get("restarted_attempt_id")
.and_then(Value::as_str)
.map(str::to_owned),
clean_boundary_available: response
.get("clean_boundary_available")
.and_then(Value::as_bool)
.ok_or_else(|| {
anyhow!("coordinator task restart response did not include clean_boundary_available")
})?,
requires_whole_process_restart: response
.get("requires_whole_process_restart")
.and_then(Value::as_bool)
.ok_or_else(|| {
anyhow!(
"coordinator task restart response did not include requires_whole_process_restart"
)
})?,
active_task: response
.get("active_task")
.and_then(Value::as_bool)
.ok_or_else(|| anyhow!("coordinator task restart response did not include active_task"))?,
completed_event_observed: response
.get("completed_event_observed")
.and_then(Value::as_bool)
.ok_or_else(|| {
anyhow!("coordinator task restart response did not include completed_event_observed")
})?,
message: response
.get("message")
.and_then(Value::as_str)
.ok_or_else(|| anyhow!("coordinator task restart response did not include message"))?
.to_owned(),
})
}
#[cfg(test)]
mod tests {
use anyhow::anyhow;
use super::debug_epoch_was_released;
use crate::virtual_model::AdapterState;
#[test]
fn accepts_epoch_release_after_an_accepted_resume() {
let state = AdapterState {
process: "vp-completed".into(),
..AdapterState::default()
};
assert!(debug_epoch_was_released(
&anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-completed"),
&state,
3,
));
assert!(!debug_epoch_was_released(
&anyhow!("coordinator protocol error: debug epoch 4 is not active for vp-completed"),
&state,
3,
));
assert!(!debug_epoch_was_released(
&anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-other"),
&state,
3,
));
}
}

Some files were not shown because too many files have changed in this diff Show more