michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-ff7f847f143d

Browse source 
Source commit: ff7f847f143dc67864bed68a86cf259114384bd5

Public tree identity: sha256:bb7dcd2ac78bdbad2f4eba0f49be649d446d67f96f4eb2796941c026412fc032
This commit is contained in:
Clusterflux release 2026-07-17 09:33:47 +02:00
commit 70319cde15
210 changed files with 78958 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

837
crates/clusterflux-node/src/assignment_runner.rs Normal file
View file

@ -0,0 +1,837 @@
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::Read;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
#[cfg(unix)]
use std::os::unix::process::CommandExt;
use crate::coordinator_session::CoordinatorSession;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
Capability, CommandInvocation, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskDispatch, TaskInstanceId, TaskJoinResult, TaskJoinState, TaskSpec, TenantId, VfsManifest,
WasmExportAbi, WasmHostDebugProbeRequest, WasmHostDebugProbeResult,
WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult, WasmHostTaskHandle,
WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation,
WasmTaskOutcome,
};
use clusterflux_node::{
BackendError, CommandOutput, LinuxRootlessPodmanBackend, LocalCheckoutTaskRequest,
LocalSourceCheckout, PodmanCommand, ProcessOutput, ProcessRunner, WasmDebugControl,
WasmTaskHost, WasmtimeTaskRuntime,
};
const MAX_CHILD_TASK_HANDLES: usize = 256;
use crate::daemon::{runtime_task_from_assignment, Args, RuntimeTask};
use crate::node_identity::signed_node_request_json;
use crate::source_snapshot::snapshot_project;
use crate::task_artifacts::{
retained_result_artifact, task_output_root, NodeArtifactStore, TaskArtifactStore,
};
mod process_runner;
use process_runner::CoordinatorControlledProcessRunner;
mod control_watcher;
use control_watcher::{spawn_task_control_watcher, spawn_worker_shutdown_watcher};
mod validation;
use validation::{
authorize_command_network, bundle_environments, capability_from_descriptor,
is_secret_environment_name, redact_configured_values, require_command_environment,
resolve_task_export, task_descriptors, verify_environment_digest, verify_source_snapshot,
};
pub(crate) fn run_verified_wasmtime_assignment(
args: &Args,
task: &RuntimeTask,
node_private_key: &str,
) -> Result<(CommandOutput, VfsManifest, Option<TaskBoundaryValue>), Box<dyn std::error::Error>> {
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!(
"clusterflux debug control: starting Wasm assignment for task {}",
task.task
);
}
let expected_bundle_digest = task.bundle_digest.as_ref().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"wasmtime task assignment includes module bytes but is missing bundle_digest",
)
})?;
let module_base64 = task
.wasm_module_base64
.as_ref()
.expect("caller checked wasm_module_base64");
let module = BASE64_STANDARD.decode(module_base64)?;
let actual_bundle_digest = Digest::sha256(&module);
if &actual_bundle_digest != expected_bundle_digest {
return Err(format!(
"bundle digest mismatch: expected {expected_bundle_digest}, received {actual_bundle_digest}"
)
.into());
}
let task_spec = task.task_spec.as_ref().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"Wasm task assignment is missing its versioned TaskSpec",
)
})?;
let (declared_export, abi) = match &task_spec.dispatch {
TaskDispatch::CoordinatorNodeWasm { export, abi } => (export.as_deref(), abi),
};
let resolved_export;
let export = match declared_export {
Some(export) => export,
None if abi == &WasmExportAbi::TaskV1 => {
resolved_export = resolve_task_export(&module, task_spec.task_definition.as_str())?;
&resolved_export
}
None => {
return Err("Wasm entrypoint assignment omitted its descriptor export".into());
}
};
let (stdout, boundary_result) = match abi {
WasmExportAbi::EntrypointV1 | WasmExportAbi::TaskV1 => {
let invocation = WasmTaskInvocation::new(
task_spec.task_definition.clone(),
task_spec.task_instance.clone(),
task_spec.args.clone(),
);
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&module,
expected_bundle_digest,
export,
&invocation,
Box::new(CoordinatorWasmTaskHost::new(
args,
task,
node_private_key,
&module,
)?),
)?;
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!(
"clusterflux debug control: Wasm assignment returned for task {}",
task.task
);
}
match result.outcome {
WasmTaskOutcome::Completed => {
let boundary = result.result.ok_or("completed Wasm task omitted result")?;
(
format!("{}\n", serde_json::to_string(&boundary)?),
Some(boundary),
)
}
WasmTaskOutcome::Failed => {
return Err(result
.error
.unwrap_or_else(|| "Wasm task failed without an error".to_owned())
.into())
}
}
}
};
let task_id = TaskInstanceId::new(task.task.clone());
let artifacts = TaskArtifactStore::new(task_id.clone(), NodeId::new(args.node.clone()));
let manifest = artifacts.flush();
Ok((
CommandOutput {
virtual_thread: task_id,
status_code: Some(0),
stdout,
stderr: String::new(),
stdout_truncated: false,
stderr_truncated: false,
log_backpressured: false,
staged_artifact: None,
},
manifest,
boundary_result,
))
}
struct CoordinatorWasmTaskHost {
args: Args,
process: String,
parent_task: String,
epoch: u64,
bundle_digest: clusterflux_core::Digest,
wasm_module_base64: String,
node_private_key: String,
allow_command: bool,
allow_network: bool,
allow_source_snapshot: bool,
environment_id: Option<String>,
environment_digest: Option<Digest>,
source_snapshot: Option<Digest>,
output_root: PathBuf,
task_descriptors: HashMap<String, serde_json::Value>,
environments: BTreeMap<String, clusterflux_core::EnvironmentResource>,
next_handle_id: u64,
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
command_status: Arc<Mutex<Option<String>>>,
cancellation_requested: Arc<AtomicBool>,
abort_requested: Arc<AtomicBool>,
debug_control: Arc<WasmDebugControl>,
stop_control_watcher: Arc<AtomicBool>,
}
impl CoordinatorWasmTaskHost {
fn new(
args: &Args,
parent: &RuntimeTask,
node_private_key: &str,
module: &[u8],
) -> Result<Self, Box<dyn std::error::Error>> {
let task_spec = parent
.task_spec
.as_ref()
.ok_or("Wasm task host requires a parent TaskSpec")?;
let cancellation_requested = Arc::new(AtomicBool::new(false));
let abort_requested = Arc::new(AtomicBool::new(false));
let debug_control = Arc::new(WasmDebugControl::default());
let stop_control_watcher = Arc::new(AtomicBool::new(false));
let handles = Arc::new(Mutex::new(HashMap::new()));
let command_status = Arc::new(Mutex::new(None));
let task_instance = TaskInstanceId::from(parent.task.as_str());
let output_root =
task_output_root(args.project_root.as_deref(), &args.node, &task_instance)?;
let task_args = task_spec
.args
.iter()
.enumerate()
.map(|(index, value)| (format!("arg_{index}"), format!("{value:?}")))
.collect();
spawn_task_control_watcher(
args.clone(),
parent.process.clone(),
parent.task.clone(),
task_spec.task_definition.as_str().to_owned(),
node_private_key.to_owned(),
Arc::clone(&cancellation_requested),
Arc::clone(&abort_requested),
Arc::clone(&debug_control),
task_args,
Arc::clone(&handles),
Arc::clone(&command_status),
Arc::clone(&stop_control_watcher),
);
spawn_worker_shutdown_watcher(
Arc::clone(&abort_requested),
Arc::clone(&debug_control),
Arc::clone(&stop_control_watcher),
);
Ok(Self {
args: args.clone(),
process: parent.process.clone(),
parent_task: parent.task.clone(),
epoch: parent.epoch.unwrap_or(task_spec.vfs_epoch),
bundle_digest: parent
.bundle_digest
.clone()
.ok_or("Wasm task host requires a bundle digest")?,
wasm_module_base64: BASE64_STANDARD.encode(module),
node_private_key: node_private_key.to_owned(),
allow_command: task_spec
.required_capabilities
.contains(&Capability::Command)
&& clusterflux_core::NodeCapabilities::detect_current()
.capabilities
.contains(&Capability::Command),
allow_network: task_spec
.required_capabilities
.contains(&Capability::Network)
&& clusterflux_core::NodeCapabilities::detect_current()
.capabilities
.contains(&Capability::Network),
allow_source_snapshot: task_spec
.required_capabilities
.contains(&Capability::SourceFilesystem)
&& clusterflux_core::NodeCapabilities::detect_current()
.capabilities
.contains(&Capability::SourceFilesystem),
environment_id: task_spec.environment_id.clone(),
environment_digest: task_spec.environment_digest.clone(),
source_snapshot: task_spec.source_snapshot.clone(),
output_root,
// Wasmtime accepts both binary Wasm and WAT in development tests. Descriptor
// discovery is required only if the guest actually invokes task_start_v1; module
// compilation/digest verification remains authoritative for malformed input.
task_descriptors: task_descriptors(module).unwrap_or_default(),
environments: bundle_environments(module)?,
next_handle_id: 1,
handles,
command_status,
cancellation_requested,
abort_requested,
debug_control,
stop_control_watcher,
})
}
fn session(&self) -> Result<CoordinatorSession, String> {
CoordinatorSession::connect(&self.args.coordinator).map_err(|error| error.to_string())
}
fn signed_request(
&self,
request_kind: &str,
payload: serde_json::Value,
) -> Result<serde_json::Value, String> {
signed_node_request_json(&self.args, &self.node_private_key, request_kind, payload)
.map_err(|error| error.to_string())
}
fn execute_local_assignment(&self, assignment: &serde_json::Value) -> Result<(), String> {
let runtime_task =
runtime_task_from_assignment(assignment).map_err(|error| error.to_string())?;
let execution =
run_verified_wasmtime_assignment(&self.args, &runtime_task, &self.node_private_key);
let (terminal_state, status_code, stdout, stderr, result, retained) = match execution {
Ok((output, _manifest, result)) => {
let retained = retained_result_artifact(
self.args.project_root.as_deref(),
&self.args.node,
result.as_ref(),
);
match retained {
Ok(retained) => (
"completed",
output.status_code,
output.stdout,
output.stderr,
result,
retained,
),
Err(error) => ("failed", Some(1), String::new(), error, None, None),
}
}
Err(error) => (
"failed",
Some(1),
String::new(),
error.to_string(),
None,
None,
),
};
let artifact_path = retained
.as_ref()
.map(|artifact| format!("/vfs/artifacts/{}", artifact.id));
let artifact_digest = retained.as_ref().map(|artifact| artifact.digest.clone());
let artifact_size_bytes = retained.as_ref().map(|artifact| artifact.size_bytes);
let mut session = self.session()?;
session
.request(self.signed_request(
"task_completed",
serde_json::json!({
"type": "task_completed",
"tenant": self.args.tenant,
"project": self.args.project,
"process": runtime_task.process,
"node": self.args.node,
"task": runtime_task.task,
"terminal_state": terminal_state,
"status_code": status_code,
"stdout_bytes": stdout.len(),
"stderr_bytes": stderr.len(),
"stdout_tail": stdout,
"stderr_tail": stderr,
"stdout_truncated": false,
"stderr_truncated": false,
"artifact_path": artifact_path,
"artifact_digest": artifact_digest,
"artifact_size_bytes": artifact_size_bytes,
"result": result,
}),
)?)
.map_err(|error| error.to_string())?;
Ok(())
}
fn poll_and_execute_assignment(&self) -> Result<bool, String> {
let mut session = self.session()?;
let response = session
.request(self.signed_request(
"poll_task_assignment",
serde_json::json!({
"type": "poll_task_assignment",
"tenant": self.args.tenant,
"project": self.args.project,
"node": self.args.node,
}),
)?)
.map_err(|error| error.to_string())?;
let Some(assignment) = response.get("assignment").filter(|value| !value.is_null()) else {
return Ok(false);
};
self.execute_local_assignment(assignment)?;
Ok(true)
}
}
impl Drop for CoordinatorWasmTaskHost {
fn drop(&mut self) {
self.stop_control_watcher.store(true, Ordering::Release);
if std::fs::symlink_metadata(&self.output_root)
.map(|metadata| metadata.is_dir() && !metadata.file_type().is_symlink())
.unwrap_or(false)
{
let _ = std::fs::remove_dir_all(&self.output_root);
}
}
}
impl WasmTaskHost for CoordinatorWasmTaskHost {
fn abort_signal(&self) -> Option<Arc<AtomicBool>> {
Some(Arc::clone(&self.abort_requested))
}
fn debug_control(&self) -> Option<Arc<WasmDebugControl>> {
Some(Arc::clone(&self.debug_control))
}
fn start_task(
&mut self,
request: WasmHostTaskStartRequest,
) -> Result<WasmHostTaskHandle, String> {
request.validate()?;
let descriptor = self
.task_descriptors
.get(request.task_definition.as_str())
.ok_or_else(|| {
format!(
"bundle has no task descriptor named `{}`",
request.task_definition
)
})?;
let export = descriptor
.get("export")
.and_then(serde_json::Value::as_str)
.filter(|export| !export.trim().is_empty())
.ok_or_else(|| {
format!(
"task `{}` descriptor omitted its Wasm export",
request.task_definition
)
})?;
let mut required_capabilities = descriptor
.get("required_capabilities")
.and_then(serde_json::Value::as_array)
.into_iter()
.flatten()
.map(|value| {
capability_from_descriptor(
value
.as_str()
.ok_or("task capability descriptor is not a string")?,
)
})
.collect::<Result<BTreeSet<_>, _>>()?;
let resolved_environment = request
.environment_id
.as_deref()
.map(|name| {
self.environments.get(name).cloned().ok_or_else(|| {
format!("bundle environment manifest has no environment `{name}`")
})
})
.transpose()?;
let environment = resolved_environment
.as_ref()
.map(|environment| environment.requirements.clone());
let environment_digest = resolved_environment
.as_ref()
.map(|environment| environment.digest.clone());
if let Some(environment) = &environment {
required_capabilities.extend(environment.capabilities.iter().cloned());
}
let handle_id = self.next_handle_id;
let task_instance = clusterflux_core::TaskInstanceId::new(format!(
"{}:child:{}",
self.parent_task, handle_id
));
let required_artifacts = request
.args
.iter()
.flat_map(TaskBoundaryValue::required_artifacts)
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let source_snapshots = request
.args
.iter()
.flat_map(TaskBoundaryValue::source_snapshots)
.collect::<BTreeSet<_>>();
if source_snapshots.len() > 1 {
return Err(
"one child task invocation cannot require multiple distinct source snapshots"
.to_owned(),
);
}
if self
.handles
.lock()
.map_err(|_| "Wasm task handle registry lock was poisoned".to_owned())?
.len()
>= MAX_CHILD_TASK_HANDLES
{
return Err(format!(
"Wasm child task-handle limit of {MAX_CHILD_TASK_HANDLES} reached"
));
}
let source_snapshot = source_snapshots.into_iter().next();
let spec = TaskSpec {
tenant: TenantId::from(self.args.tenant.as_str()),
project: ProjectId::from(self.args.project.as_str()),
process: ProcessId::from(self.process.as_str()),
task_definition: request.task_definition.clone(),
task_instance: task_instance.clone(),
dispatch: TaskDispatch::CoordinatorNodeWasm {
export: Some(export.to_owned()),
abi: WasmExportAbi::TaskV1,
},
environment_id: request.environment_id,
environment,
environment_digest,
required_capabilities,
dependency_cache: None,
source_snapshot,
required_artifacts,
args: request.args,
vfs_epoch: self.epoch,
failure_policy: request.failure_policy,
bundle_digest: Some(self.bundle_digest.clone()),
};
let mut session = self.session()?;
let response = session
.request(self.signed_request(
"launch_child_task",
serde_json::json!({
"type": "launch_child_task",
"tenant": self.args.tenant,
"project": self.args.project,
"process": self.process,
"node": self.args.node,
"parent_task": self.parent_task,
"task_spec": spec,
"wait_for_node": true,
"artifact_path": format!("/vfs/artifacts/{}-result.json", spec.task_instance),
"wasm_module_base64": self.wasm_module_base64,
}),
)?)
.map_err(|error| error.to_string())?;
match response.get("type").and_then(serde_json::Value::as_str) {
Some("task_launched" | "task_queued") => {}
other => return Err(format!("unexpected child launch response {other:?}")),
}
self.next_handle_id = self.next_handle_id.saturating_add(1);
self.handles
.lock()
.map_err(|_| "Wasm task handle registry lock was poisoned".to_owned())?
.insert(handle_id, spec.clone());
Ok(WasmHostTaskHandle {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
handle_id,
task_spec: spec,
})
}
fn join_task(
&mut self,
request: WasmHostTaskJoinRequest,
) -> Result<WasmHostTaskJoinResult, String> {
if request.abi_version != clusterflux_core::WASM_TASK_ABI_VERSION {
return Err(format!(
"unsupported Wasm task ABI version {}",
request.abi_version
));
}
let spec = self
.handles
.lock()
.map_err(|_| "Wasm task handle registry lock was poisoned".to_owned())?
.get(&request.handle_id)
.cloned()
.ok_or_else(|| format!("unknown Wasm task handle {}", request.handle_id))?;
let started = Instant::now();
loop {
let mut session = self.session()?;
let response = session
.request(self.signed_request(
"join_child_task",
serde_json::json!({
"type": "join_child_task",
"tenant": self.args.tenant,
"project": self.args.project,
"process": self.process,
"node": self.args.node,
"parent_task": self.parent_task,
"task": spec.task_instance,
}),
)?)
.map_err(|error| error.to_string())?;
let join: TaskJoinResult = serde_json::from_value(
response
.get("join")
.cloned()
.ok_or("child task join response omitted join state")?,
)
.map_err(|error| error.to_string())?;
match join.state {
TaskJoinState::Completed => {
let result = join
.result
.ok_or("completed child task omitted its boundary result")?;
self.handles
.lock()
.map_err(|_| "Wasm task handle registry lock was poisoned".to_owned())?
.remove(&request.handle_id);
return Ok(WasmHostTaskJoinResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
task_instance: spec.task_instance,
result,
});
}
TaskJoinState::Failed | TaskJoinState::Cancelled => {
self.handles
.lock()
.map_err(|_| "Wasm task handle registry lock was poisoned".to_owned())?
.remove(&request.handle_id);
return Err(join.message);
}
TaskJoinState::Pending => {
if self.cancellation_requested.load(Ordering::Acquire)
|| self.abort_requested.load(Ordering::Acquire)
|| crate::daemon::worker_shutdown_requested()
{
return Err(clusterflux_core::limits::TaskJoinError::Cancelled {
task: spec.task_instance.clone(),
}
.to_string());
}
let join_timeout = clusterflux_core::limits::task_join_timeout();
if started.elapsed() > join_timeout {
return Err(clusterflux_core::limits::TaskJoinError::timeout(
spec.task_instance.clone(),
join_timeout,
)
.to_string());
}
if !self.poll_and_execute_assignment()? {
thread::sleep(Duration::from_millis(10));
}
}
}
}
}
fn run_command(
&mut self,
request: clusterflux_core::WasmHostCommandRequest,
) -> Result<clusterflux_core::WasmHostCommandResult, String> {
request.validate()?;
if !self.allow_command {
return Err(
"Wasm task did not declare Command capability or the selected node did not grant it"
.to_owned(),
);
}
authorize_command_network(&request.network, self.allow_network)?;
let environment_id = require_command_environment(self.environment_id.as_deref())?;
let project_root = self.args.project_root.as_ref().ok_or_else(|| {
format!(
"task selected environment `{environment_id}`, but this node has no project checkout or prepared source snapshot; attach it with --project-root"
)
})?;
let expected_snapshot = self.source_snapshot.as_ref().ok_or_else(|| {
"native command task is missing an explicit SourceSnapshot handle".to_owned()
})?;
let actual_snapshot = verify_source_snapshot(project_root, expected_snapshot)?;
let expected_environment_digest = self.environment_digest.as_ref().ok_or_else(|| {
format!(
"task selected environment `{environment_id}` without a bundle-authoritative environment digest"
)
})?;
let environment =
verify_environment_digest(project_root, environment_id, expected_environment_digest)?;
let checkout = project_root
.canonicalize()
.map_err(|error| format!("resolve node project checkout: {error}"))?;
let task_id = TaskInstanceId::from(self.parent_task.as_str());
let mut artifacts =
TaskArtifactStore::new(task_id.clone(), NodeId::from(self.args.node.as_str()));
let configured_secrets = request
.environment_variables
.iter()
.filter(|(name, value)| is_secret_environment_name(name) && !value.is_empty())
.map(|(_, value)| value.clone())
.collect::<Vec<_>>();
let mut runner = CoordinatorControlledProcessRunner::new(
self,
Duration::from_millis(request.timeout_ms),
);
let output = LinuxRootlessPodmanBackend
.execute_local_checkout_task(
LocalCheckoutTaskRequest {
process: ProcessId::from(self.process.as_str()),
virtual_thread: task_id,
invocation: &CommandInvocation {
program: request.program,
args: request.args,
working_directory: request.working_directory,
environment_variables: request.environment_variables,
timeout_ms: request.timeout_ms,
network: request.network,
env: Some(environment),
},
checkout: LocalSourceCheckout {
snapshot: actual_snapshot.digest,
host_path: checkout,
},
output_root: self.output_root.clone(),
stage_stdout_as: None,
},
&mut runner,
artifacts.overlay_mut(),
)
.map_err(|error| {
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!("clusterflux command host failed: {error}");
}
error.to_string()
})?;
let stdout = redact_configured_values(output.stdout, &configured_secrets);
let stderr = redact_configured_values(output.stderr, &configured_secrets);
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!(
"clusterflux command host completed: status={:?} stdout={:?} stderr={:?}",
output.status_code, stdout, stderr
);
}
Ok(clusterflux_core::WasmHostCommandResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
status_code: output.status_code,
stdout,
stderr,
stdout_truncated: output.stdout_truncated,
stderr_truncated: output.stderr_truncated,
})
}
fn poll_task_control(
&mut self,
request: clusterflux_core::WasmHostTaskControlRequest,
) -> Result<clusterflux_core::WasmHostTaskControlResult, String> {
request.validate()?;
Ok(clusterflux_core::WasmHostTaskControlResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
cancellation_requested: self.cancellation_requested.load(Ordering::Acquire),
})
}
fn debug_probe(
&mut self,
request: WasmHostDebugProbeRequest,
) -> Result<WasmHostDebugProbeResult, String> {
request.validate()?;
let mut session = self.session()?;
let response = session
.request(self.signed_request(
"report_debug_probe_hit",
serde_json::json!({
"type": "report_debug_probe_hit",
"tenant": self.args.tenant,
"project": self.args.project,
"process": self.process,
"node": self.args.node,
"task": self.parent_task,
"probe_symbol": request.symbol,
}),
)?)
.map_err(|error| error.to_string())?;
if response.get("type").and_then(serde_json::Value::as_str) != Some("debug_probe_hit") {
return Err(format!(
"coordinator returned invalid debug probe response: {response}"
));
}
let result = WasmHostDebugProbeResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
breakpoint_matched: response
.get("breakpoint_matched")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false),
debug_epoch: response
.get("debug_epoch")
.and_then(serde_json::Value::as_u64),
};
if result.breakpoint_matched {
if let Some(epoch) = result.debug_epoch {
self.debug_control.request_freeze(epoch);
}
}
Ok(result)
}
fn vfs_operation(&mut self, request: WasmHostVfsRequest) -> Result<WasmHostVfsResult, String> {
request.validate()?;
let store =
NodeArtifactStore::for_runtime(self.args.project_root.as_deref(), &self.args.node)?;
let (retained, relative_path) = match request.operation {
WasmHostVfsOperation::FlushOutput { relative_path } => {
let retained = store.retain_output_file(&self.output_root, &relative_path)?;
(retained, relative_path)
}
WasmHostVfsOperation::MaterializeArtifact {
artifact,
relative_path,
} => {
store.materialize_into_output(&artifact, &self.output_root, &relative_path)?;
let retained = store
.metadata(&artifact.id)?
.ok_or_else(|| format!("artifact `{}` became unavailable", artifact.id))?;
(retained, relative_path)
}
};
Ok(WasmHostVfsResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
artifact: clusterflux_core::ArtifactHandle {
id: retained.id,
digest: retained.digest,
size_bytes: retained.size_bytes,
},
relative_path,
})
}
fn snapshot_source(
&mut self,
request: WasmHostSourceSnapshotRequest,
) -> Result<WasmHostSourceSnapshotResult, String> {
request.validate()?;
if !self.allow_source_snapshot {
return Err(
"Wasm task did not declare SourceFilesystem capability or the selected node did not grant it"
.to_owned(),
);
}
let project_root = self.args.project_root.as_deref().ok_or_else(|| {
"source snapshot task requires a node project checkout; attach it with --project-root"
.to_owned()
})?;
let snapshot = snapshot_project(project_root)?;
Ok(WasmHostSourceSnapshotResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
snapshot: snapshot.digest,
})
}
}
#[cfg(test)]
mod tests;

235
crates/clusterflux-node/src/assignment_runner/control_watcher.rs Normal file
View file

@ -0,0 +1,235 @@
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Duration;
use clusterflux_core::TaskSpec;
use clusterflux_node::WasmDebugControl;
use crate::coordinator_session::CoordinatorSession;
use crate::daemon::{worker_shutdown_requested, Args};
use crate::node_identity::signed_node_request_json;
const DEFAULT_DEBUG_FREEZE_TIMEOUT_MILLIS: u64 = 5_000;
fn debug_freeze_timeout() -> Duration {
std::env::var("CLUSTERFLUX_DEBUG_FREEZE_TIMEOUT_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|millis| *millis > 0)
.map(Duration::from_millis)
.unwrap_or_else(|| Duration::from_millis(DEFAULT_DEBUG_FREEZE_TIMEOUT_MILLIS))
}
fn debug_handle_snapshot(handles: &Arc<Mutex<HashMap<u64, TaskSpec>>>) -> Vec<(String, String)> {
let Ok(handles) = handles.lock() else {
return vec![(
"handle-registry-diagnostic".to_owned(),
"runtime task handle registry was unavailable".to_owned(),
)];
};
let mut snapshot = handles
.iter()
.map(|(handle_id, spec)| {
(
format!("task_handle_{handle_id}"),
format!(
"definition={} instance={} state=active",
spec.task_definition, spec.task_instance
),
)
})
.collect::<Vec<_>>();
snapshot.sort_by(|left, right| left.0.cmp(&right.0));
snapshot
}
#[allow(clippy::too_many_arguments)]
pub(super) fn spawn_task_control_watcher(
args: Args,
process: String,
task: String,
task_definition: String,
node_private_key: String,
cancellation_requested: Arc<AtomicBool>,
abort_requested: Arc<AtomicBool>,
debug_control: Arc<WasmDebugControl>,
task_args: Vec<(String, String)>,
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
command_status: Arc<Mutex<Option<String>>>,
stop: Arc<AtomicBool>,
) {
thread::spawn(move || {
let Ok(mut session) = CoordinatorSession::connect(&args.coordinator) else {
return;
};
while !stop.load(Ordering::Acquire) {
let request = signed_node_request_json(
&args,
&node_private_key,
"poll_task_control",
serde_json::json!({
"type": "poll_task_control",
"tenant": &args.tenant,
"project": &args.project,
"process": &process,
"node": &args.node,
"task": &task,
}),
);
let Ok(request) = request else {
return;
};
let Ok(response) = session.request(request) else {
return;
};
cancellation_requested.store(
response
.get("cancel_requested")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false),
Ordering::Release,
);
if response
.get("abort_requested")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
{
abort_requested.store(true, Ordering::Release);
return;
}
let debug_request = signed_node_request_json(
&args,
&node_private_key,
"poll_debug_command",
serde_json::json!({
"type": "poll_debug_command",
"tenant": &args.tenant,
"project": &args.project,
"process": &process,
"node": &args.node,
"task": &task,
}),
);
let Ok(debug_request) = debug_request else {
return;
};
let Ok(debug_response) = session.request(debug_request) else {
return;
};
if let (Some(epoch), Some(command)) = (
debug_response
.get("epoch")
.and_then(serde_json::Value::as_u64),
debug_response
.get("command")
.and_then(serde_json::Value::as_str),
) {
let freeze_timeout = debug_freeze_timeout();
let (state, message) = match command {
"freeze" => {
if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
eprintln!(
"clusterflux debug control: node received freeze for epoch {epoch} task {task} (debug={:?})",
Arc::as_ptr(&debug_control)
);
}
debug_control.request_freeze(epoch);
if debug_control.wait_until_frozen(epoch, freeze_timeout) {
("frozen", None)
} else {
debug_control.request_resume(epoch);
(
"failed",
Some(format!(
"node execution did not reach a freezeable Wasm safepoint or verified native/Podman boundary within {} ms",
freeze_timeout.as_millis()
)),
)
}
}
"resume" => {
debug_control.request_resume(epoch);
if debug_control.wait_until_running(epoch, freeze_timeout) {
("running", None)
} else {
(
"failed",
Some(format!(
"node execution did not leave its verified frozen state within {} ms",
freeze_timeout.as_millis()
)),
)
}
}
_ => (
"failed",
Some(format!("node received unknown debug command `{command}`")),
),
};
let report = signed_node_request_json(
&args,
&node_private_key,
"report_debug_state",
serde_json::json!({
"type": "report_debug_state",
"tenant": &args.tenant,
"project": &args.project,
"process": &process,
"node": &args.node,
"task": &task,
"epoch": epoch,
"state": state,
"stack_frames": if state == "frozen" {
let mut frames = debug_control.stack_frames();
if let Some(frame) = frames.first_mut() {
*frame = format!("{task_definition}::wasm / {frame}");
} else {
frames.push(format!("{task_definition}::wasm"));
}
frames
} else {
Vec::new()
},
"local_values": [],
"task_args": &task_args,
"handles": debug_handle_snapshot(&handles),
"command_status": command_status
.lock()
.ok()
.and_then(|status| status.clone()),
"recent_output": [],
"message": message,
}),
);
let Ok(report) = report else {
return;
};
if session.request(report).is_err() {
return;
}
}
thread::sleep(Duration::from_millis(50));
}
});
}
pub(super) fn spawn_worker_shutdown_watcher(
abort_requested: Arc<AtomicBool>,
debug_control: Arc<WasmDebugControl>,
stop: Arc<AtomicBool>,
) {
thread::spawn(move || {
while !stop.load(Ordering::Acquire) {
if worker_shutdown_requested() {
abort_requested.store(true, Ordering::Release);
if let Some(epoch) = debug_control.requested_epoch() {
debug_control.request_resume(epoch);
}
return;
}
thread::sleep(Duration::from_millis(10));
}
});
}

373
crates/clusterflux-node/src/assignment_runner/process_runner.rs Normal file
View file

@ -0,0 +1,373 @@
use super::*;
pub(super) struct CoordinatorControlledProcessRunner {
pub(super) args: Args,
pub(super) process: String,
pub(super) task: String,
pub(super) node_private_key: String,
pub(super) debug_control: Arc<WasmDebugControl>,
pub(super) command_status: Arc<Mutex<Option<String>>>,
pub(super) timeout: Duration,
}
impl CoordinatorControlledProcessRunner {
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
Self {
args: host.args.clone(),
process: host.process.clone(),
task: host.parent_task.clone(),
node_private_key: host.node_private_key.clone(),
debug_control: Arc::clone(&host.debug_control),
command_status: Arc::clone(&host.command_status),
timeout,
}
}
fn set_command_status(&self, status: impl Into<String>) {
if let Ok(mut current) = self.command_status.lock() {
*current = Some(status.into());
}
}
fn abort_requested(&self, session: &mut CoordinatorSession) -> Result<bool, BackendError> {
let request = signed_node_request_json(
&self.args,
&self.node_private_key,
"poll_task_control",
serde_json::json!({
"type": "poll_task_control",
"tenant": &self.args.tenant,
"project": &self.args.project,
"process": &self.process,
"node": &self.args.node,
"task": &self.task,
}),
)
.map_err(|error| BackendError::Command(error.to_string()))?;
let response = session
.request(request)
.map_err(|error| BackendError::Command(format!("poll task control: {error}")))?;
Ok(response
.get("abort_requested")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false))
}
fn terminate_process_group(child: &mut std::process::Child) {
#[cfg(unix)]
{
let process_group = -(child.id() as i32);
// The child is placed in a new process group before exec, so this reaches
// all native descendants. Podman containers are removed separately.
unsafe {
libc::kill(process_group, libc::SIGKILL);
}
}
let _ = child.kill();
let _ = child.wait();
}
fn podman_container_name(command: &PodmanCommand) -> Option<String> {
if command.program != "podman" || command.args.first().map(String::as_str) != Some("run") {
return None;
}
command
.args
.windows(2)
.find(|arguments| arguments[0] == "--name")
.map(|arguments| arguments[1].clone())
}
fn set_podman_paused(container: &str, paused: bool) -> Result<(), BackendError> {
let action = if paused { "pause" } else { "unpause" };
let output = std::process::Command::new("podman")
.args([action, container])
.output()
.map_err(|error| {
BackendError::Command(format!(
"failed to invoke `podman {action}` for container `{container}`: {error}"
))
})?;
if !output.status.success() {
return Err(BackendError::Command(format!(
"`podman {action}` failed for container `{container}`: {}",
String::from_utf8_lossy(&output.stderr).trim()
)));
}
let inspection = std::process::Command::new("podman")
.args(["inspect", "--format", "{{.State.Paused}}", container])
.output()
.map_err(|error| {
BackendError::Command(format!(
"failed to inspect Podman pause state for container `{container}`: {error}"
))
})?;
let observed = String::from_utf8_lossy(&inspection.stdout);
let expected = if paused { "true" } else { "false" };
if !inspection.status.success() || observed.trim() != expected {
return Err(BackendError::Command(format!(
"container `{container}` did not verify as {} after `podman {action}`: status={:?} stdout={} stderr={}",
if paused { "paused" } else { "running" },
inspection.status.code(),
observed.trim(),
String::from_utf8_lossy(&inspection.stderr).trim()
)));
}
Ok(())
}
fn terminate_execution(child: &mut std::process::Child, container: Option<&str>) {
if let Some(container) = container {
let _ = std::process::Command::new("podman")
.args(["rm", "--force", container])
.output();
}
Self::terminate_process_group(child);
}
fn freeze_execution(
child: &std::process::Child,
container: Option<&str>,
) -> Result<(), BackendError> {
if let Some(container) = container {
return Self::set_podman_paused(container, true);
}
#[cfg(unix)]
{
Self::freeze_process_group(child)
}
#[cfg(not(unix))]
{
let _ = child;
Err(BackendError::Command(
"native debug freeze requires Unix process groups".to_owned(),
))
}
}
fn resume_execution(
child: &std::process::Child,
container: Option<&str>,
) -> Result<(), BackendError> {
if let Some(container) = container {
return Self::set_podman_paused(container, false);
}
#[cfg(unix)]
{
Self::resume_process_group(child)
}
#[cfg(not(unix))]
{
let _ = child;
Err(BackendError::Command(
"native debug resume requires Unix process groups".to_owned(),
))
}
}
#[cfg(unix)]
fn freeze_process_group(child: &std::process::Child) -> Result<(), BackendError> {
let process_group = -(child.id() as i32);
let result = unsafe { libc::kill(process_group, libc::SIGSTOP) };
if result == 0 {
Ok(())
} else {
Err(BackendError::Command(format!(
"failed to freeze native process group {}: {}",
child.id(),
std::io::Error::last_os_error()
)))
}
}
#[cfg(unix)]
fn resume_process_group(child: &std::process::Child) -> Result<(), BackendError> {
let process_group = -(child.id() as i32);
let result = unsafe { libc::kill(process_group, libc::SIGCONT) };
if result == 0 {
Ok(())
} else {
Err(BackendError::Command(format!(
"failed to resume native process group {}: {}",
child.id(),
std::io::Error::last_os_error()
)))
}
}
fn drain_bounded(
mut reader: impl Read + Send + 'static,
maximum: usize,
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
thread::spawn(move || {
let mut captured = Vec::new();
let mut buffer = [0_u8; 16 * 1024];
loop {
let count = reader
.read(&mut buffer)
.map_err(|error| error.to_string())?;
if count == 0 {
break;
}
let remaining = maximum.saturating_sub(captured.len());
captured.extend_from_slice(&buffer[..count.min(remaining)]);
}
Ok(captured)
})
}
}
impl ProcessRunner for CoordinatorControlledProcessRunner {
fn run(&mut self, command: &PodmanCommand) -> Result<ProcessOutput, BackendError> {
let podman_container = Self::podman_container_name(command);
self.set_command_status(format!(
"starting native command: {} {}",
command.program,
command.args.join(" ")
));
let mut process = std::process::Command::new(&command.program);
process
.args(&command.args)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
process.process_group(0);
let mut child = process
.spawn()
.map_err(|error| BackendError::Command(error.to_string()))?;
self.set_command_status(format!(
"running native command pid {}: {} {}",
child.id(),
command.program,
command.args.join(" ")
));
let stdout = Self::drain_bounded(
child
.stdout
.take()
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
);
let stderr = Self::drain_bounded(
child
.stderr
.take()
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
);
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
Ok(session) => session,
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
return Err(BackendError::Command(format!(
"establish execution control channel: {error}"
)));
}
};
let mut frozen_epoch = None;
let started = Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) => {}
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
return Err(BackendError::Command(error.to_string()));
}
}
if started.elapsed() >= self.timeout {
self.set_command_status(format!(
"native command pid {} exceeded wall-clock timeout of {} ms",
child.id(),
self.timeout.as_millis()
));
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
return Err(BackendError::Command(format!(
"native command exceeded wall-clock timeout of {} ms",
self.timeout.as_millis()
)));
}
match self.abort_requested(&mut session) {
Ok(true) => {
self.set_command_status(format!(
"aborting native command pid {} at coordinator request",
child.id()
));
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
return Err(BackendError::Cancelled(
"coordinator requested cancellation or abort".to_owned(),
));
}
Ok(false) => {}
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
return Err(error);
}
}
if let Some(epoch) = self.debug_control.requested_epoch() {
if self.debug_control.resume_requested(epoch) {
if frozen_epoch == Some(epoch) {
match Self::resume_execution(&child, podman_container.as_deref()) {
Ok(()) => {
self.set_command_status(format!(
"running command pid {} after debug epoch {epoch} resumed",
child.id()
));
self.debug_control.mark_running(epoch);
frozen_epoch = None;
}
Err(error) => self.set_command_status(format!(
"debug epoch {epoch} resume is pending: {error}"
)),
}
}
} else if frozen_epoch != Some(epoch)
&& self.debug_control.frozen_epoch() != Some(epoch)
{
match Self::freeze_execution(&child, podman_container.as_deref()) {
Ok(()) => {
self.set_command_status(format!(
"frozen command pid {} for debug epoch {epoch}",
child.id()
));
self.debug_control.mark_frozen(epoch);
frozen_epoch = Some(epoch);
}
Err(error) => self.set_command_status(format!(
"debug epoch {epoch} freeze is pending: {error}"
)),
}
}
}
thread::sleep(Duration::from_millis(50));
};
let stdout = stdout
.join()
.map_err(|_| BackendError::Command("stdout reader panicked".to_owned()))?
.map_err(BackendError::Command)?;
let stderr = stderr
.join()
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
.map_err(BackendError::Command)?;
self.set_command_status(format!(
"native command exited with status {:?}",
status.code()
));
Ok(ProcessOutput {
status_code: status.code(),
stdout,
stderr,
})
}
}

205
crates/clusterflux-node/src/assignment_runner/tests.rs Normal file
View file

@ -0,0 +1,205 @@
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
use std::time::{Duration, Instant};
use super::*;
fn start_running_control_server() -> (String, thread::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let coordinator = listener.local_addr().unwrap().to_string();
let server = thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream.try_clone().unwrap());
loop {
let mut request = String::new();
match reader.read_line(&mut request) {
Ok(0) | Err(_) => break,
Ok(_) => stream
.write_all(b"{\"type\":\"task_control\",\"process\":\"vp\",\"task\":\"task\",\"cancel_requested\":false,\"abort_requested\":false}\n")
.unwrap(),
}
}
});
(coordinator, server)
}
fn test_controlled_runner(
coordinator: String,
timeout: Duration,
) -> CoordinatorControlledProcessRunner {
CoordinatorControlledProcessRunner {
args: Args {
coordinator,
tenant: "tenant".to_owned(),
project: "project".to_owned(),
project_root: None,
node: "node".to_owned(),
enrollment_grant: None,
public_key: None,
control_poll_ms: 0,
assignment_poll_ms: 1,
emit_ready: false,
worker: true,
},
process: "vp".to_owned(),
task: "task".to_owned(),
node_private_key: clusterflux_core::derive_ed25519_private_key_from_seed(
"controlled-runner-test",
),
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
timeout,
}
}
#[cfg(unix)]
#[test]
fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let coordinator = listener.local_addr().unwrap().to_string();
let server = thread::spawn(move || {
let (mut stream, _) = listener.accept().unwrap();
let mut request = String::new();
BufReader::new(stream.try_clone().unwrap())
.read_line(&mut request)
.unwrap();
assert!(!request.trim().is_empty());
stream
.write_all(b"{\"type\":\"task_control\",\"process\":\"vp\",\"task\":\"task\",\"cancel_requested\":false,\"abort_requested\":true}\n")
.unwrap();
});
let mut runner = CoordinatorControlledProcessRunner {
args: Args {
coordinator,
tenant: "tenant".to_owned(),
project: "project".to_owned(),
project_root: None,
node: "node".to_owned(),
enrollment_grant: None,
public_key: None,
control_poll_ms: 0,
assignment_poll_ms: 1,
emit_ready: false,
worker: true,
},
process: "vp".to_owned(),
task: "task".to_owned(),
node_private_key: clusterflux_core::derive_ed25519_private_key_from_seed(
"controlled-runner-test",
),
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
timeout: Duration::from_secs(30),
};
let started = Instant::now();
let error = runner
.run(&PodmanCommand {
program: "sh".to_owned(),
args: vec!["-c".to_owned(), "sleep 30 & wait".to_owned()],
})
.unwrap_err();
server.join().unwrap();
assert!(matches!(error, BackendError::Cancelled(_)));
assert!(started.elapsed() < Duration::from_secs(5));
}
#[cfg(unix)]
#[test]
fn command_timeout_is_bounded_and_a_later_command_still_runs() {
let (coordinator, server) = start_running_control_server();
let mut runner = test_controlled_runner(coordinator, Duration::from_millis(120));
let started = Instant::now();
let error = runner
.run(&PodmanCommand {
program: "sh".to_owned(),
args: vec!["-c".to_owned(), "sleep 30 & wait".to_owned()],
})
.unwrap_err();
assert!(error.to_string().contains("timeout"));
assert!(started.elapsed() < Duration::from_secs(5));
drop(runner);
server.join().unwrap();
let (coordinator, server) = start_running_control_server();
let mut runner = test_controlled_runner(coordinator, Duration::from_secs(5));
let output = runner
.run(&PodmanCommand {
program: "sh".to_owned(),
args: vec!["-c".to_owned(), "printf healthy".to_owned()],
})
.unwrap();
assert_eq!(output.status_code, Some(0));
assert_eq!(String::from_utf8(output.stdout).unwrap(), "healthy");
drop(runner);
server.join().unwrap();
}
#[test]
fn command_source_verification_rejects_a_changed_checkout() {
let checkout = tempfile::tempdir().unwrap();
std::fs::write(checkout.path().join("source.c"), "int value = 1;\n").unwrap();
let expected = snapshot_project(checkout.path()).unwrap().digest;
verify_source_snapshot(checkout.path(), &expected).unwrap();
std::fs::write(checkout.path().join("source.c"), "int value = 2;\n").unwrap();
let error = verify_source_snapshot(checkout.path(), &expected).unwrap_err();
assert!(error.contains("source snapshot mismatch"));
assert!(error.contains(expected.as_str()));
}
#[test]
fn command_environment_verification_rejects_a_changed_recipe() {
let checkout = tempfile::tempdir().unwrap();
let environment = checkout.path().join("envs/linux");
std::fs::create_dir_all(&environment).unwrap();
std::fs::write(
environment.join("Containerfile"),
"FROM docker.io/library/alpine:3.20\n",
)
.unwrap();
let expected = clusterflux_core::discover_environments(checkout.path())
.unwrap()
.into_iter()
.find(|environment| environment.name == "linux")
.unwrap()
.digest;
verify_environment_digest(checkout.path(), "linux", &expected).unwrap();
std::fs::write(
environment.join("Containerfile"),
"FROM docker.io/library/alpine:3.21\n",
)
.unwrap();
let error = verify_environment_digest(checkout.path(), "linux", &expected).unwrap_err();
assert!(error.contains("does not match the bundle"));
assert!(error.contains(expected.as_str()));
}
#[test]
fn configured_secret_values_are_redacted_before_guest_or_coordinator_logging() {
assert!(is_secret_environment_name("API_TOKEN"));
assert!(is_secret_environment_name("database_password"));
assert!(!is_secret_environment_name("SOURCE_DATE_EPOCH"));
assert_eq!(
redact_configured_values(
"token=correct-horse and again correct-horse".to_owned(),
&["correct-horse".to_owned()],
),
"token=[REDACTED] and again [REDACTED]"
);
}
#[test]
fn native_command_requires_environment_and_network_grant() {
let error = require_command_environment(None).unwrap_err();
assert!(error.contains("explicit command-capable environment"));
assert_eq!(require_command_environment(Some("linux")).unwrap(), "linux");
authorize_command_network(&clusterflux_core::CommandNetworkPolicy::Disabled, false).unwrap();
let error = authorize_command_network(&clusterflux_core::CommandNetworkPolicy::Enabled, false)
.unwrap_err();
assert!(error.contains("Network capability"));
}

190
crates/clusterflux-node/src/assignment_runner/validation.rs Normal file
View file

@ -0,0 +1,190 @@
use std::collections::{BTreeMap, HashMap};
use std::path::Path;
use clusterflux_core::{Capability, Digest};
use wasmparser::{Parser, Payload};
use crate::source_snapshot::{snapshot_project, SourceSnapshotInventory};
pub(super) fn verify_source_snapshot(
project_root: &Path,
expected: &Digest,
) -> Result<SourceSnapshotInventory, String> {
let actual = snapshot_project(project_root)?;
if &actual.digest != expected {
return Err(format!(
"node checkout source snapshot mismatch: task requires {expected}, but the current checkout is {}",
actual.digest
));
}
Ok(actual)
}
pub(super) fn authorize_command_network(
policy: &clusterflux_core::CommandNetworkPolicy,
allow_network: bool,
) -> Result<(), String> {
if policy == &clusterflux_core::CommandNetworkPolicy::Enabled && !allow_network {
return Err(
"Wasm task requested network access without an explicit granted Network capability"
.to_owned(),
);
}
Ok(())
}
pub(super) fn require_command_environment(environment_id: Option<&str>) -> Result<&str, String> {
environment_id.ok_or_else(|| {
"native commands require an explicit command-capable environment; attach one with .env(clusterflux::env!(\"name\"))"
.to_owned()
})
}
pub(super) fn verify_environment_digest(
project_root: &Path,
environment_id: &str,
expected: &Digest,
) -> Result<clusterflux_core::EnvironmentResource, String> {
let environment = clusterflux_core::discover_environments(project_root)
.map_err(|error| error.to_string())?
.into_iter()
.find(|environment| environment.name == environment_id)
.ok_or_else(|| {
format!(
"node checkout has no environment `{environment_id}` under {}/envs",
project_root.display()
)
})?;
if &environment.digest != expected {
return Err(format!(
"node environment `{environment_id}` does not match the bundle: task requires {expected}, but the current recipe is {}",
environment.digest
));
}
Ok(environment)
}
pub(super) fn is_secret_environment_name(name: &str) -> bool {
let name = name.to_ascii_uppercase();
["SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "PRIVATE_KEY"]
.iter()
.any(|marker| name.contains(marker))
}
pub(super) fn redact_configured_values(mut text: String, values: &[String]) -> String {
for value in values.iter().filter(|value| value.len() >= 4) {
text = text.replace(value, "[REDACTED]");
}
text
}
pub(super) fn capability_from_descriptor(value: &str) -> Result<Capability, String> {
match value.trim().to_ascii_lowercase().replace('-', "_").as_str() {
"command" => Ok(Capability::Command),
"containers" => Ok(Capability::Containers),
"rootless_podman" => Ok(Capability::RootlessPodman),
"source_filesystem" => Ok(Capability::SourceFilesystem),
"source_git" => Ok(Capability::SourceGit),
"host_filesystem" => Ok(Capability::HostFilesystem),
"network" => Ok(Capability::Network),
"secrets" => Ok(Capability::Secrets),
"inbound_ports" => Ok(Capability::InboundPorts),
"arbitrary_syscalls" => Ok(Capability::ArbitrarySyscalls),
"vfs_artifacts" => Ok(Capability::VfsArtifacts),
"windows_command_dev" => Ok(Capability::WindowsCommandDev),
"quic_direct" => Ok(Capability::QuicDirect),
other => Err(format!("unknown task capability `{other}`")),
}
}
pub(super) fn task_descriptors(
module: &[u8],
) -> Result<HashMap<String, serde_json::Value>, Box<dyn std::error::Error>> {
let mut descriptors = HashMap::new();
for payload in Parser::new(0).parse_all(module) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != "clusterflux.tasks" {
continue;
}
for record in section
.data()
.split(|byte| *byte == b'\n' || *byte == 0)
.filter(|record| !record.is_empty())
{
let descriptor: serde_json::Value = serde_json::from_slice(record)?;
let name = descriptor
.get("name")
.and_then(serde_json::Value::as_str)
.ok_or("task descriptor omitted name")?
.to_owned();
descriptors.insert(name, descriptor);
}
}
Ok(descriptors)
}
pub(super) fn bundle_environments(
module: &[u8],
) -> Result<BTreeMap<String, clusterflux_core::EnvironmentResource>, Box<dyn std::error::Error>> {
for payload in Parser::new(0).parse_all(module) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != "clusterflux.environments" {
continue;
}
let environments: Vec<clusterflux_core::EnvironmentResource> =
serde_json::from_slice(section.data())?;
let mut by_name = BTreeMap::new();
for environment in environments {
if by_name
.insert(environment.name.clone(), environment)
.is_some()
{
return Err("bundle environment manifest contains duplicate names".into());
}
}
return Ok(by_name);
}
Ok(BTreeMap::new())
}
pub(super) fn resolve_task_export(
module: &[u8],
task: &str,
) -> Result<String, Box<dyn std::error::Error>> {
for payload in Parser::new(0).parse_all(module) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != "clusterflux.tasks" {
continue;
}
for record in section
.data()
.split(|byte| *byte == b'\n' || *byte == 0)
.filter(|record| !record.is_empty())
{
let descriptor: serde_json::Value = serde_json::from_slice(record)?;
if descriptor.get("name").and_then(serde_json::Value::as_str) != Some(task) {
continue;
}
if descriptor
.get("abi_version")
.and_then(serde_json::Value::as_u64)
!= Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
{
return Err(format!("task `{task}` uses an unsupported Wasm ABI version").into());
}
return descriptor
.get("export")
.and_then(serde_json::Value::as_str)
.filter(|export| !export.trim().is_empty())
.map(str::to_owned)
.ok_or_else(|| format!("task `{task}` descriptor omitted its Wasm export").into());
}
}
Err(format!("bundle has no task descriptor named `{task}`").into())
}

95
crates/clusterflux-node/src/bin/clusterflux-podman-smoke.rs Normal file
View file

@ -0,0 +1,95 @@
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
CommandInvocation, Digest, EnvironmentKind, EnvironmentRequirements, EnvironmentResource,
NodeId, ProcessId, TaskInstanceId, VfsOverlay, VfsPath,
};
use clusterflux_node::{
LinuxRootlessPodmanBackend, LocalCheckoutTaskRequest, LocalSourceCheckout, StdProcessRunner,
};
use serde_json::json;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let workspace = create_workspace()?;
let env_dir = workspace.join("envs/linux");
fs::create_dir_all(&env_dir)?;
fs::write(
env_dir.join("Containerfile"),
"FROM docker.io/library/alpine:3.20\nWORKDIR /workspace\n",
)?;
fs::write(workspace.join("input.txt"), "node-local source\n")?;
let env = EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: env_dir.join("Containerfile"),
context_path: env_dir,
digest: Digest::sha256("phase2-podman-smoke-linux-env"),
requirements: EnvironmentRequirements::linux_container(),
};
let invocation = CommandInvocation {
program: "sh".to_owned(),
args: vec![
"-c".to_owned(),
"printf 'podman-ok:' && cat input.txt".to_owned(),
],
working_directory: "/workspace".to_owned(),
environment_variables: Default::default(),
timeout_ms: 60_000,
network: clusterflux_core::CommandNetworkPolicy::Disabled,
env: Some(env),
};
let checkout = LocalSourceCheckout {
host_path: workspace.clone(),
snapshot: Digest::sha256("phase2-podman-smoke-checkout"),
};
let task = TaskInstanceId::from("podman-smoke");
let output_root = workspace.join("task-output");
fs::create_dir_all(&output_root)?;
let mut overlay = VfsOverlay::new(task.clone(), NodeId::from("node-podman-smoke"));
let mut runner = StdProcessRunner;
let output = LinuxRootlessPodmanBackend.execute_local_checkout_task(
LocalCheckoutTaskRequest {
process: ProcessId::from("vp-podman-smoke"),
virtual_thread: task,
invocation: &invocation,
checkout,
output_root,
stage_stdout_as: Some(VfsPath::new("/vfs/artifacts/podman-smoke.txt")?),
},
&mut runner,
&mut overlay,
)?;
let manifest = overlay.flush();
println!(
"{}",
serde_json::to_string(&json!({
"podman_status": "completed",
"status_code": output.status_code,
"stdout": output.stdout,
"stderr": output.stderr,
"staged_artifact": output.staged_artifact,
"large_bytes_uploaded": manifest.large_bytes_uploaded,
"uses_full_repo_tarball": false,
"coordinator_routed_file_reads": false,
}))?
);
let _ = fs::remove_dir_all(workspace);
Ok(())
}
fn create_workspace() -> Result<PathBuf, std::io::Error> {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or_default();
let workspace = std::env::temp_dir().join(format!(
"clusterflux-podman-smoke-{}-{nanos}",
std::process::id()
));
fs::create_dir_all(&workspace)?;
Ok(workspace)
}

138
crates/clusterflux-node/src/bin/clusterflux-quic-smoke.rs Normal file
View file

@ -0,0 +1,138 @@
use std::{net::SocketAddr, sync::Arc};
use clusterflux_core::{
ArtifactId, DataPlaneObject, DataPlaneScope, Digest, NativeQuicTransport, NodeEndpoint, NodeId,
ProcessId, ProjectId, RendezvousRequest, TenantId, Transport,
};
use quinn::rustls::{
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},
RootCertStore,
};
use quinn::{ClientConfig, Endpoint, ServerConfig};
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
struct QuicTransferRequest {
scope: DataPlaneScope,
authorization_digest: Digest,
requested_bytes: u64,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let payload = b"artifact-bytes-over-rust-native-quic".to_vec();
let (cert, key) = self_signed_localhost_cert()?;
let server_endpoint = Endpoint::server(
ServerConfig::with_single_cert(vec![cert.clone()], key)?,
"127.0.0.1:0".parse()?,
)?;
let server_addr = server_endpoint.local_addr()?;
let scope = DataPlaneScope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("vp-quic"),
object: DataPlaneObject::Artifact(ArtifactId::from("quic-artifact")),
authorization_subject: "node-a-to-node-b".to_owned(),
};
let transport = NativeQuicTransport;
let plan = transport.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope.clone(),
source: endpoint("node-a", server_addr),
destination: endpoint("node-b", "127.0.0.1:0".parse()?),
},
true,
"",
)?;
let expected_scope = plan.scope.clone();
let expected_digest = plan.authorization_digest.clone();
let expected_payload = payload.clone();
let server = tokio::spawn(async move {
let incoming = server_endpoint
.accept()
.await
.ok_or("server endpoint closed before accepting a QUIC connection")?;
let connection = incoming.await?;
let (mut send, mut recv) = connection.accept_bi().await?;
let request_bytes = recv.read_to_end(64 * 1024).await?;
let request: QuicTransferRequest = serde_json::from_slice(&request_bytes)?;
if request.scope != expected_scope {
return Err("QUIC request scope did not match the authorized data-plane scope".into());
}
if request.authorization_digest != expected_digest {
return Err(
"QUIC request authorization digest did not match the rendezvous plan".into(),
);
}
send.write_all(&expected_payload).await?;
send.finish()?;
server_endpoint.wait_idle().await;
Ok::<usize, Box<dyn std::error::Error + Send + Sync>>(request_bytes.len())
});
let mut roots = RootCertStore::empty();
roots.add(cert)?;
let client_config = ClientConfig::with_root_certificates(Arc::new(roots))?;
let mut client_endpoint = Endpoint::client("127.0.0.1:0".parse()?)?;
client_endpoint.set_default_client_config(client_config);
let connection = client_endpoint.connect(server_addr, "localhost")?.await?;
let (mut send, mut recv) = connection.open_bi().await?;
let request = QuicTransferRequest {
scope: plan.scope.clone(),
authorization_digest: plan.authorization_digest.clone(),
requested_bytes: payload.len() as u64,
};
let request_bytes = serde_json::to_vec(&request)?;
send.write_all(&request_bytes).await?;
send.finish()?;
let received = recv.read_to_end(64 * 1024).await?;
connection.close(0u32.into(), b"done");
client_endpoint.wait_idle().await;
let server_received_request_bytes = server.await??;
if received != payload {
return Err("QUIC artifact payload did not round trip".into());
}
println!(
"{}",
json!({
"kind": "clusterflux_quic_smoke",
"transport": format!("{:?}", transport.kind()),
"rust_native_quic": true,
"authenticated_direct_connection": transport.authenticated_direct_connections(),
"coordinator_assisted_rendezvous": plan.coordinator_assisted_rendezvous,
"coordinator_bulk_relay_allowed": plan.coordinator_bulk_relay_allowed,
"source_node": plan.source.node,
"destination_node": plan.destination.node,
"scope": plan.scope,
"request_bytes": request_bytes.len(),
"server_received_request_bytes": server_received_request_bytes,
"payload_bytes": received.len(),
"authorization_digest": plan.authorization_digest,
})
);
Ok(())
}
fn self_signed_localhost_cert() -> Result<
(CertificateDer<'static>, PrivateKeyDer<'static>),
Box<dyn std::error::Error + Send + Sync>,
> {
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])?;
let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()));
Ok((cert.cert.into(), key))
}
fn endpoint(name: &str, addr: SocketAddr) -> NodeEndpoint {
NodeEndpoint {
node: NodeId::from(name),
advertised_addr: addr.to_string(),
public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
}
}

414
crates/clusterflux-node/src/bin/clusterflux-wasmtime-smoke.rs Normal file
View file

@ -0,0 +1,414 @@
use std::fs;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use clusterflux_core::{
ArtifactHandle, ArtifactId, Digest, StructuredTaskBoundary, TaskBoundaryHandle,
TaskBoundaryValue, WasmHostCommandRequest, WasmHostCommandResult,
WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult, WasmHostTaskControlRequest,
WasmHostTaskControlResult, WasmHostTaskHandle, WasmHostTaskJoinRequest, WasmHostTaskJoinResult,
WasmHostTaskStartRequest, WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult,
WasmTaskInvocation, WasmTaskOutcome,
};
use clusterflux_node::{WasmTaskHost, WasmtimeTaskRuntime};
use serde_json::json;
use wasmparser::{Parser, Payload};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = std::env::args().collect();
if args.len() == 4 && args[1] == "--host-command" {
return run_host_command_smoke(&args[2], &args[3]);
}
if args.len() == 6 && args[1] == "--debug-freeze-resume" {
return run_debug_freeze_resume_smoke(&args[2], &args[3], &args[4], &args[5]);
}
if args.len() == 6 && args[1] == "--task-abi" {
return run_task_abi_smoke(&args[2], &args[3], &args[4], &args[5]);
}
if args.len() == 4 && args[1] == "--task-artifact" {
return run_task_artifact_smoke(&args[2], &args[3]);
}
if args.len() != 5 {
return Err(
"usage: clusterflux-wasmtime-smoke <module.wasm> <export> <i32-arg> <expected-i32> | --host-command <module.wasm> <task> | --debug-freeze-resume <module.wasm> <export> <i32-arg> <expected-i32> | --task-abi <module.wasm> <export> <task> <args-json> | --task-artifact <module.wasm> <task>".into(),
);
}
let module = PathBuf::from(&args[1]);
let export = &args[2];
let arg: i32 = args[3].parse()?;
let expected: i32 = args[4].parse()?;
let wasm = fs::read(&module)?;
let runtime = WasmtimeTaskRuntime::new()?;
let result = runtime.run_i32_export(&wasm, export, arg)?;
if result != expected {
return Err(format!("expected {expected}, got {result} from export `{export}`").into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_task_smoke",
"module": module,
"export": export,
"arg": arg,
"result": result,
}))?
);
Ok(())
}
fn run_task_artifact_smoke(module: &str, task: &str) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let export = task_export(&wasm, task)?;
let published = Arc::new(Mutex::new(None));
let input_artifact = ArtifactHandle {
id: ArtifactId::from("compiler-output"),
digest: Digest::sha256("compiler-output"),
size_bytes: 15,
};
let source_snapshot = Digest::sha256("standalone-source-snapshot");
let package_input = TaskBoundaryValue::Structured(StructuredTaskBoundary {
value: serde_json::json!({
"release_name": "release.tar",
"source": {
"$task_handle": { "index": 0, "kind": "source_snapshot" }
},
"executable": {
"$task_handle": { "index": 1, "kind": "artifact" }
},
"inputs": [{
"$task_handle": { "index": 2, "kind": "artifact" }
}],
}),
handles: vec![
TaskBoundaryHandle::SourceSnapshot(source_snapshot),
TaskBoundaryHandle::Artifact(input_artifact.clone()),
TaskBoundaryHandle::Artifact(input_artifact.clone()),
],
});
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&wasm,
&Digest::sha256(&wasm),
&export,
&WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
vec![package_input],
),
Box::new(ArtifactRecordingHost {
published: Arc::clone(&published),
executed_command: Arc::new(Mutex::new(None)),
allow_command: true,
}),
)?;
if result.outcome != WasmTaskOutcome::Completed {
return Err(format!("artifact task failed: {:?}", result.error).into());
}
let Some(TaskBoundaryValue::Artifact(result_artifact)) = result.result else {
return Err("artifact task did not return an Artifact handle".into());
};
let (name, host_artifact) = published
.lock()
.map_err(|_| "artifact recording host lock was poisoned")?
.clone()
.ok_or("Wasm task did not invoke clusterflux.vfs_operation_v1 flush_output")?;
if result_artifact != host_artifact {
return Err("Wasm task returned a handle other than the host-issued artifact ID".into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_task_artifact_smoke",
"module": module,
"task": task,
"export": export,
"artifact": result_artifact,
"artifact_name": name,
"artifact_digest": host_artifact.digest,
"artifact_size_bytes": host_artifact.size_bytes,
"host_import": "clusterflux.vfs_operation_v1",
"host_issued_handle_returned": true,
}))?
);
Ok(())
}
type PublishedArtifact = (String, ArtifactHandle);
type ExecutedCommand = (WasmHostCommandRequest, WasmHostCommandResult);
struct ArtifactRecordingHost {
published: Arc<Mutex<Option<PublishedArtifact>>>,
executed_command: Arc<Mutex<Option<ExecutedCommand>>>,
allow_command: bool,
}
impl WasmTaskHost for ArtifactRecordingHost {
fn start_task(
&mut self,
_request: WasmHostTaskStartRequest,
) -> Result<WasmHostTaskHandle, String> {
Err("artifact smoke task must not spawn".to_owned())
}
fn join_task(
&mut self,
_request: WasmHostTaskJoinRequest,
) -> Result<WasmHostTaskJoinResult, String> {
Err("artifact smoke task must not join".to_owned())
}
fn run_command(
&mut self,
request: WasmHostCommandRequest,
) -> Result<WasmHostCommandResult, String> {
request.validate()?;
if !self.allow_command {
return Err("this smoke task was not granted Command capability".to_owned());
}
let result = WasmHostCommandResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
status_code: Some(0),
stdout: String::new(),
stderr: String::new(),
stdout_truncated: false,
stderr_truncated: false,
};
*self
.executed_command
.lock()
.map_err(|_| "command recording host lock was poisoned".to_owned())? =
Some((request, result.clone()));
Ok(result)
}
fn poll_task_control(
&mut self,
request: WasmHostTaskControlRequest,
) -> Result<WasmHostTaskControlResult, String> {
request.validate()?;
Ok(WasmHostTaskControlResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
cancellation_requested: false,
})
}
fn vfs_operation(&mut self, request: WasmHostVfsRequest) -> Result<WasmHostVfsResult, String> {
request.validate()?;
let (artifact, relative_path) = match request.operation {
WasmHostVfsOperation::FlushOutput { relative_path } => {
let digest = Digest::sha256(format!("standalone-vfs-smoke:{relative_path}"));
let artifact = ArtifactHandle {
id: ArtifactId::new(format!(
"{}-{}",
relative_path.replace('/', "_"),
digest.as_str().trim_start_matches("sha256:")
)),
digest,
size_bytes: relative_path.len() as u64,
};
*self
.published
.lock()
.map_err(|_| "artifact recording host lock was poisoned".to_owned())? =
Some((relative_path.clone(), artifact.clone()));
(artifact, relative_path)
}
WasmHostVfsOperation::MaterializeArtifact {
artifact,
relative_path,
} => (artifact, relative_path),
};
Ok(WasmHostVfsResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
artifact,
relative_path,
})
}
fn snapshot_source(
&mut self,
request: WasmHostSourceSnapshotRequest,
) -> Result<WasmHostSourceSnapshotResult, String> {
request.validate()?;
Ok(WasmHostSourceSnapshotResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
snapshot: Digest::sha256("standalone-source-snapshot"),
})
}
}
fn task_export(wasm: &[u8], task: &str) -> Result<String, Box<dyn std::error::Error>> {
for payload in Parser::new(0).parse_all(wasm) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != "clusterflux.tasks" {
continue;
}
for record in section
.data()
.split(|byte| *byte == b'\n' || *byte == 0)
.filter(|record| !record.is_empty())
{
let descriptor: serde_json::Value = serde_json::from_slice(record)?;
if descriptor.get("name").and_then(serde_json::Value::as_str) == Some(task) {
return descriptor
.get("export")
.and_then(serde_json::Value::as_str)
.map(str::to_owned)
.ok_or_else(|| format!("task `{task}` descriptor omitted export").into());
}
}
}
Err(format!("module has no task descriptor `{task}`").into())
}
fn run_task_abi_smoke(
module: &str,
export: &str,
task: &str,
args: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let args: Vec<TaskBoundaryValue> = serde_json::from_str(args)?;
let invocation = WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
args,
);
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified(
&wasm,
&Digest::sha256(&wasm),
export,
&invocation,
)?;
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasm_task_abi_smoke",
"module": module,
"export": export,
"invocation": invocation,
"result": result,
}))?
);
Ok(())
}
fn run_debug_freeze_resume_smoke(
module: &str,
export: &str,
arg: &str,
expected: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let arg: i32 = arg.parse()?;
let expected: i32 = expected.parse()?;
let wasm = fs::read(&module)?;
let runtime = WasmtimeTaskRuntime::new()?;
let probe = runtime.freeze_resume_i32_export_probe(&wasm, export, arg)?;
if probe.result != expected {
return Err(format!(
"expected {expected}, got {} from export `{export}`",
probe.result
)
.into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_debug_freeze_resume_smoke",
"module": module,
"export": export,
"task": probe.task,
"frozen_state": probe.frozen_state,
"resumed_state": probe.resumed_state,
"stack_frames": probe.stack_frames,
"local_values": probe.local_values,
"wasm_function": probe.wasm_function,
"wasm_pc": probe.wasm_pc,
"arg": arg,
"result": probe.result,
"node_runtime_reached_wasm_task": true,
"node_runtime_captured_wasm_locals": true,
}))?
);
Ok(())
}
fn run_host_command_smoke(module: &str, task: &str) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let export = task_export(&wasm, task)?;
let published = Arc::new(Mutex::new(None));
let executed_command = Arc::new(Mutex::new(None));
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&wasm,
&Digest::sha256(&wasm),
&export,
&WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
vec![TaskBoundaryValue::SourceSnapshot(Digest::sha256(
"standalone-source-snapshot",
))],
),
Box::new(ArtifactRecordingHost {
published: Arc::clone(&published),
executed_command: Arc::clone(&executed_command),
allow_command: true,
}),
)?;
if result.outcome != WasmTaskOutcome::Completed {
return Err(format!("command task failed: {:?}", result.error).into());
}
let Some(TaskBoundaryValue::Artifact(result_artifact)) = result.result else {
return Err("command task did not return its published Artifact handle".into());
};
let (request, command_result) = executed_command
.lock()
.map_err(|_| "command recording host lock was poisoned")?
.clone()
.ok_or("Wasm task did not invoke clusterflux.command_run_v1")?;
let (artifact_name, host_artifact) = published
.lock()
.map_err(|_| "artifact recording host lock was poisoned")?
.clone()
.ok_or("command task did not publish its command output")?;
if result_artifact != host_artifact {
return Err("command task returned a handle other than the published artifact".into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_host_command_smoke",
"module": module,
"task": task,
"export": export,
"program": request.program,
"args": request.args,
"working_directory": request.working_directory,
"environment_variables": request.environment_variables,
"timeout_ms": request.timeout_ms,
"network": request.network,
"status_code": command_result.status_code,
"stdout": command_result.stdout,
"artifact": result_artifact,
"artifact_name": artifact_name,
"artifact_digest": host_artifact.digest,
"artifact_size_bytes": host_artifact.size_bytes,
"node_host_import": "clusterflux.command_run_v1",
"artifact_host_import": "clusterflux.vfs_operation_v1",
"flagship_linux_build_task": true,
"node_executed_host_command": true,
"hosted_control_plane_ran_command": false,
}))?
);
Ok(())
}

130
crates/clusterflux-node/src/command_runner.rs Normal file
View file

@ -0,0 +1,130 @@
use clusterflux_core::{
CommandInvocation, Digest, LogBuffer, NativeCommandPolicy, NodeId, TaskInstanceId, VfsObject,
VfsOverlay, VfsPath,
};
use serde::{Deserialize, Serialize};
use super::BackendError;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CapturedCommandLogs {
pub stdout: String,
pub stderr: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub backpressured: bool,
}
pub const DEFAULT_COMMAND_LOG_LIMIT_BYTES: usize = 256 * 1024;
pub fn authorize_node_command(
hosted_control_plane: bool,
node_has_command_capability: bool,
) -> Result<(), BackendError> {
NativeCommandPolicy {
hosted_control_plane,
node_has_command_capability,
}
.authorize()
.map_err(BackendError::Denied)
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VirtualThreadCommand {
pub virtual_thread: TaskInstanceId,
pub invocation: CommandInvocation,
pub stage_stdout_as: Option<VfsPath>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CommandOutput {
pub virtual_thread: TaskInstanceId,
pub status_code: Option<i32>,
pub stdout: String,
pub stderr: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub log_backpressured: bool,
pub staged_artifact: Option<VfsObject>,
}
#[derive(Clone, Debug)]
pub struct LocalCommandExecutor {
pub node: NodeId,
pub hosted_control_plane: bool,
pub has_command_capability: bool,
}
impl LocalCommandExecutor {
pub fn run(
&self,
command: VirtualThreadCommand,
overlay: &mut VfsOverlay,
) -> Result<CommandOutput, BackendError> {
self.run_with_log_limit(command, overlay, DEFAULT_COMMAND_LOG_LIMIT_BYTES)
}
pub fn run_with_log_limit(
&self,
command: VirtualThreadCommand,
overlay: &mut VfsOverlay,
max_log_bytes: usize,
) -> Result<CommandOutput, BackendError> {
authorize_node_command(self.hosted_control_plane, self.has_command_capability)?;
let output = std::process::Command::new(&command.invocation.program)
.args(&command.invocation.args)
.output()
.map_err(|err| BackendError::Command(format!("{err:#}")))?;
let logs = capture_command_logs(
&command.virtual_thread,
&output.stdout,
&output.stderr,
max_log_bytes,
);
let staged_artifact = if let Some(path) = command.stage_stdout_as {
Some(overlay.write(
path,
Digest::sha256(&output.stdout),
output.stdout.len() as u64,
))
} else {
None
};
Ok(CommandOutput {
virtual_thread: command.virtual_thread,
status_code: output.status.code(),
stdout: logs.stdout,
stderr: logs.stderr,
stdout_truncated: logs.stdout_truncated,
stderr_truncated: logs.stderr_truncated,
log_backpressured: logs.backpressured,
staged_artifact,
})
}
}
pub(super) fn capture_command_logs(
task: &TaskInstanceId,
stdout: &[u8],
stderr: &[u8],
max_log_bytes: usize,
) -> CapturedCommandLogs {
let mut logs = LogBuffer::new(max_log_bytes);
logs.push(task.clone(), stdout);
logs.push(task.clone(), stderr);
let records = logs.records();
let stdout_record = &records[0];
let stderr_record = &records[1];
debug_assert_eq!(&stdout_record.task, task);
debug_assert_eq!(&stderr_record.task, task);
CapturedCommandLogs {
stdout: String::from_utf8_lossy(&stdout_record.bytes).into_owned(),
stderr: String::from_utf8_lossy(&stderr_record.bytes).into_owned(),
stdout_truncated: stdout_record.truncated,
stderr_truncated: stderr_record.truncated,
backpressured: logs.backpressured(),
}
}

38
crates/clusterflux-node/src/coordinator_session.rs Normal file
View file

@ -0,0 +1,38 @@
#[cfg(test)]
use clusterflux_control::endpoint_identity;
use clusterflux_control::ControlSession;
use clusterflux_core::coordinator_wire_request;
use serde_json::Value;
pub(crate) struct CoordinatorSession {
inner: ControlSession,
}
impl CoordinatorSession {
pub(crate) fn connect(addr: &str) -> Result<Self, Box<dyn std::error::Error>> {
Ok(Self {
inner: ControlSession::connect(addr)?,
})
}
pub(crate) fn request(&mut self, value: Value) -> Result<Value, Box<dyn std::error::Error>> {
let request_id = format!("node-{}", self.inner.requests() + 1);
let wire_request = coordinator_wire_request(request_id, value);
let response = self.inner.request(&wire_request)?;
if response.get("type").and_then(Value::as_str) == Some("error") {
return Err(format!("coordinator error: {response}").into());
}
Ok(response)
}
pub(crate) fn requests(&self) -> usize {
self.inner.requests() as usize
}
}
#[cfg(test)]
pub(crate) fn control_endpoint_identity(
endpoint: &str,
) -> Result<String, Box<dyn std::error::Error>> {
Ok(endpoint_identity(endpoint)?)
}

749
crates/clusterflux-node/src/daemon.rs Normal file
View file

@ -0,0 +1,749 @@
use std::collections::{BTreeMap, BTreeSet};
use std::io::Write;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
};
use serde_json::{json, Value};
use crate::assignment_runner::run_verified_wasmtime_assignment;
#[cfg(test)]
use crate::coordinator_session::control_endpoint_identity;
use crate::coordinator_session::CoordinatorSession;
use crate::debug_agent::poll_task_cancellation;
use crate::node_identity::{
establish_node_identity, node_nonce, node_private_key_for_runtime, signed_node_request_json,
unix_timestamp_seconds,
};
#[cfg(test)]
use crate::node_identity::{load_or_create_local_node_credential, unix_timestamp_nanos};
use crate::source_snapshot::snapshot_project;
use crate::task_artifacts::{
clean_stale_task_output_roots, current_epoch_seconds, NodeArtifactRetentionLimits,
NodeArtifactStore,
};
use crate::task_reports::{record_cancelled_task, record_completed_task, record_failed_task};
static WORKER_SHUTDOWN_REQUESTED: AtomicBool = AtomicBool::new(false);
pub(crate) fn worker_shutdown_requested() -> bool {
WORKER_SHUTDOWN_REQUESTED.load(Ordering::Acquire)
}
#[cfg(unix)]
extern "C" fn request_worker_shutdown(_signal: libc::c_int) {
WORKER_SHUTDOWN_REQUESTED.store(true, Ordering::SeqCst);
}
fn install_worker_shutdown_handler() {
WORKER_SHUTDOWN_REQUESTED.store(false, Ordering::SeqCst);
#[cfg(unix)]
unsafe {
libc::signal(libc::SIGINT, request_worker_shutdown as libc::sighandler_t);
libc::signal(libc::SIGTERM, request_worker_shutdown as libc::sighandler_t);
}
}
#[derive(Clone, Debug)]
pub(crate) struct Args {
pub(crate) coordinator: String,
pub(crate) tenant: String,
pub(crate) project: String,
pub(crate) project_root: Option<PathBuf>,
pub(crate) node: String,
pub(crate) enrollment_grant: Option<String>,
pub(crate) public_key: Option<String>,
pub(crate) control_poll_ms: u64,
pub(crate) assignment_poll_ms: u64,
pub(crate) emit_ready: bool,
pub(crate) worker: bool,
}
#[derive(Clone, Debug)]
pub(crate) struct RuntimeTask {
pub(crate) process: String,
pub(crate) task: String,
pub(crate) epoch: Option<u64>,
pub(crate) task_spec: Option<TaskSpec>,
pub(crate) bundle_digest: Option<Digest>,
pub(crate) wasm_module_base64: Option<String>,
pub(crate) task_assignment_response: Value,
}
pub(crate) fn run() -> Result<(), Box<dyn std::error::Error>> {
let args = parse_args()?;
let mut session = CoordinatorSession::connect(&args.coordinator)?;
let node_private_key = node_private_key_for_runtime(&args.node)?;
let registration = establish_node_identity(&mut session, &args, &node_private_key)?;
let heartbeat_request = json!({
"type": "node_heartbeat",
"node": &args.node,
});
let heartbeat_signature = sign_node_request(
&node_private_key,
&NodeId::from(args.node.as_str()),
"node_heartbeat",
&signed_request_payload_digest(&heartbeat_request),
node_nonce("node-heartbeat"),
unix_timestamp_seconds(),
)
.map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidInput, err))?;
let mut heartbeat_request = heartbeat_request;
heartbeat_request["node_signature"] = json!(heartbeat_signature);
let heartbeat = session.request(heartbeat_request)?;
clean_stale_task_output_roots(args.project_root.as_deref(), &args.node)?;
let artifact_store = NodeArtifactStore::for_runtime(args.project_root.as_deref(), &args.node)?;
let retention_limits = NodeArtifactRetentionLimits::from_environment()?;
artifact_store.garbage_collect(retention_limits, &BTreeSet::new(), current_epoch_seconds())?;
let capability_report = report_node_capabilities(
&args,
&mut session,
&node_private_key,
&artifact_store,
true,
)?;
if !args.worker {
return Err(
"one-shot native command mode was removed; run `clusterflux-node --worker` and launch a bundled Wasm task through the coordinator"
.into(),
);
}
install_worker_shutdown_handler();
worker_loop(
&args,
&mut session,
registration,
heartbeat,
capability_report,
&node_private_key,
)
}
fn worker_loop(
args: &Args,
session: &mut CoordinatorSession,
registration: Value,
heartbeat: Value,
capability_report: Value,
node_private_key: &str,
) -> Result<(), Box<dyn std::error::Error>> {
const ARTIFACT_GC_INTERVAL: Duration = Duration::from_secs(30);
let artifact_store = NodeArtifactStore::for_runtime(args.project_root.as_deref(), &args.node)?;
let retention_limits = NodeArtifactRetentionLimits::from_environment()?;
let mut last_artifact_gc = Instant::now();
let mut restart_pins = BTreeMap::<ArtifactId, Instant>::new();
if args.emit_ready {
println!(
"{}",
serde_json::to_string(&json!({
"node_status": "ready",
"mode": "worker",
"node": &args.node,
}))?
);
std::io::stdout().flush()?;
}
loop {
if worker_shutdown_requested() {
report_node_capabilities(args, session, node_private_key, &artifact_store, false)?;
return Ok(());
}
if service_pending_artifact_transfer(args, session, node_private_key)? {
continue;
}
if last_artifact_gc.elapsed() >= ARTIFACT_GC_INTERVAL {
let now = Instant::now();
restart_pins.retain(|_, expiry| *expiry > now);
let pinned = restart_pins.keys().cloned().collect::<BTreeSet<_>>();
artifact_store.garbage_collect(retention_limits, &pinned, current_epoch_seconds())?;
report_node_capabilities(args, session, node_private_key, &artifact_store, true)?;
last_artifact_gc = Instant::now();
}
let response = session.request(signed_node_request_json(
args,
node_private_key,
"poll_task_assignment",
json!({
"type": "poll_task_assignment",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
}),
)?)?;
let Some(assignment) = response.get("assignment").filter(|value| !value.is_null()) else {
std::thread::sleep(Duration::from_millis(args.assignment_poll_ms));
continue;
};
let runtime_task = runtime_task_from_assignment(assignment)?;
if args.emit_ready {
println!(
"{}",
serde_json::to_string(&json!({
"node_status": "assignment_started",
"node": &args.node,
"process": &runtime_task.process,
"virtual_thread": &runtime_task.task,
"task_assignment_response": &runtime_task.task_assignment_response,
}))?
);
std::io::stdout().flush()?;
}
if let Some(task_spec) = &runtime_task.task_spec {
let expiry = Instant::now()
.checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
.unwrap_or_else(Instant::now);
for artifact in &task_spec.required_artifacts {
restart_pins.insert(artifact.clone(), expiry);
}
}
let report = run_runtime_task(
args,
session,
runtime_task,
registration.clone(),
heartbeat.clone(),
capability_report.clone(),
node_private_key,
)?;
if let Some(artifact) = report
.pointer("/vfs_metadata_response/artifact_path")
.and_then(Value::as_str)
.and_then(|path| path.strip_prefix("/vfs/artifacts/"))
{
let expiry = Instant::now()
.checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
.unwrap_or_else(Instant::now);
restart_pins.insert(ArtifactId::new(artifact), expiry);
}
println!("{}", serde_json::to_string(&report)?);
std::io::stdout().flush()?;
}
}
fn report_node_capabilities(
args: &Args,
session: &mut CoordinatorSession,
node_private_key: &str,
artifact_store: &NodeArtifactStore,
online: bool,
) -> Result<Value, Box<dyn std::error::Error>> {
let artifact_locations = artifact_store
.artifact_ids()?
.into_iter()
.map(|artifact| artifact.as_str().to_owned())
.collect::<Vec<_>>();
let source_snapshots = args
.project_root
.as_deref()
.map(snapshot_project)
.transpose()?
.into_iter()
.map(|snapshot| snapshot.digest)
.collect::<Vec<_>>();
session.request(signed_node_request_json(
args,
node_private_key,
"report_node_capabilities",
json!({
"type": "report_node_capabilities",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"capabilities": NodeCapabilities::detect_current(),
"cached_environment_digests": [],
"dependency_cache_digests": [],
"source_snapshots": source_snapshots,
"artifact_locations": artifact_locations,
"direct_connectivity": false,
"online": online,
}),
)?)
}
fn service_pending_artifact_transfer(
args: &Args,
session: &mut CoordinatorSession,
node_private_key: &str,
) -> Result<bool, Box<dyn std::error::Error>> {
let response = session.request(signed_node_request_json(
args,
node_private_key,
"poll_artifact_transfer",
json!({
"type": "poll_artifact_transfer",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
}),
)?)?;
let Some(transfer) = response.get("transfer").filter(|value| !value.is_null()) else {
return Ok(false);
};
let transfer_id = required_string(transfer, "transfer_id")?;
let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
let expected_digest: Digest = serde_json::from_value(
transfer
.get("expected_digest")
.cloned()
.ok_or("artifact transfer omitted expected_digest")?,
)?;
let expected_size_bytes = transfer
.get("expected_size_bytes")
.and_then(Value::as_u64)
.ok_or("artifact transfer omitted expected_size_bytes")?;
let offset = transfer
.get("offset")
.and_then(Value::as_u64)
.ok_or("artifact transfer omitted offset")?;
let max_chunk_bytes = transfer
.get("max_chunk_bytes")
.and_then(Value::as_u64)
.ok_or("artifact transfer omitted max_chunk_bytes")?;
let read = NodeArtifactStore::for_runtime(args.project_root.as_deref(), &args.node).and_then(
|store| {
store.read_verified_chunk(
&artifact,
&expected_digest,
expected_size_bytes,
offset,
max_chunk_bytes,
)
},
);
let content = match read {
Ok(content) => content,
Err(message) => {
session.request(signed_node_request_json(
args,
node_private_key,
"fail_artifact_transfer",
json!({
"type": "fail_artifact_transfer",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"transfer_id": transfer_id,
"artifact": artifact,
"message": message,
}),
)?)?;
return Ok(true);
}
};
let eof = offset.saturating_add(content.len() as u64) == expected_size_bytes;
session.request(signed_node_request_json(
args,
node_private_key,
"upload_artifact_transfer_chunk",
json!({
"type": "upload_artifact_transfer_chunk",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"transfer_id": transfer_id,
"artifact": artifact,
"offset": offset,
"content_base64": BASE64_STANDARD.encode(&content),
"chunk_digest": Digest::sha256(&content),
"eof": eof,
}),
)?)?;
Ok(true)
}
pub(crate) fn runtime_task_from_assignment(
value: &Value,
) -> Result<RuntimeTask, Box<dyn std::error::Error>> {
let task_spec: TaskSpec = serde_json::from_value(
value
.get("task_spec")
.cloned()
.ok_or("task assignment missing task_spec")?,
)?;
Ok(RuntimeTask {
process: required_string(value, "process")?,
task: required_string(value, "task")?,
epoch: value.get("epoch").and_then(Value::as_u64),
bundle_digest: task_spec.bundle_digest.clone(),
task_spec: Some(task_spec),
wasm_module_base64: Some(required_string(value, "wasm_module_base64")?),
task_assignment_response: value.clone(),
})
}
fn required_string(value: &Value, field: &str) -> Result<String, Box<dyn std::error::Error>> {
value
.get(field)
.and_then(Value::as_str)
.map(str::to_owned)
.ok_or_else(|| format!("task assignment missing string field `{field}`").into())
}
fn run_runtime_task(
args: &Args,
session: &mut CoordinatorSession,
task: RuntimeTask,
registration: Value,
heartbeat: Value,
capability_report: Value,
node_private_key: &str,
) -> Result<Value, Box<dyn std::error::Error>> {
let epoch = match task.epoch {
Some(epoch) => epoch,
None => {
let started = session.request(json!({
"type": "start_process",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
}))?;
started
.get("epoch")
.and_then(Value::as_u64)
.ok_or("coordinator start_process response missing epoch")?
}
};
session.request(signed_node_request_json(
args,
node_private_key,
"reconnect_node",
json!({
"type": "reconnect_node",
"node": &args.node,
"process": &task.process,
"epoch": epoch,
}),
)?)?;
let debug_command = session.request(signed_node_request_json(
args,
node_private_key,
"poll_debug_command",
json!({
"type": "poll_debug_command",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
}),
)?)?;
if args.emit_ready && !args.worker {
println!(
"{}",
serde_json::to_string(&json!({
"node_status": "ready",
"node": &args.node,
"process": &task.process,
"task": &task.task,
}))?
);
std::io::stdout().flush()?;
}
if args.control_poll_ms > 0 && poll_task_cancellation(session, args, &task, node_private_key)? {
return record_cancelled_task(
args,
session,
&task,
registration,
heartbeat,
capability_report,
debug_command,
node_private_key,
);
}
let execution = run_verified_wasmtime_assignment(args, &task, node_private_key);
match execution {
Ok((output, manifest, result)) => match crate::task_artifacts::retained_result_artifact(
args.project_root.as_deref(),
&args.node,
result.as_ref(),
) {
Ok(retained) => record_completed_task(
args,
session,
task,
output,
manifest,
result,
retained,
registration,
heartbeat,
capability_report,
debug_command,
node_private_key,
),
Err(error) => record_failed_task(
args,
session,
&task,
registration,
heartbeat,
capability_report,
debug_command,
node_private_key,
&error,
),
},
Err(error) => {
let error = error.to_string();
if error.contains("task execution cancelled:") {
record_cancelled_task(
args,
session,
&task,
registration,
heartbeat,
capability_report,
debug_command,
node_private_key,
)
} else {
record_failed_task(
args,
session,
&task,
registration,
heartbeat,
capability_report,
debug_command,
node_private_key,
&error,
)
}
}
}
}
fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
let mut coordinator = None;
let mut tenant = "tenant".to_owned();
let mut project = "project".to_owned();
let mut project_root = None;
let mut node = "node".to_owned();
let mut enrollment_grant = None;
let mut public_key = None;
let mut control_poll_ms = 0;
let mut assignment_poll_ms = 500;
let mut emit_ready = false;
let mut worker = false;
let mut args = std::env::args().skip(1);
while let Some(arg) = args.next() {
match arg.as_str() {
"--coordinator" => coordinator = args.next(),
"--tenant" => tenant = args.next().ok_or("--tenant requires a value")?,
"--project-id" => project = args.next().ok_or("--project-id requires a value")?,
"--node" => node = args.next().ok_or("--node requires a value")?,
"--enrollment-grant" => enrollment_grant = args.next(),
"--public-key" => public_key = args.next(),
"--control-poll-ms" => {
control_poll_ms = args
.next()
.ok_or("--control-poll-ms requires a value")?
.parse()?
}
"--assignment-poll-ms" => {
assignment_poll_ms = validate_assignment_poll_ms(
args.next()
.ok_or("--assignment-poll-ms requires a value")?
.parse()?,
)?
}
"--emit-ready" => emit_ready = true,
"--worker" => worker = true,
"--project-root" => {
project_root = Some(PathBuf::from(
args.next().ok_or("--project-root requires a path")?,
));
}
other => return Err(format!("unknown argument: {other}").into()),
}
}
Ok(Args {
coordinator: coordinator.ok_or("--coordinator is required")?,
tenant,
project,
project_root,
node,
enrollment_grant,
public_key,
control_poll_ms,
assignment_poll_ms,
emit_ready,
worker,
})
}
fn validate_assignment_poll_ms(value: u64) -> Result<u64, String> {
if value < MIN_SIGNED_NODE_POLL_INTERVAL_MS {
return Err(format!(
"--assignment-poll-ms must be at least {MIN_SIGNED_NODE_POLL_INTERVAL_MS} ms so signed polling remains within the coordinator's bounded replay window"
));
}
Ok(value)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use clusterflux_core::{
Digest, ProcessId, ProjectId, TaskBoundaryValue, TaskDispatch, TaskInstanceId, TaskSpec,
TenantId, WasmExportAbi, WasmTaskResult,
};
use crate::assignment_runner::run_verified_wasmtime_assignment;
use clusterflux_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS;
use super::{
control_endpoint_identity, load_or_create_local_node_credential,
validate_assignment_poll_ms, Args, RuntimeTask,
};
#[test]
fn node_poll_interval_cannot_exhaust_the_bounded_replay_window() {
assert!(validate_assignment_poll_ms(MIN_SIGNED_NODE_POLL_INTERVAL_MS).is_ok());
let error = validate_assignment_poll_ms(MIN_SIGNED_NODE_POLL_INTERVAL_MS - 1).unwrap_err();
assert!(error.contains("bounded replay window"));
}
#[test]
fn hosted_url_remains_an_https_control_endpoint() {
assert_eq!(
control_endpoint_identity("https://clusterflux.michelpaulissen.com").unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
);
assert_eq!(
control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
.unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
);
assert_eq!(
control_endpoint_identity("127.0.0.1:7999").unwrap(),
"clusterflux+tcp://127.0.0.1:7999"
);
}
#[test]
fn daemon_local_node_credential_is_durable_between_runs() {
let temp = std::env::temp_dir().join(format!(
"clusterflux-node-credential-test-{}-{}",
std::process::id(),
super::unix_timestamp_nanos()
));
std::fs::create_dir_all(&temp).unwrap();
let first = load_or_create_local_node_credential(&temp, "daemon-node").unwrap();
let second = load_or_create_local_node_credential(&temp, "daemon-node").unwrap();
assert_eq!(first, second);
assert!(temp.join(".clusterflux").join("nodes").exists());
std::fs::remove_dir_all(&temp).unwrap();
}
#[test]
fn daemon_wasm_task_assignment_uses_abi_version_and_verifies_bundle_digest() {
let args = test_args();
let task_instance = TaskInstanceId::from("task_add_one-1");
let boundary = TaskBoundaryValue::SmallJson(serde_json::json!(2));
let result = serde_json::to_string(&WasmTaskResult::completed(
task_instance.clone(),
boundary.clone(),
))
.unwrap();
let wat_result = result.replace('\\', "\\\\").replace('"', "\\\"");
let result_length = result.len();
let packed = ((result_length as u64) << 32) | 2048;
let wasm = wat::parse_str(format!(
r#"(module
(memory (export "memory") 1)
(data (i32.const 2048) "{wat_result}")
(func (export "clusterflux_alloc_v1") (param i32) (result i32)
i32.const 1024)
(func (export "task_add_one") (param i32 i32) (result i64)
i64.const {packed}))"#
))
.unwrap();
let task = RuntimeTask {
process: "vp".to_owned(),
task: task_instance.as_str().to_owned(),
epoch: Some(7),
task_spec: Some(TaskSpec {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("vp"),
task_definition: clusterflux_core::TaskDefinitionId::from("task_add_one"),
task_instance,
dispatch: TaskDispatch::CoordinatorNodeWasm {
export: Some("task_add_one".to_owned()),
abi: WasmExportAbi::TaskV1,
},
environment_id: None,
environment: None,
environment_digest: None,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: Vec::new(),
args: Vec::new(),
vfs_epoch: 7,
failure_policy: Default::default(),
bundle_digest: Some(Digest::sha256(&wasm)),
}),
bundle_digest: Some(Digest::sha256(&wasm)),
wasm_module_base64: Some(BASE64_STANDARD.encode(&wasm)),
task_assignment_response: serde_json::json!({}),
};
let (output, manifest, result) =
run_verified_wasmtime_assignment(&args, &task, "test-node-private-key").unwrap();
assert_eq!(output.status_code, Some(0));
assert_eq!(
output.stdout,
format!("{}\n", serde_json::to_string(&boundary).unwrap())
);
assert!(output.staged_artifact.is_none());
assert!(manifest.objects.is_empty());
assert!(!manifest.large_bytes_uploaded);
assert_eq!(result, Some(boundary));
let mismatch = RuntimeTask {
bundle_digest: Some(Digest::sha256("different bundle bytes")),
wasm_module_base64: Some(BASE64_STANDARD.encode("not valid wasm")),
..task
};
let error = run_verified_wasmtime_assignment(&args, &mismatch, "test-node-private-key")
.unwrap_err();
assert!(error.to_string().contains("bundle digest mismatch"));
assert!(!error.to_string().contains("failed to parse"));
}
fn test_args() -> Args {
Args {
coordinator: "127.0.0.1:1".to_owned(),
tenant: "tenant".to_owned(),
project: "project".to_owned(),
project_root: None,
node: "node".to_owned(),
enrollment_grant: None,
public_key: None,
control_poll_ms: 0,
assignment_poll_ms: 1,
emit_ready: false,
worker: false,
}
}
}

47
crates/clusterflux-node/src/debug_agent.rs Normal file
View file

@ -0,0 +1,47 @@
use std::time::{Duration, Instant};
use serde_json::{json, Value};
use crate::coordinator_session::CoordinatorSession;
use crate::daemon::{Args, RuntimeTask};
use crate::node_identity::signed_node_request_json;
pub(crate) fn poll_task_cancellation(
session: &mut CoordinatorSession,
args: &Args,
task: &RuntimeTask,
node_private_key: &str,
) -> Result<bool, Box<dyn std::error::Error>> {
let deadline = Instant::now() + Duration::from_millis(args.control_poll_ms);
loop {
let control = session.request(signed_node_request_json(
args,
node_private_key,
"poll_task_control",
json!({
"type": "poll_task_control",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
}),
)?)?;
if control
.get("cancel_requested")
.and_then(Value::as_bool)
.unwrap_or(false)
|| control
.get("abort_requested")
.and_then(Value::as_bool)
.unwrap_or(false)
{
return Ok(true);
}
let now = Instant::now();
if now >= deadline {
return Ok(false);
}
std::thread::sleep((deadline - now).min(Duration::from_millis(100)));
}
}

1032
crates/clusterflux-node/src/lib.rs Normal file
View file

File diff suppressed because it is too large Load diff

12
crates/clusterflux-node/src/main.rs Normal file
View file

@ -0,0 +1,12 @@
mod assignment_runner;
mod coordinator_session;
mod daemon;
mod debug_agent;
mod node_identity;
mod source_snapshot;
mod task_artifacts;
mod task_reports;
fn main() -> Result<(), Box<dyn std::error::Error>> {
daemon::run()
}

224
crates/clusterflux-node/src/node_identity.rs Normal file
View file

@ -0,0 +1,224 @@
use std::path::{Path, PathBuf};
use clusterflux_core::{
generate_ed25519_private_key, node_ed25519_public_key_from_private_key, sign_node_request,
signed_request_payload_digest, NodeId,
};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::coordinator_session::CoordinatorSession;
use crate::daemon::Args;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
struct StoredNodeCredential {
kind: String,
node: String,
private_key: String,
public_key: String,
credential_scope: String,
}
pub(crate) fn establish_node_identity(
session: &mut CoordinatorSession,
args: &Args,
node_private_key: &str,
) -> Result<Value, Box<dyn std::error::Error>> {
let derived_public_key =
node_ed25519_public_key_from_private_key(node_private_key).map_err(invalid_key_error)?;
let public_key = args
.public_key
.clone()
.unwrap_or(derived_public_key.clone());
if public_key != derived_public_key {
return Err(
"node --public-key must match CLUSTERFLUX_NODE_PRIVATE_KEY or the stored local node credential"
.into(),
);
}
if let Some(grant) = &args.enrollment_grant {
session.request(json!({
"type": "exchange_node_enrollment_grant",
"tenant": &args.tenant,
"project": &args.project,
"node": &args.node,
"public_key": public_key,
"enrollment_grant": grant,
}))
} else {
Ok(json!({
"type": "node_identity_reused",
"node": &args.node,
"credential_source": "stored_project_node_identity",
"subsequent_authentication": "signed_node_requests",
}))
}
}
pub(crate) fn node_private_key_for_runtime(
node: &str,
) -> Result<String, Box<dyn std::error::Error>> {
if let Ok(private_key) = std::env::var("CLUSTERFLUX_NODE_PRIVATE_KEY") {
return Ok(private_key);
}
load_or_create_local_node_credential(&std::env::current_dir()?, node)
}
pub(crate) fn load_or_create_local_node_credential(
project: &Path,
node: &str,
) -> Result<String, Box<dyn std::error::Error>> {
let file = local_node_credential_file(project, node);
if credential_file_exists_without_symlink(&file)? {
let bytes = std::fs::read(&file)?;
let credential: StoredNodeCredential = serde_json::from_slice(&bytes)?;
if credential.node != node {
return Err(format!(
"stored node credential {} belongs to node `{}` instead of `{}`",
file.display(),
credential.node,
node
)
.into());
}
let public_key = node_ed25519_public_key_from_private_key(&credential.private_key)
.map_err(invalid_key_error)?;
if public_key != credential.public_key {
return Err(format!(
"stored node credential {} has a public key that does not match its private key",
file.display()
)
.into());
}
return Ok(credential.private_key);
}
let private_key = generate_ed25519_private_key().map_err(invalid_key_error)?;
let public_key =
node_ed25519_public_key_from_private_key(&private_key).map_err(invalid_key_error)?;
let credential = StoredNodeCredential {
kind: "clusterflux_node_credential".to_owned(),
node: node.to_owned(),
private_key: private_key.clone(),
public_key,
credential_scope: "local_project_node_identity".to_owned(),
};
persist_node_credential(&file, &credential)?;
Ok(private_key)
}
fn credential_file_exists_without_symlink(file: &Path) -> Result<bool, Box<dyn std::error::Error>> {
match std::fs::symlink_metadata(file) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(format!(
"refusing to read node credential through symbolic link {}",
file.display()
)
.into()),
Ok(metadata) if !metadata.is_file() => Err(format!(
"node credential path {} is not a regular file",
file.display()
)
.into()),
Ok(_) => Ok(true),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(error.into()),
}
}
fn persist_node_credential(
file: &Path,
credential: &StoredNodeCredential,
) -> Result<(), Box<dyn std::error::Error>> {
use std::io::Write;
let parent = file
.parent()
.ok_or_else(|| format!("node credential path {} has no parent", file.display()))?;
std::fs::create_dir_all(parent)?;
if std::fs::symlink_metadata(parent)?.file_type().is_symlink() {
return Err(format!(
"refusing to store node credential through symbolic-link directory {}",
parent.display()
)
.into());
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700))?;
}
let mut temporary = tempfile::NamedTempFile::new_in(parent)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
temporary
.as_file()
.set_permissions(std::fs::Permissions::from_mode(0o600))?;
}
temporary.write_all(&serde_json::to_vec_pretty(credential)?)?;
temporary.as_file().sync_all()?;
temporary.persist_noclobber(file).map_err(|error| {
format!(
"refusing to overwrite node credential {}: {}",
file.display(),
error.error
)
})?;
Ok(())
}
pub(crate) fn local_node_credential_file(project: &Path, node: &str) -> PathBuf {
let digest = clusterflux_core::Digest::sha256(node);
let file_stem = digest.as_str().trim_start_matches("sha256:");
project
.join(".clusterflux")
.join("nodes")
.join(format!("{file_stem}.json"))
}
pub(crate) fn node_nonce(prefix: &str) -> String {
format!("{prefix}-{}-{}", unix_timestamp_nanos(), std::process::id())
}
pub(crate) fn unix_timestamp_seconds() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_secs())
.unwrap_or(0)
}
pub(crate) fn signed_node_request_json(
args: &Args,
node_private_key: &str,
request_kind: &str,
request: Value,
) -> Result<Value, Box<dyn std::error::Error>> {
let payload_digest = signed_request_payload_digest(&request);
let node_signature = sign_node_request(
node_private_key,
&NodeId::from(args.node.as_str()),
request_kind,
&payload_digest,
node_nonce(request_kind),
unix_timestamp_seconds(),
)
.map_err(invalid_key_error)?;
Ok(json!({
"type": "signed_node",
"node": &args.node,
"node_signature": node_signature,
"request": request,
}))
}
fn invalid_key_error(error: String) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::InvalidInput, error)
}
pub(crate) fn unix_timestamp_nanos() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or(0)
}

424
crates/clusterflux-node/src/source_snapshot.rs Normal file
View file

@ -0,0 +1,424 @@
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;
use clusterflux_core::Digest;
use sha2::{Digest as _, Sha256};
const MAX_SNAPSHOT_FILES: usize = 20_000;
const MAX_SNAPSHOT_FILE_BYTES: u64 = 256 * 1024 * 1024;
const MAX_SNAPSHOT_TOTAL_BYTES: u64 = 1024 * 1024 * 1024;
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct SourceSnapshotInventory {
pub(crate) digest: Digest,
pub(crate) provider: &'static str,
pub(crate) file_count: usize,
pub(crate) total_bytes: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct SnapshotFile {
path: String,
kind: &'static str,
executable: bool,
digest: Digest,
size_bytes: u64,
}
pub(crate) fn snapshot_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
let project_root = project_root
.canonicalize()
.map_err(|error| format!("resolve source checkout: {error}"))?;
if !project_root.is_dir() {
return Err("source checkout root is not a directory".to_owned());
}
if let Some(repository_root) = git_repository_root(&project_root)? {
snapshot_git_project(&repository_root, &project_root)
} else {
snapshot_filesystem_project(&project_root)
}
}
fn snapshot_git_project(
repository_root: &Path,
project_root: &Path,
) -> Result<SourceSnapshotInventory, String> {
let project_prefix = project_root
.strip_prefix(repository_root)
.map_err(|_| "source checkout is outside its discovered Git repository".to_owned())?;
let project_prefix = if project_prefix.as_os_str().is_empty() {
".".to_owned()
} else {
slash_path(project_prefix)?
};
let mut command = git_command(repository_root);
command
.args([
"ls-files",
"-z",
"--cached",
"--others",
"--exclude-standard",
"--",
])
.arg(&project_prefix);
let output = command
.output()
.map_err(|error| format!("enumerate Git source snapshot: {error}"))?;
if !output.status.success() {
return Err(format!(
"enumerate Git source snapshot failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
));
}
let mut files = Vec::new();
for raw in output
.stdout
.split(|byte| *byte == 0)
.filter(|raw| !raw.is_empty())
{
let repository_relative =
std::str::from_utf8(raw).map_err(|_| "Git source path is not UTF-8".to_owned())?;
let absolute = repository_root.join(repository_relative);
let relative = absolute
.strip_prefix(project_root)
.map_err(|_| "Git enumerated source outside the selected project".to_owned())?;
files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
}
files.sort_by(|left, right| left.path.cmp(&right.path));
files.dedup_by(|left, right| left.path == right.path);
validate_snapshot_bounds(&files)?;
let head =
git_text(repository_root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unborn".to_owned());
let roots = git_text(repository_root, &["rev-list", "--max-parents=0", "HEAD"])
.unwrap_or_else(|| "unborn".to_owned());
let remote = git_text(repository_root, &["config", "--get", "remote.origin.url"])
.unwrap_or_else(|| "no-origin".to_owned());
let submodules = git_text(
repository_root,
&["submodule", "status", "--recursive", "--", &project_prefix],
)
.unwrap_or_else(|| "no-submodules".to_owned());
finish_snapshot(
"git",
[
b"node-source-snapshot:git:v1".to_vec(),
head.into_bytes(),
roots.into_bytes(),
remote.into_bytes(),
submodules.into_bytes(),
project_prefix.into_bytes(),
],
files,
)
}
fn snapshot_filesystem_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
let mut paths = Vec::new();
collect_filesystem_paths(project_root, project_root, &mut paths)?;
paths.sort();
if paths.len() > MAX_SNAPSHOT_FILES {
return Err(format!(
"source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
paths.len()
));
}
let mut files = Vec::with_capacity(paths.len());
for absolute in paths {
let relative = absolute
.strip_prefix(project_root)
.map_err(|_| "filesystem source escaped its project root".to_owned())?;
files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
}
validate_snapshot_bounds(&files)?;
finish_snapshot(
"filesystem",
[b"node-source-snapshot:filesystem:v1".to_vec()],
files,
)
}
fn finish_snapshot<const N: usize>(
provider: &'static str,
identity_parts: [Vec<u8>; N],
files: Vec<SnapshotFile>,
) -> Result<SourceSnapshotInventory, String> {
let mut parts = identity_parts.into_iter().collect::<Vec<_>>();
let mut total_bytes = 0_u64;
for file in &files {
total_bytes = total_bytes
.checked_add(file.size_bytes)
.ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
parts.push(file.path.as_bytes().to_vec());
parts.push(file.kind.as_bytes().to_vec());
parts.push(if file.executable { b"x" } else { b"-" }.to_vec());
parts.push(file.digest.as_str().as_bytes().to_vec());
parts.push(file.size_bytes.to_string().into_bytes());
}
Ok(SourceSnapshotInventory {
digest: Digest::from_parts(parts),
provider,
file_count: files.len(),
total_bytes,
})
}
fn snapshot_file(absolute: &Path, relative: &str) -> Result<SnapshotFile, String> {
validate_relative_source_path(relative)?;
let metadata = match fs::symlink_metadata(absolute) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(SnapshotFile {
path: relative.to_owned(),
kind: "deleted",
executable: false,
digest: Digest::sha256("deleted"),
size_bytes: 0,
})
}
Err(error) => return Err(error.to_string()),
};
if metadata.file_type().is_symlink() {
let target = fs::read_link(absolute).map_err(|error| error.to_string())?;
let target = target
.to_str()
.ok_or_else(|| "source symlink target is not UTF-8".to_owned())?;
return Ok(SnapshotFile {
path: relative.to_owned(),
kind: "symlink",
executable: false,
digest: Digest::from_parts([b"source-symlink:v1".as_slice(), target.as_bytes()]),
size_bytes: target.len() as u64,
});
}
if !metadata.is_file() {
return Err(format!("source snapshot entry `{relative}` is not a file"));
}
if metadata.len() > MAX_SNAPSHOT_FILE_BYTES {
return Err(format!(
"source file `{relative}` is {} bytes; per-file limit is {MAX_SNAPSHOT_FILE_BYTES}",
metadata.len()
));
}
let mut file = fs::File::open(absolute).map_err(|error| error.to_string())?;
let mut hasher = Sha256::new();
let mut size_bytes = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = file.read(&mut buffer).map_err(|error| error.to_string())?;
if read == 0 {
break;
}
size_bytes = size_bytes
.checked_add(read as u64)
.ok_or_else(|| "source file size overflowed".to_owned())?;
if size_bytes > MAX_SNAPSHOT_FILE_BYTES {
return Err(format!(
"source file `{relative}` grew beyond {MAX_SNAPSHOT_FILE_BYTES} bytes while hashing"
));
}
hasher.update(&buffer[..read]);
}
let digest = Digest::from_sha256_hex(format!("{:x}", hasher.finalize()))?;
#[cfg(unix)]
let executable = {
use std::os::unix::fs::PermissionsExt;
metadata.permissions().mode() & 0o111 != 0
};
#[cfg(not(unix))]
let executable = false;
Ok(SnapshotFile {
path: relative.to_owned(),
kind: "file",
executable,
digest,
size_bytes,
})
}
fn validate_snapshot_bounds(files: &[SnapshotFile]) -> Result<(), String> {
if files.len() > MAX_SNAPSHOT_FILES {
return Err(format!(
"source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
files.len()
));
}
let total = files
.iter()
.try_fold(0_u64, |total, file| total.checked_add(file.size_bytes))
.ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
if total > MAX_SNAPSHOT_TOTAL_BYTES {
return Err(format!(
"source snapshot is {total} bytes; limit is {MAX_SNAPSHOT_TOTAL_BYTES}"
));
}
Ok(())
}
fn collect_filesystem_paths(
root: &Path,
directory: &Path,
paths: &mut Vec<PathBuf>,
) -> Result<(), String> {
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let name = entry
.file_name()
.into_string()
.map_err(|_| "source path is not UTF-8".to_owned())?;
if directory == root && matches!(name.as_str(), ".git" | ".clusterflux" | "target") {
continue;
}
let file_type = entry.file_type().map_err(|error| error.to_string())?;
if file_type.is_dir() {
collect_filesystem_paths(root, &entry.path(), paths)?;
} else if file_type.is_file() || file_type.is_symlink() {
paths.push(entry.path());
if paths.len() > MAX_SNAPSHOT_FILES {
return Err(format!(
"source snapshot exceeds file-count limit of {MAX_SNAPSHOT_FILES}"
));
}
}
}
Ok(())
}
fn git_repository_root(project_root: &Path) -> Result<Option<PathBuf>, String> {
let output = git_command(project_root)
.args(["rev-parse", "--show-toplevel"])
.output();
let Ok(output) = output else {
return Ok(None);
};
if !output.status.success() {
return Ok(None);
}
let root = String::from_utf8(output.stdout)
.map_err(|_| "Git repository root is not UTF-8".to_owned())?;
let root = PathBuf::from(root.trim())
.canonicalize()
.map_err(|error| error.to_string())?;
Ok(Some(root))
}
fn git_text(repository_root: &Path, args: &[&str]) -> Option<String> {
let output = git_command(repository_root).args(args).output().ok()?;
if !output.status.success() {
return None;
}
String::from_utf8(output.stdout)
.ok()
.map(|value| value.trim().to_owned())
}
fn git_command(repository_root: &Path) -> Command {
let mut command = Command::new("git");
command
.arg("-C")
.arg(repository_root)
.env("GIT_OPTIONAL_LOCKS", "0")
.env("GIT_TERMINAL_PROMPT", "0");
command
}
fn slash_path(path: &Path) -> Result<String, String> {
let mut result = String::new();
for component in path.components() {
let component = component
.as_os_str()
.to_str()
.ok_or_else(|| "source path is not UTF-8".to_owned())?;
if !result.is_empty() {
result.push('/');
}
result.push_str(component);
}
Ok(result)
}
fn validate_relative_source_path(path: &str) -> Result<(), String> {
if path.is_empty()
|| path.starts_with('/')
|| path.split('/').any(|component| {
component.is_empty()
|| component == "."
|| component == ".."
|| component.contains('\0')
})
{
return Err("source snapshot path must be a safe relative path".to_owned());
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn git(root: &Path, args: &[&str]) {
let output = Command::new("git")
.arg("-C")
.arg(root)
.args(args)
.env("GIT_AUTHOR_NAME", "Clusterflux Test")
.env("GIT_AUTHOR_EMAIL", "test@clusterflux.invalid")
.env("GIT_COMMITTER_NAME", "Clusterflux Test")
.env("GIT_COMMITTER_EMAIL", "test@clusterflux.invalid")
.output()
.unwrap();
assert!(
output.status.success(),
"git {args:?} failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
fn initialize_repository(root: &Path, message: &str) {
git(root, &["init", "--quiet"]);
git(root, &["add", "."]);
git(root, &["commit", "--quiet", "-m", message]);
}
#[test]
fn dirty_and_untracked_content_changes_snapshot_identity() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 1 }").unwrap();
initialize_repository(temp.path(), "initial source");
let first = snapshot_project(temp.path()).unwrap();
fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 2 }").unwrap();
let dirty = snapshot_project(temp.path()).unwrap();
fs::write(
temp.path().join("fixture.c"),
"int main(void) { return 0; }",
)
.unwrap();
let untracked = snapshot_project(temp.path()).unwrap();
assert_ne!(first.digest, dirty.digest);
assert_ne!(dirty.digest, untracked.digest);
assert_eq!(untracked.provider, "git");
assert_eq!(untracked.file_count, 2);
}
#[test]
fn different_repositories_at_the_same_path_are_not_identified_by_path() {
let temp = tempfile::tempdir().unwrap();
fs::write(temp.path().join("source.txt"), "identical source content").unwrap();
initialize_repository(temp.path(), "first repository identity");
let first = snapshot_project(temp.path()).unwrap();
fs::remove_dir_all(temp.path().join(".git")).unwrap();
initialize_repository(temp.path(), "second repository identity");
let second = snapshot_project(temp.path()).unwrap();
assert_ne!(first.digest, second.digest);
assert_eq!(first.provider, "git");
assert_eq!(second.provider, "git");
}
}

987
crates/clusterflux-node/src/task_artifacts.rs Normal file
View file

@ -0,0 +1,987 @@
use std::collections::BTreeSet;
use std::fs::{self, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{ArtifactId, Digest, NodeId, TaskInstanceId, VfsManifest, VfsOverlay};
use sha2::{Digest as _, Sha256};
const MAX_NODE_ARTIFACT_BYTES: u64 = 256 * 1024 * 1024;
const MAX_NODE_ARTIFACT_CHUNK_BYTES: u64 = 256 * 1024;
const DEFAULT_MAX_NODE_ARTIFACT_STORE_BYTES: u64 = 4 * 1024 * 1024 * 1024;
const DEFAULT_MAX_NODE_ARTIFACT_COUNT: usize = 1_024;
const DEFAULT_MAX_NODE_ARTIFACT_AGE_SECONDS: u64 = 7 * 24 * 60 * 60;
const DEFAULT_NODE_ARTIFACT_RESTART_PIN_SECONDS: u64 = 60 * 60;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct NodeArtifactRetentionLimits {
pub(crate) max_bytes: u64,
pub(crate) max_count: usize,
pub(crate) max_age_seconds: u64,
pub(crate) restart_pin_seconds: u64,
}
impl Default for NodeArtifactRetentionLimits {
fn default() -> Self {
Self {
max_bytes: DEFAULT_MAX_NODE_ARTIFACT_STORE_BYTES,
max_count: DEFAULT_MAX_NODE_ARTIFACT_COUNT,
max_age_seconds: DEFAULT_MAX_NODE_ARTIFACT_AGE_SECONDS,
restart_pin_seconds: DEFAULT_NODE_ARTIFACT_RESTART_PIN_SECONDS,
}
}
}
impl NodeArtifactRetentionLimits {
pub(crate) fn from_environment() -> Result<Self, String> {
Ok(Self {
max_bytes: environment_u64(
"CLUSTERFLUX_NODE_ARTIFACT_MAX_BYTES",
DEFAULT_MAX_NODE_ARTIFACT_STORE_BYTES,
)?,
max_count: usize::try_from(environment_u64(
"CLUSTERFLUX_NODE_ARTIFACT_MAX_COUNT",
DEFAULT_MAX_NODE_ARTIFACT_COUNT as u64,
)?)
.map_err(|_| "CLUSTERFLUX_NODE_ARTIFACT_MAX_COUNT does not fit usize".to_owned())?,
max_age_seconds: environment_u64(
"CLUSTERFLUX_NODE_ARTIFACT_MAX_AGE_SECONDS",
DEFAULT_MAX_NODE_ARTIFACT_AGE_SECONDS,
)?,
restart_pin_seconds: environment_u64(
"CLUSTERFLUX_NODE_ARTIFACT_RESTART_PIN_SECONDS",
DEFAULT_NODE_ARTIFACT_RESTART_PIN_SECONDS,
)?,
})
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct NodeArtifactGcReport {
pub(crate) retained_count: usize,
pub(crate) retained_bytes: u64,
pub(crate) evicted: Vec<ArtifactId>,
pub(crate) limits_satisfied: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct RetainedArtifact {
pub(crate) id: ArtifactId,
pub(crate) digest: Digest,
pub(crate) size_bytes: u64,
pub(crate) path: PathBuf,
}
#[derive(Clone, Debug)]
pub(crate) struct NodeArtifactStore {
root: PathBuf,
}
impl NodeArtifactStore {
pub(crate) fn for_runtime(project_root: Option<&Path>, node: &str) -> Result<Self, String> {
let base = project_root
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("."));
let node = node
.chars()
.map(|character| {
if character.is_ascii_alphanumeric() || "._-".contains(character) {
character
} else {
'_'
}
})
.collect::<String>();
let root = base.join(".clusterflux").join("node-artifacts").join(node);
if fs::symlink_metadata(&root)
.map(|metadata| metadata.file_type().is_symlink())
.unwrap_or(false)
{
return Err("node artifact store root must not be a symbolic link".to_owned());
}
fs::create_dir_all(&root).map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700))
.map_err(|error| error.to_string())?;
}
let root = root.canonicalize().map_err(|error| error.to_string())?;
Ok(Self { root })
}
pub(crate) fn retain_output_file(
&self,
output_root: &Path,
relative_path: &str,
) -> Result<RetainedArtifact, String> {
let source = confined_output_file(output_root, relative_path)?;
let name = relative_path.replace('/', "_");
validate_artifact_component(&name, "artifact name", 240)?;
let temporary = self.root.join(format!(
".flush.{}.{}.tmp",
std::process::id(),
NEXT_FILE_ID.fetch_add(1, Ordering::Relaxed)
));
let retained = (|| {
let mut input = open_regular_readonly(&source, "task output")?;
let metadata = input.metadata().map_err(|error| error.to_string())?;
if metadata.len() > MAX_NODE_ARTIFACT_BYTES {
return Err(format!(
"task output is {} bytes; node artifact limit is {MAX_NODE_ARTIFACT_BYTES} bytes",
metadata.len()
));
}
let mut output = create_new_private_file(&temporary)?;
let (digest, size) = copy_and_hash(&mut input, &mut output, MAX_NODE_ARTIFACT_BYTES)?;
output.sync_all().map_err(|error| error.to_string())?;
let digest_hex = digest
.as_str()
.strip_prefix("sha256:")
.ok_or_else(|| "task output digest is not SHA-256".to_owned())?;
let id = ArtifactId::new(format!("{name}-{digest_hex}"));
let path = self.root.join(id.as_str());
match fs::hard_link(&temporary, &path) {
Ok(()) => {
fs::remove_file(&temporary).map_err(|error| error.to_string())?;
Ok(RetainedArtifact {
id,
digest,
size_bytes: size,
path,
})
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
let existing = self
.metadata(&id)?
.ok_or_else(|| "retained artifact disappeared during flush".to_owned())?;
if existing.digest != digest || existing.size_bytes != size {
return Err("retained artifact path has conflicting bytes".to_owned());
}
Ok(existing)
}
Err(error) => Err(error.to_string()),
}
})();
let _ = fs::remove_file(&temporary);
retained
}
pub(crate) fn materialize_into_output(
&self,
artifact: &clusterflux_core::ArtifactHandle,
output_root: &Path,
relative_path: &str,
) -> Result<PathBuf, String> {
let retained = self
.metadata(&artifact.id)?
.ok_or_else(|| format!("retained artifact `{}` is unavailable", artifact.id))?;
if retained.digest != artifact.digest || retained.size_bytes != artifact.size_bytes {
return Err(
"artifact handle digest/size does not match retained node metadata".to_owned(),
);
}
let target = confined_output_target(output_root, relative_path)?;
let copied = (|| {
let mut output = create_new_private_file(&target)?;
let mut input = open_regular_readonly(&retained.path, "retained artifact")?;
let (digest, size_bytes) =
copy_and_hash(&mut input, &mut output, MAX_NODE_ARTIFACT_BYTES)?;
output.sync_all().map_err(|error| error.to_string())?;
Ok::<_, String>((digest, size_bytes))
})();
let (digest, size_bytes) = match copied {
Ok(copied) => copied,
Err(error) => {
let _ = fs::remove_file(&target);
return Err(error);
}
};
if size_bytes != artifact.size_bytes || digest != artifact.digest {
let _ = fs::remove_file(&target);
return Err("materialized artifact digest/size changed during local copy".to_owned());
}
Ok(target)
}
pub(crate) fn metadata(&self, id: &ArtifactId) -> Result<Option<RetainedArtifact>, String> {
validate_artifact_component(id.as_str(), "artifact id", 256)?;
let path = self.root.join(id.as_str());
let metadata = match fs::symlink_metadata(&path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error.to_string()),
};
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err("retained artifact path is not a regular non-symlink file".to_owned());
}
if metadata.len() > MAX_NODE_ARTIFACT_BYTES {
return Err(format!(
"retained artifact exceeds node artifact limit of {MAX_NODE_ARTIFACT_BYTES} bytes"
));
}
let mut file = open_regular_readonly(&path, "retained artifact")?;
let (digest, size_bytes) = hash_reader(&mut file, MAX_NODE_ARTIFACT_BYTES)?;
Ok(Some(RetainedArtifact {
id: id.clone(),
digest,
size_bytes,
path,
}))
}
#[cfg(test)]
pub(crate) fn read_verified(
&self,
id: &ArtifactId,
expected_digest: &Digest,
expected_size_bytes: u64,
) -> Result<Vec<u8>, String> {
let retained = self
.metadata(id)?
.ok_or_else(|| format!("retained artifact `{id}` no longer exists on this node"))?;
if &retained.digest != expected_digest || retained.size_bytes != expected_size_bytes {
return Err(format!(
"retained artifact `{id}` does not match coordinator digest/size metadata"
));
}
fs::read(retained.path).map_err(|error| error.to_string())
}
pub(crate) fn read_verified_chunk(
&self,
id: &ArtifactId,
expected_digest: &Digest,
expected_size_bytes: u64,
offset: u64,
max_chunk_bytes: u64,
) -> Result<Vec<u8>, String> {
validate_artifact_component(id.as_str(), "artifact id", 256)?;
let digest_hex = expected_digest
.as_str()
.strip_prefix("sha256:")
.ok_or_else(|| "artifact transfer expected digest is not SHA-256".to_owned())?;
if !id.as_str().ends_with(digest_hex) {
return Err(format!(
"retained artifact `{id}` is not the content-addressed object requested by the coordinator"
));
}
let path = self.root.join(id.as_str());
let mut file = open_regular_readonly(&path, "retained artifact")
.map_err(|_| format!("retained artifact `{id}` no longer exists on this node"))?;
let actual_size = file.metadata().map_err(|error| error.to_string())?.len();
if actual_size != expected_size_bytes {
return Err(format!(
"retained artifact `{id}` does not match coordinator size metadata"
));
}
if offset > actual_size {
return Err(format!(
"artifact transfer offset {offset} exceeds retained artifact size {actual_size}"
));
}
file.seek(SeekFrom::Start(offset))
.map_err(|error| error.to_string())?;
let length = max_chunk_bytes
.min(MAX_NODE_ARTIFACT_CHUNK_BYTES)
.min(actual_size.saturating_sub(offset));
let mut bytes = vec![0; usize::try_from(length).map_err(|error| error.to_string())?];
file.read_exact(&mut bytes)
.map_err(|error| error.to_string())?;
Ok(bytes)
}
pub(crate) fn garbage_collect(
&self,
limits: NodeArtifactRetentionLimits,
pinned: &BTreeSet<ArtifactId>,
now_epoch_seconds: u64,
) -> Result<NodeArtifactGcReport, String> {
let mut entries = Vec::new();
for entry in fs::read_dir(&self.root).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let name = entry
.file_name()
.into_string()
.map_err(|_| "retained artifact filename is not UTF-8".to_owned())?;
if name.starts_with('.') {
continue;
}
validate_artifact_component(&name, "artifact id", 256)?;
let metadata = entry.metadata().map_err(|error| error.to_string())?;
if !entry
.file_type()
.map_err(|error| error.to_string())?
.is_file()
|| !metadata.is_file()
{
continue;
}
entries.push(StoredArtifactEntry {
id: ArtifactId::new(name),
path: entry.path(),
size_bytes: metadata.len(),
modified_epoch_seconds: metadata
.modified()
.ok()
.and_then(system_time_epoch_seconds)
.unwrap_or_default(),
});
}
entries.sort_by(|left, right| {
left.modified_epoch_seconds
.cmp(&right.modified_epoch_seconds)
.then_with(|| left.id.cmp(&right.id))
});
let age_cutoff = now_epoch_seconds.saturating_sub(limits.max_age_seconds);
let mut retained_count = entries.len();
let mut retained_bytes = entries
.iter()
.try_fold(0_u64, |total, entry| total.checked_add(entry.size_bytes))
.ok_or_else(|| "node artifact byte accounting overflowed".to_owned())?;
let mut evicted = Vec::new();
for entry in &entries {
let expired = entry.modified_epoch_seconds <= age_cutoff;
let over_count = retained_count > limits.max_count;
let over_bytes = retained_bytes > limits.max_bytes;
if !(expired || over_count || over_bytes) || pinned.contains(&entry.id) {
continue;
}
match fs::remove_file(&entry.path) {
Ok(()) => {
retained_count = retained_count.saturating_sub(1);
retained_bytes = retained_bytes.saturating_sub(entry.size_bytes);
evicted.push(entry.id.clone());
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
retained_count = retained_count.saturating_sub(1);
retained_bytes = retained_bytes.saturating_sub(entry.size_bytes);
evicted.push(entry.id.clone());
}
Err(error) => return Err(error.to_string()),
}
}
let retained_ids = self.artifact_ids()?.into_iter().collect::<BTreeSet<_>>();
let retained_has_expired = entries.iter().any(|entry| {
retained_ids.contains(&entry.id) && entry.modified_epoch_seconds <= age_cutoff
});
Ok(NodeArtifactGcReport {
retained_count,
retained_bytes,
evicted,
limits_satisfied: retained_count <= limits.max_count
&& retained_bytes <= limits.max_bytes
&& !retained_has_expired,
})
}
pub(crate) fn artifact_ids(&self) -> Result<Vec<ArtifactId>, String> {
let mut ids = Vec::new();
for entry in fs::read_dir(&self.root).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
if !entry
.file_type()
.map_err(|error| error.to_string())?
.is_file()
{
continue;
}
let name = entry
.file_name()
.into_string()
.map_err(|_| "retained artifact filename is not UTF-8".to_owned())?;
if name.starts_with('.') {
continue;
}
validate_artifact_component(&name, "artifact id", 256)?;
ids.push(ArtifactId::new(name));
}
ids.sort();
Ok(ids)
}
}
#[derive(Clone, Debug)]
struct StoredArtifactEntry {
id: ArtifactId,
path: PathBuf,
size_bytes: u64,
modified_epoch_seconds: u64,
}
pub(crate) fn retained_result_artifact(
project_root: Option<&Path>,
node: &str,
result: Option<&clusterflux_core::TaskBoundaryValue>,
) -> Result<Option<RetainedArtifact>, String> {
let Some(clusterflux_core::TaskBoundaryValue::Artifact(handle)) = result else {
return Ok(None);
};
NodeArtifactStore::for_runtime(project_root, node)?
.metadata(&handle.id)?
.ok_or_else(|| {
format!(
"task returned artifact `{}` without flushing a file on this node",
handle.id
)
})
.and_then(|retained| {
if retained.digest != handle.digest || retained.size_bytes != handle.size_bytes {
return Err("task artifact handle does not match retained file metadata".to_owned());
}
Ok(Some(retained))
})
}
static NEXT_FILE_ID: AtomicU64 = AtomicU64::new(1);
static NEXT_OUTPUT_ROOT_ID: AtomicU64 = AtomicU64::new(1);
pub(crate) fn task_output_root(
project_root: Option<&Path>,
node: &str,
task: &TaskInstanceId,
) -> Result<PathBuf, String> {
let base = project_root
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("."));
let safe = |value: &str| {
value
.chars()
.map(|character| {
if character.is_ascii_alphanumeric() || "._-".contains(character) {
character
} else {
'_'
}
})
.collect::<String>()
};
let parent = base
.join(".clusterflux")
.join("task-outputs")
.join(safe(node));
fs::create_dir_all(&parent).map_err(|error| error.to_string())?;
let now_nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or_default();
let root = parent.join(format!(
"{}.{}.{}.{}",
safe(task.as_str()),
std::process::id(),
now_nanos,
NEXT_OUTPUT_ROOT_ID.fetch_add(1, Ordering::Relaxed)
));
fs::create_dir(&root).map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&root, fs::Permissions::from_mode(0o700))
.map_err(|error| error.to_string())?;
}
root.canonicalize().map_err(|error| error.to_string())
}
pub(crate) fn clean_stale_task_output_roots(
project_root: Option<&Path>,
node: &str,
) -> Result<usize, String> {
let base = project_root
.map(Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("."));
let node = node
.chars()
.map(|character| {
if character.is_ascii_alphanumeric() || "._-".contains(character) {
character
} else {
'_'
}
})
.collect::<String>();
let parent = base.join(".clusterflux").join("task-outputs").join(node);
let parent_metadata = match fs::symlink_metadata(&parent) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0),
Err(error) => return Err(error.to_string()),
};
if parent_metadata.file_type().is_symlink() || !parent_metadata.is_dir() {
return Err("task-output store root must be a non-symlink directory".to_owned());
}
let mut removed = 0;
for entry in fs::read_dir(&parent).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let file_type = entry.file_type().map_err(|error| error.to_string())?;
if file_type.is_dir() {
fs::remove_dir_all(entry.path()).map_err(|error| error.to_string())?;
} else {
fs::remove_file(entry.path()).map_err(|error| error.to_string())?;
}
removed += 1;
}
Ok(removed)
}
fn confined_output_file(output_root: &Path, relative_path: &str) -> Result<PathBuf, String> {
validate_relative_path(relative_path)?;
let root = output_root
.canonicalize()
.map_err(|error| error.to_string())?;
reject_symlink_components(&root, relative_path, true)?;
let path = root.join(relative_path);
let metadata = fs::symlink_metadata(&path).map_err(|error| error.to_string())?;
if metadata.file_type().is_symlink() {
return Err("task output publication rejects symbolic links".to_owned());
}
let canonical = path.canonicalize().map_err(|error| error.to_string())?;
if !canonical.starts_with(&root) {
return Err("task output path escapes its bounded output root".to_owned());
}
Ok(canonical)
}
fn confined_output_target(output_root: &Path, relative_path: &str) -> Result<PathBuf, String> {
validate_relative_path(relative_path)?;
let root = output_root
.canonicalize()
.map_err(|error| error.to_string())?;
let path = root.join(relative_path);
let parent = path
.parent()
.ok_or_else(|| "task output target has no parent".to_owned())?;
create_confined_directories(&root, relative_path)?;
reject_symlink_components(&root, relative_path, false)?;
let parent = parent.canonicalize().map_err(|error| error.to_string())?;
if !parent.starts_with(&root) {
return Err("task output target escapes its bounded output root".to_owned());
}
Ok(path)
}
fn create_confined_directories(root: &Path, relative_path: &str) -> Result<(), String> {
let components = relative_path.split('/').collect::<Vec<_>>();
let mut current = root.to_path_buf();
for component in components.iter().take(components.len().saturating_sub(1)) {
current.push(component);
match fs::symlink_metadata(&current) {
Ok(metadata) if metadata.file_type().is_symlink() => {
return Err("task output target rejects symbolic-link directories".to_owned())
}
Ok(metadata) if !metadata.is_dir() => {
return Err("task output target parent is not a directory".to_owned())
}
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
fs::create_dir(&current).map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&current, fs::Permissions::from_mode(0o700))
.map_err(|error| error.to_string())?;
}
}
Err(error) => return Err(error.to_string()),
}
}
Ok(())
}
fn reject_symlink_components(
root: &Path,
relative_path: &str,
include_leaf: bool,
) -> Result<(), String> {
let components = relative_path.split('/').collect::<Vec<_>>();
let count = if include_leaf {
components.len()
} else {
components.len().saturating_sub(1)
};
let mut current = root.to_path_buf();
for component in components.iter().take(count) {
current.push(component);
let metadata = fs::symlink_metadata(&current).map_err(|error| error.to_string())?;
if metadata.file_type().is_symlink() {
return Err("task output path rejects symbolic-link components".to_owned());
}
}
Ok(())
}
fn open_regular_readonly(path: &Path, label: &str) -> Result<fs::File, String> {
let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(format!("{label} must be a regular non-symlink file"));
}
let mut options = OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC);
}
let file = options.open(path).map_err(|error| error.to_string())?;
if !file
.metadata()
.map_err(|error| error.to_string())?
.is_file()
{
return Err(format!("{label} must remain a regular file while open"));
}
Ok(file)
}
fn create_new_private_file(path: &Path) -> Result<fs::File, String> {
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options
.mode(0o600)
.custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC);
}
options.open(path).map_err(|error| error.to_string())
}
fn hash_reader(reader: &mut impl Read, limit: u64) -> Result<(Digest, u64), String> {
let mut sink = std::io::sink();
copy_and_hash(reader, &mut sink, limit)
}
fn copy_and_hash(
reader: &mut impl Read,
writer: &mut impl Write,
limit: u64,
) -> Result<(Digest, u64), String> {
let mut hasher = Sha256::new();
let mut size_bytes = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let read = reader
.read(&mut buffer)
.map_err(|error| error.to_string())?;
if read == 0 {
break;
}
size_bytes = size_bytes
.checked_add(read as u64)
.ok_or_else(|| "artifact size accounting overflowed".to_owned())?;
if size_bytes > limit {
return Err(format!(
"artifact exceeds bounded size limit of {limit} bytes"
));
}
hasher.update(&buffer[..read]);
writer
.write_all(&buffer[..read])
.map_err(|error| error.to_string())?;
}
let digest_hex = format!("{:x}", hasher.finalize());
Ok((Digest::from_sha256_hex(&digest_hex)?, size_bytes))
}
fn environment_u64(name: &str, default: u64) -> Result<u64, String> {
match std::env::var(name) {
Ok(value) => value
.parse::<u64>()
.map_err(|_| format!("{name} must be an unsigned integer")),
Err(std::env::VarError::NotPresent) => Ok(default),
Err(error) => Err(format!("read {name}: {error}")),
}
}
fn system_time_epoch_seconds(time: SystemTime) -> Option<u64> {
time.duration_since(UNIX_EPOCH)
.ok()
.map(|duration| duration.as_secs())
}
pub(crate) fn current_epoch_seconds() -> u64 {
system_time_epoch_seconds(SystemTime::now()).unwrap_or_default()
}
fn validate_relative_path(path: &str) -> Result<(), String> {
if path.is_empty()
|| path.len() > 240
|| path.starts_with('/')
|| path.starts_with('\\')
|| path.split('/').any(|component| {
component.is_empty()
|| component == "."
|| component == ".."
|| !component
.chars()
.all(|character| character.is_ascii_alphanumeric() || "._-".contains(character))
})
{
return Err("task output path must be a safe relative path".to_owned());
}
Ok(())
}
fn validate_artifact_component(value: &str, label: &str, max_len: usize) -> Result<(), String> {
if value.trim().is_empty()
|| value.len() > max_len
|| !value
.chars()
.all(|character| character.is_ascii_alphanumeric() || "._-".contains(character))
{
return Err(format!("{label} is invalid"));
}
Ok(())
}
pub(crate) struct TaskArtifactStore {
overlay: VfsOverlay,
}
impl TaskArtifactStore {
pub(crate) fn new(task: TaskInstanceId, node: NodeId) -> Self {
Self {
overlay: VfsOverlay::new(task, node),
}
}
pub(crate) fn overlay_mut(&mut self) -> &mut VfsOverlay {
&mut self.overlay
}
pub(crate) fn flush(mut self) -> VfsManifest {
self.overlay.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn retain_fixture(
store: &NodeArtifactStore,
workspace: &Path,
name: &str,
bytes: &[u8],
) -> RetainedArtifact {
let output = workspace.join("output");
fs::create_dir_all(&output).unwrap();
let path = output.join(name);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, bytes).unwrap();
store.retain_output_file(&output, name).unwrap()
}
#[test]
fn node_artifact_store_retains_real_content_addressed_bytes() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node/a").unwrap();
let retained = retain_fixture(&store, temp.path(), "app.tar.zst", b"real artifact bytes");
assert_eq!(retained.digest, Digest::sha256(b"real artifact bytes"));
assert_eq!(retained.size_bytes, 19);
assert_eq!(fs::read(&retained.path).unwrap(), b"real artifact bytes");
assert_eq!(store.metadata(&retained.id).unwrap().unwrap(), retained);
}
#[test]
fn node_artifact_store_rejects_path_traversal() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let output = temp.path().join("output");
fs::create_dir_all(&output).unwrap();
assert!(store.retain_output_file(&output, "../escape").is_err());
assert!(store.metadata(&ArtifactId::from("../escape")).is_err());
}
#[test]
fn node_artifact_store_reads_only_digest_verified_bytes() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let retained = retain_fixture(&store, temp.path(), "result.bin", b"verified bytes");
assert_eq!(
store
.read_verified(&retained.id, &retained.digest, retained.size_bytes)
.unwrap(),
b"verified bytes"
);
assert!(store
.read_verified(&retained.id, &Digest::sha256("wrong"), retained.size_bytes)
.is_err());
assert_eq!(store.artifact_ids().unwrap(), vec![retained.id]);
}
#[test]
fn node_artifact_store_reads_bounded_transfer_chunks() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let retained = retain_fixture(&store, temp.path(), "large.bin", b"0123456789");
assert_eq!(
store
.read_verified_chunk(&retained.id, &retained.digest, retained.size_bytes, 3, 4)
.unwrap(),
b"3456"
);
assert!(store
.read_verified_chunk(&retained.id, &retained.digest, retained.size_bytes, 11, 4)
.is_err());
}
#[test]
fn downstream_materialization_copies_exact_digest_verified_bytes() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let retained = retain_fixture(&store, temp.path(), "build/app", b"real compiler output");
let consumer = temp.path().join("consumer");
fs::create_dir_all(&consumer).unwrap();
let handle = clusterflux_core::ArtifactHandle {
id: retained.id,
digest: retained.digest,
size_bytes: retained.size_bytes,
};
let path = store
.materialize_into_output(&handle, &consumer, "package/app")
.unwrap();
assert_eq!(path, consumer.join("package/app"));
assert_eq!(fs::read(path).unwrap(), b"real compiler output");
}
#[test]
fn materialization_rejects_a_corrupt_handle_and_removes_partial_output() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let retained = retain_fixture(&store, temp.path(), "app", b"artifact");
let consumer = temp.path().join("consumer");
fs::create_dir_all(&consumer).unwrap();
let handle = clusterflux_core::ArtifactHandle {
id: retained.id,
digest: Digest::sha256("forged"),
size_bytes: retained.size_bytes,
};
assert!(store
.materialize_into_output(&handle, &consumer, "app")
.is_err());
assert!(!consumer.join("app").exists());
}
#[cfg(unix)]
#[test]
fn publication_and_materialization_reject_symlink_components() {
use std::os::unix::fs::symlink;
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let output = temp.path().join("output");
let outside = temp.path().join("outside");
fs::create_dir_all(&output).unwrap();
fs::create_dir_all(&outside).unwrap();
fs::write(outside.join("secret"), b"outside").unwrap();
symlink(outside.join("secret"), output.join("linked-file")).unwrap();
symlink(&outside, output.join("linked-dir")).unwrap();
assert!(store.retain_output_file(&output, "linked-file").is_err());
assert!(store
.retain_output_file(&output, "linked-dir/secret")
.is_err());
let retained = retain_fixture(&store, temp.path(), "safe", b"safe");
let handle = clusterflux_core::ArtifactHandle {
id: retained.id,
digest: retained.digest,
size_bytes: retained.size_bytes,
};
assert!(store
.materialize_into_output(&handle, &output, "linked-dir/copied")
.is_err());
assert!(!outside.join("copied").exists());
}
#[test]
fn transfer_chunks_are_hard_bounded_by_the_node() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let bytes = vec![7_u8; (MAX_NODE_ARTIFACT_CHUNK_BYTES + 10) as usize];
let retained = retain_fixture(&store, temp.path(), "large", &bytes);
let chunk = store
.read_verified_chunk(
&retained.id,
&retained.digest,
retained.size_bytes,
0,
u64::MAX,
)
.unwrap();
assert_eq!(chunk.len() as u64, MAX_NODE_ARTIFACT_CHUNK_BYTES);
}
#[test]
fn garbage_collection_obeys_count_bytes_age_and_pins() {
let temp = tempfile::tempdir().unwrap();
let store = NodeArtifactStore::for_runtime(Some(temp.path()), "node-a").unwrap();
let pinned = retain_fixture(&store, temp.path(), "pinned", b"12345");
let disposable = retain_fixture(&store, temp.path(), "disposable", b"67890");
let pins = BTreeSet::from([pinned.id.clone()]);
let report = store
.garbage_collect(
NodeArtifactRetentionLimits {
max_bytes: 5,
max_count: 1,
max_age_seconds: u64::MAX,
restart_pin_seconds: 60,
},
&pins,
current_epoch_seconds(),
)
.unwrap();
assert_eq!(report.evicted, vec![disposable.id]);
assert!(report.limits_satisfied);
assert_eq!(store.artifact_ids().unwrap(), vec![pinned.id.clone()]);
let pinned_over_limit = store
.garbage_collect(
NodeArtifactRetentionLimits {
max_bytes: 0,
max_count: 0,
max_age_seconds: 0,
restart_pin_seconds: 60,
},
&pins,
current_epoch_seconds().saturating_add(1),
)
.unwrap();
assert!(pinned_over_limit.evicted.is_empty());
assert!(!pinned_over_limit.limits_satisfied);
assert!(pinned.path.exists());
}
#[test]
fn task_output_roots_are_unique_and_stale_roots_are_cleaned_on_node_start() {
let temp = tempfile::tempdir().unwrap();
let task = TaskInstanceId::from("task:1");
let first = task_output_root(Some(temp.path()), "node-a", &task).unwrap();
let second = task_output_root(Some(temp.path()), "node-a", &task).unwrap();
fs::write(first.join("stale"), b"one").unwrap();
fs::write(second.join("stale"), b"two").unwrap();
assert_ne!(first, second);
assert_eq!(
clean_stale_task_output_roots(Some(temp.path()), "node-a").unwrap(),
2
);
assert!(!first.exists());
assert!(!second.exists());
}
}

379
crates/clusterflux-node/src/task_reports.rs Normal file
View file

@ -0,0 +1,379 @@
use clusterflux_core::{TaskBoundaryValue, VfsManifest};
use clusterflux_node::CommandOutput;
use serde_json::{json, Value};
use crate::daemon::RuntimeTask;
use crate::{
coordinator_session::CoordinatorSession, daemon::Args, node_identity::signed_node_request_json,
task_artifacts::RetainedArtifact,
};
#[allow(clippy::too_many_arguments)]
pub(crate) fn record_completed_task(
args: &Args,
session: &mut CoordinatorSession,
task: RuntimeTask,
output: CommandOutput,
manifest: VfsManifest,
result: Option<TaskBoundaryValue>,
retained: Option<RetainedArtifact>,
registration: Value,
heartbeat: Value,
capability_report: Value,
debug_command: Value,
node_private_key: &str,
) -> Result<Value, Box<dyn std::error::Error>> {
let staged = output.staged_artifact.as_ref();
let artifact_digest = retained
.as_ref()
.map(|artifact| artifact.digest.clone())
.or_else(|| staged.map(|artifact| artifact.digest.clone()));
let artifact_path = retained
.as_ref()
.map(|artifact| format!("/vfs/artifacts/{}", artifact.id))
.or_else(|| staged.map(|artifact| artifact.path.as_str().to_owned()));
let artifact_size_bytes = retained
.as_ref()
.map(|artifact| artifact.size_bytes)
.or_else(|| staged.map(|artifact| artifact.size));
let log_event = session.request(signed_node_request_json(
args,
node_private_key,
"report_task_log",
json!({
"type": "report_task_log",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
"stderr_truncated": output.stderr_truncated,
"backpressured": output.log_backpressured,
}),
)?)?;
let vfs_metadata = session.request(signed_node_request_json(
args,
node_private_key,
"report_vfs_metadata",
json!({
"type": "report_vfs_metadata",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"artifact_path": artifact_path,
"artifact_digest": artifact_digest,
"artifact_size_bytes": artifact_size_bytes,
"large_bytes_uploaded": manifest.large_bytes_uploaded,
}),
)?)?;
let recorded = session.request(signed_node_request_json(
args,
node_private_key,
"task_completed",
json!({
"type": "task_completed",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"status_code": output.status_code,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
"stderr_truncated": output.stderr_truncated,
"artifact_path": artifact_path,
"artifact_digest": artifact_digest,
"artifact_size_bytes": artifact_size_bytes,
"result": result,
}),
)?)?;
Ok(completed_node_report(
output,
manifest.large_bytes_uploaded,
registration,
heartbeat,
capability_report,
task.task_assignment_response,
debug_command,
log_event,
vfs_metadata,
recorded,
session.requests(),
))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn record_failed_task(
args: &Args,
session: &mut CoordinatorSession,
task: &RuntimeTask,
registration: Value,
heartbeat: Value,
capability_report: Value,
debug_command: Value,
node_private_key: &str,
error: &str,
) -> Result<Value, Box<dyn std::error::Error>> {
let error = bounded_runtime_error(error);
let log_event = session.request(signed_node_request_json(
args,
node_private_key,
"report_task_log",
json!({
"type": "report_task_log",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_tail": "",
"stderr_tail": &error,
"stdout_truncated": false,
"stderr_truncated": false,
"backpressured": false,
}),
)?)?;
let vfs_metadata = session.request(signed_node_request_json(
args,
node_private_key,
"report_vfs_metadata",
json!({
"type": "report_vfs_metadata",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"artifact_path": null,
"artifact_digest": null,
"artifact_size_bytes": null,
"large_bytes_uploaded": false,
}),
)?)?;
let recorded = session.request(signed_node_request_json(
args,
node_private_key,
"task_completed",
json!({
"type": "task_completed",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"terminal_state": "failed",
"status_code": -1,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_tail": "",
"stderr_tail": &error,
"stdout_truncated": false,
"stderr_truncated": false,
"artifact_path": null,
"artifact_digest": null,
"artifact_size_bytes": null,
}),
)?)?;
Ok(failed_node_report(
task,
&error,
registration,
heartbeat,
capability_report,
debug_command,
log_event,
vfs_metadata,
recorded,
session.requests(),
))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn record_cancelled_task(
args: &Args,
session: &mut CoordinatorSession,
task: &RuntimeTask,
registration: Value,
heartbeat: Value,
capability_report: Value,
debug_command: Value,
node_private_key: &str,
) -> Result<Value, Box<dyn std::error::Error>> {
let recorded = session.request(signed_node_request_json(
args,
node_private_key,
"task_completed",
json!({
"type": "task_completed",
"tenant": &args.tenant,
"project": &args.project,
"process": &task.process,
"node": &args.node,
"task": &task.task,
"terminal_state": "cancelled",
"status_code": null,
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout_tail": "",
"stderr_tail": "",
"stdout_truncated": false,
"stderr_truncated": false,
"artifact_path": null,
"artifact_digest": null,
"artifact_size_bytes": null,
"result": null,
}),
)?)?;
Ok(cancelled_node_report(
task,
registration,
heartbeat,
capability_report,
task.task_assignment_response.clone(),
debug_command,
recorded,
session.requests(),
))
}
fn bounded_runtime_error(error: &str) -> String {
const MAX_BYTES: usize = 16 * 1024;
if error.len() <= MAX_BYTES {
return error.to_owned();
}
let boundary = error
.char_indices()
.take_while(|(index, _)| *index <= MAX_BYTES)
.map(|(index, _)| index)
.last()
.unwrap_or(0);
format!("{}\n<truncated>", &error[..boundary])
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn completed_node_report(
output: CommandOutput,
large_bytes_uploaded: bool,
registration_response: Value,
heartbeat_response: Value,
capability_response: Value,
task_assignment_response: Value,
debug_command_response: Value,
log_event_response: Value,
vfs_metadata_response: Value,
coordinator_response: Value,
session_requests: usize,
) -> Value {
json!({
"node_status": "completed",
"virtual_thread": output.virtual_thread,
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
"status_code": output.status_code,
"stdout_bytes": output.stdout.len(),
"stderr_bytes": output.stderr.len(),
"stdout_tail": &output.stdout,
"stderr_tail": &output.stderr,
"stdout_truncated": output.stdout_truncated,
"stderr_truncated": output.stderr_truncated,
"log_backpressured": output.log_backpressured,
"staged_artifact": output.staged_artifact,
"large_bytes_uploaded": large_bytes_uploaded,
"registration_response": registration_response,
"heartbeat_response": heartbeat_response,
"capability_response": capability_response,
"task_assignment_response": task_assignment_response,
"debug_command_response": debug_command_response,
"log_event_response": log_event_response,
"vfs_metadata_response": vfs_metadata_response,
"session_requests": session_requests,
"coordinator_response": coordinator_response,
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn cancelled_node_report(
task: &RuntimeTask,
registration_response: Value,
heartbeat_response: Value,
capability_response: Value,
task_assignment_response: Value,
debug_command_response: Value,
coordinator_response: Value,
session_requests: usize,
) -> Value {
json!({
"node_status": "cancelled",
"virtual_thread": &task.task,
"terminal_state": "cancelled",
"status_code": null,
"stdout_bytes": 0,
"stderr_bytes": 0,
"stdout_tail": "",
"stderr_tail": "",
"stdout_truncated": false,
"stderr_truncated": false,
"log_backpressured": false,
"staged_artifact": null,
"large_bytes_uploaded": false,
"registration_response": registration_response,
"heartbeat_response": heartbeat_response,
"capability_response": capability_response,
"task_assignment_response": task_assignment_response,
"debug_command_response": debug_command_response,
"log_event_response": null,
"vfs_metadata_response": null,
"session_requests": session_requests,
"coordinator_response": coordinator_response,
})
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn failed_node_report(
task: &RuntimeTask,
error: &str,
registration_response: Value,
heartbeat_response: Value,
capability_response: Value,
debug_command_response: Value,
log_event_response: Value,
vfs_metadata_response: Value,
coordinator_response: Value,
session_requests: usize,
) -> Value {
json!({
"node_status": "failed",
"virtual_thread": &task.task,
"terminal_state": "failed",
"status_code": -1,
"stdout_bytes": 0,
"stderr_bytes": error.len(),
"stdout_tail": "",
"stderr_tail": error,
"stdout_truncated": false,
"stderr_truncated": false,
"log_backpressured": false,
"staged_artifact": null,
"large_bytes_uploaded": false,
"registration_response": registration_response,
"heartbeat_response": heartbeat_response,
"capability_response": capability_response,
"task_assignment_response": &task.task_assignment_response,
"debug_command_response": debug_command_response,
"log_event_response": log_event_response,
"vfs_metadata_response": vfs_metadata_response,
"session_requests": session_requests,
"coordinator_response": coordinator_response,
})
}

39
crates/clusterflux-node/src/windows_dev.rs Normal file
View file

@ -0,0 +1,39 @@
use clusterflux_core::{
Capability, CommandBackendKind, CommandInvocation, CommandPlan, GuestRuntimeKind,
};
use crate::{BackendError, CommandBackend};
#[derive(Clone, Debug, Default)]
pub struct WindowsCommandDevBackend;
impl CommandBackend for WindowsCommandDevBackend {
fn kind(&self) -> CommandBackendKind {
CommandBackendKind::WindowsCommandDev
}
fn plan(&self, _invocation: &CommandInvocation) -> Result<CommandPlan, BackendError> {
Ok(CommandPlan {
guest_runtime: GuestRuntimeKind::Wasmtime,
backend: CommandBackendKind::WindowsCommandDev,
required_capability: Capability::WindowsCommandDev,
user_attached_development_execution: true,
})
}
}
#[derive(Clone, Debug, Default)]
pub struct WindowsSandboxStubBackend;
impl CommandBackend for WindowsSandboxStubBackend {
fn kind(&self) -> CommandBackendKind {
CommandBackendKind::StubbedWindowsSandbox
}
fn plan(&self, _invocation: &CommandInvocation) -> Result<CommandPlan, BackendError> {
Err(BackendError::Denied(
"Windows sandbox backend is an explicit stub for MVP; use windows-command-dev only for user-attached development execution"
.to_owned(),
))
}
}