Publish Clusterflux 1d0b6fa filtered source

crates/clusterflux-node/src/assignment_runner/control_watcher.rs

@@ -0,0 +1,235 @@
+use std::collections::HashMap;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex};
+use std::thread;
+use std::time::Duration;
+
+use clusterflux_core::TaskSpec;
+use clusterflux_node::WasmDebugControl;
+
+use crate::coordinator_session::CoordinatorSession;
+use crate::daemon::{worker_shutdown_requested, Args};
+use crate::node_identity::signed_node_request_json;
+
+const DEFAULT_DEBUG_FREEZE_TIMEOUT_MILLIS: u64 = 5_000;
+
+fn debug_freeze_timeout() -> Duration {
+    std::env::var("CLUSTERFLUX_DEBUG_FREEZE_TIMEOUT_MS")
+        .ok()
+        .and_then(|value| value.parse::<u64>().ok())
+        .filter(|millis| *millis > 0)
+        .map(Duration::from_millis)
+        .unwrap_or_else(|| Duration::from_millis(DEFAULT_DEBUG_FREEZE_TIMEOUT_MILLIS))
+}
+
+fn debug_handle_snapshot(handles: &Arc<Mutex<HashMap<u64, TaskSpec>>>) -> Vec<(String, String)> {
+    let Ok(handles) = handles.lock() else {
+        return vec![(
+            "handle-registry-diagnostic".to_owned(),
+            "runtime task handle registry was unavailable".to_owned(),
+        )];
+    };
+    let mut snapshot = handles
+        .iter()
+        .map(|(handle_id, spec)| {
+            (
+                format!("task_handle_{handle_id}"),
+                format!(
+                    "definition={} instance={} state=active",
+                    spec.task_definition, spec.task_instance
+                ),
+            )
+        })
+        .collect::<Vec<_>>();
+    snapshot.sort_by(|left, right| left.0.cmp(&right.0));
+    snapshot
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(super) fn spawn_task_control_watcher(
+    args: Args,
+    process: String,
+    task: String,
+    task_definition: String,
+    node_private_key: String,
+    cancellation_requested: Arc<AtomicBool>,
+    abort_requested: Arc<AtomicBool>,
+    debug_control: Arc<WasmDebugControl>,
+    task_args: Vec<(String, String)>,
+    handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
+    command_status: Arc<Mutex<Option<String>>>,
+    stop: Arc<AtomicBool>,
+) {
+    thread::spawn(move || {
+        let Ok(mut session) = CoordinatorSession::connect(&args.coordinator) else {
+            return;
+        };
+        while !stop.load(Ordering::Acquire) {
+            let request = signed_node_request_json(
+                &args,
+                &node_private_key,
+                "poll_task_control",
+                serde_json::json!({
+                    "type": "poll_task_control",
+                    "tenant": &args.tenant,
+                    "project": &args.project,
+                    "process": &process,
+                    "node": &args.node,
+                    "task": &task,
+                }),
+            );
+            let Ok(request) = request else {
+                return;
+            };
+            let Ok(response) = session.request(request) else {
+                return;
+            };
+            cancellation_requested.store(
+                response
+                    .get("cancel_requested")
+                    .and_then(serde_json::Value::as_bool)
+                    .unwrap_or(false),
+                Ordering::Release,
+            );
+            if response
+                .get("abort_requested")
+                .and_then(serde_json::Value::as_bool)
+                .unwrap_or(false)
+            {
+                abort_requested.store(true, Ordering::Release);
+                return;
+            }
+            let debug_request = signed_node_request_json(
+                &args,
+                &node_private_key,
+                "poll_debug_command",
+                serde_json::json!({
+                    "type": "poll_debug_command",
+                    "tenant": &args.tenant,
+                    "project": &args.project,
+                    "process": &process,
+                    "node": &args.node,
+                    "task": &task,
+                }),
+            );
+            let Ok(debug_request) = debug_request else {
+                return;
+            };
+            let Ok(debug_response) = session.request(debug_request) else {
+                return;
+            };
+            if let (Some(epoch), Some(command)) = (
+                debug_response
+                    .get("epoch")
+                    .and_then(serde_json::Value::as_u64),
+                debug_response
+                    .get("command")
+                    .and_then(serde_json::Value::as_str),
+            ) {
+                let freeze_timeout = debug_freeze_timeout();
+                let (state, message) = match command {
+                    "freeze" => {
+                        if std::env::var_os("CLUSTERFLUX_DEBUG_CONTROL_TRACE").is_some() {
+                            eprintln!(
+                                "clusterflux debug control: node received freeze for epoch {epoch} task {task} (debug={:?})",
+                                Arc::as_ptr(&debug_control)
+                            );
+                        }
+                        debug_control.request_freeze(epoch);
+                        if debug_control.wait_until_frozen(epoch, freeze_timeout) {
+                            ("frozen", None)
+                        } else {
+                            debug_control.request_resume(epoch);
+                            (
+                                "failed",
+                                Some(format!(
+                                    "node execution did not reach a freezeable Wasm safepoint or verified native/Podman boundary within {} ms",
+                                    freeze_timeout.as_millis()
+                                )),
+                            )
+                        }
+                    }
+                    "resume" => {
+                        debug_control.request_resume(epoch);
+                        if debug_control.wait_until_running(epoch, freeze_timeout) {
+                            ("running", None)
+                        } else {
+                            (
+                                "failed",
+                                Some(format!(
+                                    "node execution did not leave its verified frozen state within {} ms",
+                                    freeze_timeout.as_millis()
+                                )),
+                            )
+                        }
+                    }
+                    _ => (
+                        "failed",
+                        Some(format!("node received unknown debug command `{command}`")),
+                    ),
+                };
+                let report = signed_node_request_json(
+                    &args,
+                    &node_private_key,
+                    "report_debug_state",
+                    serde_json::json!({
+                        "type": "report_debug_state",
+                        "tenant": &args.tenant,
+                        "project": &args.project,
+                        "process": &process,
+                        "node": &args.node,
+                        "task": &task,
+                        "epoch": epoch,
+                        "state": state,
+                        "stack_frames": if state == "frozen" {
+                            let mut frames = debug_control.stack_frames();
+                            if let Some(frame) = frames.first_mut() {
+                                *frame = format!("{task_definition}::wasm / {frame}");
+                            } else {
+                                frames.push(format!("{task_definition}::wasm"));
+                            }
+                            frames
+                        } else {
+                            Vec::new()
+                        },
+                        "local_values": [],
+                        "task_args": &task_args,
+                        "handles": debug_handle_snapshot(&handles),
+                        "command_status": command_status
+                            .lock()
+                            .ok()
+                            .and_then(|status| status.clone()),
+                        "recent_output": [],
+                        "message": message,
+                    }),
+                );
+                let Ok(report) = report else {
+                    return;
+                };
+                if session.request(report).is_err() {
+                    return;
+                }
+            }
+            thread::sleep(Duration::from_millis(50));
+        }
+    });
+}
+
+pub(super) fn spawn_worker_shutdown_watcher(
+    abort_requested: Arc<AtomicBool>,
+    debug_control: Arc<WasmDebugControl>,
+    stop: Arc<AtomicBool>,
+) {
+    thread::spawn(move || {
+        while !stop.load(Ordering::Acquire) {
+            if worker_shutdown_requested() {
+                abort_requested.store(true, Ordering::Release);
+                if let Some(epoch) = debug_control.requested_epoch() {
+                    debug_control.request_resume(epoch);
+                }
+                return;
+            }
+            thread::sleep(Duration::from_millis(10));
+        }
+    });
+}

crates/clusterflux-node/src/assignment_runner/process_runner.rs

@@ -0,0 +1,373 @@
+use super::*;
+
+pub(super) struct CoordinatorControlledProcessRunner {
+    pub(super) args: Args,
+    pub(super) process: String,
+    pub(super) task: String,
+    pub(super) node_private_key: String,
+    pub(super) debug_control: Arc<WasmDebugControl>,
+    pub(super) command_status: Arc<Mutex<Option<String>>>,
+    pub(super) timeout: Duration,
+}
+
+impl CoordinatorControlledProcessRunner {
+    const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
+
+    pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
+        Self {
+            args: host.args.clone(),
+            process: host.process.clone(),
+            task: host.parent_task.clone(),
+            node_private_key: host.node_private_key.clone(),
+            debug_control: Arc::clone(&host.debug_control),
+            command_status: Arc::clone(&host.command_status),
+            timeout,
+        }
+    }
+
+    fn set_command_status(&self, status: impl Into<String>) {
+        if let Ok(mut current) = self.command_status.lock() {
+            *current = Some(status.into());
+        }
+    }
+
+    fn abort_requested(&self, session: &mut CoordinatorSession) -> Result<bool, BackendError> {
+        let request = signed_node_request_json(
+            &self.args,
+            &self.node_private_key,
+            "poll_task_control",
+            serde_json::json!({
+                "type": "poll_task_control",
+                "tenant": &self.args.tenant,
+                "project": &self.args.project,
+                "process": &self.process,
+                "node": &self.args.node,
+                "task": &self.task,
+            }),
+        )
+        .map_err(|error| BackendError::Command(error.to_string()))?;
+        let response = session
+            .request(request)
+            .map_err(|error| BackendError::Command(format!("poll task control: {error}")))?;
+        Ok(response
+            .get("abort_requested")
+            .and_then(serde_json::Value::as_bool)
+            .unwrap_or(false))
+    }
+
+    fn terminate_process_group(child: &mut std::process::Child) {
+        #[cfg(unix)]
+        {
+            let process_group = -(child.id() as i32);
+            // The child is placed in a new process group before exec, so this reaches
+            // all native descendants. Podman containers are removed separately.
+            unsafe {
+                libc::kill(process_group, libc::SIGKILL);
+            }
+        }
+        let _ = child.kill();
+        let _ = child.wait();
+    }
+
+    fn podman_container_name(command: &PodmanCommand) -> Option<String> {
+        if command.program != "podman" || command.args.first().map(String::as_str) != Some("run") {
+            return None;
+        }
+        command
+            .args
+            .windows(2)
+            .find(|arguments| arguments[0] == "--name")
+            .map(|arguments| arguments[1].clone())
+    }
+
+    fn set_podman_paused(container: &str, paused: bool) -> Result<(), BackendError> {
+        let action = if paused { "pause" } else { "unpause" };
+        let output = std::process::Command::new("podman")
+            .args([action, container])
+            .output()
+            .map_err(|error| {
+                BackendError::Command(format!(
+                    "failed to invoke `podman {action}` for container `{container}`: {error}"
+                ))
+            })?;
+        if !output.status.success() {
+            return Err(BackendError::Command(format!(
+                "`podman {action}` failed for container `{container}`: {}",
+                String::from_utf8_lossy(&output.stderr).trim()
+            )));
+        }
+
+        let inspection = std::process::Command::new("podman")
+            .args(["inspect", "--format", "{{.State.Paused}}", container])
+            .output()
+            .map_err(|error| {
+                BackendError::Command(format!(
+                    "failed to inspect Podman pause state for container `{container}`: {error}"
+                ))
+            })?;
+        let observed = String::from_utf8_lossy(&inspection.stdout);
+        let expected = if paused { "true" } else { "false" };
+        if !inspection.status.success() || observed.trim() != expected {
+            return Err(BackendError::Command(format!(
+                "container `{container}` did not verify as {} after `podman {action}`: status={:?} stdout={} stderr={}",
+                if paused { "paused" } else { "running" },
+                inspection.status.code(),
+                observed.trim(),
+                String::from_utf8_lossy(&inspection.stderr).trim()
+            )));
+        }
+        Ok(())
+    }
+
+    fn terminate_execution(child: &mut std::process::Child, container: Option<&str>) {
+        if let Some(container) = container {
+            let _ = std::process::Command::new("podman")
+                .args(["rm", "--force", container])
+                .output();
+        }
+        Self::terminate_process_group(child);
+    }
+
+    fn freeze_execution(
+        child: &std::process::Child,
+        container: Option<&str>,
+    ) -> Result<(), BackendError> {
+        if let Some(container) = container {
+            return Self::set_podman_paused(container, true);
+        }
+        #[cfg(unix)]
+        {
+            Self::freeze_process_group(child)
+        }
+        #[cfg(not(unix))]
+        {
+            let _ = child;
+            Err(BackendError::Command(
+                "native debug freeze requires Unix process groups".to_owned(),
+            ))
+        }
+    }
+
+    fn resume_execution(
+        child: &std::process::Child,
+        container: Option<&str>,
+    ) -> Result<(), BackendError> {
+        if let Some(container) = container {
+            return Self::set_podman_paused(container, false);
+        }
+        #[cfg(unix)]
+        {
+            Self::resume_process_group(child)
+        }
+        #[cfg(not(unix))]
+        {
+            let _ = child;
+            Err(BackendError::Command(
+                "native debug resume requires Unix process groups".to_owned(),
+            ))
+        }
+    }
+
+    #[cfg(unix)]
+    fn freeze_process_group(child: &std::process::Child) -> Result<(), BackendError> {
+        let process_group = -(child.id() as i32);
+        let result = unsafe { libc::kill(process_group, libc::SIGSTOP) };
+        if result == 0 {
+            Ok(())
+        } else {
+            Err(BackendError::Command(format!(
+                "failed to freeze native process group {}: {}",
+                child.id(),
+                std::io::Error::last_os_error()
+            )))
+        }
+    }
+
+    #[cfg(unix)]
+    fn resume_process_group(child: &std::process::Child) -> Result<(), BackendError> {
+        let process_group = -(child.id() as i32);
+        let result = unsafe { libc::kill(process_group, libc::SIGCONT) };
+        if result == 0 {
+            Ok(())
+        } else {
+            Err(BackendError::Command(format!(
+                "failed to resume native process group {}: {}",
+                child.id(),
+                std::io::Error::last_os_error()
+            )))
+        }
+    }
+
+    fn drain_bounded(
+        mut reader: impl Read + Send + 'static,
+        maximum: usize,
+    ) -> thread::JoinHandle<Result<Vec<u8>, String>> {
+        thread::spawn(move || {
+            let mut captured = Vec::new();
+            let mut buffer = [0_u8; 16 * 1024];
+            loop {
+                let count = reader
+                    .read(&mut buffer)
+                    .map_err(|error| error.to_string())?;
+                if count == 0 {
+                    break;
+                }
+                let remaining = maximum.saturating_sub(captured.len());
+                captured.extend_from_slice(&buffer[..count.min(remaining)]);
+            }
+            Ok(captured)
+        })
+    }
+}
+
+impl ProcessRunner for CoordinatorControlledProcessRunner {
+    fn run(&mut self, command: &PodmanCommand) -> Result<ProcessOutput, BackendError> {
+        let podman_container = Self::podman_container_name(command);
+        self.set_command_status(format!(
+            "starting native command: {} {}",
+            command.program,
+            command.args.join(" ")
+        ));
+        let mut process = std::process::Command::new(&command.program);
+        process
+            .args(&command.args)
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        #[cfg(unix)]
+        process.process_group(0);
+        let mut child = process
+            .spawn()
+            .map_err(|error| BackendError::Command(error.to_string()))?;
+        self.set_command_status(format!(
+            "running native command pid {}: {} {}",
+            child.id(),
+            command.program,
+            command.args.join(" ")
+        ));
+        let stdout = Self::drain_bounded(
+            child
+                .stdout
+                .take()
+                .ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
+            Self::MAX_CAPTURE_BYTES,
+        );
+        let stderr = Self::drain_bounded(
+            child
+                .stderr
+                .take()
+                .ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
+            Self::MAX_CAPTURE_BYTES,
+        );
+        let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
+            Ok(session) => session,
+            Err(error) => {
+                Self::terminate_execution(&mut child, podman_container.as_deref());
+                return Err(BackendError::Command(format!(
+                    "establish execution control channel: {error}"
+                )));
+            }
+        };
+
+        let mut frozen_epoch = None;
+        let started = Instant::now();
+        let status = loop {
+            match child.try_wait() {
+                Ok(Some(status)) => break status,
+                Ok(None) => {}
+                Err(error) => {
+                    Self::terminate_execution(&mut child, podman_container.as_deref());
+                    return Err(BackendError::Command(error.to_string()));
+                }
+            }
+            if started.elapsed() >= self.timeout {
+                self.set_command_status(format!(
+                    "native command pid {} exceeded wall-clock timeout of {} ms",
+                    child.id(),
+                    self.timeout.as_millis()
+                ));
+                Self::terminate_execution(&mut child, podman_container.as_deref());
+                let _ = stdout.join();
+                let _ = stderr.join();
+                return Err(BackendError::Command(format!(
+                    "native command exceeded wall-clock timeout of {} ms",
+                    self.timeout.as_millis()
+                )));
+            }
+            match self.abort_requested(&mut session) {
+                Ok(true) => {
+                    self.set_command_status(format!(
+                        "aborting native command pid {} at coordinator request",
+                        child.id()
+                    ));
+                    Self::terminate_execution(&mut child, podman_container.as_deref());
+                    let _ = stdout.join();
+                    let _ = stderr.join();
+                    return Err(BackendError::Cancelled(
+                        "coordinator requested cancellation or abort".to_owned(),
+                    ));
+                }
+                Ok(false) => {}
+                Err(error) => {
+                    Self::terminate_execution(&mut child, podman_container.as_deref());
+                    let _ = stdout.join();
+                    let _ = stderr.join();
+                    return Err(error);
+                }
+            }
+            if let Some(epoch) = self.debug_control.requested_epoch() {
+                if self.debug_control.resume_requested(epoch) {
+                    if frozen_epoch == Some(epoch) {
+                        match Self::resume_execution(&child, podman_container.as_deref()) {
+                            Ok(()) => {
+                                self.set_command_status(format!(
+                                    "running command pid {} after debug epoch {epoch} resumed",
+                                    child.id()
+                                ));
+                                self.debug_control.mark_running(epoch);
+                                frozen_epoch = None;
+                            }
+                            Err(error) => self.set_command_status(format!(
+                                "debug epoch {epoch} resume is pending: {error}"
+                            )),
+                        }
+                    }
+                } else if frozen_epoch != Some(epoch)
+                    && self.debug_control.frozen_epoch() != Some(epoch)
+                {
+                    match Self::freeze_execution(&child, podman_container.as_deref()) {
+                        Ok(()) => {
+                            self.set_command_status(format!(
+                                "frozen command pid {} for debug epoch {epoch}",
+                                child.id()
+                            ));
+                            self.debug_control.mark_frozen(epoch);
+                            frozen_epoch = Some(epoch);
+                        }
+                        Err(error) => self.set_command_status(format!(
+                            "debug epoch {epoch} freeze is pending: {error}"
+                        )),
+                    }
+                }
+            }
+            thread::sleep(Duration::from_millis(50));
+        };
+        let stdout = stdout
+            .join()
+            .map_err(|_| BackendError::Command("stdout reader panicked".to_owned()))?
+            .map_err(BackendError::Command)?;
+        let stderr = stderr
+            .join()
+            .map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
+            .map_err(BackendError::Command)?;
+        self.set_command_status(format!(
+            "native command exited with status {:?}",
+            status.code()
+        ));
+        Ok(ProcessOutput {
+            status_code: status.code(),
+            stdout,
+            stderr,
+        })
+    }
+}

crates/clusterflux-node/src/assignment_runner/tests.rs

@@ -0,0 +1,205 @@
+use std::io::{BufRead, BufReader, Write};
+use std::net::TcpListener;
+use std::time::{Duration, Instant};
+
+use super::*;
+
+fn start_running_control_server() -> (String, thread::JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let coordinator = listener.local_addr().unwrap().to_string();
+    let server = thread::spawn(move || {
+        let (mut stream, _) = listener.accept().unwrap();
+        let mut reader = BufReader::new(stream.try_clone().unwrap());
+        loop {
+            let mut request = String::new();
+            match reader.read_line(&mut request) {
+                Ok(0) | Err(_) => break,
+                Ok(_) => stream
+                    .write_all(b"{\"type\":\"task_control\",\"process\":\"vp\",\"task\":\"task\",\"cancel_requested\":false,\"abort_requested\":false}\n")
+                    .unwrap(),
+            }
+        }
+    });
+    (coordinator, server)
+}
+
+fn test_controlled_runner(
+    coordinator: String,
+    timeout: Duration,
+) -> CoordinatorControlledProcessRunner {
+    CoordinatorControlledProcessRunner {
+        args: Args {
+            coordinator,
+            tenant: "tenant".to_owned(),
+            project: "project".to_owned(),
+            project_root: None,
+            node: "node".to_owned(),
+            enrollment_grant: None,
+            public_key: None,
+            control_poll_ms: 0,
+            assignment_poll_ms: 1,
+            emit_ready: false,
+            worker: true,
+        },
+        process: "vp".to_owned(),
+        task: "task".to_owned(),
+        node_private_key: clusterflux_core::derive_ed25519_private_key_from_seed(
+            "controlled-runner-test",
+        ),
+        debug_control: Arc::new(WasmDebugControl::default()),
+        command_status: Arc::new(Mutex::new(None)),
+        timeout,
+    }
+}
+
+#[cfg(unix)]
+#[test]
+fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
+    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+    let coordinator = listener.local_addr().unwrap().to_string();
+    let server = thread::spawn(move || {
+        let (mut stream, _) = listener.accept().unwrap();
+        let mut request = String::new();
+        BufReader::new(stream.try_clone().unwrap())
+            .read_line(&mut request)
+            .unwrap();
+        assert!(!request.trim().is_empty());
+        stream
+                .write_all(b"{\"type\":\"task_control\",\"process\":\"vp\",\"task\":\"task\",\"cancel_requested\":false,\"abort_requested\":true}\n")
+                .unwrap();
+    });
+    let mut runner = CoordinatorControlledProcessRunner {
+        args: Args {
+            coordinator,
+            tenant: "tenant".to_owned(),
+            project: "project".to_owned(),
+            project_root: None,
+            node: "node".to_owned(),
+            enrollment_grant: None,
+            public_key: None,
+            control_poll_ms: 0,
+            assignment_poll_ms: 1,
+            emit_ready: false,
+            worker: true,
+        },
+        process: "vp".to_owned(),
+        task: "task".to_owned(),
+        node_private_key: clusterflux_core::derive_ed25519_private_key_from_seed(
+            "controlled-runner-test",
+        ),
+        debug_control: Arc::new(WasmDebugControl::default()),
+        command_status: Arc::new(Mutex::new(None)),
+        timeout: Duration::from_secs(30),
+    };
+    let started = Instant::now();
+    let error = runner
+        .run(&PodmanCommand {
+            program: "sh".to_owned(),
+            args: vec!["-c".to_owned(), "sleep 30 & wait".to_owned()],
+        })
+        .unwrap_err();
+    server.join().unwrap();
+
+    assert!(matches!(error, BackendError::Cancelled(_)));
+    assert!(started.elapsed() < Duration::from_secs(5));
+}
+
+#[cfg(unix)]
+#[test]
+fn command_timeout_is_bounded_and_a_later_command_still_runs() {
+    let (coordinator, server) = start_running_control_server();
+    let mut runner = test_controlled_runner(coordinator, Duration::from_millis(120));
+    let started = Instant::now();
+    let error = runner
+        .run(&PodmanCommand {
+            program: "sh".to_owned(),
+            args: vec!["-c".to_owned(), "sleep 30 & wait".to_owned()],
+        })
+        .unwrap_err();
+    assert!(error.to_string().contains("timeout"));
+    assert!(started.elapsed() < Duration::from_secs(5));
+    drop(runner);
+    server.join().unwrap();
+
+    let (coordinator, server) = start_running_control_server();
+    let mut runner = test_controlled_runner(coordinator, Duration::from_secs(5));
+    let output = runner
+        .run(&PodmanCommand {
+            program: "sh".to_owned(),
+            args: vec!["-c".to_owned(), "printf healthy".to_owned()],
+        })
+        .unwrap();
+    assert_eq!(output.status_code, Some(0));
+    assert_eq!(String::from_utf8(output.stdout).unwrap(), "healthy");
+    drop(runner);
+    server.join().unwrap();
+}
+
+#[test]
+fn command_source_verification_rejects_a_changed_checkout() {
+    let checkout = tempfile::tempdir().unwrap();
+    std::fs::write(checkout.path().join("source.c"), "int value = 1;\n").unwrap();
+    let expected = snapshot_project(checkout.path()).unwrap().digest;
+    verify_source_snapshot(checkout.path(), &expected).unwrap();
+
+    std::fs::write(checkout.path().join("source.c"), "int value = 2;\n").unwrap();
+    let error = verify_source_snapshot(checkout.path(), &expected).unwrap_err();
+
+    assert!(error.contains("source snapshot mismatch"));
+    assert!(error.contains(expected.as_str()));
+}
+
+#[test]
+fn command_environment_verification_rejects_a_changed_recipe() {
+    let checkout = tempfile::tempdir().unwrap();
+    let environment = checkout.path().join("envs/linux");
+    std::fs::create_dir_all(&environment).unwrap();
+    std::fs::write(
+        environment.join("Containerfile"),
+        "FROM docker.io/library/alpine:3.20\n",
+    )
+    .unwrap();
+    let expected = clusterflux_core::discover_environments(checkout.path())
+        .unwrap()
+        .into_iter()
+        .find(|environment| environment.name == "linux")
+        .unwrap()
+        .digest;
+    verify_environment_digest(checkout.path(), "linux", &expected).unwrap();
+
+    std::fs::write(
+        environment.join("Containerfile"),
+        "FROM docker.io/library/alpine:3.21\n",
+    )
+    .unwrap();
+    let error = verify_environment_digest(checkout.path(), "linux", &expected).unwrap_err();
+
+    assert!(error.contains("does not match the bundle"));
+    assert!(error.contains(expected.as_str()));
+}
+
+#[test]
+fn configured_secret_values_are_redacted_before_guest_or_coordinator_logging() {
+    assert!(is_secret_environment_name("API_TOKEN"));
+    assert!(is_secret_environment_name("database_password"));
+    assert!(!is_secret_environment_name("SOURCE_DATE_EPOCH"));
+    assert_eq!(
+        redact_configured_values(
+            "token=correct-horse and again correct-horse".to_owned(),
+            &["correct-horse".to_owned()],
+        ),
+        "token=[REDACTED] and again [REDACTED]"
+    );
+}
+
+#[test]
+fn native_command_requires_environment_and_network_grant() {
+    let error = require_command_environment(None).unwrap_err();
+    assert!(error.contains("explicit command-capable environment"));
+    assert_eq!(require_command_environment(Some("linux")).unwrap(), "linux");
+
+    authorize_command_network(&clusterflux_core::CommandNetworkPolicy::Disabled, false).unwrap();
+    let error = authorize_command_network(&clusterflux_core::CommandNetworkPolicy::Enabled, false)
+        .unwrap_err();
+    assert!(error.contains("Network capability"));
+}

crates/clusterflux-node/src/assignment_runner/validation.rs

@@ -0,0 +1,190 @@
+use std::collections::{BTreeMap, HashMap};
+use std::path::Path;
+
+use clusterflux_core::{Capability, Digest};
+use wasmparser::{Parser, Payload};
+
+use crate::source_snapshot::{snapshot_project, SourceSnapshotInventory};
+
+pub(super) fn verify_source_snapshot(
+    project_root: &Path,
+    expected: &Digest,
+) -> Result<SourceSnapshotInventory, String> {
+    let actual = snapshot_project(project_root)?;
+    if &actual.digest != expected {
+        return Err(format!(
+            "node checkout source snapshot mismatch: task requires {expected}, but the current checkout is {}",
+            actual.digest
+        ));
+    }
+    Ok(actual)
+}
+
+pub(super) fn authorize_command_network(
+    policy: &clusterflux_core::CommandNetworkPolicy,
+    allow_network: bool,
+) -> Result<(), String> {
+    if policy == &clusterflux_core::CommandNetworkPolicy::Enabled && !allow_network {
+        return Err(
+            "Wasm task requested network access without an explicit granted Network capability"
+                .to_owned(),
+        );
+    }
+    Ok(())
+}
+
+pub(super) fn require_command_environment(environment_id: Option<&str>) -> Result<&str, String> {
+    environment_id.ok_or_else(|| {
+        "native commands require an explicit command-capable environment; attach one with .env(clusterflux::env!(\"name\"))"
+            .to_owned()
+    })
+}
+
+pub(super) fn verify_environment_digest(
+    project_root: &Path,
+    environment_id: &str,
+    expected: &Digest,
+) -> Result<clusterflux_core::EnvironmentResource, String> {
+    let environment = clusterflux_core::discover_environments(project_root)
+        .map_err(|error| error.to_string())?
+        .into_iter()
+        .find(|environment| environment.name == environment_id)
+        .ok_or_else(|| {
+            format!(
+                "node checkout has no environment `{environment_id}` under {}/envs",
+                project_root.display()
+            )
+        })?;
+    if &environment.digest != expected {
+        return Err(format!(
+            "node environment `{environment_id}` does not match the bundle: task requires {expected}, but the current recipe is {}",
+            environment.digest
+        ));
+    }
+    Ok(environment)
+}
+
+pub(super) fn is_secret_environment_name(name: &str) -> bool {
+    let name = name.to_ascii_uppercase();
+    ["SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "PRIVATE_KEY"]
+        .iter()
+        .any(|marker| name.contains(marker))
+}
+
+pub(super) fn redact_configured_values(mut text: String, values: &[String]) -> String {
+    for value in values.iter().filter(|value| value.len() >= 4) {
+        text = text.replace(value, "[REDACTED]");
+    }
+    text
+}
+
+pub(super) fn capability_from_descriptor(value: &str) -> Result<Capability, String> {
+    match value.trim().to_ascii_lowercase().replace('-', "_").as_str() {
+        "command" => Ok(Capability::Command),
+        "containers" => Ok(Capability::Containers),
+        "rootless_podman" => Ok(Capability::RootlessPodman),
+        "source_filesystem" => Ok(Capability::SourceFilesystem),
+        "source_git" => Ok(Capability::SourceGit),
+        "host_filesystem" => Ok(Capability::HostFilesystem),
+        "network" => Ok(Capability::Network),
+        "secrets" => Ok(Capability::Secrets),
+        "inbound_ports" => Ok(Capability::InboundPorts),
+        "arbitrary_syscalls" => Ok(Capability::ArbitrarySyscalls),
+        "vfs_artifacts" => Ok(Capability::VfsArtifacts),
+        "windows_command_dev" => Ok(Capability::WindowsCommandDev),
+        "quic_direct" => Ok(Capability::QuicDirect),
+        other => Err(format!("unknown task capability `{other}`")),
+    }
+}
+
+pub(super) fn task_descriptors(
+    module: &[u8],
+) -> Result<HashMap<String, serde_json::Value>, Box<dyn std::error::Error>> {
+    let mut descriptors = HashMap::new();
+    for payload in Parser::new(0).parse_all(module) {
+        let Payload::CustomSection(section) = payload? else {
+            continue;
+        };
+        if section.name() != "clusterflux.tasks" {
+            continue;
+        }
+        for record in section
+            .data()
+            .split(|byte| *byte == b'\n' || *byte == 0)
+            .filter(|record| !record.is_empty())
+        {
+            let descriptor: serde_json::Value = serde_json::from_slice(record)?;
+            let name = descriptor
+                .get("name")
+                .and_then(serde_json::Value::as_str)
+                .ok_or("task descriptor omitted name")?
+                .to_owned();
+            descriptors.insert(name, descriptor);
+        }
+    }
+    Ok(descriptors)
+}
+
+pub(super) fn bundle_environments(
+    module: &[u8],
+) -> Result<BTreeMap<String, clusterflux_core::EnvironmentResource>, Box<dyn std::error::Error>> {
+    for payload in Parser::new(0).parse_all(module) {
+        let Payload::CustomSection(section) = payload? else {
+            continue;
+        };
+        if section.name() != "clusterflux.environments" {
+            continue;
+        }
+        let environments: Vec<clusterflux_core::EnvironmentResource> =
+            serde_json::from_slice(section.data())?;
+        let mut by_name = BTreeMap::new();
+        for environment in environments {
+            if by_name
+                .insert(environment.name.clone(), environment)
+                .is_some()
+            {
+                return Err("bundle environment manifest contains duplicate names".into());
+            }
+        }
+        return Ok(by_name);
+    }
+    Ok(BTreeMap::new())
+}
+
+pub(super) fn resolve_task_export(
+    module: &[u8],
+    task: &str,
+) -> Result<String, Box<dyn std::error::Error>> {
+    for payload in Parser::new(0).parse_all(module) {
+        let Payload::CustomSection(section) = payload? else {
+            continue;
+        };
+        if section.name() != "clusterflux.tasks" {
+            continue;
+        }
+        for record in section
+            .data()
+            .split(|byte| *byte == b'\n' || *byte == 0)
+            .filter(|record| !record.is_empty())
+        {
+            let descriptor: serde_json::Value = serde_json::from_slice(record)?;
+            if descriptor.get("name").and_then(serde_json::Value::as_str) != Some(task) {
+                continue;
+            }
+            if descriptor
+                .get("abi_version")
+                .and_then(serde_json::Value::as_u64)
+                != Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
+            {
+                return Err(format!("task `{task}` uses an unsupported Wasm ABI version").into());
+            }
+            return descriptor
+                .get("export")
+                .and_then(serde_json::Value::as_str)
+                .filter(|export| !export.trim().is_empty())
+                .map(str::to_owned)
+                .ok_or_else(|| format!("task `{task}` descriptor omitted its Wasm export").into());
+        }
+    }
+    Err(format!("bundle has no task descriptor named `{task}`").into())
+}