michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Update public backend API surface

Browse source 
Private source commit: ba3f7ce2b6d9
This commit is contained in:
Clusterflux Release 2026-07-26 17:22:33 +02:00
parent 784463622c
commit 26fdcb9d84
34 changed files with 5473 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

234
crates/clusterflux-node/src/assignment_runner/process_runner.rs
View file

@ -1,4 +1,63 @@
use super::*;
use std::sync::mpsc::{self, Receiver, SyncSender};
struct LiveLogChunk {
stream: &'static str,
offset: u64,
source_bytes: u64,
bytes: Vec<u8>,
truncated: bool,
}
fn redact_safe_live_log_prefix(
pending: &[u8],
configured_secrets: &[String],
final_chunk: bool,
) -> Option<(usize, String)> {
if pending.is_empty() {
return None;
}
let secret_bytes = configured_secrets
.iter()
.filter(|secret| secret.len() >= 4)
.map(String::as_bytes)
.collect::<Vec<_>>();
let maximum_secret_bytes = secret_bytes
.iter()
.map(|secret| secret.len())
.max()
.unwrap_or(0);
if !final_chunk && pending.len() <= maximum_secret_bytes {
return None;
}
let mut consumed = if final_chunk {
pending.len()
} else {
pending.len() - maximum_secret_bytes
};
loop {
let previous = consumed;
for secret in &secret_bytes {
for start in 0..=pending.len().saturating_sub(secret.len()) {
let end = start + secret.len();
if start < consumed && end > consumed && &pending[start..end] == *secret {
consumed = end;
}
}
}
if consumed == previous {
break;
}
}
if consumed == 0 {
return None;
}
let text = redact_configured_values(
String::from_utf8_lossy(&pending[..consumed]).into_owned(),
configured_secrets,
);
Some((consumed, text))
}
pub(super) struct CoordinatorControlledProcessRunner {
pub(super) args: Args,
@ -8,12 +67,37 @@ pub(super) struct CoordinatorControlledProcessRunner {
pub(super) debug_control: Arc<WasmDebugControl>,
pub(super) command_status: Arc<Mutex<Option<String>>>,
pub(super) timeout: Duration,
pub(super) configured_secrets: Vec<String>,
}
#[cfg(test)]
mod tests {
use super::redact_safe_live_log_prefix;
#[test]
fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
let secrets = vec!["correct-horse".to_owned()];
let mut pending = b"prefix correct-".to_vec();
let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
pending.drain(..consumed);
pending.extend_from_slice(b"horse suffix");
let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
let combined = format!("{first}{second}");
assert_eq!(combined, "prefix [REDACTED] suffix");
assert!(!combined.contains("correct-"));
assert!(!combined.contains("horse"));
}
}
impl CoordinatorControlledProcessRunner {
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
pub(super) fn new(
host: &CoordinatorWasmTaskHost,
timeout: Duration,
configured_secrets: Vec<String>,
) -> Self {
Self {
args: host.args.clone(),
process: host.process.clone(),
@ -22,6 +106,7 @@ impl CoordinatorControlledProcessRunner {
debug_control: Arc::clone(&host.debug_control),
command_status: Arc::clone(&host.command_status),
timeout,
configured_secrets,
}
}
@ -201,10 +286,17 @@ impl CoordinatorControlledProcessRunner {
fn drain_bounded(
mut reader: impl Read + Send + 'static,
maximum: usize,
stream: &'static str,
sender: SyncSender<LiveLogChunk>,
configured_secrets: Vec<String>,
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
thread::spawn(move || {
let mut captured = Vec::new();
let mut buffer = [0_u8; 16 * 1024];
let mut source_offset = 0_u64;
let mut pending_offset = 0_u64;
let mut pending = Vec::new();
let mut discarded = false;
loop {
let count = reader
.read(&mut buffer)
@ -213,11 +305,129 @@ impl CoordinatorControlledProcessRunner {
break;
}
let remaining = maximum.saturating_sub(captured.len());
captured.extend_from_slice(&buffer[..count.min(remaining)]);
let retained = count.min(remaining);
if retained > 0 {
captured.extend_from_slice(&buffer[..retained]);
pending.extend_from_slice(&buffer[..retained]);
if let Some((consumed, text)) =
redact_safe_live_log_prefix(&pending, &configured_secrets, false)
{
let _ = sender.try_send(LiveLogChunk {
stream,
offset: pending_offset,
source_bytes: consumed as u64,
bytes: text.into_bytes(),
truncated: false,
});
pending.drain(..consumed);
pending_offset = pending_offset.saturating_add(consumed as u64);
}
}
if retained < count {
discarded = true;
}
source_offset = source_offset.saturating_add(count as u64);
}
if let Some((consumed, text)) =
redact_safe_live_log_prefix(&pending, &configured_secrets, true)
{
let _ = sender.try_send(LiveLogChunk {
stream,
offset: pending_offset,
source_bytes: consumed as u64,
bytes: text.into_bytes(),
truncated: false,
});
}
if discarded {
let _ = sender.try_send(LiveLogChunk {
stream,
offset: maximum as u64,
source_bytes: 0,
bytes: b"[log output truncated at node capture limit]".to_vec(),
truncated: true,
});
}
Ok(captured)
})
}
fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
let args = self.args.clone();
let process = self.process.clone();
let task = self.task.clone();
let node_private_key = self.node_private_key.clone();
let configured_secrets = self.configured_secrets.clone();
let command_status = Arc::clone(&self.command_status);
thread::spawn(move || {
let mut log_session = None;
let mut delivery_available = true;
while let Ok(chunk) = receiver.recv() {
if !delivery_available {
continue;
}
let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
text = redact_configured_values(text, &configured_secrets);
let mut delivered = false;
for _ in 0..2 {
if log_session.is_none() {
log_session = CoordinatorSession::connect_with_timeouts(
&args.coordinator,
Duration::from_millis(500),
Duration::from_millis(500),
)
.ok();
}
let Some(session) = log_session.as_mut() else {
continue;
};
let request = signed_node_request_json(
&args,
&node_private_key,
"report_task_log_chunk",
serde_json::json!({
"type": "report_task_log_chunk",
"tenant": &args.tenant,
"project": &args.project,
"process": &process,
"node": &args.node,
"task": &task,
"stream": chunk.stream,
"offset": chunk.offset,
"source_bytes": chunk.source_bytes,
"text": &text,
"truncated": chunk.truncated,
}),
);
let result = match request {
Ok(request) => session
.request(request)
.map(|_| ())
.map_err(|error| error.to_string()),
Err(error) => Err(error.to_string()),
};
match result {
Ok(()) => {
delivered = true;
break;
}
Err(_) => {
log_session = None;
}
}
}
if !delivered {
delivery_available = false;
if let Ok(mut current) = command_status.lock() {
*current = Some(
"live log delivery was interrupted; final bounded output remains available"
.to_owned(),
);
}
}
}
})
}
}
impl ProcessRunner for CoordinatorControlledProcessRunner {
@ -245,12 +455,16 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
command.program,
command.args.join(" ")
));
let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
let stdout = Self::drain_bounded(
child
.stdout
.take()
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
"stdout",
live_log_sender.clone(),
self.configured_secrets.clone(),
);
let stderr = Self::drain_bounded(
child
@ -258,11 +472,18 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
.take()
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
Self::MAX_CAPTURE_BYTES,
"stderr",
live_log_sender,
self.configured_secrets.clone(),
);
let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
Ok(session) => session,
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(format!(
"establish execution control channel: {error}"
)));
@ -277,6 +498,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Ok(None) => {}
Err(error) => {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(error.to_string()));
}
}
@ -289,6 +513,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Command(format!(
"native command exceeded wall-clock timeout of {} ms",
self.timeout.as_millis()
@ -303,6 +528,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(BackendError::Cancelled(
"coordinator requested cancellation or abort".to_owned(),
));
@ -312,6 +538,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
Self::terminate_execution(&mut child, podman_container.as_deref());
let _ = stdout.join();
let _ = stderr.join();
let _ = live_log_reporter.join();
return Err(error);
}
}
@ -360,6 +587,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
.join()
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
.map_err(BackendError::Command)?;
live_log_reporter
.join()
.map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
self.set_command_status(format!(
"native command exited with status {:?}",
status.code()

2
crates/clusterflux-node/src/assignment_runner/tests.rs
View file

@ -49,6 +49,7 @@ fn test_controlled_runner(
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
timeout,
configured_secrets: Vec::new(),
}
}
@ -90,6 +91,7 @@ fn controlled_process_runner_kills_running_group_when_abort_is_polled() {
debug_control: Arc::new(WasmDebugControl::default()),
command_status: Arc::new(Mutex::new(None)),
timeout: Duration::from_secs(30),
configured_secrets: Vec::new(),
};
let started = Instant::now();
let error = runner