Update public backend API surface
Private source commit: ba3f7ce2b6d9
This commit is contained in:
Clusterflux Release
parent
784463622c
commit
26fdcb9d84
34 changed files with 5473 additions and 72 deletions
|
|
@ -1,4 +1,63 @@
|
|||
use super::*;
|
||||
use std::sync::mpsc::{self, Receiver, SyncSender};
|
||||
|
||||
struct LiveLogChunk {
|
||||
stream: &'static str,
|
||||
offset: u64,
|
||||
source_bytes: u64,
|
||||
bytes: Vec<u8>,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
fn redact_safe_live_log_prefix(
|
||||
pending: &[u8],
|
||||
configured_secrets: &[String],
|
||||
final_chunk: bool,
|
||||
) -> Option<(usize, String)> {
|
||||
if pending.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let secret_bytes = configured_secrets
|
||||
.iter()
|
||||
.filter(|secret| secret.len() >= 4)
|
||||
.map(String::as_bytes)
|
||||
.collect::<Vec<_>>();
|
||||
let maximum_secret_bytes = secret_bytes
|
||||
.iter()
|
||||
.map(|secret| secret.len())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
if !final_chunk && pending.len() <= maximum_secret_bytes {
|
||||
return None;
|
||||
}
|
||||
let mut consumed = if final_chunk {
|
||||
pending.len()
|
||||
} else {
|
||||
pending.len() - maximum_secret_bytes
|
||||
};
|
||||
loop {
|
||||
let previous = consumed;
|
||||
for secret in &secret_bytes {
|
||||
for start in 0..=pending.len().saturating_sub(secret.len()) {
|
||||
let end = start + secret.len();
|
||||
if start < consumed && end > consumed && &pending[start..end] == *secret {
|
||||
consumed = end;
|
||||
}
|
||||
}
|
||||
}
|
||||
if consumed == previous {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if consumed == 0 {
|
||||
return None;
|
||||
}
|
||||
let text = redact_configured_values(
|
||||
String::from_utf8_lossy(&pending[..consumed]).into_owned(),
|
||||
configured_secrets,
|
||||
);
|
||||
Some((consumed, text))
|
||||
}
|
||||
|
||||
pub(super) struct CoordinatorControlledProcessRunner {
|
||||
pub(super) args: Args,
|
||||
|
|
@ -8,12 +67,37 @@ pub(super) struct CoordinatorControlledProcessRunner {
|
|||
pub(super) debug_control: Arc<WasmDebugControl>,
|
||||
pub(super) command_status: Arc<Mutex<Option<String>>>,
|
||||
pub(super) timeout: Duration,
|
||||
pub(super) configured_secrets: Vec<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::redact_safe_live_log_prefix;
|
||||
|
||||
#[test]
|
||||
fn live_log_redaction_holds_boundaries_until_split_secrets_are_complete() {
|
||||
let secrets = vec!["correct-horse".to_owned()];
|
||||
let mut pending = b"prefix correct-".to_vec();
|
||||
let (consumed, first) = redact_safe_live_log_prefix(&pending, &secrets, false).unwrap();
|
||||
pending.drain(..consumed);
|
||||
pending.extend_from_slice(b"horse suffix");
|
||||
let (_, second) = redact_safe_live_log_prefix(&pending, &secrets, true).unwrap();
|
||||
|
||||
let combined = format!("{first}{second}");
|
||||
assert_eq!(combined, "prefix [REDACTED] suffix");
|
||||
assert!(!combined.contains("correct-"));
|
||||
assert!(!combined.contains("horse"));
|
||||
}
|
||||
}
|
||||
|
||||
impl CoordinatorControlledProcessRunner {
|
||||
const MAX_CAPTURE_BYTES: usize = 256 * 1024 + 1;
|
||||
|
||||
pub(super) fn new(host: &CoordinatorWasmTaskHost, timeout: Duration) -> Self {
|
||||
pub(super) fn new(
|
||||
host: &CoordinatorWasmTaskHost,
|
||||
timeout: Duration,
|
||||
configured_secrets: Vec<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
args: host.args.clone(),
|
||||
process: host.process.clone(),
|
||||
|
|
@ -22,6 +106,7 @@ impl CoordinatorControlledProcessRunner {
|
|||
debug_control: Arc::clone(&host.debug_control),
|
||||
command_status: Arc::clone(&host.command_status),
|
||||
timeout,
|
||||
configured_secrets,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -201,10 +286,17 @@ impl CoordinatorControlledProcessRunner {
|
|||
fn drain_bounded(
|
||||
mut reader: impl Read + Send + 'static,
|
||||
maximum: usize,
|
||||
stream: &'static str,
|
||||
sender: SyncSender<LiveLogChunk>,
|
||||
configured_secrets: Vec<String>,
|
||||
) -> thread::JoinHandle<Result<Vec<u8>, String>> {
|
||||
thread::spawn(move || {
|
||||
let mut captured = Vec::new();
|
||||
let mut buffer = [0_u8; 16 * 1024];
|
||||
let mut source_offset = 0_u64;
|
||||
let mut pending_offset = 0_u64;
|
||||
let mut pending = Vec::new();
|
||||
let mut discarded = false;
|
||||
loop {
|
||||
let count = reader
|
||||
.read(&mut buffer)
|
||||
|
|
@ -213,11 +305,129 @@ impl CoordinatorControlledProcessRunner {
|
|||
break;
|
||||
}
|
||||
let remaining = maximum.saturating_sub(captured.len());
|
||||
captured.extend_from_slice(&buffer[..count.min(remaining)]);
|
||||
let retained = count.min(remaining);
|
||||
if retained > 0 {
|
||||
captured.extend_from_slice(&buffer[..retained]);
|
||||
pending.extend_from_slice(&buffer[..retained]);
|
||||
if let Some((consumed, text)) =
|
||||
redact_safe_live_log_prefix(&pending, &configured_secrets, false)
|
||||
{
|
||||
let _ = sender.try_send(LiveLogChunk {
|
||||
stream,
|
||||
offset: pending_offset,
|
||||
source_bytes: consumed as u64,
|
||||
bytes: text.into_bytes(),
|
||||
truncated: false,
|
||||
});
|
||||
pending.drain(..consumed);
|
||||
pending_offset = pending_offset.saturating_add(consumed as u64);
|
||||
}
|
||||
}
|
||||
if retained < count {
|
||||
discarded = true;
|
||||
}
|
||||
source_offset = source_offset.saturating_add(count as u64);
|
||||
}
|
||||
if let Some((consumed, text)) =
|
||||
redact_safe_live_log_prefix(&pending, &configured_secrets, true)
|
||||
{
|
||||
let _ = sender.try_send(LiveLogChunk {
|
||||
stream,
|
||||
offset: pending_offset,
|
||||
source_bytes: consumed as u64,
|
||||
bytes: text.into_bytes(),
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
if discarded {
|
||||
let _ = sender.try_send(LiveLogChunk {
|
||||
stream,
|
||||
offset: maximum as u64,
|
||||
source_bytes: 0,
|
||||
bytes: b"[log output truncated at node capture limit]".to_vec(),
|
||||
truncated: true,
|
||||
});
|
||||
}
|
||||
Ok(captured)
|
||||
})
|
||||
}
|
||||
|
||||
fn spawn_live_log_reporter(&self, receiver: Receiver<LiveLogChunk>) -> thread::JoinHandle<()> {
|
||||
let args = self.args.clone();
|
||||
let process = self.process.clone();
|
||||
let task = self.task.clone();
|
||||
let node_private_key = self.node_private_key.clone();
|
||||
let configured_secrets = self.configured_secrets.clone();
|
||||
let command_status = Arc::clone(&self.command_status);
|
||||
thread::spawn(move || {
|
||||
let mut log_session = None;
|
||||
let mut delivery_available = true;
|
||||
while let Ok(chunk) = receiver.recv() {
|
||||
if !delivery_available {
|
||||
continue;
|
||||
}
|
||||
let mut text = String::from_utf8_lossy(&chunk.bytes).into_owned();
|
||||
text = redact_configured_values(text, &configured_secrets);
|
||||
let mut delivered = false;
|
||||
for _ in 0..2 {
|
||||
if log_session.is_none() {
|
||||
log_session = CoordinatorSession::connect_with_timeouts(
|
||||
&args.coordinator,
|
||||
Duration::from_millis(500),
|
||||
Duration::from_millis(500),
|
||||
)
|
||||
.ok();
|
||||
}
|
||||
let Some(session) = log_session.as_mut() else {
|
||||
continue;
|
||||
};
|
||||
let request = signed_node_request_json(
|
||||
&args,
|
||||
&node_private_key,
|
||||
"report_task_log_chunk",
|
||||
serde_json::json!({
|
||||
"type": "report_task_log_chunk",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &process,
|
||||
"node": &args.node,
|
||||
"task": &task,
|
||||
"stream": chunk.stream,
|
||||
"offset": chunk.offset,
|
||||
"source_bytes": chunk.source_bytes,
|
||||
"text": &text,
|
||||
"truncated": chunk.truncated,
|
||||
}),
|
||||
);
|
||||
let result = match request {
|
||||
Ok(request) => session
|
||||
.request(request)
|
||||
.map(|_| ())
|
||||
.map_err(|error| error.to_string()),
|
||||
Err(error) => Err(error.to_string()),
|
||||
};
|
||||
match result {
|
||||
Ok(()) => {
|
||||
delivered = true;
|
||||
break;
|
||||
}
|
||||
Err(_) => {
|
||||
log_session = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !delivered {
|
||||
delivery_available = false;
|
||||
if let Ok(mut current) = command_status.lock() {
|
||||
*current = Some(
|
||||
"live log delivery was interrupted; final bounded output remains available"
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ProcessRunner for CoordinatorControlledProcessRunner {
|
||||
|
|
@ -245,12 +455,16 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
command.program,
|
||||
command.args.join(" ")
|
||||
));
|
||||
let (live_log_sender, live_log_receiver) = mpsc::sync_channel(64);
|
||||
let stdout = Self::drain_bounded(
|
||||
child
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or_else(|| BackendError::Command("command stdout pipe missing".to_owned()))?,
|
||||
Self::MAX_CAPTURE_BYTES,
|
||||
"stdout",
|
||||
live_log_sender.clone(),
|
||||
self.configured_secrets.clone(),
|
||||
);
|
||||
let stderr = Self::drain_bounded(
|
||||
child
|
||||
|
|
@ -258,11 +472,18 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
.take()
|
||||
.ok_or_else(|| BackendError::Command("command stderr pipe missing".to_owned()))?,
|
||||
Self::MAX_CAPTURE_BYTES,
|
||||
"stderr",
|
||||
live_log_sender,
|
||||
self.configured_secrets.clone(),
|
||||
);
|
||||
let live_log_reporter = self.spawn_live_log_reporter(live_log_receiver);
|
||||
let mut session = match CoordinatorSession::connect(&self.args.coordinator) {
|
||||
Ok(session) => session,
|
||||
Err(error) => {
|
||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||
let _ = stdout.join();
|
||||
let _ = stderr.join();
|
||||
let _ = live_log_reporter.join();
|
||||
return Err(BackendError::Command(format!(
|
||||
"establish execution control channel: {error}"
|
||||
)));
|
||||
|
|
@ -277,6 +498,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||
let _ = stdout.join();
|
||||
let _ = stderr.join();
|
||||
let _ = live_log_reporter.join();
|
||||
return Err(BackendError::Command(error.to_string()));
|
||||
}
|
||||
}
|
||||
|
|
@ -289,6 +513,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||
let _ = stdout.join();
|
||||
let _ = stderr.join();
|
||||
let _ = live_log_reporter.join();
|
||||
return Err(BackendError::Command(format!(
|
||||
"native command exceeded wall-clock timeout of {} ms",
|
||||
self.timeout.as_millis()
|
||||
|
|
@ -303,6 +528,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||
let _ = stdout.join();
|
||||
let _ = stderr.join();
|
||||
let _ = live_log_reporter.join();
|
||||
return Err(BackendError::Cancelled(
|
||||
"coordinator requested cancellation or abort".to_owned(),
|
||||
));
|
||||
|
|
@ -312,6 +538,7 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
Self::terminate_execution(&mut child, podman_container.as_deref());
|
||||
let _ = stdout.join();
|
||||
let _ = stderr.join();
|
||||
let _ = live_log_reporter.join();
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
|
|
@ -360,6 +587,9 @@ impl ProcessRunner for CoordinatorControlledProcessRunner {
|
|||
.join()
|
||||
.map_err(|_| BackendError::Command("stderr reader panicked".to_owned()))?
|
||||
.map_err(BackendError::Command)?;
|
||||
live_log_reporter
|
||||
.join()
|
||||
.map_err(|_| BackendError::Command("live log reporter panicked".to_owned()))?;
|
||||
self.set_command_status(format!(
|
||||
"native command exited with status {:?}",
|
||||
status.code()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue