0cf66aaa65
Source commit: e570b80c3a0f4b152a9aa0e7bb8635b3eebcec27 Public tree identity: sha256:cff3388143eb2e9476804cfa1c1246e3a46d485d8aef90f2cdd4df45f2b69abf
379 lines
12 KiB
Rust
379 lines
12 KiB
Rust
use clusterflux_core::{TaskBoundaryValue, VfsManifest};
|
|
use clusterflux_node::CommandOutput;
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::daemon::RuntimeTask;
|
|
use crate::{
|
|
coordinator_session::CoordinatorSession, daemon::Args, node_identity::signed_node_request_json,
|
|
task_artifacts::RetainedArtifact,
|
|
};
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn record_completed_task(
|
|
args: &Args,
|
|
session: &mut CoordinatorSession,
|
|
task: RuntimeTask,
|
|
output: CommandOutput,
|
|
manifest: VfsManifest,
|
|
result: Option<TaskBoundaryValue>,
|
|
retained: Option<RetainedArtifact>,
|
|
registration: Value,
|
|
heartbeat: Value,
|
|
capability_report: Value,
|
|
debug_command: Value,
|
|
node_private_key: &str,
|
|
) -> Result<Value, Box<dyn std::error::Error>> {
|
|
let staged = output.staged_artifact.as_ref();
|
|
let artifact_digest = retained
|
|
.as_ref()
|
|
.map(|artifact| artifact.digest.clone())
|
|
.or_else(|| staged.map(|artifact| artifact.digest.clone()));
|
|
let artifact_path = retained
|
|
.as_ref()
|
|
.map(|artifact| format!("/vfs/artifacts/{}", artifact.id))
|
|
.or_else(|| staged.map(|artifact| artifact.path.as_str().to_owned()));
|
|
let artifact_size_bytes = retained
|
|
.as_ref()
|
|
.map(|artifact| artifact.size_bytes)
|
|
.or_else(|| staged.map(|artifact| artifact.size));
|
|
let log_event = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"report_task_log",
|
|
json!({
|
|
"type": "report_task_log",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"stdout_bytes": output.stdout.len(),
|
|
"stderr_bytes": output.stderr.len(),
|
|
"stdout_tail": &output.stdout,
|
|
"stderr_tail": &output.stderr,
|
|
"stdout_truncated": output.stdout_truncated,
|
|
"stderr_truncated": output.stderr_truncated,
|
|
"backpressured": output.log_backpressured,
|
|
}),
|
|
)?)?;
|
|
let vfs_metadata = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"report_vfs_metadata",
|
|
json!({
|
|
"type": "report_vfs_metadata",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"artifact_path": artifact_path,
|
|
"artifact_digest": artifact_digest,
|
|
"artifact_size_bytes": artifact_size_bytes,
|
|
"large_bytes_uploaded": manifest.large_bytes_uploaded,
|
|
}),
|
|
)?)?;
|
|
let recorded = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"task_completed",
|
|
json!({
|
|
"type": "task_completed",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"status_code": output.status_code,
|
|
"stdout_bytes": output.stdout.len(),
|
|
"stderr_bytes": output.stderr.len(),
|
|
"stdout_tail": &output.stdout,
|
|
"stderr_tail": &output.stderr,
|
|
"stdout_truncated": output.stdout_truncated,
|
|
"stderr_truncated": output.stderr_truncated,
|
|
"artifact_path": artifact_path,
|
|
"artifact_digest": artifact_digest,
|
|
"artifact_size_bytes": artifact_size_bytes,
|
|
"result": result,
|
|
}),
|
|
)?)?;
|
|
Ok(completed_node_report(
|
|
output,
|
|
manifest.large_bytes_uploaded,
|
|
registration,
|
|
heartbeat,
|
|
capability_report,
|
|
task.task_assignment_response,
|
|
debug_command,
|
|
log_event,
|
|
vfs_metadata,
|
|
recorded,
|
|
session.requests(),
|
|
))
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn record_failed_task(
|
|
args: &Args,
|
|
session: &mut CoordinatorSession,
|
|
task: &RuntimeTask,
|
|
registration: Value,
|
|
heartbeat: Value,
|
|
capability_report: Value,
|
|
debug_command: Value,
|
|
node_private_key: &str,
|
|
error: &str,
|
|
) -> Result<Value, Box<dyn std::error::Error>> {
|
|
let error = bounded_runtime_error(error);
|
|
let log_event = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"report_task_log",
|
|
json!({
|
|
"type": "report_task_log",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"stdout_bytes": 0,
|
|
"stderr_bytes": error.len(),
|
|
"stdout_tail": "",
|
|
"stderr_tail": &error,
|
|
"stdout_truncated": false,
|
|
"stderr_truncated": false,
|
|
"backpressured": false,
|
|
}),
|
|
)?)?;
|
|
let vfs_metadata = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"report_vfs_metadata",
|
|
json!({
|
|
"type": "report_vfs_metadata",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"artifact_path": null,
|
|
"artifact_digest": null,
|
|
"artifact_size_bytes": null,
|
|
"large_bytes_uploaded": false,
|
|
}),
|
|
)?)?;
|
|
let recorded = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"task_completed",
|
|
json!({
|
|
"type": "task_completed",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"terminal_state": "failed",
|
|
"status_code": -1,
|
|
"stdout_bytes": 0,
|
|
"stderr_bytes": error.len(),
|
|
"stdout_tail": "",
|
|
"stderr_tail": &error,
|
|
"stdout_truncated": false,
|
|
"stderr_truncated": false,
|
|
"artifact_path": null,
|
|
"artifact_digest": null,
|
|
"artifact_size_bytes": null,
|
|
}),
|
|
)?)?;
|
|
Ok(failed_node_report(
|
|
task,
|
|
&error,
|
|
registration,
|
|
heartbeat,
|
|
capability_report,
|
|
debug_command,
|
|
log_event,
|
|
vfs_metadata,
|
|
recorded,
|
|
session.requests(),
|
|
))
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn record_cancelled_task(
|
|
args: &Args,
|
|
session: &mut CoordinatorSession,
|
|
task: &RuntimeTask,
|
|
registration: Value,
|
|
heartbeat: Value,
|
|
capability_report: Value,
|
|
debug_command: Value,
|
|
node_private_key: &str,
|
|
) -> Result<Value, Box<dyn std::error::Error>> {
|
|
let recorded = session.request(signed_node_request_json(
|
|
args,
|
|
node_private_key,
|
|
"task_completed",
|
|
json!({
|
|
"type": "task_completed",
|
|
"tenant": &args.tenant,
|
|
"project": &args.project,
|
|
"process": &task.process,
|
|
"node": &args.node,
|
|
"task": &task.task,
|
|
"terminal_state": "cancelled",
|
|
"status_code": null,
|
|
"stdout_bytes": 0,
|
|
"stderr_bytes": 0,
|
|
"stdout_tail": "",
|
|
"stderr_tail": "",
|
|
"stdout_truncated": false,
|
|
"stderr_truncated": false,
|
|
"artifact_path": null,
|
|
"artifact_digest": null,
|
|
"artifact_size_bytes": null,
|
|
"result": null,
|
|
}),
|
|
)?)?;
|
|
Ok(cancelled_node_report(
|
|
task,
|
|
registration,
|
|
heartbeat,
|
|
capability_report,
|
|
task.task_assignment_response.clone(),
|
|
debug_command,
|
|
recorded,
|
|
session.requests(),
|
|
))
|
|
}
|
|
|
|
fn bounded_runtime_error(error: &str) -> String {
|
|
const MAX_BYTES: usize = 16 * 1024;
|
|
if error.len() <= MAX_BYTES {
|
|
return error.to_owned();
|
|
}
|
|
let boundary = error
|
|
.char_indices()
|
|
.take_while(|(index, _)| *index <= MAX_BYTES)
|
|
.map(|(index, _)| index)
|
|
.last()
|
|
.unwrap_or(0);
|
|
format!("{}\n<truncated>", &error[..boundary])
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn completed_node_report(
|
|
output: CommandOutput,
|
|
large_bytes_uploaded: bool,
|
|
registration_response: Value,
|
|
heartbeat_response: Value,
|
|
capability_response: Value,
|
|
task_assignment_response: Value,
|
|
debug_command_response: Value,
|
|
log_event_response: Value,
|
|
vfs_metadata_response: Value,
|
|
coordinator_response: Value,
|
|
session_requests: usize,
|
|
) -> Value {
|
|
json!({
|
|
"node_status": "completed",
|
|
"virtual_thread": output.virtual_thread,
|
|
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
|
|
"status_code": output.status_code,
|
|
"stdout_bytes": output.stdout.len(),
|
|
"stderr_bytes": output.stderr.len(),
|
|
"stdout_tail": &output.stdout,
|
|
"stderr_tail": &output.stderr,
|
|
"stdout_truncated": output.stdout_truncated,
|
|
"stderr_truncated": output.stderr_truncated,
|
|
"log_backpressured": output.log_backpressured,
|
|
"staged_artifact": output.staged_artifact,
|
|
"large_bytes_uploaded": large_bytes_uploaded,
|
|
"registration_response": registration_response,
|
|
"heartbeat_response": heartbeat_response,
|
|
"capability_response": capability_response,
|
|
"task_assignment_response": task_assignment_response,
|
|
"debug_command_response": debug_command_response,
|
|
"log_event_response": log_event_response,
|
|
"vfs_metadata_response": vfs_metadata_response,
|
|
"session_requests": session_requests,
|
|
"coordinator_response": coordinator_response,
|
|
})
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn cancelled_node_report(
|
|
task: &RuntimeTask,
|
|
registration_response: Value,
|
|
heartbeat_response: Value,
|
|
capability_response: Value,
|
|
task_assignment_response: Value,
|
|
debug_command_response: Value,
|
|
coordinator_response: Value,
|
|
session_requests: usize,
|
|
) -> Value {
|
|
json!({
|
|
"node_status": "cancelled",
|
|
"virtual_thread": &task.task,
|
|
"terminal_state": "cancelled",
|
|
"status_code": null,
|
|
"stdout_bytes": 0,
|
|
"stderr_bytes": 0,
|
|
"stdout_tail": "",
|
|
"stderr_tail": "",
|
|
"stdout_truncated": false,
|
|
"stderr_truncated": false,
|
|
"log_backpressured": false,
|
|
"staged_artifact": null,
|
|
"large_bytes_uploaded": false,
|
|
"registration_response": registration_response,
|
|
"heartbeat_response": heartbeat_response,
|
|
"capability_response": capability_response,
|
|
"task_assignment_response": task_assignment_response,
|
|
"debug_command_response": debug_command_response,
|
|
"log_event_response": null,
|
|
"vfs_metadata_response": null,
|
|
"session_requests": session_requests,
|
|
"coordinator_response": coordinator_response,
|
|
})
|
|
}
|
|
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub(crate) fn failed_node_report(
|
|
task: &RuntimeTask,
|
|
error: &str,
|
|
registration_response: Value,
|
|
heartbeat_response: Value,
|
|
capability_response: Value,
|
|
debug_command_response: Value,
|
|
log_event_response: Value,
|
|
vfs_metadata_response: Value,
|
|
coordinator_response: Value,
|
|
session_requests: usize,
|
|
) -> Value {
|
|
json!({
|
|
"node_status": "failed",
|
|
"virtual_thread": &task.task,
|
|
"terminal_state": "failed",
|
|
"status_code": -1,
|
|
"stdout_bytes": 0,
|
|
"stderr_bytes": error.len(),
|
|
"stdout_tail": "",
|
|
"stderr_tail": error,
|
|
"stdout_truncated": false,
|
|
"stderr_truncated": false,
|
|
"log_backpressured": false,
|
|
"staged_artifact": null,
|
|
"large_bytes_uploaded": false,
|
|
"registration_response": registration_response,
|
|
"heartbeat_response": heartbeat_response,
|
|
"capability_response": capability_response,
|
|
"task_assignment_response": &task.task_assignment_response,
|
|
"debug_command_response": debug_command_response,
|
|
"log_event_response": log_event_response,
|
|
"vfs_metadata_response": vfs_metadata_response,
|
|
"session_requests": session_requests,
|
|
"coordinator_response": coordinator_response,
|
|
})
|
|
}
|