Public release release-e570b80c3a0f
Source commit: e570b80c3a0f4b152a9aa0e7bb8635b3eebcec27 Public tree identity: sha256:cff3388143eb2e9476804cfa1c1246e3a46d485d8aef90f2cdd4df45f2b69abf
This commit is contained in:
Clusterflux release
commit
0cf66aaa65
210 changed files with 78651 additions and 0 deletions
379
crates/clusterflux-node/src/task_reports.rs
Normal file
379
crates/clusterflux-node/src/task_reports.rs
Normal file
|
|
@ -0,0 +1,379 @@
|
|||
use clusterflux_core::{TaskBoundaryValue, VfsManifest};
|
||||
use clusterflux_node::CommandOutput;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::daemon::RuntimeTask;
|
||||
use crate::{
|
||||
coordinator_session::CoordinatorSession, daemon::Args, node_identity::signed_node_request_json,
|
||||
task_artifacts::RetainedArtifact,
|
||||
};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn record_completed_task(
|
||||
args: &Args,
|
||||
session: &mut CoordinatorSession,
|
||||
task: RuntimeTask,
|
||||
output: CommandOutput,
|
||||
manifest: VfsManifest,
|
||||
result: Option<TaskBoundaryValue>,
|
||||
retained: Option<RetainedArtifact>,
|
||||
registration: Value,
|
||||
heartbeat: Value,
|
||||
capability_report: Value,
|
||||
debug_command: Value,
|
||||
node_private_key: &str,
|
||||
) -> Result<Value, Box<dyn std::error::Error>> {
|
||||
let staged = output.staged_artifact.as_ref();
|
||||
let artifact_digest = retained
|
||||
.as_ref()
|
||||
.map(|artifact| artifact.digest.clone())
|
||||
.or_else(|| staged.map(|artifact| artifact.digest.clone()));
|
||||
let artifact_path = retained
|
||||
.as_ref()
|
||||
.map(|artifact| format!("/vfs/artifacts/{}", artifact.id))
|
||||
.or_else(|| staged.map(|artifact| artifact.path.as_str().to_owned()));
|
||||
let artifact_size_bytes = retained
|
||||
.as_ref()
|
||||
.map(|artifact| artifact.size_bytes)
|
||||
.or_else(|| staged.map(|artifact| artifact.size));
|
||||
let log_event = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"report_task_log",
|
||||
json!({
|
||||
"type": "report_task_log",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"stdout_bytes": output.stdout.len(),
|
||||
"stderr_bytes": output.stderr.len(),
|
||||
"stdout_tail": &output.stdout,
|
||||
"stderr_tail": &output.stderr,
|
||||
"stdout_truncated": output.stdout_truncated,
|
||||
"stderr_truncated": output.stderr_truncated,
|
||||
"backpressured": output.log_backpressured,
|
||||
}),
|
||||
)?)?;
|
||||
let vfs_metadata = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"report_vfs_metadata",
|
||||
json!({
|
||||
"type": "report_vfs_metadata",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"artifact_path": artifact_path,
|
||||
"artifact_digest": artifact_digest,
|
||||
"artifact_size_bytes": artifact_size_bytes,
|
||||
"large_bytes_uploaded": manifest.large_bytes_uploaded,
|
||||
}),
|
||||
)?)?;
|
||||
let recorded = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"task_completed",
|
||||
json!({
|
||||
"type": "task_completed",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"status_code": output.status_code,
|
||||
"stdout_bytes": output.stdout.len(),
|
||||
"stderr_bytes": output.stderr.len(),
|
||||
"stdout_tail": &output.stdout,
|
||||
"stderr_tail": &output.stderr,
|
||||
"stdout_truncated": output.stdout_truncated,
|
||||
"stderr_truncated": output.stderr_truncated,
|
||||
"artifact_path": artifact_path,
|
||||
"artifact_digest": artifact_digest,
|
||||
"artifact_size_bytes": artifact_size_bytes,
|
||||
"result": result,
|
||||
}),
|
||||
)?)?;
|
||||
Ok(completed_node_report(
|
||||
output,
|
||||
manifest.large_bytes_uploaded,
|
||||
registration,
|
||||
heartbeat,
|
||||
capability_report,
|
||||
task.task_assignment_response,
|
||||
debug_command,
|
||||
log_event,
|
||||
vfs_metadata,
|
||||
recorded,
|
||||
session.requests(),
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn record_failed_task(
|
||||
args: &Args,
|
||||
session: &mut CoordinatorSession,
|
||||
task: &RuntimeTask,
|
||||
registration: Value,
|
||||
heartbeat: Value,
|
||||
capability_report: Value,
|
||||
debug_command: Value,
|
||||
node_private_key: &str,
|
||||
error: &str,
|
||||
) -> Result<Value, Box<dyn std::error::Error>> {
|
||||
let error = bounded_runtime_error(error);
|
||||
let log_event = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"report_task_log",
|
||||
json!({
|
||||
"type": "report_task_log",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": error.len(),
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": &error,
|
||||
"stdout_truncated": false,
|
||||
"stderr_truncated": false,
|
||||
"backpressured": false,
|
||||
}),
|
||||
)?)?;
|
||||
let vfs_metadata = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"report_vfs_metadata",
|
||||
json!({
|
||||
"type": "report_vfs_metadata",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"artifact_path": null,
|
||||
"artifact_digest": null,
|
||||
"artifact_size_bytes": null,
|
||||
"large_bytes_uploaded": false,
|
||||
}),
|
||||
)?)?;
|
||||
let recorded = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"task_completed",
|
||||
json!({
|
||||
"type": "task_completed",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"terminal_state": "failed",
|
||||
"status_code": -1,
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": error.len(),
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": &error,
|
||||
"stdout_truncated": false,
|
||||
"stderr_truncated": false,
|
||||
"artifact_path": null,
|
||||
"artifact_digest": null,
|
||||
"artifact_size_bytes": null,
|
||||
}),
|
||||
)?)?;
|
||||
Ok(failed_node_report(
|
||||
task,
|
||||
&error,
|
||||
registration,
|
||||
heartbeat,
|
||||
capability_report,
|
||||
debug_command,
|
||||
log_event,
|
||||
vfs_metadata,
|
||||
recorded,
|
||||
session.requests(),
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn record_cancelled_task(
|
||||
args: &Args,
|
||||
session: &mut CoordinatorSession,
|
||||
task: &RuntimeTask,
|
||||
registration: Value,
|
||||
heartbeat: Value,
|
||||
capability_report: Value,
|
||||
debug_command: Value,
|
||||
node_private_key: &str,
|
||||
) -> Result<Value, Box<dyn std::error::Error>> {
|
||||
let recorded = session.request(signed_node_request_json(
|
||||
args,
|
||||
node_private_key,
|
||||
"task_completed",
|
||||
json!({
|
||||
"type": "task_completed",
|
||||
"tenant": &args.tenant,
|
||||
"project": &args.project,
|
||||
"process": &task.process,
|
||||
"node": &args.node,
|
||||
"task": &task.task,
|
||||
"terminal_state": "cancelled",
|
||||
"status_code": null,
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": 0,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"stdout_truncated": false,
|
||||
"stderr_truncated": false,
|
||||
"artifact_path": null,
|
||||
"artifact_digest": null,
|
||||
"artifact_size_bytes": null,
|
||||
"result": null,
|
||||
}),
|
||||
)?)?;
|
||||
Ok(cancelled_node_report(
|
||||
task,
|
||||
registration,
|
||||
heartbeat,
|
||||
capability_report,
|
||||
task.task_assignment_response.clone(),
|
||||
debug_command,
|
||||
recorded,
|
||||
session.requests(),
|
||||
))
|
||||
}
|
||||
|
||||
fn bounded_runtime_error(error: &str) -> String {
|
||||
const MAX_BYTES: usize = 16 * 1024;
|
||||
if error.len() <= MAX_BYTES {
|
||||
return error.to_owned();
|
||||
}
|
||||
let boundary = error
|
||||
.char_indices()
|
||||
.take_while(|(index, _)| *index <= MAX_BYTES)
|
||||
.map(|(index, _)| index)
|
||||
.last()
|
||||
.unwrap_or(0);
|
||||
format!("{}\n<truncated>", &error[..boundary])
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn completed_node_report(
|
||||
output: CommandOutput,
|
||||
large_bytes_uploaded: bool,
|
||||
registration_response: Value,
|
||||
heartbeat_response: Value,
|
||||
capability_response: Value,
|
||||
task_assignment_response: Value,
|
||||
debug_command_response: Value,
|
||||
log_event_response: Value,
|
||||
vfs_metadata_response: Value,
|
||||
coordinator_response: Value,
|
||||
session_requests: usize,
|
||||
) -> Value {
|
||||
json!({
|
||||
"node_status": "completed",
|
||||
"virtual_thread": output.virtual_thread,
|
||||
"terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" },
|
||||
"status_code": output.status_code,
|
||||
"stdout_bytes": output.stdout.len(),
|
||||
"stderr_bytes": output.stderr.len(),
|
||||
"stdout_tail": &output.stdout,
|
||||
"stderr_tail": &output.stderr,
|
||||
"stdout_truncated": output.stdout_truncated,
|
||||
"stderr_truncated": output.stderr_truncated,
|
||||
"log_backpressured": output.log_backpressured,
|
||||
"staged_artifact": output.staged_artifact,
|
||||
"large_bytes_uploaded": large_bytes_uploaded,
|
||||
"registration_response": registration_response,
|
||||
"heartbeat_response": heartbeat_response,
|
||||
"capability_response": capability_response,
|
||||
"task_assignment_response": task_assignment_response,
|
||||
"debug_command_response": debug_command_response,
|
||||
"log_event_response": log_event_response,
|
||||
"vfs_metadata_response": vfs_metadata_response,
|
||||
"session_requests": session_requests,
|
||||
"coordinator_response": coordinator_response,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn cancelled_node_report(
|
||||
task: &RuntimeTask,
|
||||
registration_response: Value,
|
||||
heartbeat_response: Value,
|
||||
capability_response: Value,
|
||||
task_assignment_response: Value,
|
||||
debug_command_response: Value,
|
||||
coordinator_response: Value,
|
||||
session_requests: usize,
|
||||
) -> Value {
|
||||
json!({
|
||||
"node_status": "cancelled",
|
||||
"virtual_thread": &task.task,
|
||||
"terminal_state": "cancelled",
|
||||
"status_code": null,
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": 0,
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": "",
|
||||
"stdout_truncated": false,
|
||||
"stderr_truncated": false,
|
||||
"log_backpressured": false,
|
||||
"staged_artifact": null,
|
||||
"large_bytes_uploaded": false,
|
||||
"registration_response": registration_response,
|
||||
"heartbeat_response": heartbeat_response,
|
||||
"capability_response": capability_response,
|
||||
"task_assignment_response": task_assignment_response,
|
||||
"debug_command_response": debug_command_response,
|
||||
"log_event_response": null,
|
||||
"vfs_metadata_response": null,
|
||||
"session_requests": session_requests,
|
||||
"coordinator_response": coordinator_response,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn failed_node_report(
|
||||
task: &RuntimeTask,
|
||||
error: &str,
|
||||
registration_response: Value,
|
||||
heartbeat_response: Value,
|
||||
capability_response: Value,
|
||||
debug_command_response: Value,
|
||||
log_event_response: Value,
|
||||
vfs_metadata_response: Value,
|
||||
coordinator_response: Value,
|
||||
session_requests: usize,
|
||||
) -> Value {
|
||||
json!({
|
||||
"node_status": "failed",
|
||||
"virtual_thread": &task.task,
|
||||
"terminal_state": "failed",
|
||||
"status_code": -1,
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": error.len(),
|
||||
"stdout_tail": "",
|
||||
"stderr_tail": error,
|
||||
"stdout_truncated": false,
|
||||
"stderr_truncated": false,
|
||||
"log_backpressured": false,
|
||||
"staged_artifact": null,
|
||||
"large_bytes_uploaded": false,
|
||||
"registration_response": registration_response,
|
||||
"heartbeat_response": heartbeat_response,
|
||||
"capability_response": capability_response,
|
||||
"task_assignment_response": &task.task_assignment_response,
|
||||
"debug_command_response": debug_command_response,
|
||||
"log_event_response": log_event_response,
|
||||
"vfs_metadata_response": vfs_metadata_response,
|
||||
"session_requests": session_requests,
|
||||
"coordinator_response": coordinator_response,
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue