use clusterflux_core::{TaskBoundaryValue, VfsManifest}; use clusterflux_node::CommandOutput; use serde_json::{json, Value}; use crate::daemon::RuntimeTask; use crate::{ coordinator_session::CoordinatorSession, daemon::Args, node_identity::signed_node_request_json, task_artifacts::RetainedArtifact, }; #[allow(clippy::too_many_arguments)] pub(crate) fn record_completed_task( args: &Args, session: &mut CoordinatorSession, task: RuntimeTask, output: CommandOutput, manifest: VfsManifest, result: Option, retained: Option, registration: Value, heartbeat: Value, capability_report: Value, debug_command: Value, node_private_key: &str, ) -> Result> { let staged = output.staged_artifact.as_ref(); let artifact_digest = retained .as_ref() .map(|artifact| artifact.digest.clone()) .or_else(|| staged.map(|artifact| artifact.digest.clone())); let artifact_path = retained .as_ref() .map(|artifact| format!("/vfs/artifacts/{}", artifact.id)) .or_else(|| staged.map(|artifact| artifact.path.as_str().to_owned())); let artifact_size_bytes = retained .as_ref() .map(|artifact| artifact.size_bytes) .or_else(|| staged.map(|artifact| artifact.size)); let log_event = session.request(signed_node_request_json( args, node_private_key, "report_task_log", json!({ "type": "report_task_log", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "stdout_bytes": output.stdout.len(), "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, "stderr_truncated": output.stderr_truncated, "backpressured": output.log_backpressured, }), )?)?; let vfs_metadata = session.request(signed_node_request_json( args, node_private_key, "report_vfs_metadata", json!({ "type": "report_vfs_metadata", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "artifact_path": artifact_path, "artifact_digest": artifact_digest, "artifact_size_bytes": artifact_size_bytes, "large_bytes_uploaded": manifest.large_bytes_uploaded, }), )?)?; let recorded = session.request(signed_node_request_json( args, node_private_key, "task_completed", json!({ "type": "task_completed", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "status_code": output.status_code, "stdout_bytes": output.stdout.len(), "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, "stderr_truncated": output.stderr_truncated, "artifact_path": artifact_path, "artifact_digest": artifact_digest, "artifact_size_bytes": artifact_size_bytes, "result": result, }), )?)?; Ok(completed_node_report( output, manifest.large_bytes_uploaded, registration, heartbeat, capability_report, task.task_assignment_response, debug_command, log_event, vfs_metadata, recorded, session.requests(), )) } #[allow(clippy::too_many_arguments)] pub(crate) fn record_failed_task( args: &Args, session: &mut CoordinatorSession, task: &RuntimeTask, registration: Value, heartbeat: Value, capability_report: Value, debug_command: Value, node_private_key: &str, error: &str, ) -> Result> { let error = bounded_runtime_error(error); let log_event = session.request(signed_node_request_json( args, node_private_key, "report_task_log", json!({ "type": "report_task_log", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "stdout_bytes": 0, "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, "stderr_truncated": false, "backpressured": false, }), )?)?; let vfs_metadata = session.request(signed_node_request_json( args, node_private_key, "report_vfs_metadata", json!({ "type": "report_vfs_metadata", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, "large_bytes_uploaded": false, }), )?)?; let recorded = session.request(signed_node_request_json( args, node_private_key, "task_completed", json!({ "type": "task_completed", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "terminal_state": "failed", "status_code": -1, "stdout_bytes": 0, "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": &error, "stdout_truncated": false, "stderr_truncated": false, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, }), )?)?; Ok(failed_node_report( task, &error, registration, heartbeat, capability_report, debug_command, log_event, vfs_metadata, recorded, session.requests(), )) } #[allow(clippy::too_many_arguments)] pub(crate) fn record_cancelled_task( args: &Args, session: &mut CoordinatorSession, task: &RuntimeTask, registration: Value, heartbeat: Value, capability_report: Value, debug_command: Value, node_private_key: &str, ) -> Result> { let recorded = session.request(signed_node_request_json( args, node_private_key, "task_completed", json!({ "type": "task_completed", "tenant": &args.tenant, "project": &args.project, "process": &task.process, "node": &args.node, "task": &task.task, "terminal_state": "cancelled", "status_code": null, "stdout_bytes": 0, "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", "stdout_truncated": false, "stderr_truncated": false, "artifact_path": null, "artifact_digest": null, "artifact_size_bytes": null, "result": null, }), )?)?; Ok(cancelled_node_report( task, registration, heartbeat, capability_report, task.task_assignment_response.clone(), debug_command, recorded, session.requests(), )) } fn bounded_runtime_error(error: &str) -> String { const MAX_BYTES: usize = 16 * 1024; if error.len() <= MAX_BYTES { return error.to_owned(); } let boundary = error .char_indices() .take_while(|(index, _)| *index <= MAX_BYTES) .map(|(index, _)| index) .last() .unwrap_or(0); format!("{}\n", &error[..boundary]) } #[allow(clippy::too_many_arguments)] pub(crate) fn completed_node_report( output: CommandOutput, large_bytes_uploaded: bool, registration_response: Value, heartbeat_response: Value, capability_response: Value, task_assignment_response: Value, debug_command_response: Value, log_event_response: Value, vfs_metadata_response: Value, coordinator_response: Value, session_requests: usize, ) -> Value { json!({ "node_status": "completed", "virtual_thread": output.virtual_thread, "terminal_state": if output.status_code == Some(0) { "completed" } else { "failed" }, "status_code": output.status_code, "stdout_bytes": output.stdout.len(), "stderr_bytes": output.stderr.len(), "stdout_tail": &output.stdout, "stderr_tail": &output.stderr, "stdout_truncated": output.stdout_truncated, "stderr_truncated": output.stderr_truncated, "log_backpressured": output.log_backpressured, "staged_artifact": output.staged_artifact, "large_bytes_uploaded": large_bytes_uploaded, "registration_response": registration_response, "heartbeat_response": heartbeat_response, "capability_response": capability_response, "task_assignment_response": task_assignment_response, "debug_command_response": debug_command_response, "log_event_response": log_event_response, "vfs_metadata_response": vfs_metadata_response, "session_requests": session_requests, "coordinator_response": coordinator_response, }) } #[allow(clippy::too_many_arguments)] pub(crate) fn cancelled_node_report( task: &RuntimeTask, registration_response: Value, heartbeat_response: Value, capability_response: Value, task_assignment_response: Value, debug_command_response: Value, coordinator_response: Value, session_requests: usize, ) -> Value { json!({ "node_status": "cancelled", "virtual_thread": &task.task, "terminal_state": "cancelled", "status_code": null, "stdout_bytes": 0, "stderr_bytes": 0, "stdout_tail": "", "stderr_tail": "", "stdout_truncated": false, "stderr_truncated": false, "log_backpressured": false, "staged_artifact": null, "large_bytes_uploaded": false, "registration_response": registration_response, "heartbeat_response": heartbeat_response, "capability_response": capability_response, "task_assignment_response": task_assignment_response, "debug_command_response": debug_command_response, "log_event_response": null, "vfs_metadata_response": null, "session_requests": session_requests, "coordinator_response": coordinator_response, }) } #[allow(clippy::too_many_arguments)] pub(crate) fn failed_node_report( task: &RuntimeTask, error: &str, registration_response: Value, heartbeat_response: Value, capability_response: Value, debug_command_response: Value, log_event_response: Value, vfs_metadata_response: Value, coordinator_response: Value, session_requests: usize, ) -> Value { json!({ "node_status": "failed", "virtual_thread": &task.task, "terminal_state": "failed", "status_code": -1, "stdout_bytes": 0, "stderr_bytes": error.len(), "stdout_tail": "", "stderr_tail": error, "stdout_truncated": false, "stderr_truncated": false, "log_backpressured": false, "staged_artifact": null, "large_bytes_uploaded": false, "registration_response": registration_response, "heartbeat_response": heartbeat_response, "capability_response": capability_response, "task_assignment_response": &task.task_assignment_response, "debug_command_response": debug_command_response, "log_event_response": log_event_response, "vfs_metadata_response": vfs_metadata_response, "session_requests": session_requests, "coordinator_response": coordinator_response, }) }