use std::thread; use std::time::{Duration, Instant}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use clusterflux_control::ControlSession; #[cfg(target_arch = "wasm32")] use clusterflux_core::TaskDefinitionId; use clusterflux_core::{ coordinator_wire_request, Digest, TaskBoundaryValue, TaskFailurePolicy, TaskJoinResult, TaskJoinState, TaskSpec, }; use serde::de::DeserializeOwned; use serde_json::{json, Value}; use crate::EnvRef; const COORDINATOR_ENV: &str = "CLUSTERFLUX_SDK_COORDINATOR"; const TENANT_ENV: &str = "CLUSTERFLUX_SDK_TENANT"; const PROJECT_ENV: &str = "CLUSTERFLUX_SDK_PROJECT"; const PROCESS_ENV: &str = "CLUSTERFLUX_SDK_PROCESS"; const USER_ENV: &str = "CLUSTERFLUX_SDK_USER"; const SESSION_SECRET_ENV: &str = "CLUSTERFLUX_SDK_SESSION_SECRET"; const BUNDLE_DIGEST_ENV: &str = "CLUSTERFLUX_SDK_BUNDLE_DIGEST"; const WASM_MODULE_ENV: &str = "CLUSTERFLUX_SDK_WASM_MODULE_BASE64"; const WASM_MODULE_PATH_ENV: &str = "CLUSTERFLUX_SDK_WASM_MODULE_PATH"; const VFS_EPOCH_ENV: &str = "CLUSTERFLUX_SDK_VFS_EPOCH"; const JOIN_TIMEOUT_SECONDS_ENV: &str = clusterflux_core::limits::TASK_JOIN_TIMEOUT_SECONDS_ENV; const DEFAULT_JOIN_TIMEOUT_SECONDS: u64 = clusterflux_core::limits::DEFAULT_TASK_JOIN_TIMEOUT_SECONDS; #[derive(Clone, Debug, PartialEq, Eq)] pub struct ProductRuntimeConfig { pub coordinator: String, pub tenant: String, pub project: String, pub process: String, pub actor_user: String, pub session_secret: Option, pub bundle_digest: Digest, pub wasm_module_base64: String, pub vfs_epoch: u64, pub join_poll_interval: Duration, pub join_timeout: Duration, } impl ProductRuntimeConfig { pub fn from_env() -> Result, TaskRuntimeError> { let Some(coordinator) = nonempty_env(COORDINATOR_ENV) else { return Ok(None); }; let tenant = required_env(TENANT_ENV)?; let project = required_env(PROJECT_ENV)?; let process = required_env(PROCESS_ENV)?; let actor_user = required_env(USER_ENV)?; let bundle_digest = parse_digest(&required_env(BUNDLE_DIGEST_ENV)?)?; let wasm_module_base64 = if let Some(module) = nonempty_env(WASM_MODULE_ENV) { module } else { let module_path = required_env(WASM_MODULE_PATH_ENV)?; let module = std::fs::read(&module_path).map_err(|error| { TaskRuntimeError::Configuration(format!( "read {WASM_MODULE_PATH_ENV} `{module_path}`: {error}" )) })?; BASE64_STANDARD.encode(module) }; let vfs_epoch = required_env(VFS_EPOCH_ENV)?.parse::().map_err(|_| { TaskRuntimeError::Configuration(format!("{VFS_EPOCH_ENV} must be an unsigned integer")) })?; Ok(Some(Self { coordinator, tenant, project, process, actor_user, session_secret: nonempty_env(SESSION_SECRET_ENV), bundle_digest, wasm_module_base64, vfs_epoch, join_poll_interval: Duration::from_millis(25), join_timeout: Duration::from_secs(optional_positive_u64_env( JOIN_TIMEOUT_SECONDS_ENV, DEFAULT_JOIN_TIMEOUT_SECONDS, )?), })) } } #[derive(Clone, Debug)] pub(crate) struct RemoteTaskHandle { pub config: Option, pub spec: TaskSpec, #[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))] pub host_handle_id: Option, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum TaskRuntimeError { Argument(String), Configuration(String), Transport(String), Protocol(String), RemoteTask(String), JoinTimeout { task: clusterflux_core::TaskInstanceId, waited: Duration, }, ResultDecode(String), CommandFailed { program: String, status_code: Option, stdout: String, stderr: String, stdout_truncated: bool, stderr_truncated: bool, }, } impl std::fmt::Display for TaskRuntimeError { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let (kind, message) = match self { Self::Argument(message) => ("task argument", message), Self::Configuration(message) => ("product runtime configuration", message), Self::Transport(message) => ("coordinator transport", message), Self::Protocol(message) => ("coordinator protocol", message), Self::RemoteTask(message) => ("remote task", message), Self::JoinTimeout { task, waited } => { return write!( formatter, "task join timeout: {task} remained pending for {} seconds; the child task was left running", waited.as_secs() ); } Self::ResultDecode(message) => ("remote result", message), Self::CommandFailed { program, status_code, stdout, stderr, stdout_truncated, stderr_truncated, } => { return write!( formatter, "command `{program}` failed with status {status_code:?}; stdout{}: {stdout:?}; stderr{}: {stderr:?}", if *stdout_truncated { " (truncated)" } else { "" }, if *stderr_truncated { " (truncated)" } else { "" }, ); } }; write!(formatter, "{kind} error: {message}") } } impl std::error::Error for TaskRuntimeError {} #[cfg(not(target_arch = "wasm32"))] pub(crate) fn start_remote_task( config: ProductRuntimeConfig, task_definition: String, environment: Option, args: Vec, failure_policy: TaskFailurePolicy, ) -> Result { let _ = (config, task_definition, environment, args, failure_policy); Err(TaskRuntimeError::Configuration( "native SDK task spawning requires coordinator EntrypointV1 execution; external TaskV1 launch is forbidden" .to_owned(), )) } pub(crate) fn join_remote_task(handle: RemoteTaskHandle) -> Result where R: DeserializeOwned, { #[cfg(target_arch = "wasm32")] if let Some(handle_id) = handle.host_handle_id { let response: Result = guest_host_call( &clusterflux_core::WasmHostTaskJoinRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, handle_id, }, GuestHostCall::Join, )?; let response = response.map_err(TaskRuntimeError::RemoteTask)?; return decode_boundary_value(response.result); } let config = handle.config.as_ref().ok_or_else(|| { TaskRuntimeError::Protocol("remote task handle has no coordinator authority".to_owned()) })?; let started = Instant::now(); loop { let response = coordinator_request( config, json!({ "type": "join_task", "tenant": config.tenant, "project": config.project, "actor_user": config.actor_user, "process": config.process, "task": handle.spec.task_instance, }), )?; if response.get("type").and_then(Value::as_str) == Some("error") { return Err(remote_error(&response)); } if response.get("type").and_then(Value::as_str) != Some("task_joined") { return Err(TaskRuntimeError::Protocol( "join response was not task_joined".to_owned(), )); } let join: TaskJoinResult = serde_json::from_value( response .get("join") .cloned() .ok_or_else(|| TaskRuntimeError::Protocol("join result missing".to_owned()))?, ) .map_err(|error| TaskRuntimeError::Protocol(error.to_string()))?; match join.state { TaskJoinState::Pending => { if started.elapsed() >= config.join_timeout { return Err(TaskRuntimeError::JoinTimeout { task: handle.spec.task_instance.clone(), waited: config.join_timeout, }); } thread::sleep(config.join_poll_interval); } TaskJoinState::Completed => { if !join.remote_completion_observed { return Err(TaskRuntimeError::Protocol( "completed join did not prove a signed node completion".to_owned(), )); } let boundary = join.result.ok_or_else(|| { TaskRuntimeError::ResultDecode( "completed remote task did not return a boundary value".to_owned(), ) })?; return decode_boundary_value(boundary); } TaskJoinState::Failed | TaskJoinState::Cancelled => { return Err(TaskRuntimeError::RemoteTask(join.message)); } } } } #[cfg(target_arch = "wasm32")] pub(crate) fn start_guest_host_task( task_definition: TaskDefinitionId, environment: Option, args: Vec, failure_policy: TaskFailurePolicy, ) -> Result { let request = clusterflux_core::WasmHostTaskStartRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, task_definition, environment_id: environment.map(|environment| environment.name.to_owned()), args, failure_policy, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::Start)?; let response = response.map_err(TaskRuntimeError::RemoteTask)?; Ok(RemoteTaskHandle { config: None, spec: response.task_spec, host_handle_id: Some(response.handle_id), }) } #[cfg(target_arch = "wasm32")] enum GuestHostCall { Start, Join, Command, TaskControl, DebugProbe, SourceSnapshot, Vfs, } #[cfg(target_arch = "wasm32")] pub(crate) fn run_guest_host_command( program: String, args: Vec, working_directory: String, environment_variables: std::collections::BTreeMap, timeout_ms: u64, network: clusterflux_core::CommandNetworkPolicy, ) -> Result { let request = clusterflux_core::WasmHostCommandRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, program, args, working_directory, environment_variables, timeout_ms, network, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::Command)?; response.map_err(TaskRuntimeError::RemoteTask) } #[cfg(target_arch = "wasm32")] pub(crate) fn guest_cancellation_requested() -> Result { let request = clusterflux_core::WasmHostTaskControlRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::TaskControl)?; response .map(|result| result.cancellation_requested) .map_err(TaskRuntimeError::RemoteTask) } #[cfg(target_arch = "wasm32")] pub(crate) fn guest_debug_probe( symbol: String, ) -> Result { let request = clusterflux_core::WasmHostDebugProbeRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, symbol, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::DebugProbe)?; response.map_err(TaskRuntimeError::RemoteTask) } #[cfg(target_arch = "wasm32")] pub(crate) fn guest_vfs_operation( operation: clusterflux_core::WasmHostVfsOperation, ) -> Result { let request = clusterflux_core::WasmHostVfsRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, operation, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::Vfs)?; response.map_err(TaskRuntimeError::RemoteTask) } #[cfg(target_arch = "wasm32")] pub(crate) fn guest_source_snapshot( ) -> Result { let request = clusterflux_core::WasmHostSourceSnapshotRequest { abi_version: clusterflux_core::WASM_TASK_ABI_VERSION, }; request.validate().map_err(TaskRuntimeError::Argument)?; let response: Result = guest_host_call(&request, GuestHostCall::SourceSnapshot)?; response.map_err(TaskRuntimeError::RemoteTask) } #[cfg(target_arch = "wasm32")] fn guest_host_call(input: &I, call: GuestHostCall) -> Result where I: serde::Serialize, O: DeserializeOwned, { #[link(wasm_import_module = "clusterflux")] unsafe extern "C" { #[link_name = "task_start_v1"] fn task_start_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "task_join_v1"] fn task_join_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "command_run_v1"] fn command_run_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "vfs_operation_v1"] fn vfs_operation_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "source_snapshot_v1"] fn source_snapshot_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "task_control_v1"] fn task_control_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; #[link_name = "debug_probe_v1"] fn debug_probe_v1( input_pointer: u32, input_length: u32, output_pointer: u32, output_capacity: u32, ) -> i32; } let input = serde_json::to_vec(input).map_err(|error| TaskRuntimeError::Protocol(error.to_string()))?; if input.len() > clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES { return Err(TaskRuntimeError::Argument( "Wasm host-call request exceeds the task ABI limit".to_owned(), )); } let mut output = vec![0_u8; clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES]; let written = unsafe { match call { GuestHostCall::Start => task_start_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::Join => task_join_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::Command => command_run_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::TaskControl => task_control_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::DebugProbe => debug_probe_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::Vfs => vfs_operation_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), GuestHostCall::SourceSnapshot => source_snapshot_v1( input.as_ptr() as u32, input.len() as u32, output.as_mut_ptr() as u32, output.len() as u32, ), } }; if written <= 0 || written as usize > output.len() { return Err(TaskRuntimeError::Protocol(format!( "Wasm host-call returned invalid response length {written}" ))); } serde_json::from_slice(&output[..written as usize]) .map_err(|error| TaskRuntimeError::Protocol(error.to_string())) } fn coordinator_request( config: &ProductRuntimeConfig, payload: Value, ) -> Result { let payload = if let Some(session_secret) = &config.session_secret { let request = payload .as_object() .cloned() .ok_or_else(|| TaskRuntimeError::Protocol("request must be an object".to_owned()))?; let request = request .into_iter() .filter(|(key, _)| !matches!(key.as_str(), "tenant" | "project" | "actor_user")) .collect::>(); json!({ "type": "authenticated", "session_secret": session_secret, "request": request, }) } else { payload }; let wire = coordinator_wire_request("sdk-product-runtime", payload); let mut session = ControlSession::connect(&config.coordinator) .map_err(|error| TaskRuntimeError::Transport(error.to_string()))?; session .request(&wire) .map_err(|error| TaskRuntimeError::Transport(error.to_string())) } fn decode_boundary_value(value: TaskBoundaryValue) -> Result where R: DeserializeOwned, { let value = value .materialize() .map_err(TaskRuntimeError::ResultDecode)?; serde_json::from_value(value).map_err(|error| TaskRuntimeError::ResultDecode(error.to_string())) } fn optional_positive_u64_env(name: &str, default: u64) -> Result { let Some(value) = nonempty_env(name) else { return Ok(default); }; value .parse::() .ok() .filter(|value| *value > 0) .ok_or_else(|| { TaskRuntimeError::Configuration(format!("{name} must be a positive integer")) }) } fn remote_error(response: &Value) -> TaskRuntimeError { TaskRuntimeError::RemoteTask( response .get("message") .and_then(Value::as_str) .unwrap_or("coordinator rejected task request") .to_owned(), ) } fn parse_digest(value: &str) -> Result { let digest: Digest = serde_json::from_value(Value::String(value.to_owned())) .map_err(|error| TaskRuntimeError::Configuration(error.to_string()))?; if !digest.is_valid_sha256() { return Err(TaskRuntimeError::Configuration(format!( "{BUNDLE_DIGEST_ENV} must be a lowercase sha256 digest" ))); } Ok(digest) } fn required_env(name: &str) -> Result { nonempty_env(name).ok_or_else(|| { TaskRuntimeError::Configuration(format!( "{name} is required when {COORDINATOR_ENV} enables product mode" )) }) } fn nonempty_env(name: &str) -> Option { std::env::var(name) .ok() .map(|value| value.trim().to_owned()) .filter(|value| !value.is_empty()) }