Public release release-55b5a563d642
Source commit: 55b5a563d6421f49b5f5e77bc3c1f29752da8801 Public tree identity: sha256:52790e23e2b1806b00cc220c92edcfeed445600dcde620342af2b3369e7883fa
This commit is contained in:
Clusterflux release
commit
6acd2d6eb7
210 changed files with 76963 additions and 0 deletions
18
crates/clusterflux-sdk/Cargo.toml
Normal file
18
crates/clusterflux-sdk/Cargo.toml
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
[package]
|
||||
name = "clusterflux-sdk"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "clusterflux"
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
clusterflux-core = { path = "../clusterflux-core" }
|
||||
clusterflux-control = { path = "../clusterflux-control" }
|
||||
clusterflux-macros = { path = "../clusterflux-macros" }
|
||||
futures-executor.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
477
crates/clusterflux-sdk/src/__private.rs
Normal file
477
crates/clusterflux-sdk/src/__private.rs
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
#[cfg(target_arch = "wasm32")]
|
||||
use serde::de::DeserializeOwned;
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
use crate::TaskArg;
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn clusterflux_alloc_v1(length: u32) -> u32 {
|
||||
if length as usize > clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES {
|
||||
return 0;
|
||||
}
|
||||
let mut bytes = Vec::<u8>::with_capacity(length as usize);
|
||||
let pointer = bytes.as_mut_ptr() as usize;
|
||||
std::mem::forget(bytes);
|
||||
u32::try_from(pointer).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task0_v1<R, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
R: TaskArg,
|
||||
F: FnOnce() -> R,
|
||||
{
|
||||
invoke_task0_output(
|
||||
expected_task,
|
||||
input_pointer,
|
||||
input_length,
|
||||
|| Ok(function()),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task0_result_v1<R, E, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
R: TaskArg,
|
||||
E: std::fmt::Display,
|
||||
F: FnOnce() -> Result<R, E>,
|
||||
{
|
||||
invoke_task0_output(expected_task, input_pointer, input_length, || {
|
||||
function().map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task0_async_v1<R, F, Fut>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
R: TaskArg,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = R>,
|
||||
{
|
||||
invoke_task0_output(expected_task, input_pointer, input_length, || {
|
||||
Ok(futures_executor::block_on(function()))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task0_async_result_v1<R, E, F, Fut>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
R: TaskArg,
|
||||
E: std::fmt::Display,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<R, E>>,
|
||||
{
|
||||
invoke_task0_output(expected_task, input_pointer, input_length, || {
|
||||
futures_executor::block_on(function()).map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn invoke_task0_output<R, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
R: TaskArg,
|
||||
F: FnOnce() -> Result<R, String>,
|
||||
{
|
||||
let invocation = match decode_invocation(expected_task, input_pointer, input_length) {
|
||||
Ok(invocation) => invocation,
|
||||
Err(error) => return encode_result(failed_result(expected_task, error)),
|
||||
};
|
||||
if !invocation.args.is_empty() {
|
||||
return encode_result(failed_result(
|
||||
invocation.task_instance.clone(),
|
||||
format!(
|
||||
"task expected zero arguments but received {}",
|
||||
invocation.args.len()
|
||||
),
|
||||
));
|
||||
}
|
||||
let result = match function() {
|
||||
Ok(output) => match output.task_boundary_value() {
|
||||
Ok(result) => {
|
||||
clusterflux_core::WasmTaskResult::completed(invocation.task_instance, result)
|
||||
}
|
||||
Err(error) => clusterflux_core::WasmTaskResult::failed(
|
||||
invocation.task_instance,
|
||||
bounded_task_error(error.to_string()),
|
||||
),
|
||||
},
|
||||
Err(error) => clusterflux_core::WasmTaskResult::failed(
|
||||
invocation.task_instance,
|
||||
bounded_task_error(error),
|
||||
),
|
||||
};
|
||||
encode_result(result)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task1_v1<A, R, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
A: TaskArg + DeserializeOwned,
|
||||
R: TaskArg,
|
||||
F: FnOnce(A) -> R,
|
||||
{
|
||||
invoke_task1_output(expected_task, input_pointer, input_length, |argument| {
|
||||
Ok(function(argument))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task1_result_v1<A, R, E, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
A: TaskArg + DeserializeOwned,
|
||||
R: TaskArg,
|
||||
E: std::fmt::Display,
|
||||
F: FnOnce(A) -> Result<R, E>,
|
||||
{
|
||||
invoke_task1_output(expected_task, input_pointer, input_length, |argument| {
|
||||
function(argument).map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task1_async_v1<A, R, F, Fut>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
A: TaskArg + DeserializeOwned,
|
||||
R: TaskArg,
|
||||
F: FnOnce(A) -> Fut,
|
||||
Fut: std::future::Future<Output = R>,
|
||||
{
|
||||
invoke_task1_output(expected_task, input_pointer, input_length, |argument| {
|
||||
Ok(futures_executor::block_on(function(argument)))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub fn invoke_task1_async_result_v1<A, R, E, F, Fut>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
A: TaskArg + DeserializeOwned,
|
||||
R: TaskArg,
|
||||
E: std::fmt::Display,
|
||||
F: FnOnce(A) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<R, E>>,
|
||||
{
|
||||
invoke_task1_output(expected_task, input_pointer, input_length, |argument| {
|
||||
futures_executor::block_on(function(argument)).map_err(|error| error.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn invoke_task1_output<A, R, F>(
|
||||
expected_task: &str,
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
function: F,
|
||||
) -> u64
|
||||
where
|
||||
A: TaskArg + DeserializeOwned,
|
||||
R: TaskArg,
|
||||
F: FnOnce(A) -> Result<R, String>,
|
||||
{
|
||||
let invocation = match decode_invocation(expected_task, input_pointer, input_length) {
|
||||
Ok(invocation) => invocation,
|
||||
Err(error) => return encode_result(failed_result(expected_task, error)),
|
||||
};
|
||||
let [argument] = invocation.args.as_slice() else {
|
||||
return encode_result(failed_result(
|
||||
invocation.task_instance.clone(),
|
||||
format!(
|
||||
"task expected one argument but received {}",
|
||||
invocation.args.len()
|
||||
),
|
||||
));
|
||||
};
|
||||
let argument = match decode_boundary_value(argument.clone()) {
|
||||
Ok(argument) => argument,
|
||||
Err(error) => {
|
||||
return encode_result(clusterflux_core::WasmTaskResult::failed(
|
||||
invocation.task_instance,
|
||||
error,
|
||||
))
|
||||
}
|
||||
};
|
||||
let result = match function(argument) {
|
||||
Ok(output) => match output.task_boundary_value() {
|
||||
Ok(result) => {
|
||||
clusterflux_core::WasmTaskResult::completed(invocation.task_instance, result)
|
||||
}
|
||||
Err(error) => clusterflux_core::WasmTaskResult::failed(
|
||||
invocation.task_instance,
|
||||
bounded_task_error(error.to_string()),
|
||||
),
|
||||
},
|
||||
Err(error) => clusterflux_core::WasmTaskResult::failed(
|
||||
invocation.task_instance,
|
||||
bounded_task_error(error),
|
||||
),
|
||||
};
|
||||
encode_result(result)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn bounded_task_error(mut error: String) -> String {
|
||||
const LIMIT: usize = 4 * 1024;
|
||||
const SUFFIX: &str = "… [truncated]";
|
||||
if error.len() <= LIMIT {
|
||||
return error;
|
||||
}
|
||||
let mut end = LIMIT.saturating_sub(SUFFIX.len());
|
||||
while !error.is_char_boundary(end) {
|
||||
end = end.saturating_sub(1);
|
||||
}
|
||||
error.truncate(end);
|
||||
error.push_str(SUFFIX);
|
||||
error
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn decode_invocation(
|
||||
expected_task: &str,
|
||||
pointer: u32,
|
||||
length: u32,
|
||||
) -> Result<clusterflux_core::WasmTaskInvocation, String> {
|
||||
if length as usize > clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES {
|
||||
return Err(format!(
|
||||
"task invocation exceeds {} byte ABI limit",
|
||||
clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES
|
||||
));
|
||||
}
|
||||
if pointer == 0 && length != 0 {
|
||||
return Err("task invocation pointer is null".to_owned());
|
||||
}
|
||||
let bytes = unsafe { std::slice::from_raw_parts(pointer as *const u8, length as usize) };
|
||||
let invocation: clusterflux_core::WasmTaskInvocation =
|
||||
serde_json::from_slice(bytes).map_err(|error| error.to_string())?;
|
||||
invocation.validate()?;
|
||||
if invocation.task_definition.as_str() != expected_task {
|
||||
return Err(format!(
|
||||
"task export `{expected_task}` received invocation for `{}`",
|
||||
invocation.task_definition
|
||||
));
|
||||
}
|
||||
Ok(invocation)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn decode_boundary_value<T>(value: clusterflux_core::TaskBoundaryValue) -> Result<T, String>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
let value = match value {
|
||||
clusterflux_core::TaskBoundaryValue::SmallJson(value) => value,
|
||||
clusterflux_core::TaskBoundaryValue::Structured(structured) => structured.materialize()?,
|
||||
clusterflux_core::TaskBoundaryValue::SourceSnapshot(digest)
|
||||
| clusterflux_core::TaskBoundaryValue::Blob(digest)
|
||||
| clusterflux_core::TaskBoundaryValue::VfsManifest(digest) => {
|
||||
serde_json::json!({ "digest": digest })
|
||||
}
|
||||
clusterflux_core::TaskBoundaryValue::Artifact(artifact) => {
|
||||
serde_json::json!({
|
||||
"id": artifact.id,
|
||||
"digest": artifact.digest,
|
||||
"size_bytes": artifact.size_bytes,
|
||||
})
|
||||
}
|
||||
};
|
||||
serde_json::from_value(value).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn failed_result(
|
||||
task_instance: impl Into<FailedTaskInstance>,
|
||||
error: impl Into<String>,
|
||||
) -> clusterflux_core::WasmTaskResult {
|
||||
clusterflux_core::WasmTaskResult::failed(task_instance.into().0, error)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
struct FailedTaskInstance(clusterflux_core::TaskInstanceId);
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
impl From<&str> for FailedTaskInstance {
|
||||
fn from(task_definition: &str) -> Self {
|
||||
Self(clusterflux_core::TaskInstanceId::new(format!(
|
||||
"invalid-invocation:{task_definition}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
impl From<clusterflux_core::TaskInstanceId> for FailedTaskInstance {
|
||||
fn from(task_instance: clusterflux_core::TaskInstanceId) -> Self {
|
||||
Self(task_instance)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn encode_result(mut result: clusterflux_core::WasmTaskResult) -> u64 {
|
||||
let mut bytes = serde_json::to_vec(&result).unwrap_or_default();
|
||||
if bytes.len() > clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES {
|
||||
result = clusterflux_core::WasmTaskResult::failed(
|
||||
result.task_instance,
|
||||
format!(
|
||||
"task result exceeds {} byte ABI limit",
|
||||
clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES
|
||||
),
|
||||
);
|
||||
bytes = serde_json::to_vec(&result).unwrap_or_default();
|
||||
}
|
||||
let length = bytes.len() as u32;
|
||||
let pointer = bytes.as_mut_ptr() as usize;
|
||||
let pointer = u32::try_from(pointer).unwrap_or(0);
|
||||
std::mem::forget(bytes);
|
||||
((length as u64) << 32) | pointer as u64
|
||||
}
|
||||
std::thread_local! {
|
||||
static TASK_HANDLE_ENCODING: std::cell::RefCell<Option<Vec<clusterflux_core::TaskBoundaryHandle>>> =
|
||||
const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub trait CollectTaskHandles {}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn serialize_task_handle<S, F>(
|
||||
handle: clusterflux_core::TaskBoundaryHandle,
|
||||
serializer: S,
|
||||
serialize_plain: F,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
F: FnOnce(S) -> Result<S::Ok, S::Error>,
|
||||
{
|
||||
TASK_HANDLE_ENCODING.with(|encoding| {
|
||||
let mut encoding = encoding.borrow_mut();
|
||||
let Some(handles) = encoding.as_mut() else {
|
||||
drop(encoding);
|
||||
return serialize_plain(serializer);
|
||||
};
|
||||
let index = handles.len();
|
||||
let kind = match &handle {
|
||||
clusterflux_core::TaskBoundaryHandle::SourceSnapshot(_) => "source_snapshot",
|
||||
clusterflux_core::TaskBoundaryHandle::Blob(_) => "blob",
|
||||
clusterflux_core::TaskBoundaryHandle::Artifact(_) => "artifact",
|
||||
clusterflux_core::TaskBoundaryHandle::VfsManifest(_) => "vfs_manifest",
|
||||
};
|
||||
handles.push(handle);
|
||||
serde::Serialize::serialize(
|
||||
&serde_json::json!({
|
||||
"$task_handle": {
|
||||
"index": index,
|
||||
"kind": kind,
|
||||
}
|
||||
}),
|
||||
serializer,
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn structured_task_boundary<T>(
|
||||
value: &T,
|
||||
) -> Result<clusterflux_core::TaskBoundaryValue, crate::TaskArgError>
|
||||
where
|
||||
T: serde::Serialize + CollectTaskHandles + ?Sized,
|
||||
{
|
||||
TASK_HANDLE_ENCODING.with(|encoding| {
|
||||
if encoding.borrow().is_some() {
|
||||
return Err(crate::TaskArgError::Serialization(
|
||||
"nested structured task argument encoding is not allowed".to_owned(),
|
||||
));
|
||||
}
|
||||
*encoding.borrow_mut() = Some(Vec::new());
|
||||
let serialized = serde_json::to_value(value)
|
||||
.map_err(|error| crate::TaskArgError::Serialization(error.to_string()));
|
||||
let handles = encoding.borrow_mut().take().unwrap_or_default();
|
||||
let serialized = serialized?;
|
||||
let boundary = clusterflux_core::StructuredTaskBoundary {
|
||||
value: serialized,
|
||||
handles,
|
||||
};
|
||||
boundary
|
||||
.validate()
|
||||
.map_err(crate::TaskArgError::Serialization)?;
|
||||
Ok(clusterflux_core::TaskBoundaryValue::Structured(boundary))
|
||||
})
|
||||
}
|
||||
|
||||
macro_rules! no_task_handles {
|
||||
($($ty:ty),+ $(,)?) => {
|
||||
$(
|
||||
impl CollectTaskHandles for $ty {}
|
||||
)+
|
||||
};
|
||||
}
|
||||
|
||||
no_task_handles!(
|
||||
(),
|
||||
bool,
|
||||
char,
|
||||
String,
|
||||
i8,
|
||||
i16,
|
||||
i32,
|
||||
i64,
|
||||
isize,
|
||||
u8,
|
||||
u16,
|
||||
u32,
|
||||
u64,
|
||||
usize,
|
||||
f32,
|
||||
f64,
|
||||
);
|
||||
|
||||
impl CollectTaskHandles for crate::SourceSnapshot {}
|
||||
impl CollectTaskHandles for crate::Blob {}
|
||||
impl CollectTaskHandles for crate::Artifact {}
|
||||
impl CollectTaskHandles for crate::Vfs {}
|
||||
|
||||
impl<T> CollectTaskHandles for Option<T> where T: CollectTaskHandles {}
|
||||
|
||||
impl<T> CollectTaskHandles for Vec<T> where T: CollectTaskHandles {}
|
||||
1058
crates/clusterflux-sdk/src/lib.rs
Normal file
1058
crates/clusterflux-sdk/src/lib.rs
Normal file
File diff suppressed because it is too large
Load diff
547
crates/clusterflux-sdk/src/sdk_runtime.rs
Normal file
547
crates/clusterflux-sdk/src/sdk_runtime.rs
Normal file
|
|
@ -0,0 +1,547 @@
|
|||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
||||
use clusterflux_control::ControlSession;
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
use clusterflux_core::TaskDefinitionId;
|
||||
use clusterflux_core::{
|
||||
coordinator_wire_request, Digest, TaskBoundaryValue, TaskFailurePolicy, TaskJoinResult,
|
||||
TaskJoinState, TaskSpec,
|
||||
};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::EnvRef;
|
||||
|
||||
const COORDINATOR_ENV: &str = "CLUSTERFLUX_SDK_COORDINATOR";
|
||||
const TENANT_ENV: &str = "CLUSTERFLUX_SDK_TENANT";
|
||||
const PROJECT_ENV: &str = "CLUSTERFLUX_SDK_PROJECT";
|
||||
const PROCESS_ENV: &str = "CLUSTERFLUX_SDK_PROCESS";
|
||||
const USER_ENV: &str = "CLUSTERFLUX_SDK_USER";
|
||||
const SESSION_SECRET_ENV: &str = "CLUSTERFLUX_SDK_SESSION_SECRET";
|
||||
const BUNDLE_DIGEST_ENV: &str = "CLUSTERFLUX_SDK_BUNDLE_DIGEST";
|
||||
const WASM_MODULE_ENV: &str = "CLUSTERFLUX_SDK_WASM_MODULE_BASE64";
|
||||
const WASM_MODULE_PATH_ENV: &str = "CLUSTERFLUX_SDK_WASM_MODULE_PATH";
|
||||
const VFS_EPOCH_ENV: &str = "CLUSTERFLUX_SDK_VFS_EPOCH";
|
||||
const JOIN_TIMEOUT_SECONDS_ENV: &str = clusterflux_core::limits::TASK_JOIN_TIMEOUT_SECONDS_ENV;
|
||||
const DEFAULT_JOIN_TIMEOUT_SECONDS: u64 =
|
||||
clusterflux_core::limits::DEFAULT_TASK_JOIN_TIMEOUT_SECONDS;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ProductRuntimeConfig {
|
||||
pub coordinator: String,
|
||||
pub tenant: String,
|
||||
pub project: String,
|
||||
pub process: String,
|
||||
pub actor_user: String,
|
||||
pub session_secret: Option<String>,
|
||||
pub bundle_digest: Digest,
|
||||
pub wasm_module_base64: String,
|
||||
pub vfs_epoch: u64,
|
||||
pub join_poll_interval: Duration,
|
||||
pub join_timeout: Duration,
|
||||
}
|
||||
|
||||
impl ProductRuntimeConfig {
|
||||
pub fn from_env() -> Result<Option<Self>, TaskRuntimeError> {
|
||||
let Some(coordinator) = nonempty_env(COORDINATOR_ENV) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let tenant = required_env(TENANT_ENV)?;
|
||||
let project = required_env(PROJECT_ENV)?;
|
||||
let process = required_env(PROCESS_ENV)?;
|
||||
let actor_user = required_env(USER_ENV)?;
|
||||
let bundle_digest = parse_digest(&required_env(BUNDLE_DIGEST_ENV)?)?;
|
||||
let wasm_module_base64 = if let Some(module) = nonempty_env(WASM_MODULE_ENV) {
|
||||
module
|
||||
} else {
|
||||
let module_path = required_env(WASM_MODULE_PATH_ENV)?;
|
||||
let module = std::fs::read(&module_path).map_err(|error| {
|
||||
TaskRuntimeError::Configuration(format!(
|
||||
"read {WASM_MODULE_PATH_ENV} `{module_path}`: {error}"
|
||||
))
|
||||
})?;
|
||||
BASE64_STANDARD.encode(module)
|
||||
};
|
||||
let vfs_epoch = required_env(VFS_EPOCH_ENV)?.parse::<u64>().map_err(|_| {
|
||||
TaskRuntimeError::Configuration(format!("{VFS_EPOCH_ENV} must be an unsigned integer"))
|
||||
})?;
|
||||
Ok(Some(Self {
|
||||
coordinator,
|
||||
tenant,
|
||||
project,
|
||||
process,
|
||||
actor_user,
|
||||
session_secret: nonempty_env(SESSION_SECRET_ENV),
|
||||
bundle_digest,
|
||||
wasm_module_base64,
|
||||
vfs_epoch,
|
||||
join_poll_interval: Duration::from_millis(25),
|
||||
join_timeout: Duration::from_secs(optional_positive_u64_env(
|
||||
JOIN_TIMEOUT_SECONDS_ENV,
|
||||
DEFAULT_JOIN_TIMEOUT_SECONDS,
|
||||
)?),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct RemoteTaskHandle {
|
||||
pub config: Option<ProductRuntimeConfig>,
|
||||
pub spec: TaskSpec,
|
||||
#[cfg_attr(not(target_arch = "wasm32"), allow(dead_code))]
|
||||
pub host_handle_id: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum TaskRuntimeError {
|
||||
Argument(String),
|
||||
Configuration(String),
|
||||
Transport(String),
|
||||
Protocol(String),
|
||||
RemoteTask(String),
|
||||
JoinTimeout {
|
||||
task: clusterflux_core::TaskInstanceId,
|
||||
waited: Duration,
|
||||
},
|
||||
ResultDecode(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TaskRuntimeError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let (kind, message) = match self {
|
||||
Self::Argument(message) => ("task argument", message),
|
||||
Self::Configuration(message) => ("product runtime configuration", message),
|
||||
Self::Transport(message) => ("coordinator transport", message),
|
||||
Self::Protocol(message) => ("coordinator protocol", message),
|
||||
Self::RemoteTask(message) => ("remote task", message),
|
||||
Self::JoinTimeout { task, waited } => {
|
||||
return write!(
|
||||
formatter,
|
||||
"task join timeout: {task} remained pending for {} seconds; the child task was left running",
|
||||
waited.as_secs()
|
||||
);
|
||||
}
|
||||
Self::ResultDecode(message) => ("remote result", message),
|
||||
};
|
||||
write!(formatter, "{kind} error: {message}")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TaskRuntimeError {}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub(crate) fn start_remote_task(
|
||||
config: ProductRuntimeConfig,
|
||||
task_definition: String,
|
||||
environment: Option<EnvRef>,
|
||||
args: Vec<TaskBoundaryValue>,
|
||||
failure_policy: TaskFailurePolicy,
|
||||
) -> Result<RemoteTaskHandle, TaskRuntimeError> {
|
||||
let _ = (config, task_definition, environment, args, failure_policy);
|
||||
Err(TaskRuntimeError::Configuration(
|
||||
"native SDK task spawning requires coordinator EntrypointV1 execution; external TaskV1 launch is forbidden"
|
||||
.to_owned(),
|
||||
))
|
||||
}
|
||||
|
||||
pub(crate) fn join_remote_task<R>(handle: RemoteTaskHandle) -> Result<R, TaskRuntimeError>
|
||||
where
|
||||
R: DeserializeOwned,
|
||||
{
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
if let Some(handle_id) = handle.host_handle_id {
|
||||
let response: Result<clusterflux_core::WasmHostTaskJoinResult, String> = guest_host_call(
|
||||
&clusterflux_core::WasmHostTaskJoinRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
handle_id,
|
||||
},
|
||||
GuestHostCall::Join,
|
||||
)?;
|
||||
let response = response.map_err(TaskRuntimeError::RemoteTask)?;
|
||||
return decode_boundary_value(response.result);
|
||||
}
|
||||
let config = handle.config.as_ref().ok_or_else(|| {
|
||||
TaskRuntimeError::Protocol("remote task handle has no coordinator authority".to_owned())
|
||||
})?;
|
||||
let started = Instant::now();
|
||||
loop {
|
||||
let response = coordinator_request(
|
||||
config,
|
||||
json!({
|
||||
"type": "join_task",
|
||||
"tenant": config.tenant,
|
||||
"project": config.project,
|
||||
"actor_user": config.actor_user,
|
||||
"process": config.process,
|
||||
"task": handle.spec.task_instance,
|
||||
}),
|
||||
)?;
|
||||
if response.get("type").and_then(Value::as_str) == Some("error") {
|
||||
return Err(remote_error(&response));
|
||||
}
|
||||
if response.get("type").and_then(Value::as_str) != Some("task_joined") {
|
||||
return Err(TaskRuntimeError::Protocol(
|
||||
"join response was not task_joined".to_owned(),
|
||||
));
|
||||
}
|
||||
let join: TaskJoinResult = serde_json::from_value(
|
||||
response
|
||||
.get("join")
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskRuntimeError::Protocol("join result missing".to_owned()))?,
|
||||
)
|
||||
.map_err(|error| TaskRuntimeError::Protocol(error.to_string()))?;
|
||||
match join.state {
|
||||
TaskJoinState::Pending => {
|
||||
if started.elapsed() >= config.join_timeout {
|
||||
return Err(TaskRuntimeError::JoinTimeout {
|
||||
task: handle.spec.task_instance.clone(),
|
||||
waited: config.join_timeout,
|
||||
});
|
||||
}
|
||||
thread::sleep(config.join_poll_interval);
|
||||
}
|
||||
TaskJoinState::Completed => {
|
||||
if !join.remote_completion_observed {
|
||||
return Err(TaskRuntimeError::Protocol(
|
||||
"completed join did not prove a signed node completion".to_owned(),
|
||||
));
|
||||
}
|
||||
let boundary = join.result.ok_or_else(|| {
|
||||
TaskRuntimeError::ResultDecode(
|
||||
"completed remote task did not return a boundary value".to_owned(),
|
||||
)
|
||||
})?;
|
||||
return decode_boundary_value(boundary);
|
||||
}
|
||||
TaskJoinState::Failed | TaskJoinState::Cancelled => {
|
||||
return Err(TaskRuntimeError::RemoteTask(join.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn start_guest_host_task(
|
||||
task_definition: TaskDefinitionId,
|
||||
environment: Option<EnvRef>,
|
||||
args: Vec<TaskBoundaryValue>,
|
||||
failure_policy: TaskFailurePolicy,
|
||||
) -> Result<RemoteTaskHandle, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostTaskStartRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
task_definition,
|
||||
environment_id: environment.map(|environment| environment.name.to_owned()),
|
||||
args,
|
||||
failure_policy,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostTaskHandle, String> =
|
||||
guest_host_call(&request, GuestHostCall::Start)?;
|
||||
let response = response.map_err(TaskRuntimeError::RemoteTask)?;
|
||||
Ok(RemoteTaskHandle {
|
||||
config: None,
|
||||
spec: response.task_spec,
|
||||
host_handle_id: Some(response.handle_id),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
enum GuestHostCall {
|
||||
Start,
|
||||
Join,
|
||||
Command,
|
||||
TaskControl,
|
||||
DebugProbe,
|
||||
SourceSnapshot,
|
||||
Vfs,
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn run_guest_host_command(
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
working_directory: String,
|
||||
environment_variables: std::collections::BTreeMap<String, String>,
|
||||
timeout_ms: u64,
|
||||
network: clusterflux_core::CommandNetworkPolicy,
|
||||
) -> Result<clusterflux_core::WasmHostCommandResult, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostCommandRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
program,
|
||||
args,
|
||||
working_directory,
|
||||
environment_variables,
|
||||
timeout_ms,
|
||||
network,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostCommandResult, String> =
|
||||
guest_host_call(&request, GuestHostCall::Command)?;
|
||||
response.map_err(TaskRuntimeError::RemoteTask)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn guest_cancellation_requested() -> Result<bool, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostTaskControlRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostTaskControlResult, String> =
|
||||
guest_host_call(&request, GuestHostCall::TaskControl)?;
|
||||
response
|
||||
.map(|result| result.cancellation_requested)
|
||||
.map_err(TaskRuntimeError::RemoteTask)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn guest_debug_probe(
|
||||
symbol: String,
|
||||
) -> Result<clusterflux_core::WasmHostDebugProbeResult, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostDebugProbeRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
symbol,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostDebugProbeResult, String> =
|
||||
guest_host_call(&request, GuestHostCall::DebugProbe)?;
|
||||
response.map_err(TaskRuntimeError::RemoteTask)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn guest_vfs_operation(
|
||||
operation: clusterflux_core::WasmHostVfsOperation,
|
||||
) -> Result<clusterflux_core::WasmHostVfsResult, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostVfsRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
operation,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostVfsResult, String> =
|
||||
guest_host_call(&request, GuestHostCall::Vfs)?;
|
||||
response.map_err(TaskRuntimeError::RemoteTask)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
pub(crate) fn guest_source_snapshot(
|
||||
) -> Result<clusterflux_core::WasmHostSourceSnapshotResult, TaskRuntimeError> {
|
||||
let request = clusterflux_core::WasmHostSourceSnapshotRequest {
|
||||
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
|
||||
};
|
||||
request.validate().map_err(TaskRuntimeError::Argument)?;
|
||||
let response: Result<clusterflux_core::WasmHostSourceSnapshotResult, String> =
|
||||
guest_host_call(&request, GuestHostCall::SourceSnapshot)?;
|
||||
response.map_err(TaskRuntimeError::RemoteTask)
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn guest_host_call<I, O>(input: &I, call: GuestHostCall) -> Result<O, TaskRuntimeError>
|
||||
where
|
||||
I: serde::Serialize,
|
||||
O: DeserializeOwned,
|
||||
{
|
||||
#[link(wasm_import_module = "clusterflux")]
|
||||
unsafe extern "C" {
|
||||
#[link_name = "task_start_v1"]
|
||||
fn task_start_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "task_join_v1"]
|
||||
fn task_join_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "command_run_v1"]
|
||||
fn command_run_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "vfs_operation_v1"]
|
||||
fn vfs_operation_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "source_snapshot_v1"]
|
||||
fn source_snapshot_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "task_control_v1"]
|
||||
fn task_control_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
#[link_name = "debug_probe_v1"]
|
||||
fn debug_probe_v1(
|
||||
input_pointer: u32,
|
||||
input_length: u32,
|
||||
output_pointer: u32,
|
||||
output_capacity: u32,
|
||||
) -> i32;
|
||||
}
|
||||
|
||||
let input =
|
||||
serde_json::to_vec(input).map_err(|error| TaskRuntimeError::Protocol(error.to_string()))?;
|
||||
if input.len() > clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES {
|
||||
return Err(TaskRuntimeError::Argument(
|
||||
"Wasm host-call request exceeds the task ABI limit".to_owned(),
|
||||
));
|
||||
}
|
||||
let mut output = vec![0_u8; clusterflux_core::MAX_WASM_TASK_ENVELOPE_BYTES];
|
||||
let written = unsafe {
|
||||
match call {
|
||||
GuestHostCall::Start => task_start_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::Join => task_join_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::Command => command_run_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::TaskControl => task_control_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::DebugProbe => debug_probe_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::Vfs => vfs_operation_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
GuestHostCall::SourceSnapshot => source_snapshot_v1(
|
||||
input.as_ptr() as u32,
|
||||
input.len() as u32,
|
||||
output.as_mut_ptr() as u32,
|
||||
output.len() as u32,
|
||||
),
|
||||
}
|
||||
};
|
||||
if written <= 0 || written as usize > output.len() {
|
||||
return Err(TaskRuntimeError::Protocol(format!(
|
||||
"Wasm host-call returned invalid response length {written}"
|
||||
)));
|
||||
}
|
||||
serde_json::from_slice(&output[..written as usize])
|
||||
.map_err(|error| TaskRuntimeError::Protocol(error.to_string()))
|
||||
}
|
||||
|
||||
fn coordinator_request(
|
||||
config: &ProductRuntimeConfig,
|
||||
payload: Value,
|
||||
) -> Result<Value, TaskRuntimeError> {
|
||||
let payload = if let Some(session_secret) = &config.session_secret {
|
||||
let request = payload
|
||||
.as_object()
|
||||
.cloned()
|
||||
.ok_or_else(|| TaskRuntimeError::Protocol("request must be an object".to_owned()))?;
|
||||
let request = request
|
||||
.into_iter()
|
||||
.filter(|(key, _)| !matches!(key.as_str(), "tenant" | "project" | "actor_user"))
|
||||
.collect::<serde_json::Map<_, _>>();
|
||||
json!({
|
||||
"type": "authenticated",
|
||||
"session_secret": session_secret,
|
||||
"request": request,
|
||||
})
|
||||
} else {
|
||||
payload
|
||||
};
|
||||
let wire = coordinator_wire_request("sdk-product-runtime", payload);
|
||||
let mut session = ControlSession::connect(&config.coordinator)
|
||||
.map_err(|error| TaskRuntimeError::Transport(error.to_string()))?;
|
||||
session
|
||||
.request(&wire)
|
||||
.map_err(|error| TaskRuntimeError::Transport(error.to_string()))
|
||||
}
|
||||
|
||||
fn decode_boundary_value<R>(value: TaskBoundaryValue) -> Result<R, TaskRuntimeError>
|
||||
where
|
||||
R: DeserializeOwned,
|
||||
{
|
||||
let value = value
|
||||
.materialize()
|
||||
.map_err(TaskRuntimeError::ResultDecode)?;
|
||||
serde_json::from_value(value).map_err(|error| TaskRuntimeError::ResultDecode(error.to_string()))
|
||||
}
|
||||
|
||||
fn optional_positive_u64_env(name: &str, default: u64) -> Result<u64, TaskRuntimeError> {
|
||||
let Some(value) = nonempty_env(name) else {
|
||||
return Ok(default);
|
||||
};
|
||||
value
|
||||
.parse::<u64>()
|
||||
.ok()
|
||||
.filter(|value| *value > 0)
|
||||
.ok_or_else(|| {
|
||||
TaskRuntimeError::Configuration(format!("{name} must be a positive integer"))
|
||||
})
|
||||
}
|
||||
|
||||
fn remote_error(response: &Value) -> TaskRuntimeError {
|
||||
TaskRuntimeError::RemoteTask(
|
||||
response
|
||||
.get("message")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("coordinator rejected task request")
|
||||
.to_owned(),
|
||||
)
|
||||
}
|
||||
|
||||
fn parse_digest(value: &str) -> Result<Digest, TaskRuntimeError> {
|
||||
let digest: Digest = serde_json::from_value(Value::String(value.to_owned()))
|
||||
.map_err(|error| TaskRuntimeError::Configuration(error.to_string()))?;
|
||||
if !digest.is_valid_sha256() {
|
||||
return Err(TaskRuntimeError::Configuration(format!(
|
||||
"{BUNDLE_DIGEST_ENV} must be a lowercase sha256 digest"
|
||||
)));
|
||||
}
|
||||
Ok(digest)
|
||||
}
|
||||
|
||||
fn required_env(name: &str) -> Result<String, TaskRuntimeError> {
|
||||
nonempty_env(name).ok_or_else(|| {
|
||||
TaskRuntimeError::Configuration(format!(
|
||||
"{name} is required when {COORDINATOR_ENV} enables product mode"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
fn nonempty_env(name: &str) -> Option<String> {
|
||||
std::env::var(name)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_owned())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
352
crates/clusterflux-sdk/src/task_args.rs
Normal file
352
crates/clusterflux-sdk/src/task_args.rs
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
use serde::{ser::SerializeStruct, Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
pub struct SourceSnapshot {
|
||||
pub digest: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
pub struct Blob {
|
||||
pub digest: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
pub struct Artifact {
|
||||
pub id: String,
|
||||
pub digest: String,
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
|
||||
pub struct Vfs {
|
||||
pub digest: String,
|
||||
}
|
||||
|
||||
impl Serialize for SourceSnapshot {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
crate::__private::serialize_task_handle(
|
||||
clusterflux_core::TaskBoundaryHandle::SourceSnapshot(
|
||||
parse_handle_digest(&self.digest).map_err(serde::ser::Error::custom)?,
|
||||
),
|
||||
serializer,
|
||||
|serializer| serialize_digest_struct("SourceSnapshot", &self.digest, serializer),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for Blob {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
crate::__private::serialize_task_handle(
|
||||
clusterflux_core::TaskBoundaryHandle::Blob(
|
||||
parse_handle_digest(&self.digest).map_err(serde::ser::Error::custom)?,
|
||||
),
|
||||
serializer,
|
||||
|serializer| serialize_digest_struct("Blob", &self.digest, serializer),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for Vfs {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
crate::__private::serialize_task_handle(
|
||||
clusterflux_core::TaskBoundaryHandle::VfsManifest(
|
||||
parse_handle_digest(&self.digest).map_err(serde::ser::Error::custom)?,
|
||||
),
|
||||
serializer,
|
||||
|serializer| serialize_digest_struct("Vfs", &self.digest, serializer),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for Artifact {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
if self.id.trim().is_empty() || self.id.len() > 256 {
|
||||
return Err(serde::ser::Error::custom(
|
||||
"artifact handle ID must be non-empty and at most 256 bytes",
|
||||
));
|
||||
}
|
||||
let handle = clusterflux_core::ArtifactHandle {
|
||||
id: clusterflux_core::ArtifactId::from(self.id.as_str()),
|
||||
digest: parse_handle_digest(&self.digest).map_err(serde::ser::Error::custom)?,
|
||||
size_bytes: self.size_bytes,
|
||||
};
|
||||
crate::__private::serialize_task_handle(
|
||||
clusterflux_core::TaskBoundaryHandle::Artifact(handle),
|
||||
serializer,
|
||||
|serializer| {
|
||||
let mut state = serializer.serialize_struct("Artifact", 3)?;
|
||||
state.serialize_field("id", &self.id)?;
|
||||
state.serialize_field("digest", &self.digest)?;
|
||||
state.serialize_field("size_bytes", &self.size_bytes)?;
|
||||
state.end()
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_digest_struct<S>(
|
||||
name: &'static str,
|
||||
digest: &str,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let mut state = serializer.serialize_struct(name, 1)?;
|
||||
state.serialize_field("digest", digest)?;
|
||||
state.end()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct EnvRef {
|
||||
pub name: &'static str,
|
||||
}
|
||||
|
||||
impl EnvRef {
|
||||
pub const fn new_static(name: &'static str) -> Self {
|
||||
Self { name }
|
||||
}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! env {
|
||||
($name:literal) => {
|
||||
$crate::EnvRef::new_static($name)
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum TaskArgKind {
|
||||
SmallSerialized,
|
||||
Structured,
|
||||
Handle,
|
||||
}
|
||||
|
||||
/// A value allowed to cross a Clusterflux task boundary.
|
||||
///
|
||||
/// Implementations are deliberately limited to small owned values and explicit
|
||||
/// Clusterflux handles. Host-only values fail at compile time because they do not
|
||||
/// implement `TaskArg`:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let borrowed = String::from("host-owned");
|
||||
/// let _ = clusterflux::spawn::task_with_arg(borrowed.as_str(), |_| 1_u32);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let pointer = std::ptr::null::<u8>();
|
||||
/// let _ = clusterflux::spawn::task_with_arg(pointer, |_| 1_u32);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let file = std::fs::File::open("Cargo.toml").unwrap();
|
||||
/// let _ = clusterflux::spawn::task_with_arg(file, |_| 1_u32);
|
||||
/// ```
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let lock = std::sync::Mutex::new(1_u32);
|
||||
/// let guard = lock.lock().unwrap();
|
||||
/// let _ = clusterflux::spawn::task_with_arg(guard, |_| 1_u32);
|
||||
/// ```
|
||||
pub trait TaskArg: Serialize {
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::SmallSerialized
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
serde_json::to_value(self)
|
||||
.map(clusterflux_core::TaskBoundaryValue::SmallJson)
|
||||
.map_err(|error| TaskArgError::Serialization(error.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! small_task_arg {
|
||||
($($ty:ty),+ $(,)?) => {
|
||||
$(
|
||||
impl TaskArg for $ty {}
|
||||
)+
|
||||
};
|
||||
}
|
||||
|
||||
small_task_arg!(
|
||||
(),
|
||||
bool,
|
||||
char,
|
||||
String,
|
||||
i8,
|
||||
i16,
|
||||
i32,
|
||||
i64,
|
||||
isize,
|
||||
u8,
|
||||
u16,
|
||||
u32,
|
||||
u64,
|
||||
usize,
|
||||
f32,
|
||||
f64,
|
||||
);
|
||||
|
||||
impl TaskArg for SourceSnapshot {
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Handle
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
parse_handle_digest(&self.digest).map(clusterflux_core::TaskBoundaryValue::SourceSnapshot)
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskArg for Blob {
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Handle
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
parse_handle_digest(&self.digest).map(clusterflux_core::TaskBoundaryValue::Blob)
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskArg for Artifact {
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Handle
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
Ok(clusterflux_core::TaskBoundaryValue::Artifact(
|
||||
clusterflux_core::ArtifactHandle {
|
||||
id: clusterflux_core::ArtifactId::from(self.id.as_str()),
|
||||
digest: parse_handle_digest(&self.digest)?,
|
||||
size_bytes: self.size_bytes,
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl TaskArg for Vfs {
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Handle
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
parse_handle_digest(&self.digest).map(clusterflux_core::TaskBoundaryValue::VfsManifest)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> TaskArg for Option<T>
|
||||
where
|
||||
T: TaskArg + crate::__private::CollectTaskHandles,
|
||||
{
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Structured
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
crate::__private::structured_task_boundary(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> TaskArg for Vec<T>
|
||||
where
|
||||
T: TaskArg + crate::__private::CollectTaskHandles,
|
||||
{
|
||||
fn task_arg_kind(&self) -> TaskArgKind {
|
||||
TaskArgKind::Structured
|
||||
}
|
||||
|
||||
fn task_boundary_value(&self) -> Result<clusterflux_core::TaskBoundaryValue, TaskArgError> {
|
||||
crate::__private::structured_task_boundary(self)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct TaskArgBudget {
|
||||
pub max_inline_bytes: usize,
|
||||
}
|
||||
|
||||
impl Default for TaskArgBudget {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_inline_bytes: 64 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct TaskArgValidation {
|
||||
pub inline_bytes: usize,
|
||||
pub kind: TaskArgKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum TaskArgError {
|
||||
Serialization(String),
|
||||
TooLarge { size: usize, limit: usize },
|
||||
HostOnly { type_name: &'static str },
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TaskArgError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Serialization(error) => write!(f, "task argument could not serialize: {error}"),
|
||||
Self::TooLarge { size, limit } => write!(
|
||||
f,
|
||||
"task argument is {size} bytes; inline task arguments are limited to {limit} bytes, use SourceSnapshot, Blob, Artifact, or VFS handles"
|
||||
),
|
||||
Self::HostOnly { type_name } => write!(
|
||||
f,
|
||||
"task boundary value `{type_name}` is host-only; use small serialized data or handles"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TaskArgError {}
|
||||
|
||||
pub fn validate_task_arg<T>(
|
||||
value: &T,
|
||||
budget: TaskArgBudget,
|
||||
) -> Result<TaskArgValidation, TaskArgError>
|
||||
where
|
||||
T: TaskArg + ?Sized,
|
||||
{
|
||||
let bytes = serde_json::to_vec(value)
|
||||
.map_err(|error| TaskArgError::Serialization(error.to_string()))?;
|
||||
let kind = value.task_arg_kind();
|
||||
if kind != TaskArgKind::Handle && bytes.len() > budget.max_inline_bytes {
|
||||
return Err(TaskArgError::TooLarge {
|
||||
size: bytes.len(),
|
||||
limit: budget.max_inline_bytes,
|
||||
});
|
||||
}
|
||||
Ok(TaskArgValidation {
|
||||
inline_bytes: bytes.len(),
|
||||
kind,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn reject_host_only_task_arg<T: ?Sized>() -> TaskArgError {
|
||||
TaskArgError::HostOnly {
|
||||
type_name: std::any::type_name::<T>(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn parse_handle_digest(value: &str) -> Result<clusterflux_core::Digest, TaskArgError> {
|
||||
serde_json::from_value(serde_json::Value::String(value.to_owned())).map_err(|_| {
|
||||
TaskArgError::Serialization(
|
||||
"handle digest must be a serialized sha256 digest issued by Clusterflux".to_owned(),
|
||||
)
|
||||
})
|
||||
}
|
||||
196
crates/clusterflux-sdk/tests/macros.rs
Normal file
196
crates/clusterflux-sdk/tests/macros.rs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
use futures_executor::block_on;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, clusterflux::TaskArg)]
|
||||
struct CompoundBoundary {
|
||||
label: String,
|
||||
source: clusterflux::SourceSnapshot,
|
||||
optional_artifact: Option<clusterflux::Artifact>,
|
||||
artifacts: Vec<clusterflux::Artifact>,
|
||||
blobs: Vec<Option<clusterflux::Blob>>,
|
||||
vfs: clusterflux::Vfs,
|
||||
}
|
||||
|
||||
#[clusterflux::main]
|
||||
fn build_main() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
#[clusterflux::main(name = "release")]
|
||||
fn release_main() -> u32 {
|
||||
2
|
||||
}
|
||||
|
||||
#[clusterflux::task(name = "compile-linux", capabilities = "Command")]
|
||||
fn compile_linux() -> u32 {
|
||||
41
|
||||
}
|
||||
|
||||
#[clusterflux::task]
|
||||
fn package_release() -> clusterflux::Artifact {
|
||||
clusterflux::Artifact {
|
||||
id: "artifact://package/release.tar.zst".to_owned(),
|
||||
digest: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
.to_owned(),
|
||||
size_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[clusterflux::main(name = "async-build")]
|
||||
async fn async_build_main() -> Result<(), &'static str> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[clusterflux::task(name = "async-compile")]
|
||||
async fn async_compile(input: u32) -> Result<u32, &'static str> {
|
||||
Ok(input + 1)
|
||||
}
|
||||
|
||||
#[clusterflux::task(name = "roundtrip-compound")]
|
||||
async fn roundtrip_compound(input: CompoundBoundary) -> CompoundBoundary {
|
||||
input
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clusterflux_attributes_and_spawn_api_compile_for_rust_build_workflow() {
|
||||
let program = clusterflux::RegisteredProgram {
|
||||
entrypoints: &[
|
||||
__CLUSTERFLUX_ENTRYPOINT_BUILD_MAIN,
|
||||
__CLUSTERFLUX_ENTRYPOINT_RELEASE_MAIN,
|
||||
],
|
||||
tasks: &[
|
||||
__CLUSTERFLUX_TASK_COMPILE_LINUX,
|
||||
__CLUSTERFLUX_TASK_PACKAGE_RELEASE,
|
||||
],
|
||||
};
|
||||
assert_eq!(
|
||||
program.select_entrypoint("build").unwrap().function,
|
||||
"build_main"
|
||||
);
|
||||
assert_eq!(
|
||||
program.select_entrypoint("release").unwrap().function,
|
||||
"release_main"
|
||||
);
|
||||
assert_eq!(
|
||||
program.task("compile-linux").unwrap().function,
|
||||
"compile_linux"
|
||||
);
|
||||
let task = program.task("compile-linux").unwrap();
|
||||
assert!(task.stable_id.starts_with("sha256:"));
|
||||
assert_eq!(task.argument_schema, "");
|
||||
assert_eq!(task.result_schema, "u32");
|
||||
assert_eq!(task.required_capabilities, ["Command"]);
|
||||
assert!(task.restart_compatibility_hash.starts_with("sha256:"));
|
||||
assert_eq!(task.abi_version, 1);
|
||||
assert!(task.source_file.ends_with("macros.rs"));
|
||||
assert!(task.source_line > 0);
|
||||
assert_eq!(task.probe_symbol, "clusterflux.probe.compile_linux");
|
||||
assert!(task.bundle_manifest_entry);
|
||||
assert!(task.remotely_startable);
|
||||
let entrypoint = program.select_entrypoint("build").unwrap();
|
||||
assert!(entrypoint.stable_id.starts_with("sha256:"));
|
||||
assert_eq!(entrypoint.result_schema, "u32");
|
||||
assert_eq!(entrypoint.abi_version, 1);
|
||||
assert!(entrypoint.bundle_manifest_entry);
|
||||
|
||||
let result = block_on(async {
|
||||
let handle = clusterflux::spawn::task(|| compile_linux() + build_main())
|
||||
.name("compile linux")
|
||||
.env(clusterflux::env!("linux"))
|
||||
.start()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(handle.virtual_thread_id() > 0);
|
||||
handle.join().await.unwrap()
|
||||
});
|
||||
|
||||
assert_eq!(result, 42);
|
||||
assert_eq!(release_main(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn task_boundaries_accept_small_values_and_runtime_handles() {
|
||||
let small =
|
||||
clusterflux::validate_task_arg(&7_u32, clusterflux::TaskArgBudget::default()).unwrap();
|
||||
assert_eq!(small.kind, clusterflux::TaskArgKind::SmallSerialized);
|
||||
|
||||
let artifact = package_release();
|
||||
let artifact = clusterflux::validate_task_arg(
|
||||
&artifact,
|
||||
clusterflux::TaskArgBudget {
|
||||
max_inline_bytes: 4,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(artifact.kind, clusterflux::TaskArgKind::Handle);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn async_main_and_task_attributes_preserve_real_rust_futures_and_results() {
|
||||
assert_eq!(block_on(async_build_main()), Ok(()));
|
||||
assert_eq!(block_on(async_compile(41)), Ok(42));
|
||||
assert_eq!(
|
||||
__CLUSTERFLUX_ENTRYPOINT_ASYNC_BUILD_MAIN.name,
|
||||
"async-build"
|
||||
);
|
||||
assert_eq!(__CLUSTERFLUX_TASK_ASYNC_COMPILE.name, "async-compile");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_compound_boundary_keeps_nested_handles_out_of_opaque_json() {
|
||||
let source = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
||||
let value = CompoundBoundary {
|
||||
label: "release".to_owned(),
|
||||
source: clusterflux::SourceSnapshot {
|
||||
digest: source.to_owned(),
|
||||
},
|
||||
optional_artifact: Some(clusterflux::Artifact {
|
||||
id: "optional.bin".to_owned(),
|
||||
digest: source.to_owned(),
|
||||
size_bytes: 0,
|
||||
}),
|
||||
artifacts: vec![
|
||||
clusterflux::Artifact {
|
||||
id: "one.bin".to_owned(),
|
||||
digest: source.to_owned(),
|
||||
size_bytes: 0,
|
||||
},
|
||||
clusterflux::Artifact {
|
||||
id: "two.bin".to_owned(),
|
||||
digest: source.to_owned(),
|
||||
size_bytes: 0,
|
||||
},
|
||||
],
|
||||
blobs: vec![Some(clusterflux::Blob {
|
||||
digest: source.to_owned(),
|
||||
})],
|
||||
vfs: clusterflux::Vfs {
|
||||
digest: source.to_owned(),
|
||||
},
|
||||
};
|
||||
assert_eq!(block_on(roundtrip_compound(value.clone())), value);
|
||||
let boundary = clusterflux::TaskArg::task_boundary_value(&value).unwrap();
|
||||
let clusterflux::core::TaskBoundaryValue::Structured(structured) = boundary else {
|
||||
panic!("derived boundary must be structured");
|
||||
};
|
||||
assert_eq!(structured.handles.len(), 6);
|
||||
let inline = structured.value.to_string();
|
||||
assert!(!inline.contains(source));
|
||||
assert!(!inline.contains("optional.bin"));
|
||||
assert_eq!(
|
||||
clusterflux::core::TaskBoundaryValue::Structured(structured.clone()).source_snapshots(),
|
||||
vec![clusterflux::core::Digest::sha256([])]
|
||||
);
|
||||
assert_eq!(
|
||||
clusterflux::core::TaskBoundaryValue::Structured(structured.clone()).required_artifacts(),
|
||||
vec![
|
||||
clusterflux::core::ArtifactId::from("optional.bin"),
|
||||
clusterflux::core::ArtifactId::from("one.bin"),
|
||||
clusterflux::core::ArtifactId::from("two.bin"),
|
||||
]
|
||||
);
|
||||
let decoded: CompoundBoundary =
|
||||
serde_json::from_value(structured.materialize().unwrap()).unwrap();
|
||||
assert_eq!(decoded, value);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue