Publish Clusterflux 1d0b6fa filtered source

This commit is contained in:
Michel Paulissen 2026-07-21 20:36:04 +02:00
commit e5d609cfb2
226 changed files with 86613 additions and 0 deletions

View file

@ -0,0 +1,95 @@
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
use clusterflux_core::{
CommandInvocation, Digest, EnvironmentKind, EnvironmentRequirements, EnvironmentResource,
NodeId, ProcessId, TaskInstanceId, VfsOverlay, VfsPath,
};
use clusterflux_node::{
LinuxRootlessPodmanBackend, LocalCheckoutTaskRequest, LocalSourceCheckout, StdProcessRunner,
};
use serde_json::json;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let workspace = create_workspace()?;
let env_dir = workspace.join("envs/linux");
fs::create_dir_all(&env_dir)?;
fs::write(
env_dir.join("Containerfile"),
"FROM docker.io/library/alpine:3.20\nWORKDIR /workspace\n",
)?;
fs::write(workspace.join("input.txt"), "node-local source\n")?;
let env = EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: env_dir.join("Containerfile"),
context_path: env_dir,
digest: Digest::sha256("phase2-podman-smoke-linux-env"),
requirements: EnvironmentRequirements::linux_container(),
};
let invocation = CommandInvocation {
program: "sh".to_owned(),
args: vec![
"-c".to_owned(),
"printf 'podman-ok:' && cat input.txt".to_owned(),
],
working_directory: "/workspace".to_owned(),
environment_variables: Default::default(),
timeout_ms: 60_000,
network: clusterflux_core::CommandNetworkPolicy::Disabled,
env: Some(env),
};
let checkout = LocalSourceCheckout {
host_path: workspace.clone(),
snapshot: Digest::sha256("phase2-podman-smoke-checkout"),
};
let task = TaskInstanceId::from("podman-smoke");
let output_root = workspace.join("task-output");
fs::create_dir_all(&output_root)?;
let mut overlay = VfsOverlay::new(task.clone(), NodeId::from("node-podman-smoke"));
let mut runner = StdProcessRunner;
let output = LinuxRootlessPodmanBackend.execute_local_checkout_task(
LocalCheckoutTaskRequest {
process: ProcessId::from("vp-podman-smoke"),
virtual_thread: task,
invocation: &invocation,
checkout,
output_root,
stage_stdout_as: Some(VfsPath::new("/vfs/artifacts/podman-smoke.txt")?),
},
&mut runner,
&mut overlay,
)?;
let manifest = overlay.flush();
println!(
"{}",
serde_json::to_string(&json!({
"podman_status": "completed",
"status_code": output.status_code,
"stdout": output.stdout,
"stderr": output.stderr,
"staged_artifact": output.staged_artifact,
"large_bytes_uploaded": manifest.large_bytes_uploaded,
"uses_full_repo_tarball": false,
"coordinator_routed_file_reads": false,
}))?
);
let _ = fs::remove_dir_all(workspace);
Ok(())
}
fn create_workspace() -> Result<PathBuf, std::io::Error> {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.unwrap_or_default();
let workspace = std::env::temp_dir().join(format!(
"clusterflux-podman-smoke-{}-{nanos}",
std::process::id()
));
fs::create_dir_all(&workspace)?;
Ok(workspace)
}

View file

@ -0,0 +1,138 @@
use std::{net::SocketAddr, sync::Arc};
use clusterflux_core::{
ArtifactId, DataPlaneObject, DataPlaneScope, Digest, NativeQuicTransport, NodeEndpoint, NodeId,
ProcessId, ProjectId, RendezvousRequest, TenantId, Transport,
};
use quinn::rustls::{
pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer},
RootCertStore,
};
use quinn::{ClientConfig, Endpoint, ServerConfig};
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
struct QuicTransferRequest {
scope: DataPlaneScope,
authorization_digest: Digest,
requested_bytes: u64,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let payload = b"artifact-bytes-over-rust-native-quic".to_vec();
let (cert, key) = self_signed_localhost_cert()?;
let server_endpoint = Endpoint::server(
ServerConfig::with_single_cert(vec![cert.clone()], key)?,
"127.0.0.1:0".parse()?,
)?;
let server_addr = server_endpoint.local_addr()?;
let scope = DataPlaneScope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("vp-quic"),
object: DataPlaneObject::Artifact(ArtifactId::from("quic-artifact")),
authorization_subject: "node-a-to-node-b".to_owned(),
};
let transport = NativeQuicTransport;
let plan = transport.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope.clone(),
source: endpoint("node-a", server_addr),
destination: endpoint("node-b", "127.0.0.1:0".parse()?),
},
true,
"",
)?;
let expected_scope = plan.scope.clone();
let expected_digest = plan.authorization_digest.clone();
let expected_payload = payload.clone();
let server = tokio::spawn(async move {
let incoming = server_endpoint
.accept()
.await
.ok_or("server endpoint closed before accepting a QUIC connection")?;
let connection = incoming.await?;
let (mut send, mut recv) = connection.accept_bi().await?;
let request_bytes = recv.read_to_end(64 * 1024).await?;
let request: QuicTransferRequest = serde_json::from_slice(&request_bytes)?;
if request.scope != expected_scope {
return Err("QUIC request scope did not match the authorized data-plane scope".into());
}
if request.authorization_digest != expected_digest {
return Err(
"QUIC request authorization digest did not match the rendezvous plan".into(),
);
}
send.write_all(&expected_payload).await?;
send.finish()?;
server_endpoint.wait_idle().await;
Ok::<usize, Box<dyn std::error::Error + Send + Sync>>(request_bytes.len())
});
let mut roots = RootCertStore::empty();
roots.add(cert)?;
let client_config = ClientConfig::with_root_certificates(Arc::new(roots))?;
let mut client_endpoint = Endpoint::client("127.0.0.1:0".parse()?)?;
client_endpoint.set_default_client_config(client_config);
let connection = client_endpoint.connect(server_addr, "localhost")?.await?;
let (mut send, mut recv) = connection.open_bi().await?;
let request = QuicTransferRequest {
scope: plan.scope.clone(),
authorization_digest: plan.authorization_digest.clone(),
requested_bytes: payload.len() as u64,
};
let request_bytes = serde_json::to_vec(&request)?;
send.write_all(&request_bytes).await?;
send.finish()?;
let received = recv.read_to_end(64 * 1024).await?;
connection.close(0u32.into(), b"done");
client_endpoint.wait_idle().await;
let server_received_request_bytes = server.await??;
if received != payload {
return Err("QUIC artifact payload did not round trip".into());
}
println!(
"{}",
json!({
"kind": "clusterflux_quic_smoke",
"transport": format!("{:?}", transport.kind()),
"rust_native_quic": true,
"authenticated_direct_connection": transport.authenticated_direct_connections(),
"coordinator_assisted_rendezvous": plan.coordinator_assisted_rendezvous,
"coordinator_bulk_relay_allowed": plan.coordinator_bulk_relay_allowed,
"source_node": plan.source.node,
"destination_node": plan.destination.node,
"scope": plan.scope,
"request_bytes": request_bytes.len(),
"server_received_request_bytes": server_received_request_bytes,
"payload_bytes": received.len(),
"authorization_digest": plan.authorization_digest,
})
);
Ok(())
}
fn self_signed_localhost_cert() -> Result<
(CertificateDer<'static>, PrivateKeyDer<'static>),
Box<dyn std::error::Error + Send + Sync>,
> {
let cert = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()])?;
let key = PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()));
Ok((cert.cert.into(), key))
}
fn endpoint(name: &str, addr: SocketAddr) -> NodeEndpoint {
NodeEndpoint {
node: NodeId::from(name),
advertised_addr: addr.to_string(),
public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
}
}

View file

@ -0,0 +1,414 @@
use std::fs;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use clusterflux_core::{
ArtifactHandle, ArtifactId, Digest, StructuredTaskBoundary, TaskBoundaryHandle,
TaskBoundaryValue, WasmHostCommandRequest, WasmHostCommandResult,
WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult, WasmHostTaskControlRequest,
WasmHostTaskControlResult, WasmHostTaskHandle, WasmHostTaskJoinRequest, WasmHostTaskJoinResult,
WasmHostTaskStartRequest, WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult,
WasmTaskInvocation, WasmTaskOutcome,
};
use clusterflux_node::{WasmTaskHost, WasmtimeTaskRuntime};
use serde_json::json;
use wasmparser::{Parser, Payload};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = std::env::args().collect();
if args.len() == 4 && args[1] == "--host-command" {
return run_host_command_smoke(&args[2], &args[3]);
}
if args.len() == 6 && args[1] == "--debug-freeze-resume" {
return run_debug_freeze_resume_smoke(&args[2], &args[3], &args[4], &args[5]);
}
if args.len() == 6 && args[1] == "--task-abi" {
return run_task_abi_smoke(&args[2], &args[3], &args[4], &args[5]);
}
if args.len() == 4 && args[1] == "--task-artifact" {
return run_task_artifact_smoke(&args[2], &args[3]);
}
if args.len() != 5 {
return Err(
"usage: clusterflux-wasmtime-smoke <module.wasm> <export> <i32-arg> <expected-i32> | --host-command <module.wasm> <task> | --debug-freeze-resume <module.wasm> <export> <i32-arg> <expected-i32> | --task-abi <module.wasm> <export> <task> <args-json> | --task-artifact <module.wasm> <task>".into(),
);
}
let module = PathBuf::from(&args[1]);
let export = &args[2];
let arg: i32 = args[3].parse()?;
let expected: i32 = args[4].parse()?;
let wasm = fs::read(&module)?;
let runtime = WasmtimeTaskRuntime::new()?;
let result = runtime.run_i32_export(&wasm, export, arg)?;
if result != expected {
return Err(format!("expected {expected}, got {result} from export `{export}`").into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_task_smoke",
"module": module,
"export": export,
"arg": arg,
"result": result,
}))?
);
Ok(())
}
fn run_task_artifact_smoke(module: &str, task: &str) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let export = task_export(&wasm, task)?;
let published = Arc::new(Mutex::new(None));
let input_artifact = ArtifactHandle {
id: ArtifactId::from("compiler-output"),
digest: Digest::sha256("compiler-output"),
size_bytes: 15,
};
let source_snapshot = Digest::sha256("standalone-source-snapshot");
let package_input = TaskBoundaryValue::Structured(StructuredTaskBoundary {
value: serde_json::json!({
"release_name": "release.tar",
"source": {
"$task_handle": { "index": 0, "kind": "source_snapshot" }
},
"executable": {
"$task_handle": { "index": 1, "kind": "artifact" }
},
"inputs": [{
"$task_handle": { "index": 2, "kind": "artifact" }
}],
}),
handles: vec![
TaskBoundaryHandle::SourceSnapshot(source_snapshot),
TaskBoundaryHandle::Artifact(input_artifact.clone()),
TaskBoundaryHandle::Artifact(input_artifact.clone()),
],
});
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&wasm,
&Digest::sha256(&wasm),
&export,
&WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
vec![package_input],
),
Box::new(ArtifactRecordingHost {
published: Arc::clone(&published),
executed_command: Arc::new(Mutex::new(None)),
allow_command: true,
}),
)?;
if result.outcome != WasmTaskOutcome::Completed {
return Err(format!("artifact task failed: {:?}", result.error).into());
}
let Some(TaskBoundaryValue::Artifact(result_artifact)) = result.result else {
return Err("artifact task did not return an Artifact handle".into());
};
let (name, host_artifact) = published
.lock()
.map_err(|_| "artifact recording host lock was poisoned")?
.clone()
.ok_or("Wasm task did not invoke clusterflux.vfs_operation_v1 flush_output")?;
if result_artifact != host_artifact {
return Err("Wasm task returned a handle other than the host-issued artifact ID".into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_task_artifact_smoke",
"module": module,
"task": task,
"export": export,
"artifact": result_artifact,
"artifact_name": name,
"artifact_digest": host_artifact.digest,
"artifact_size_bytes": host_artifact.size_bytes,
"host_import": "clusterflux.vfs_operation_v1",
"host_issued_handle_returned": true,
}))?
);
Ok(())
}
type PublishedArtifact = (String, ArtifactHandle);
type ExecutedCommand = (WasmHostCommandRequest, WasmHostCommandResult);
struct ArtifactRecordingHost {
published: Arc<Mutex<Option<PublishedArtifact>>>,
executed_command: Arc<Mutex<Option<ExecutedCommand>>>,
allow_command: bool,
}
impl WasmTaskHost for ArtifactRecordingHost {
fn start_task(
&mut self,
_request: WasmHostTaskStartRequest,
) -> Result<WasmHostTaskHandle, String> {
Err("artifact smoke task must not spawn".to_owned())
}
fn join_task(
&mut self,
_request: WasmHostTaskJoinRequest,
) -> Result<WasmHostTaskJoinResult, String> {
Err("artifact smoke task must not join".to_owned())
}
fn run_command(
&mut self,
request: WasmHostCommandRequest,
) -> Result<WasmHostCommandResult, String> {
request.validate()?;
if !self.allow_command {
return Err("this smoke task was not granted Command capability".to_owned());
}
let result = WasmHostCommandResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
status_code: Some(0),
stdout: String::new(),
stderr: String::new(),
stdout_truncated: false,
stderr_truncated: false,
};
*self
.executed_command
.lock()
.map_err(|_| "command recording host lock was poisoned".to_owned())? =
Some((request, result.clone()));
Ok(result)
}
fn poll_task_control(
&mut self,
request: WasmHostTaskControlRequest,
) -> Result<WasmHostTaskControlResult, String> {
request.validate()?;
Ok(WasmHostTaskControlResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
cancellation_requested: false,
})
}
fn vfs_operation(&mut self, request: WasmHostVfsRequest) -> Result<WasmHostVfsResult, String> {
request.validate()?;
let (artifact, relative_path) = match request.operation {
WasmHostVfsOperation::FlushOutput { relative_path } => {
let digest = Digest::sha256(format!("standalone-vfs-smoke:{relative_path}"));
let artifact = ArtifactHandle {
id: ArtifactId::new(format!(
"{}-{}",
relative_path.replace('/', "_"),
digest.as_str().trim_start_matches("sha256:")
)),
digest,
size_bytes: relative_path.len() as u64,
};
*self
.published
.lock()
.map_err(|_| "artifact recording host lock was poisoned".to_owned())? =
Some((relative_path.clone(), artifact.clone()));
(artifact, relative_path)
}
WasmHostVfsOperation::MaterializeArtifact {
artifact,
relative_path,
} => (artifact, relative_path),
};
Ok(WasmHostVfsResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
artifact,
relative_path,
})
}
fn snapshot_source(
&mut self,
request: WasmHostSourceSnapshotRequest,
) -> Result<WasmHostSourceSnapshotResult, String> {
request.validate()?;
Ok(WasmHostSourceSnapshotResult {
abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
snapshot: Digest::sha256("standalone-source-snapshot"),
})
}
}
fn task_export(wasm: &[u8], task: &str) -> Result<String, Box<dyn std::error::Error>> {
for payload in Parser::new(0).parse_all(wasm) {
let Payload::CustomSection(section) = payload? else {
continue;
};
if section.name() != "clusterflux.tasks" {
continue;
}
for record in section
.data()
.split(|byte| *byte == b'\n' || *byte == 0)
.filter(|record| !record.is_empty())
{
let descriptor: serde_json::Value = serde_json::from_slice(record)?;
if descriptor.get("name").and_then(serde_json::Value::as_str) == Some(task) {
return descriptor
.get("export")
.and_then(serde_json::Value::as_str)
.map(str::to_owned)
.ok_or_else(|| format!("task `{task}` descriptor omitted export").into());
}
}
}
Err(format!("module has no task descriptor `{task}`").into())
}
fn run_task_abi_smoke(
module: &str,
export: &str,
task: &str,
args: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let args: Vec<TaskBoundaryValue> = serde_json::from_str(args)?;
let invocation = WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
args,
);
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified(
&wasm,
&Digest::sha256(&wasm),
export,
&invocation,
)?;
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasm_task_abi_smoke",
"module": module,
"export": export,
"invocation": invocation,
"result": result,
}))?
);
Ok(())
}
fn run_debug_freeze_resume_smoke(
module: &str,
export: &str,
arg: &str,
expected: &str,
) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let arg: i32 = arg.parse()?;
let expected: i32 = expected.parse()?;
let wasm = fs::read(&module)?;
let runtime = WasmtimeTaskRuntime::new()?;
let probe = runtime.freeze_resume_i32_export_probe(&wasm, export, arg)?;
if probe.result != expected {
return Err(format!(
"expected {expected}, got {} from export `{export}`",
probe.result
)
.into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_debug_freeze_resume_smoke",
"module": module,
"export": export,
"task": probe.task,
"frozen_state": probe.frozen_state,
"resumed_state": probe.resumed_state,
"stack_frames": probe.stack_frames,
"local_values": probe.local_values,
"wasm_function": probe.wasm_function,
"wasm_pc": probe.wasm_pc,
"arg": arg,
"result": probe.result,
"node_runtime_reached_wasm_task": true,
"node_runtime_captured_wasm_locals": true,
}))?
);
Ok(())
}
fn run_host_command_smoke(module: &str, task: &str) -> Result<(), Box<dyn std::error::Error>> {
let module = PathBuf::from(module);
let wasm = fs::read(&module)?;
let export = task_export(&wasm, task)?;
let published = Arc::new(Mutex::new(None));
let executed_command = Arc::new(Mutex::new(None));
let result = WasmtimeTaskRuntime::new()?.run_task_export_verified_with_task_host(
&wasm,
&Digest::sha256(&wasm),
&export,
&WasmTaskInvocation::new(
clusterflux_core::TaskDefinitionId::from(task),
clusterflux_core::TaskInstanceId::new(format!("{task}-1")),
vec![TaskBoundaryValue::SourceSnapshot(Digest::sha256(
"standalone-source-snapshot",
))],
),
Box::new(ArtifactRecordingHost {
published: Arc::clone(&published),
executed_command: Arc::clone(&executed_command),
allow_command: true,
}),
)?;
if result.outcome != WasmTaskOutcome::Completed {
return Err(format!("command task failed: {:?}", result.error).into());
}
let Some(TaskBoundaryValue::Artifact(result_artifact)) = result.result else {
return Err("command task did not return its published Artifact handle".into());
};
let (request, command_result) = executed_command
.lock()
.map_err(|_| "command recording host lock was poisoned")?
.clone()
.ok_or("Wasm task did not invoke clusterflux.command_run_v1")?;
let (artifact_name, host_artifact) = published
.lock()
.map_err(|_| "artifact recording host lock was poisoned")?
.clone()
.ok_or("command task did not publish its command output")?;
if result_artifact != host_artifact {
return Err("command task returned a handle other than the published artifact".into());
}
println!(
"{}",
serde_json::to_string(&json!({
"type": "wasmtime_host_command_smoke",
"module": module,
"task": task,
"export": export,
"program": request.program,
"args": request.args,
"working_directory": request.working_directory,
"environment_variables": request.environment_variables,
"timeout_ms": request.timeout_ms,
"network": request.network,
"status_code": command_result.status_code,
"stdout": command_result.stdout,
"artifact": result_artifact,
"artifact_name": artifact_name,
"artifact_digest": host_artifact.digest,
"artifact_size_bytes": host_artifact.size_bytes,
"node_host_import": "clusterflux.command_run_v1",
"artifact_host_import": "clusterflux.vfs_operation_v1",
"flagship_linux_build_task": true,
"node_executed_host_command": true,
"hosted_control_plane_ran_command": false,
}))?
);
Ok(())
}