michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-6756e5208c8b

Browse source 
Source commit: 6756e5208c8b79e83d55610251430bc1baef53a3

Public tree identity: sha256:2f58837c3759b4f466cbc274571ee71bc0d24c7552dc009c186394e99123da87
This commit is contained in:
Clusterflux release 2026-07-17 04:13:46 +02:00
commit 18cba9c609
210 changed files with 78616 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

22
crates/clusterflux-core/Cargo.toml Normal file
View file

@ -0,0 +1,22 @@
[package]
name = "clusterflux-core"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
ed25519-dalek.workspace = true
serde.workspace = true
serde_json.workspace = true
hex.workspace = true
sha2.workspace = true
syn.workspace = true
thiserror.workspace = true
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
getrandom.workspace = true
[dev-dependencies]
tempfile.workspace = true

1178
crates/clusterflux-core/src/artifact.rs Normal file
View file

File diff suppressed because it is too large Load diff

829
crates/clusterflux-core/src/auth.rs Normal file
View file

@ -0,0 +1,829 @@
#[cfg(not(target_arch = "wasm32"))]
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use base64::{engine::general_purpose::STANDARD, Engine as _};
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
AgentId, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId,
};
pub fn admin_request_proof(
admin_token: &str,
operation: &str,
tenant: &str,
actor_user: &str,
target_tenant: &str,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Digest {
admin_request_proof_from_token_digest(
&Digest::sha256(admin_token),
operation,
tenant,
actor_user,
target_tenant,
nonce,
issued_at_epoch_seconds,
)
}
pub fn admin_request_proof_from_token_digest(
admin_token_digest: &Digest,
operation: &str,
tenant: &str,
actor_user: &str,
target_tenant: &str,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Digest {
let issued_at_epoch_seconds = issued_at_epoch_seconds.to_string();
Digest::from_parts([
b"clusterflux-admin-request-proof:v1".as_slice(),
admin_token_digest.as_str().as_bytes(),
operation.as_bytes(),
tenant.as_bytes(),
actor_user.as_bytes(),
target_tenant.as_bytes(),
nonce.as_bytes(),
issued_at_epoch_seconds.as_bytes(),
])
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Actor {
User(UserId),
Agent(AgentId),
Node(NodeId),
Task(TaskInstanceId),
}
impl Actor {
pub fn kind(&self) -> IdentityKind {
match self {
Self::User(_) => IdentityKind::User,
Self::Agent(_) => IdentityKind::Agent,
Self::Node(_) => IdentityKind::Node,
Self::Task(_) => IdentityKind::Task,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum IdentityKind {
User,
Agent,
Node,
Project,
Task,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum CredentialKind {
BrowserSession,
CliDeviceSession,
PublicKey,
NodeCredential,
TaskCredential,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AuthContext {
pub tenant: TenantId,
pub project: ProjectId,
pub actor: Actor,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Action {
CreateProject,
AttachNode,
CreateNodeEnrollmentGrant,
ExchangeNodeEnrollmentGrant,
LoginBrowser,
LoginCli,
EnrollAgent,
List,
Inspect,
Mutate,
ClaimTask,
DebugAttach,
DebugRead,
DownloadArtifact,
PublishArtifact,
RunNativeCommand,
RunContainer,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowserLoginFlow {
pub authorization_url: String,
pub callback_path: String,
pub state: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PublicKeyIdentity {
pub subject: Actor,
pub public_key: String,
pub fingerprint: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgentSignedRequest {
pub nonce: String,
pub issued_at_epoch_seconds: u64,
pub signature: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeSignedRequest {
pub nonce: String,
pub issued_at_epoch_seconds: u64,
pub signature: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnrollmentGrant {
pub tenant: TenantId,
pub project: ProjectId,
pub grant_id: String,
pub scope: String,
pub expires_at_epoch_seconds: u64,
pub consumed: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeCredential {
pub node: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub public_key_fingerprint: Digest,
pub scope: String,
pub capability_policy_digest: Digest,
pub credential_kind: CredentialKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum EnrollmentError {
Expired,
AlreadyConsumed,
WrongScope,
}
impl EnrollmentGrant {
pub fn exchange_for_node_identity(
&mut self,
node: NodeId,
public_key: &str,
requested_scope: &str,
now_epoch_seconds: u64,
) -> Result<NodeCredential, EnrollmentError> {
if self.consumed {
return Err(EnrollmentError::AlreadyConsumed);
}
if now_epoch_seconds > self.expires_at_epoch_seconds {
return Err(EnrollmentError::Expired);
}
if requested_scope != self.scope {
return Err(EnrollmentError::WrongScope);
}
self.consumed = true;
let capability_policy_digest =
node_capability_policy_digest(&self.tenant, &self.project, &self.scope);
Ok(NodeCredential {
node,
tenant: self.tenant.clone(),
project: self.project.clone(),
public_key_fingerprint: Digest::sha256(public_key),
scope: self.scope.clone(),
capability_policy_digest,
credential_kind: CredentialKind::NodeCredential,
})
}
}
pub fn node_capability_policy_digest(
tenant: &TenantId,
project: &ProjectId,
scope: &str,
) -> Digest {
Digest::from_parts([
b"node-capability-policy:v1".as_slice(),
tenant.as_str().as_bytes(),
project.as_str().as_bytes(),
scope.as_bytes(),
])
}
pub fn agent_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "agent private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
Ok(format!(
"ed25519:{}",
STANDARD.encode(signing_key.verifying_key().to_bytes())
))
}
pub fn node_ed25519_public_key_from_private_key(private_key: &str) -> Result<String, String> {
agent_ed25519_public_key_from_private_key(private_key)
}
pub fn derive_ed25519_private_key_from_seed(seed: &str) -> String {
let digest = Digest::sha256(seed);
let hex = digest.as_str().trim_start_matches("sha256:");
let bytes = hex::decode(hex).expect("sha256 digest hex should decode");
format!("ed25519:{}", STANDARD.encode(bytes))
}
/// Generates a new Ed25519 private key from the operating system CSPRNG.
///
/// Seed-derived keys remain available for deterministic test fixtures, but
/// must not be used for persisted user, Agent, or Node credentials.
#[cfg(not(target_arch = "wasm32"))]
pub fn generate_ed25519_private_key() -> Result<String, String> {
let mut bytes = [0_u8; 32];
getrandom::fill(&mut bytes)
.map_err(|err| format!("operating system random source failed: {err}"))?;
Ok(format!("ed25519:{}", STANDARD.encode(bytes)))
}
/// Generates an opaque, URL-safe 256-bit token suitable for one-time grants,
/// session secrets, and nonces. The label is non-secret domain separation.
#[cfg(not(target_arch = "wasm32"))]
pub fn generate_opaque_token(label: &str) -> Result<String, String> {
let mut bytes = [0_u8; 32];
getrandom::fill(&mut bytes)
.map_err(|err| format!("operating system random source failed: {err}"))?;
Ok(format!("{label}_{}", URL_SAFE_NO_PAD.encode(bytes)))
}
#[derive(Clone, Copy, Debug)]
pub struct AgentWorkflowScope<'a> {
pub tenant: &'a TenantId,
pub project: &'a ProjectId,
pub agent: &'a AgentId,
pub request_kind: &'a str,
pub process: &'a ProcessId,
pub task: Option<&'a TaskInstanceId>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AgentWorkflowRequestScope {
pub tenant: TenantId,
pub project: ProjectId,
pub request_kind: String,
pub process: ProcessId,
pub task: Option<TaskInstanceId>,
}
impl AgentWorkflowRequestScope {
pub fn new(
tenant: TenantId,
project: ProjectId,
request_kind: impl Into<String>,
process: ProcessId,
task: Option<TaskInstanceId>,
) -> Result<Self, String> {
let request_kind = request_kind.into();
match request_kind.as_str() {
"start_process" if task.is_some() => {
return Err("start_process agent scope must not contain a task instance".to_owned())
}
"launch_task" if task.is_none() => {
return Err("launch_task agent scope requires a task instance".to_owned())
}
"start_process" | "launch_task" => {}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
}
Ok(Self {
tenant,
project,
request_kind,
process,
task,
})
}
pub fn for_agent<'a>(&'a self, agent: &'a AgentId) -> AgentWorkflowScope<'a> {
AgentWorkflowScope {
tenant: &self.tenant,
project: &self.project,
agent,
request_kind: &self.request_kind,
process: &self.process,
task: self.task.as_ref(),
}
}
}
pub fn agent_workflow_request_scope_from_payload(
payload: &Value,
) -> Result<AgentWorkflowRequestScope, String> {
let object = payload
.as_object()
.ok_or_else(|| "agent workflow request payload must be an object".to_owned())?;
let request_kind = object
.get("type")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow request type is missing".to_owned())?;
let tenant = object
.get("tenant")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow tenant is missing".to_owned())?;
let project = object
.get("project")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow project is missing".to_owned())?;
let (process, task) = match request_kind {
"start_process" => (
object
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "start_process agent scope is missing process".to_owned())?,
None,
),
"launch_task" => {
let task_spec = object
.get("task_spec")
.and_then(Value::as_object)
.ok_or_else(|| "launch_task agent scope is missing task_spec".to_owned())?;
let process = task_spec
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing process".to_owned())?;
let task = task_spec
.get("task_instance")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
(process, Some(TaskInstanceId::from(task)))
}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
};
AgentWorkflowRequestScope::new(
TenantId::from(tenant),
ProjectId::from(project),
request_kind,
ProcessId::from(process),
task,
)
}
pub fn sign_agent_workflow_request(
private_key: &str,
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<AgentSignedRequest, String> {
let private_key = decode_ed25519_key(private_key, 32, "agent private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "agent private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
let message =
agent_workflow_signature_message(scope, payload_digest, &nonce, issued_at_epoch_seconds);
let signature: Signature = signing_key.sign(&message);
Ok(AgentSignedRequest {
nonce,
issued_at_epoch_seconds,
signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
})
}
pub fn verify_agent_workflow_signature(
public_key: &str,
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
signed_request: &AgentSignedRequest,
) -> Result<(), String> {
let public_key = decode_ed25519_key(public_key, 32, "agent public key")?;
let public_key: [u8; 32] = public_key
.try_into()
.map_err(|_| "agent public key must be 32 bytes".to_owned())?;
let verifying_key = VerifyingKey::from_bytes(&public_key)
.map_err(|_| "agent public key is not a valid Ed25519 verifying key".to_owned())?;
let signature = decode_ed25519_key(&signed_request.signature, 64, "agent signature")?;
let signature: [u8; 64] = signature
.try_into()
.map_err(|_| "agent signature must be 64 bytes".to_owned())?;
let signature = Signature::from_bytes(&signature);
let message = agent_workflow_signature_message(
scope,
payload_digest,
&signed_request.nonce,
signed_request.issued_at_epoch_seconds,
);
verifying_key
.verify(&message, &signature)
.map_err(|_| "agent signature does not verify against the registered public key".to_owned())
}
pub fn sign_node_request(
private_key: &str,
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
nonce: String,
issued_at_epoch_seconds: u64,
) -> Result<NodeSignedRequest, String> {
let private_key = decode_ed25519_key(private_key, 32, "node private key")?;
let private_key: [u8; 32] = private_key
.try_into()
.map_err(|_| "node private key must be 32 bytes".to_owned())?;
let signing_key = SigningKey::from_bytes(&private_key);
let message = node_request_signature_message(
node,
request_kind,
payload_digest,
&nonce,
issued_at_epoch_seconds,
);
let signature: Signature = signing_key.sign(&message);
Ok(NodeSignedRequest {
nonce,
issued_at_epoch_seconds,
signature: format!("ed25519:{}", STANDARD.encode(signature.to_bytes())),
})
}
pub fn verify_node_request_signature(
public_key: &str,
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
signed_request: &NodeSignedRequest,
) -> Result<(), String> {
let public_key = decode_ed25519_key(public_key, 32, "node public key")?;
let public_key: [u8; 32] = public_key
.try_into()
.map_err(|_| "node public key must be 32 bytes".to_owned())?;
let verifying_key = VerifyingKey::from_bytes(&public_key)
.map_err(|_| "node public key is not a valid Ed25519 verifying key".to_owned())?;
let signature = decode_ed25519_key(&signed_request.signature, 64, "node signature")?;
let signature: [u8; 64] = signature
.try_into()
.map_err(|_| "node signature must be 64 bytes".to_owned())?;
let signature = Signature::from_bytes(&signature);
let message = node_request_signature_message(
node,
request_kind,
payload_digest,
&signed_request.nonce,
signed_request.issued_at_epoch_seconds,
);
verifying_key
.verify(&message, &signature)
.map_err(|_| "node signature does not verify against the enrolled public key".to_owned())
}
fn decode_ed25519_key(value: &str, expected_len: usize, kind: &str) -> Result<Vec<u8>, String> {
let encoded = value
.strip_prefix("ed25519:")
.ok_or_else(|| format!("{kind} must use ed25519:<base64> encoding"))?;
let bytes = STANDARD
.decode(encoded)
.map_err(|_| format!("{kind} is not valid base64"))?;
if bytes.len() != expected_len {
return Err(format!("{kind} must be {expected_len} bytes"));
}
Ok(bytes)
}
fn agent_workflow_signature_message(
scope: AgentWorkflowScope<'_>,
payload_digest: &Digest,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Vec<u8> {
let issued_at = issued_at_epoch_seconds.to_string();
let task = scope.task.map(TaskInstanceId::as_str).unwrap_or("");
let parts = [
"clusterflux-agent-workflow-signature:v2",
scope.tenant.as_str(),
scope.project.as_str(),
scope.agent.as_str(),
scope.request_kind,
scope.process.as_str(),
task,
payload_digest.as_str(),
nonce,
&issued_at,
];
let mut message = Vec::new();
for part in parts {
message.extend_from_slice(part.len().to_string().as_bytes());
message.push(b':');
message.extend_from_slice(part.as_bytes());
message.push(b'\n');
}
message
}
fn node_request_signature_message(
node: &NodeId,
request_kind: &str,
payload_digest: &Digest,
nonce: &str,
issued_at_epoch_seconds: u64,
) -> Vec<u8> {
let issued_at = issued_at_epoch_seconds.to_string();
let parts = [
"clusterflux-node-request-signature:v2",
node.as_str(),
request_kind,
payload_digest.as_str(),
nonce,
&issued_at,
];
let mut message = Vec::new();
for part in parts {
message.extend_from_slice(part.len().to_string().as_bytes());
message.push(b':');
message.extend_from_slice(part.as_bytes());
message.push(b'\n');
}
message
}
/// Computes the stable digest covered by node and agent request signatures.
///
/// The proof field itself is excluded, and explicit JSON nulls are normalized
/// with omitted optional fields because the wire protocol deserializes both to
/// the same request. Every semantically meaningful key and value remains bound
/// by the signature.
pub fn signed_request_payload_digest(value: &Value) -> Digest {
fn canonicalize(value: &Value, top_level: bool) -> Value {
match value {
Value::Object(object) => Value::Object(
object
.iter()
.filter(|(key, value)| {
!value.is_null()
&& (!top_level
|| !matches!(key.as_str(), "agent_signature" | "node_signature"))
})
.map(|(key, value)| (key.clone(), canonicalize(value, false)))
.collect(),
),
Value::Array(values) => Value::Array(
values
.iter()
.map(|value| canonicalize(value, false))
.collect(),
),
value => value.clone(),
}
}
let canonical = canonicalize(value, true);
let bytes = serde_json::to_vec(&canonical)
.expect("canonical JSON request values are always serializable");
Digest::sha256(bytes)
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Scope {
pub tenant: TenantId,
pub project: ProjectId,
pub process: Option<ProcessId>,
pub task: Option<TaskInstanceId>,
pub node: Option<NodeId>,
pub artifact: Option<ArtifactId>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Authorization {
pub allowed: bool,
pub reason: String,
}
impl Authorization {
pub fn allow(reason: impl Into<String>) -> Self {
Self {
allowed: true,
reason: reason.into(),
}
}
pub fn deny(reason: impl Into<String>) -> Self {
Self {
allowed: false,
reason: reason.into(),
}
}
}
pub fn same_tenant_project(context: &AuthContext, scope: &Scope) -> Authorization {
if context.tenant != scope.tenant {
return Authorization::deny("tenant mismatch");
}
if context.project != scope.project {
return Authorization::deny("project mismatch");
}
Authorization::allow("same tenant and project")
}
pub fn task_credentials_do_not_contain_user_session(
task: &Actor,
credentials: &[CredentialKind],
) -> Authorization {
if !matches!(task, Actor::Task(_)) {
return Authorization::deny("credential check requires task actor");
}
if credentials.iter().any(|credential| {
matches!(
credential,
CredentialKind::BrowserSession | CredentialKind::CliDeviceSession
)
}) {
return Authorization::deny(
"user OAuth/session tokens must not be passed to nodes as task credentials",
);
}
Authorization::allow("task credentials are scoped runtime credentials")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tenant_project_scope_denies_cross_tenant_access() {
let context = AuthContext {
tenant: TenantId::from("tenant-a"),
project: ProjectId::from("project-a"),
actor: Actor::User(UserId::from("user-a")),
};
let scope = Scope {
tenant: TenantId::from("tenant-b"),
project: ProjectId::from("project-a"),
process: None,
task: None,
node: None,
artifact: None,
};
assert!(!same_tenant_project(&context, &scope).allowed);
}
#[test]
fn node_enrollment_exchanges_short_lived_grant_once() {
let mut grant = EnrollmentGrant {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
grant_id: "grant".to_owned(),
scope: "node:attach".to_owned(),
expires_at_epoch_seconds: 100,
consumed: false,
};
let credential = grant
.exchange_for_node_identity(NodeId::from("node"), "public-key", "node:attach", 99)
.unwrap();
assert_eq!(credential.credential_kind, CredentialKind::NodeCredential);
assert_eq!(credential.tenant, TenantId::from("tenant"));
assert_eq!(credential.project, ProjectId::from("project"));
assert_eq!(credential.node, NodeId::from("node"));
assert_eq!(credential.scope, "node:attach");
assert_eq!(
credential.capability_policy_digest,
node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:attach"
)
);
assert_eq!(
grant.exchange_for_node_identity(
NodeId::from("node2"),
"public-key",
"node:attach",
99
),
Err(EnrollmentError::AlreadyConsumed)
);
}
#[test]
fn node_capability_policy_digest_is_scoped() {
let base = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:attach",
);
let other_project = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("other"),
"node:attach",
);
let other_scope = node_capability_policy_digest(
&TenantId::from("tenant"),
&ProjectId::from("project"),
"node:limited",
);
assert!(base.is_valid_sha256());
assert_ne!(base, other_project);
assert_ne!(base, other_scope);
}
#[test]
fn generated_ed25519_private_keys_are_random_and_valid() {
let first = generate_ed25519_private_key().unwrap();
let second = generate_ed25519_private_key().unwrap();
assert_ne!(first, second);
assert!(agent_ed25519_public_key_from_private_key(&first)
.unwrap()
.starts_with("ed25519:"));
assert!(node_ed25519_public_key_from_private_key(&second)
.unwrap()
.starts_with("ed25519:"));
}
#[test]
fn generated_opaque_tokens_are_random_and_url_safe() {
let first = generate_opaque_token("grant").unwrap();
let second = generate_opaque_token("grant").unwrap();
assert_ne!(first, second);
assert!(first.starts_with("grant_"));
assert!(first
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-')));
}
#[test]
fn task_credentials_reject_user_session_tokens() {
for credential in [
CredentialKind::BrowserSession,
CredentialKind::CliDeviceSession,
] {
let authz = task_credentials_do_not_contain_user_session(
&Actor::Task(TaskInstanceId::from("task")),
&[CredentialKind::TaskCredential, credential],
);
assert!(!authz.allowed);
assert!(authz.reason.contains("must not be passed"));
}
let scoped = task_credentials_do_not_contain_user_session(
&Actor::Task(TaskInstanceId::from("task")),
&[
CredentialKind::TaskCredential,
CredentialKind::NodeCredential,
],
);
assert!(scoped.allowed);
}
#[test]
fn identities_remain_distinct_for_authorization() {
assert_eq!(Actor::User(UserId::from("user")).kind(), IdentityKind::User);
assert_eq!(
Actor::Agent(AgentId::from("agent")).kind(),
IdentityKind::Agent
);
assert_eq!(Actor::Node(NodeId::from("node")).kind(), IdentityKind::Node);
assert_eq!(
Actor::Task(TaskInstanceId::from("task")).kind(),
IdentityKind::Task
);
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: Some(ProcessId::from("process")),
task: Some(TaskInstanceId::from("task")),
node: Some(NodeId::from("node")),
artifact: Some(ArtifactId::from("artifact")),
};
assert_eq!(scope.process, Some(ProcessId::from("process")));
assert_eq!(scope.artifact, Some(ArtifactId::from("artifact")));
assert_ne!(
CredentialKind::BrowserSession,
CredentialKind::CliDeviceSession
);
assert_ne!(CredentialKind::PublicKey, CredentialKind::NodeCredential);
assert_ne!(
CredentialKind::NodeCredential,
CredentialKind::TaskCredential
);
}
}

459
crates/clusterflux-core/src/bundle.rs Normal file
View file

@ -0,0 +1,459 @@
use serde::{Deserialize, Serialize};
use syn::parse::Parser;
use syn::{Expr, Item, Lit, Meta, Token};
use crate::{Digest, EnvironmentResource, SourceTransferPolicy, TaskDefinitionId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SelectedInput {
pub path: String,
pub digest: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleIdentityInputs {
pub wasm_code: Digest,
pub task_abi: Digest,
pub entrypoints: Vec<String>,
pub default_entrypoint: String,
pub environments: Vec<EnvironmentResource>,
pub source_provider_manifest: Digest,
pub source_transfer_policy: SourceTransferPolicy,
pub selected_inputs: Vec<SelectedInput>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleMetadata {
pub identity: Digest,
pub wasm_code: Digest,
pub task_metadata: BundleTaskMetadata,
pub source_metadata: BundleSourceMetadata,
pub debug_metadata: BundleDebugMetadata,
pub large_input_policy: BundleLargeInputPolicy,
pub restart_compatibility: BundleRestartCompatibility,
pub environments: Vec<EnvironmentResource>,
pub selected_inputs: Vec<SelectedInput>,
pub embeds_full_container_images: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleTaskMetadata {
pub task_abi: Digest,
pub entrypoints: Vec<String>,
pub default_entrypoint: String,
pub boundary: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleSourceMetadata {
pub source_provider_manifest: Digest,
pub transfer_policy: SourceTransferPolicy,
pub selected_inputs: Vec<SelectedInput>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleDebugMetadata {
pub available: bool,
pub source_level_breakpoints: bool,
pub dap_virtual_process: bool,
pub variables_pane_supported: bool,
pub probes: Vec<BundleDebugProbe>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleDebugProbe {
pub id: String,
pub source_path: String,
pub line_start: u32,
pub line_end: u32,
pub function: String,
pub task: TaskDefinitionId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleLargeInputPolicy {
pub selected_inputs_are_content_digests: bool,
pub selected_input_bytes_included: bool,
pub full_repository_bytes_included: bool,
pub silent_task_argument_serialization: bool,
pub supported_handle_types: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BundleRestartCompatibility {
pub source_edits_can_restart_from_clean_task_boundary: bool,
pub requires_clean_checkpoint_boundary: bool,
pub compares_task_abi: Digest,
pub compares_environment_digests: bool,
pub compares_serialized_args: bool,
pub discards_unflushed_task_local_changes: bool,
pub incompatible_changes_require_whole_process_restart: bool,
}
impl BundleIdentityInputs {
pub fn identity(&self) -> Digest {
let mut parts = vec![
b"bundle:v1".to_vec(),
self.wasm_code.as_str().as_bytes().to_vec(),
self.task_abi.as_str().as_bytes().to_vec(),
self.source_provider_manifest.as_str().as_bytes().to_vec(),
self.default_entrypoint.as_bytes().to_vec(),
format!("{:?}", self.source_transfer_policy).into_bytes(),
];
let mut entrypoints = self.entrypoints.clone();
entrypoints.sort();
for entrypoint in entrypoints {
parts.push(entrypoint.into_bytes());
}
let mut environments = self.environments.clone();
environments.sort_by(|left, right| left.name.cmp(&right.name));
for environment in environments {
parts.push(environment.name.as_bytes().to_vec());
parts.push(format!("{:?}", environment.kind).into_bytes());
parts.push(environment.digest.as_str().as_bytes().to_vec());
}
let mut inputs = self.selected_inputs.clone();
inputs.sort_by(|left, right| left.path.cmp(&right.path));
for input in inputs {
parts.push(input.path.into_bytes());
parts.push(input.digest.as_str().as_bytes().to_vec());
}
Digest::from_parts(parts)
}
pub fn inspectable_metadata(&self) -> BundleMetadata {
let mut entrypoints = self.entrypoints.clone();
entrypoints.sort();
BundleMetadata {
identity: self.identity(),
wasm_code: self.wasm_code.clone(),
task_metadata: BundleTaskMetadata {
task_abi: self.task_abi.clone(),
entrypoints,
default_entrypoint: self.default_entrypoint.clone(),
boundary: "clusterflux_task_exports".to_owned(),
},
source_metadata: BundleSourceMetadata {
source_provider_manifest: self.source_provider_manifest.clone(),
transfer_policy: self.source_transfer_policy.clone(),
selected_inputs: self.selected_inputs.clone(),
},
debug_metadata: BundleDebugMetadata {
available: true,
source_level_breakpoints: true,
dap_virtual_process: true,
variables_pane_supported: true,
probes: Vec::new(),
},
large_input_policy: BundleLargeInputPolicy {
selected_inputs_are_content_digests: true,
selected_input_bytes_included: false,
full_repository_bytes_included: false,
silent_task_argument_serialization: false,
supported_handle_types: vec![
"SourceSnapshot".to_owned(),
"Blob".to_owned(),
"Artifact".to_owned(),
"VFS".to_owned(),
],
},
restart_compatibility: BundleRestartCompatibility {
source_edits_can_restart_from_clean_task_boundary: true,
requires_clean_checkpoint_boundary: true,
compares_task_abi: self.task_abi.clone(),
compares_environment_digests: true,
compares_serialized_args: true,
discards_unflushed_task_local_changes: true,
incompatible_changes_require_whole_process_restart: true,
},
environments: self.environments.clone(),
selected_inputs: self.selected_inputs.clone(),
embeds_full_container_images: false,
}
}
}
pub fn discover_source_debug_probes(
source_path: impl Into<String>,
source: &str,
) -> Vec<BundleDebugProbe> {
let source_path = source_path.into();
let lines = source.lines().collect::<Vec<_>>();
let function_starts = lines
.iter()
.enumerate()
.filter_map(|(index, line)| {
parse_rust_function_name(line).map(|function| (index, function))
})
.collect::<Vec<_>>();
let Ok(file) = syn::parse_file(source) else {
return Vec::new();
};
file.items
.iter()
.filter_map(|item| {
let Item::Fn(function) = item else {
return None;
};
let function_name = function.sig.ident.to_string();
let task = clusterflux_probe_task(function)?;
let (function_index, (line_index, _)) = function_starts
.iter()
.enumerate()
.find(|(_, (_, candidate))| candidate == &function_name)?;
let line_start = (*line_index + 1) as u32;
let next_start = function_starts
.get(function_index + 1)
.map(|(next_index, _)| *next_index)
.unwrap_or(lines.len());
let line_end = next_start.max(*line_index + 1) as u32;
Some(debug_probe(
&source_path,
line_start,
line_end,
&function_name,
task,
))
})
.collect()
}
fn debug_probe(
source_path: &str,
line_start: u32,
line_end: u32,
function: &str,
task: TaskDefinitionId,
) -> BundleDebugProbe {
let id = Digest::from_parts([
b"bundle-debug-probe:v1".as_slice(),
source_path.as_bytes(),
function.as_bytes(),
task.as_str().as_bytes(),
line_start.to_string().as_bytes(),
line_end.to_string().as_bytes(),
])
.as_str()
.to_owned();
BundleDebugProbe {
id,
source_path: source_path.to_owned(),
line_start,
line_end,
function: function.to_owned(),
task,
}
}
fn parse_rust_function_name(line: &str) -> Option<String> {
let start = line.find("fn ")? + 3;
let rest = &line[start..];
let name = rest
.chars()
.take_while(|ch| ch.is_ascii_alphanumeric() || *ch == '_')
.collect::<String>();
(!name.is_empty()).then_some(name)
}
fn clusterflux_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> {
for attribute in &function.attrs {
let mut segments = attribute.path().segments.iter();
let Some(namespace) = segments.next() else {
continue;
};
let Some(kind) = segments.next() else {
continue;
};
if namespace.ident != "clusterflux" || segments.next().is_some() {
continue;
}
let function_name = function.sig.ident.to_string();
let default_name = match kind.ident.to_string().as_str() {
"main" => function_name
.strip_suffix("_main")
.unwrap_or(&function_name)
.to_owned(),
"task" => function_name,
_ => continue,
};
let declared_name = match &attribute.meta {
Meta::List(list) => {
let parser = syn::punctuated::Punctuated::<Meta, Token![,]>::parse_terminated;
parser
.parse2(list.tokens.clone())
.ok()
.and_then(|items| {
items.into_iter().find_map(|item| {
let Meta::NameValue(name_value) = item else {
return None;
};
if !name_value.path.is_ident("name") {
return None;
}
let Expr::Lit(value) = name_value.value else {
return None;
};
let Lit::Str(value) = value.lit else {
return None;
};
Some(value.value())
})
})
.unwrap_or(default_name)
}
_ => default_name,
};
return Some(TaskDefinitionId::new(declared_name));
}
None
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::{EnvironmentKind, EnvironmentRequirements};
use super::*;
fn env(digest: &str) -> EnvironmentResource {
EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: PathBuf::from("envs/linux/Containerfile"),
context_path: PathBuf::from("envs/linux"),
digest: Digest::sha256(digest),
requirements: EnvironmentRequirements::linux_container(),
}
}
#[test]
fn bundle_identity_changes_when_environment_recipe_changes() {
let base = BundleIdentityInputs {
wasm_code: Digest::sha256("wasm"),
task_abi: Digest::sha256("abi"),
entrypoints: vec!["build".to_owned()],
default_entrypoint: "build".to_owned(),
environments: vec![env("recipe-a")],
source_provider_manifest: Digest::sha256("source"),
source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
selected_inputs: vec![],
};
let mut changed = base.clone();
changed.environments = vec![env("recipe-b")];
assert_ne!(base.identity(), changed.identity());
}
#[test]
fn bundle_metadata_is_inspectable_and_does_not_vendor_images_by_default() {
let inputs = BundleIdentityInputs {
wasm_code: Digest::sha256("wasm"),
task_abi: Digest::sha256("abi"),
entrypoints: vec!["build".to_owned(), "test".to_owned()],
default_entrypoint: "build".to_owned(),
environments: vec![env("recipe")],
source_provider_manifest: Digest::sha256("source"),
source_transfer_policy: SourceTransferPolicy::local_first_snapshot_chunks(),
selected_inputs: vec![SelectedInput {
path: "inputs/config.json".to_owned(),
digest: Digest::sha256("config"),
}],
};
let metadata = inputs.inspectable_metadata();
assert!(metadata.wasm_code.as_str().starts_with("sha256:"));
assert_eq!(metadata.task_metadata.default_entrypoint, "build");
assert!(metadata
.task_metadata
.entrypoints
.contains(&"test".to_owned()));
assert!(metadata.debug_metadata.dap_virtual_process);
assert!(
metadata
.source_metadata
.transfer_policy
.local_source_bytes_remain_node_local
);
assert!(
metadata
.large_input_policy
.selected_inputs_are_content_digests
);
assert!(!metadata.large_input_policy.selected_input_bytes_included);
assert!(!metadata.large_input_policy.full_repository_bytes_included);
assert!(
!metadata
.large_input_policy
.silent_task_argument_serialization
);
assert!(metadata
.large_input_policy
.supported_handle_types
.contains(&"Artifact".to_owned()));
assert!(
metadata
.restart_compatibility
.source_edits_can_restart_from_clean_task_boundary
);
assert!(
metadata
.restart_compatibility
.requires_clean_checkpoint_boundary
);
assert_eq!(
metadata.restart_compatibility.compares_task_abi,
inputs.task_abi
);
assert!(
metadata
.restart_compatibility
.incompatible_changes_require_whole_process_restart
);
assert_eq!(metadata.environments.len(), 1);
assert!(!metadata.embeds_full_container_images);
}
#[test]
fn source_debug_probe_metadata_maps_function_ranges_to_tasks() {
let probes = discover_source_debug_probes(
"src/build.rs",
r#"#[clusterflux::main]
fn build_main() {
let linux = compile_linux();
}
#[clusterflux::task]
fn compile_linux() {
println!("linux");
}
fn helper_without_runtime_probe() {}
#[clusterflux::task(name = "release")]
fn package_release() {
println!("package");
}
"#,
);
assert_eq!(probes.len(), 3);
assert_eq!(probes[0].source_path, "src/build.rs");
assert_eq!(probes[0].function, "build_main");
assert_eq!(probes[0].task, TaskDefinitionId::from("build"));
assert_eq!(probes[0].line_start, 2);
assert_eq!(probes[0].line_end, 6);
assert_eq!(probes[1].function, "compile_linux");
assert_eq!(probes[1].task, TaskDefinitionId::from("compile_linux"));
assert_eq!(probes[2].function, "package_release");
assert_eq!(probes[2].task, TaskDefinitionId::from("release"));
assert!(probes.iter().all(|probe| probe.id.starts_with("sha256:")));
}
}

218
crates/clusterflux-core/src/capability.rs Normal file
View file

@ -0,0 +1,218 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum Capability {
Command,
Containers,
RootlessPodman,
SourceFilesystem,
SourceGit,
HostFilesystem,
Network,
Secrets,
InboundPorts,
ArbitrarySyscalls,
VfsArtifacts,
WindowsCommandDev,
QuicDirect,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum EnvironmentBackend {
Container,
NixFlake,
WindowsCommandDev,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum Os {
Linux,
Windows,
Macos,
Other(String),
}
impl Os {
pub fn current() -> Self {
match std::env::consts::OS {
"linux" => Self::Linux,
"windows" => Self::Windows,
"macos" => Self::Macos,
other => Self::Other(other.to_owned()),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeCapabilities {
pub os: Os,
pub arch: String,
pub capabilities: BTreeSet<Capability>,
pub environment_backends: BTreeSet<EnvironmentBackend>,
pub source_providers: BTreeSet<String>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum CapabilityReportError {
#[error("node architecture `{0}` is invalid")]
InvalidArchitecture(String),
#[error("node OS label `{0}` is invalid")]
InvalidOsLabel(String),
#[error("source provider id `{0}` is invalid")]
InvalidSourceProvider(String),
}
impl NodeCapabilities {
pub fn detect_current() -> Self {
let os = Os::current();
let mut capabilities = BTreeSet::from([
Capability::Command,
Capability::SourceFilesystem,
Capability::VfsArtifacts,
]);
let mut environment_backends = BTreeSet::new();
match os {
Os::Linux => {
if rootless_podman_available() {
capabilities.insert(Capability::Containers);
capabilities.insert(Capability::RootlessPodman);
environment_backends.insert(EnvironmentBackend::Container);
}
}
Os::Windows => {
capabilities.insert(Capability::WindowsCommandDev);
environment_backends.insert(EnvironmentBackend::WindowsCommandDev);
}
Os::Macos | Os::Other(_) => {}
}
Self {
os,
arch: std::env::consts::ARCH.to_owned(),
capabilities,
environment_backends,
source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
}
}
pub fn with_capability(mut self, capability: Capability) -> Self {
self.capabilities.insert(capability);
self
}
pub fn has_all(&self, required: &BTreeSet<Capability>) -> bool {
required
.iter()
.all(|capability| self.capabilities.contains(capability))
}
pub fn validate_public_report(&self) -> Result<(), CapabilityReportError> {
if !valid_capability_label(&self.arch) {
return Err(CapabilityReportError::InvalidArchitecture(
self.arch.clone(),
));
}
if let Os::Other(label) = &self.os {
if !valid_capability_label(label) {
return Err(CapabilityReportError::InvalidOsLabel(label.clone()));
}
}
for provider in &self.source_providers {
if !valid_source_provider_id(provider) {
return Err(CapabilityReportError::InvalidSourceProvider(
provider.clone(),
));
}
}
Ok(())
}
}
#[cfg(not(target_arch = "wasm32"))]
fn rootless_podman_available() -> bool {
const ATTEMPTS: usize = 3;
for attempt in 0..ATTEMPTS {
match std::process::Command::new("podman")
.args(["info", "--format", "{{.Host.Security.Rootless}}"])
.output()
{
Ok(output) if rootless_podman_probe_succeeded(&output) => return true,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return false,
Ok(_) | Err(_) if attempt + 1 < ATTEMPTS => {
std::thread::sleep(std::time::Duration::from_millis(250));
}
Ok(_) | Err(_) => {}
}
}
false
}
#[cfg(not(target_arch = "wasm32"))]
fn rootless_podman_probe_succeeded(output: &std::process::Output) -> bool {
output.status.success() && String::from_utf8_lossy(&output.stdout).trim() == "true"
}
#[cfg(target_arch = "wasm32")]
fn rootless_podman_available() -> bool {
false
}
fn valid_capability_label(label: &str) -> bool {
!label.is_empty()
&& label.len() <= 64
&& label.bytes().all(
|byte| matches!(byte, b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'-' | b'_' | b'.'),
)
}
fn valid_source_provider_id(provider: &str) -> bool {
!provider.is_empty()
&& provider.len() <= 64
&& provider
.bytes()
.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
}
#[cfg(test)]
mod tests {
use super::*;
fn capabilities() -> NodeCapabilities {
NodeCapabilities {
os: Os::Linux,
arch: "x86_64".to_owned(),
capabilities: BTreeSet::from([Capability::Command]),
environment_backends: BTreeSet::new(),
source_providers: BTreeSet::from(["filesystem".to_owned(), "git".to_owned()]),
}
}
#[test]
fn capability_reports_validate_hostile_strings() {
assert!(capabilities().validate_public_report().is_ok());
let mut invalid_arch = capabilities();
invalid_arch.arch = "x86_64\nmalicious".to_owned();
assert_eq!(
invalid_arch.validate_public_report(),
Err(CapabilityReportError::InvalidArchitecture(
"x86_64\nmalicious".to_owned()
))
);
let mut invalid_provider = capabilities();
invalid_provider
.source_providers
.insert("../checkout".to_owned());
assert_eq!(
invalid_provider.validate_public_report(),
Err(CapabilityReportError::InvalidSourceProvider(
"../checkout".to_owned()
))
);
}
}

284
crates/clusterflux-core/src/checkpoint.rs Normal file
View file

@ -0,0 +1,284 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Digest, TaskInstanceId, VfsManifest};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CheckpointBoundary {
pub task_entrypoint: String,
pub serialized_args: Digest,
pub environment_digest: Digest,
pub vfs_epoch: u64,
pub task_abi: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskCheckpoint {
pub task: TaskInstanceId,
pub boundary: CheckpointBoundary,
pub vfs_manifest: VfsManifest,
pub depends_on_live_stack: bool,
pub depends_on_live_socket: bool,
pub depends_on_ephemeral_artifact_durability: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RestartRequest {
pub task: TaskInstanceId,
pub entrypoint: String,
pub serialized_args: Digest,
pub environment_digest: Digest,
pub task_abi: Digest,
pub source_edited: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum RestartDecision {
RestartTask {
task: TaskInstanceId,
from_vfs_epoch: u64,
discard_unflushed_changes: bool,
},
RestartWholeVirtualProcess {
message: String,
},
}
#[derive(Clone, Debug, Default)]
pub struct RestartPolicy;
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum CompatibilityFailure {
#[error("task entrypoint changed")]
Entrypoint,
#[error("serialized task arguments changed")]
Args,
#[error("environment digest changed")]
Environment,
#[error("task ABI changed")]
TaskAbi,
#[error("checkpoint depends on unsupported live stack migration")]
LiveStack,
#[error("checkpoint depends on unsupported live socket checkpointing")]
LiveSocket,
#[error("checkpoint incorrectly treats ephemeral artifacts as durable")]
EphemeralArtifactDurability,
}
impl RestartPolicy {
pub fn decide(&self, checkpoint: &TaskCheckpoint, request: &RestartRequest) -> RestartDecision {
match compatibility_failure(checkpoint, request) {
None => RestartDecision::RestartTask {
task: checkpoint.task.clone(),
from_vfs_epoch: checkpoint.boundary.vfs_epoch,
discard_unflushed_changes: true,
},
Some(failure) => RestartDecision::RestartWholeVirtualProcess {
message: format!(
"cannot restart selected task `{}` from checkpoint: {failure}; restart the whole virtual process",
checkpoint.task
),
},
}
}
}
fn compatibility_failure(
checkpoint: &TaskCheckpoint,
request: &RestartRequest,
) -> Option<CompatibilityFailure> {
if checkpoint.depends_on_live_stack {
return Some(CompatibilityFailure::LiveStack);
}
if checkpoint.depends_on_live_socket {
return Some(CompatibilityFailure::LiveSocket);
}
if checkpoint.depends_on_ephemeral_artifact_durability {
return Some(CompatibilityFailure::EphemeralArtifactDurability);
}
if checkpoint.boundary.task_entrypoint != request.entrypoint {
return Some(CompatibilityFailure::Entrypoint);
}
if checkpoint.boundary.serialized_args != request.serialized_args {
return Some(CompatibilityFailure::Args);
}
if checkpoint.boundary.environment_digest != request.environment_digest {
return Some(CompatibilityFailure::Environment);
}
if checkpoint.boundary.task_abi != request.task_abi {
return Some(CompatibilityFailure::TaskAbi);
}
None
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::{
EnvironmentKind, EnvironmentRequirements, EnvironmentResource, NodeId, VfsOverlay, VfsPath,
};
use super::*;
fn env(digest_input: &str) -> EnvironmentResource {
EnvironmentResource {
name: "linux".to_owned(),
kind: EnvironmentKind::Containerfile,
recipe_path: PathBuf::from("envs/linux/Containerfile"),
context_path: PathBuf::from("envs/linux"),
digest: Digest::sha256(digest_input),
requirements: EnvironmentRequirements::linux_container(),
}
}
fn checkpoint() -> (TaskCheckpoint, EnvironmentResource) {
let environment = env("env");
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node"));
overlay.write(
VfsPath::new("/vfs/artifacts/app").unwrap(),
Digest::sha256("app"),
3,
);
let manifest = overlay.flush();
(
TaskCheckpoint {
task: TaskInstanceId::from("task"),
boundary: CheckpointBoundary {
task_entrypoint: "compile_linux".to_owned(),
serialized_args: Digest::sha256("args"),
environment_digest: environment.digest.clone(),
vfs_epoch: manifest.epoch,
task_abi: Digest::sha256("abi"),
},
vfs_manifest: manifest,
depends_on_live_stack: false,
depends_on_live_socket: false,
depends_on_ephemeral_artifact_durability: false,
},
environment,
)
}
fn restart_request(environment: EnvironmentResource) -> RestartRequest {
RestartRequest {
task: TaskInstanceId::from("task"),
entrypoint: "compile_linux".to_owned(),
serialized_args: Digest::sha256("args"),
environment_digest: environment.digest,
task_abi: Digest::sha256("abi"),
source_edited: true,
}
}
fn assert_whole_process_restart(decision: RestartDecision, expected_reason: &str) {
match decision {
RestartDecision::RestartWholeVirtualProcess { message } => {
assert!(
message.contains(expected_reason),
"restart message `{message}` did not include `{expected_reason}`"
);
assert!(
message.contains("restart the whole virtual process"),
"restart message `{message}` did not direct a whole-process restart"
);
}
RestartDecision::RestartTask { .. } => {
panic!("incompatible checkpoint unexpectedly restarted selected task")
}
}
}
#[test]
fn compatible_restart_uses_task_boundary_and_discards_unflushed_changes() {
let (checkpoint, environment) = checkpoint();
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_eq!(
decision,
RestartDecision::RestartTask {
task: TaskInstanceId::from("task"),
from_vfs_epoch: 1,
discard_unflushed_changes: true
}
);
}
#[test]
fn incompatible_environment_requires_whole_process_restart() {
let (checkpoint, _) = checkpoint();
let request = restart_request(env("changed-env"));
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "environment digest changed");
}
#[test]
fn incompatible_entrypoint_requires_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.entrypoint = "package_linux".to_owned();
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "task entrypoint changed");
}
#[test]
fn incompatible_serialized_args_require_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.serialized_args = Digest::sha256("changed-args");
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "serialized task arguments changed");
}
#[test]
fn incompatible_task_abi_requires_whole_process_restart() {
let (checkpoint, environment) = checkpoint();
let mut request = restart_request(environment);
request.task_abi = Digest::sha256("changed-abi");
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "task ABI changed");
}
#[test]
fn restart_never_claims_live_stack_migration() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_live_stack = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "live stack");
}
#[test]
fn restart_never_claims_live_socket_checkpointing() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_live_socket = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "live socket");
}
#[test]
fn restart_never_depends_on_ephemeral_artifact_durability() {
let (mut checkpoint, environment) = checkpoint();
checkpoint.depends_on_ephemeral_artifact_durability = true;
let request = restart_request(environment);
let decision = RestartPolicy.decide(&checkpoint, &request);
assert_whole_process_restart(decision, "ephemeral artifacts");
}
}

337
crates/clusterflux-core/src/debug.rs Normal file
View file

@ -0,0 +1,337 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{ProcessId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugParticipantKind {
WasmTask,
ControlledNativeCommand,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugRuntimeState {
Running,
Frozen,
Completed,
Failed(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugParticipant {
pub task: TaskInstanceId,
pub name: String,
pub kind: DebugParticipantKind,
pub can_freeze: bool,
pub state: DebugRuntimeState,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DebugStopReason {
Breakpoint { task: TaskInstanceId, line: u32 },
PauseRequest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ThreadInspection {
pub task: TaskInstanceId,
pub name: String,
pub stack_frames: Vec<String>,
pub local_values: Vec<(String, String)>,
pub task_args: Vec<(String, String)>,
pub handles: Vec<(String, String)>,
pub command_status: Option<String>,
pub recent_output: Vec<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugEpoch {
pub process: ProcessId,
pub epoch: u64,
pub reason: DebugStopReason,
participants: BTreeMap<TaskInstanceId, DebugParticipant>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum DebugEpochError {
#[error("no participant could freeze; `{task}` rejected the freeze request")]
CannotFreeze { task: TaskInstanceId },
#[error("participant `{0}` is not part of this debug epoch")]
UnknownParticipant(TaskInstanceId),
#[error("participant `{0}` did not acknowledge frozen state in this debug epoch")]
ParticipantNotFrozen(TaskInstanceId),
}
impl DebugEpoch {
pub fn all_stop(
process: ProcessId,
epoch: u64,
reason: DebugStopReason,
participants: Vec<DebugParticipant>,
) -> Result<Self, DebugEpochError> {
let first_rejected = participants
.iter()
.find(|participant| {
matches!(participant.state, DebugRuntimeState::Running) && !participant.can_freeze
})
.map(|participant| participant.task.clone());
let participants: BTreeMap<_, _> = participants
.into_iter()
.map(|mut participant| {
if matches!(participant.state, DebugRuntimeState::Running) {
participant.state = if participant.can_freeze {
DebugRuntimeState::Frozen
} else {
DebugRuntimeState::Failed(
"participant did not acknowledge frozen state before the debug deadline"
.to_owned(),
)
};
}
(participant.task.clone(), participant)
})
.collect();
if !participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
{
return Err(DebugEpochError::CannotFreeze {
task: first_rejected.unwrap_or_else(|| TaskInstanceId::from("debug-epoch")),
});
}
Ok(Self {
process,
epoch,
reason,
participants,
})
}
pub fn pause(
process: ProcessId,
epoch: u64,
participants: Vec<DebugParticipant>,
) -> Result<Self, DebugEpochError> {
Self::all_stop(process, epoch, DebugStopReason::PauseRequest, participants)
}
pub fn continue_all(&mut self) {
for participant in self.participants.values_mut() {
if participant.state == DebugRuntimeState::Frozen {
participant.state = DebugRuntimeState::Running;
}
}
}
pub fn inspection(&self, task: &TaskInstanceId) -> Result<ThreadInspection, DebugEpochError> {
let participant = self
.participants
.get(task)
.ok_or_else(|| DebugEpochError::UnknownParticipant(task.clone()))?;
if participant.state != DebugRuntimeState::Frozen {
return Err(DebugEpochError::ParticipantNotFrozen(task.clone()));
}
Ok(ThreadInspection {
task: participant.task.clone(),
name: participant.name.clone(),
stack_frames: participant.stack_frames.clone(),
local_values: participant.local_values.clone(),
task_args: participant.task_args.clone(),
handles: participant.handles.clone(),
command_status: participant.command_status.clone(),
recent_output: participant.recent_output.clone(),
})
}
pub fn participant_state(&self, task: &TaskInstanceId) -> Option<&DebugRuntimeState> {
self.participants
.get(task)
.map(|participant| &participant.state)
}
pub fn thread_names(&self) -> Vec<String> {
self.participants
.values()
.map(|participant| participant.name.clone())
.collect()
}
pub fn all_threads_stopped(&self) -> bool {
!self.participants.is_empty()
&& self
.participants
.values()
.all(|participant| participant.state == DebugRuntimeState::Frozen)
}
pub fn partially_frozen(&self) -> bool {
!self.all_threads_stopped()
&& self
.participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn participant(task: &str, kind: DebugParticipantKind, can_freeze: bool) -> DebugParticipant {
DebugParticipant {
task: TaskInstanceId::from(task),
name: task.to_owned(),
kind,
can_freeze,
state: DebugRuntimeState::Running,
stack_frames: vec![format!("{task}::run")],
local_values: vec![("wasm_local_0".to_owned(), "I32(41)".to_owned())],
task_args: vec![("target".to_owned(), "linux".to_owned())],
handles: vec![("artifact".to_owned(), "artifact-1".to_owned())],
command_status: Some("running".to_owned()),
recent_output: vec!["building".to_owned()],
}
}
#[test]
fn breakpoint_creates_all_stop_debug_epoch_for_wasm_and_command_tasks() {
let epoch = DebugEpoch::all_stop(
ProcessId::from("process"),
1,
DebugStopReason::Breakpoint {
task: TaskInstanceId::from("compile-linux"),
line: 42,
},
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
true,
),
],
)
.unwrap();
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("main")),
Some(&DebugRuntimeState::Frozen)
);
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("compile-linux")),
Some(&DebugRuntimeState::Frozen)
);
}
#[test]
fn debug_epoch_reports_failure_when_no_participant_can_freeze() {
let error = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
false,
)],
)
.unwrap_err();
assert!(matches!(error, DebugEpochError::CannotFreeze { .. }));
}
#[test]
fn debug_epoch_keeps_frozen_participants_when_another_participant_fails() {
let epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant(
"native",
DebugParticipantKind::ControlledNativeCommand,
false,
),
],
)
.unwrap();
assert!(epoch.partially_frozen());
assert!(!epoch.all_threads_stopped());
assert!(matches!(
epoch.participant_state(&TaskInstanceId::from("native")),
Some(DebugRuntimeState::Failed(_))
));
assert!(matches!(
epoch.inspection(&TaskInstanceId::from("native")),
Err(DebugEpochError::ParticipantNotFrozen(_))
));
}
#[test]
fn continue_resumes_every_frozen_participant() {
let mut epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant("task", DebugParticipantKind::WasmTask, true),
],
)
.unwrap();
epoch.continue_all();
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("main")),
Some(&DebugRuntimeState::Running)
);
assert_eq!(
epoch.participant_state(&TaskInstanceId::from("task")),
Some(&DebugRuntimeState::Running)
);
}
#[test]
fn inspection_exposes_stack_args_handles_command_status_and_output() {
let epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![participant(
"compile-linux",
DebugParticipantKind::ControlledNativeCommand,
true,
)],
)
.unwrap();
let inspection = epoch
.inspection(&TaskInstanceId::from("compile-linux"))
.unwrap();
assert_eq!(inspection.stack_frames, vec!["compile-linux::run"]);
assert_eq!(
inspection.local_values[0],
("wasm_local_0".to_owned(), "I32(41)".to_owned())
);
assert_eq!(
inspection.task_args[0],
("target".to_owned(), "linux".to_owned())
);
assert_eq!(
inspection.handles[0],
("artifact".to_owned(), "artifact-1".to_owned())
);
assert_eq!(inspection.command_status, Some("running".to_owned()));
assert_eq!(inspection.recent_output, vec!["building"]);
}
}

68
crates/clusterflux-core/src/digest.rs Normal file
View file

@ -0,0 +1,68 @@
use serde::{Deserialize, Serialize};
use sha2::{Digest as ShaDigest, Sha256};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct Digest(String);
impl Digest {
pub fn sha256(bytes: impl AsRef<[u8]>) -> Self {
let mut hasher = Sha256::new();
hasher.update(bytes.as_ref());
Self(format!("sha256:{}", hex::encode(hasher.finalize())))
}
pub fn from_parts(parts: impl IntoIterator<Item = impl AsRef<[u8]>>) -> Self {
let mut hasher = Sha256::new();
for part in parts {
let part = part.as_ref();
hasher.update((part.len() as u64).to_be_bytes());
hasher.update(part);
}
Self(format!("sha256:{}", hex::encode(hasher.finalize())))
}
pub fn from_sha256_hex(hex_digest: impl Into<String>) -> Result<Self, String> {
let digest = Self(format!("sha256:{}", hex_digest.into()));
if digest.is_valid_sha256() {
Ok(digest)
} else {
Err(
"SHA-256 digest must contain exactly 64 lowercase hexadecimal characters"
.to_owned(),
)
}
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn is_valid_sha256(&self) -> bool {
let Some(hex) = self.0.strip_prefix("sha256:") else {
return false;
};
hex.len() == 64
&& hex
.bytes()
.all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'))
}
}
impl std::fmt::Display for Digest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn digest_validates_strict_sha256_syntax() {
assert!(Digest::sha256("bytes").is_valid_sha256());
assert!(!Digest("sha1:abc".to_owned()).is_valid_sha256());
assert!(!Digest("sha256:ABCDEF".to_owned()).is_valid_sha256());
assert!(!Digest("sha256:not-hex".to_owned()).is_valid_sha256());
}
}

288
crates/clusterflux-core/src/environment.rs Normal file
View file

@ -0,0 +1,288 @@
use std::collections::BTreeSet;
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Capability, Digest, Os};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum EnvironmentKind {
Containerfile,
Dockerfile,
NixFlake,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentRequirements {
pub os: Option<Os>,
pub arch: Option<String>,
pub capabilities: BTreeSet<Capability>,
}
impl EnvironmentRequirements {
pub fn linux_container() -> Self {
Self {
os: Some(Os::Linux),
arch: None,
capabilities: BTreeSet::from([Capability::Containers, Capability::RootlessPodman]),
}
}
pub fn windows_command_dev() -> Self {
Self {
os: Some(Os::Windows),
arch: None,
capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
}
}
pub fn unconstrained() -> Self {
Self {
os: None,
arch: None,
capabilities: BTreeSet::new(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentResource {
pub name: String,
pub kind: EnvironmentKind,
pub recipe_path: PathBuf,
pub context_path: PathBuf,
pub digest: Digest,
pub requirements: EnvironmentRequirements,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentReference {
pub name: String,
pub byte_offset: usize,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnvironmentDiagnostic {
pub reference: EnvironmentReference,
pub message: String,
}
#[derive(Debug, Error)]
pub enum EnvironmentError {
#[error("failed to read environment resources under {path}: {source}")]
Read {
path: PathBuf,
#[source]
source: std::io::Error,
},
}
pub fn discover_environments(
project_root: &Path,
) -> Result<Vec<EnvironmentResource>, EnvironmentError> {
let envs_dir = project_root.join("envs");
if !envs_dir.exists() {
return Ok(Vec::new());
}
let mut resources = Vec::new();
let entries = fs::read_dir(&envs_dir).map_err(|source| EnvironmentError::Read {
path: envs_dir.clone(),
source,
})?;
for entry in entries {
let entry = entry.map_err(|source| EnvironmentError::Read {
path: envs_dir.clone(),
source,
})?;
let path = entry.path();
if !path.is_dir() {
continue;
}
let Some(name) = path.file_name().and_then(|name| name.to_str()) else {
continue;
};
if let Some(resource) = discover_one(project_root, name, &path)? {
resources.push(resource);
}
}
resources.sort_by(|left, right| left.name.cmp(&right.name));
Ok(resources)
}
pub fn diagnose_environment_references(
source: &str,
environments: &[EnvironmentResource],
) -> Vec<EnvironmentDiagnostic> {
let known = environments
.iter()
.map(|environment| environment.name.as_str())
.collect::<BTreeSet<_>>();
find_env_macro_references(source)
.into_iter()
.filter(|reference| !known.contains(reference.name.as_str()))
.map(|reference| EnvironmentDiagnostic {
message: format!(
"missing Clusterflux environment `{}`; expected envs/{}/Containerfile or envs/{}/Dockerfile",
reference.name, reference.name, reference.name
),
reference,
})
.collect()
}
fn discover_one(
project_root: &Path,
name: &str,
env_dir: &Path,
) -> Result<Option<EnvironmentResource>, EnvironmentError> {
let candidates = [
("Containerfile", EnvironmentKind::Containerfile),
("Dockerfile", EnvironmentKind::Dockerfile),
("flake.nix", EnvironmentKind::NixFlake),
];
for (file_name, kind) in candidates {
let recipe_path = env_dir.join(file_name);
if !recipe_path.exists() {
continue;
}
let recipe_bytes = fs::read(&recipe_path).map_err(|source| EnvironmentError::Read {
path: recipe_path.clone(),
source,
})?;
let relative_recipe = recipe_path
.strip_prefix(project_root)
.unwrap_or(&recipe_path)
.to_string_lossy();
let digest = Digest::from_parts([
b"environment:v1".as_slice(),
name.as_bytes(),
format!("{kind:?}").as_bytes(),
relative_recipe.as_bytes(),
recipe_bytes.as_slice(),
]);
let requirements = match kind {
EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile
if name.eq_ignore_ascii_case("windows") =>
{
EnvironmentRequirements::windows_command_dev()
}
EnvironmentKind::Containerfile | EnvironmentKind::Dockerfile => {
EnvironmentRequirements::linux_container()
}
EnvironmentKind::NixFlake => EnvironmentRequirements::unconstrained(),
};
return Ok(Some(EnvironmentResource {
name: name.to_owned(),
kind,
recipe_path,
context_path: env_dir.to_path_buf(),
digest,
requirements,
}));
}
Ok(None)
}
fn find_env_macro_references(source: &str) -> Vec<EnvironmentReference> {
let mut references = Vec::new();
let mut cursor = 0;
while let Some(index) = source[cursor..].find("env!(") {
let start = cursor + index;
let mut pos = start + "env!(".len();
while source[pos..].starts_with(char::is_whitespace) {
pos += source[pos..]
.chars()
.next()
.map(char::len_utf8)
.unwrap_or(1);
}
if !source[pos..].starts_with('"') {
cursor = pos;
continue;
}
pos += 1;
let name_start = pos;
while pos < source.len() && !source[pos..].starts_with('"') {
pos += source[pos..]
.chars()
.next()
.map(char::len_utf8)
.unwrap_or(1);
}
if pos < source.len() {
references.push(EnvironmentReference {
name: source[name_start..pos].to_owned(),
byte_offset: start,
});
}
cursor = pos.saturating_add(1);
}
references
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn discovers_containerfile_environments_by_logical_name() {
let temp = tempfile::tempdir().unwrap();
let linux = temp.path().join("envs/linux");
fs::create_dir_all(&linux).unwrap();
fs::write(linux.join("Containerfile"), "FROM alpine\n").unwrap();
let envs = discover_environments(temp.path()).unwrap();
assert_eq!(envs.len(), 1);
assert_eq!(envs[0].name, "linux");
assert_eq!(envs[0].kind, EnvironmentKind::Containerfile);
assert!(!envs[0].digest.as_str().is_empty());
}
#[test]
fn missing_env_macro_reference_reports_clear_diagnostic() {
let source = r#"fn main() { let _ = env!("windows"); }"#;
let diagnostics = diagnose_environment_references(source, &[]);
assert_eq!(diagnostics.len(), 1);
assert!(diagnostics[0]
.message
.contains("envs/windows/Containerfile"));
}
#[test]
fn windows_environment_name_uses_windows_development_requirements() {
let temp = tempfile::tempdir().unwrap();
let windows = temp.path().join("envs/windows");
fs::create_dir_all(&windows).unwrap();
fs::write(
windows.join("Dockerfile"),
"# user-attached windows dev contract\n",
)
.unwrap();
let envs = discover_environments(temp.path()).unwrap();
assert_eq!(envs[0].name, "windows");
assert_eq!(envs[0].requirements.os, Some(Os::Windows));
assert!(envs[0]
.requirements
.capabilities
.contains(&Capability::WindowsCommandDev));
}
}

1133
crates/clusterflux-core/src/execution.rs Normal file
View file

File diff suppressed because it is too large Load diff

45
crates/clusterflux-core/src/ids.rs Normal file
View file

@ -0,0 +1,45 @@
use serde::{Deserialize, Serialize};
macro_rules! id_type {
($name:ident) => {
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct $name(String);
impl $name {
pub fn new(value: impl Into<String>) -> Self {
let value = value.into();
assert!(
!value.trim().is_empty(),
concat!(stringify!($name), " cannot be empty")
);
Self(value)
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl From<&str> for $name {
fn from(value: &str) -> Self {
Self::new(value)
}
}
impl std::fmt::Display for $name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
};
}
id_type!(AgentId);
id_type!(ArtifactId);
id_type!(NodeId);
id_type!(ProcessId);
id_type!(ProjectId);
id_type!(TaskDefinitionId);
id_type!(TaskInstanceId);
id_type!(TenantId);
id_type!(UserId);

102
crates/clusterflux-core/src/lib.rs Normal file
View file

@ -0,0 +1,102 @@
pub mod artifact;
pub mod auth;
pub mod bundle;
pub mod capability;
pub mod checkpoint;
pub mod debug;
pub mod digest;
pub mod environment;
pub mod execution;
pub mod ids;
pub mod limits;
pub mod operator_panel;
pub mod policy;
pub mod project;
pub mod scheduler;
pub mod source;
pub mod transport;
pub mod vfs;
pub mod wire;
pub use artifact::{
ArtifactDownloadStream, ArtifactFlush, ArtifactHandle, ArtifactMetadata, ArtifactRegistry,
ArtifactUnavailable, DownloadAction, DownloadError, DownloadLink, DownloadPolicy,
DownloadStreamRequest, RetentionPolicy, StorageLocation,
};
pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest,
agent_ed25519_public_key_from_private_key, agent_workflow_request_scope_from_payload,
derive_ed25519_private_key_from_seed, node_capability_policy_digest,
node_ed25519_public_key_from_private_key, sign_agent_workflow_request, sign_node_request,
signed_request_payload_digest, verify_agent_workflow_signature, verify_node_request_signature,
Action, Actor, AgentSignedRequest, AgentWorkflowRequestScope, AgentWorkflowScope, AuthContext,
Authorization, BrowserLoginFlow, CredentialKind, EnrollmentError, EnrollmentGrant,
IdentityKind, NodeCredential, NodeSignedRequest, PublicKeyIdentity, Scope,
};
#[cfg(not(target_arch = "wasm32"))]
pub use auth::{generate_ed25519_private_key, generate_opaque_token};
pub use bundle::{
discover_source_debug_probes, BundleDebugMetadata, BundleDebugProbe, BundleIdentityInputs,
BundleLargeInputPolicy, BundleMetadata, BundleRestartCompatibility, BundleSourceMetadata,
BundleTaskMetadata, SelectedInput,
};
pub use capability::{Capability, CapabilityReportError, EnvironmentBackend, NodeCapabilities, Os};
pub use checkpoint::{
CheckpointBoundary, CompatibilityFailure, RestartDecision, RestartPolicy, RestartRequest,
TaskCheckpoint,
};
pub use debug::{
DebugEpoch, DebugEpochError, DebugParticipant, DebugParticipantKind, DebugRuntimeState,
DebugStopReason, ThreadInspection,
};
pub use digest::Digest;
pub use environment::{
diagnose_environment_references, discover_environments, EnvironmentDiagnostic, EnvironmentKind,
EnvironmentReference, EnvironmentRequirements, EnvironmentResource,
};
pub use execution::{
CommandBackendKind, CommandInvocation, CommandNetworkPolicy, CommandPlan, GuestRuntimeKind,
NativeCommandPolicy, StructuredTaskBoundary, TaskBoundaryHandle, TaskBoundaryValue,
TaskDispatch, TaskFailurePolicy, TaskJoinResult, TaskJoinState, TaskSpec, WasmExportAbi,
WasmHostCommandRequest, WasmHostCommandResult, WasmHostDebugProbeRequest,
WasmHostDebugProbeResult, WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult,
WasmHostTaskControlRequest, WasmHostTaskControlResult, WasmHostTaskHandle,
WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation,
WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
};
pub use ids::{
AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,
UserId,
};
pub use limits::{
LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,
ResourceMeter, TaskArgumentBudget, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
};
pub use operator_panel::{
ControlPlaneAction, PanelError, PanelEvent, PanelEventKind, PanelState, PanelWidget,
PanelWidgetKind, RateLimit,
};
pub use policy::{
CapabilityPolicy, Decision, LocalTrustedPolicy, PolicyReason, ResourceRequest, ServicePolicy,
};
pub use project::{Entrypoint, ProjectModel, ProjectModelError};
pub use scheduler::{
DefaultScheduler, NodeDescriptor, Placement, PlacementError, PlacementRequest, Scheduler,
};
pub use source::{
SourceManifestError, SourcePreparation, SourceProviderKind, SourceProviderManifest,
SourceProviderModule, SourceTransferMode, SourceTransferPolicy,
};
pub use transport::{
BulkTransferDecision, DataPlaneObject, DataPlaneScope, DirectBulkTransferPlan,
NativeQuicTransport, NodeEndpoint, RendezvousRequest, Transport, TransportError, TransportKind,
};
pub use vfs::{
ReuseDecision, SyncPolicy, VfsError, VfsManifest, VfsObject, VfsOverlay, VfsPath,
VfsSyncDecision,
};
pub use wire::{
coordinator_authentication_metadata, coordinator_payload_operation, coordinator_wire_request,
COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE,
};

300
crates/clusterflux-core/src/limits.rs Normal file
View file

@ -0,0 +1,300 @@
use std::collections::BTreeMap;
use std::time::Duration;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::TaskInstanceId;
/// Fastest supported interval for a node's signed artifact/assignment polling loop.
/// The coordinator's bounded replay window is sized against this protocol limit.
pub const MIN_SIGNED_NODE_POLL_INTERVAL_MS: u64 = 20;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum LimitKind {
ApiCall,
Spawn,
LogBytes,
MetadataBytes,
DebugReadBytes,
UiEvent,
RendezvousAttempt,
ArtifactDownloadBytes,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceLimits {
pub limits: BTreeMap<LimitKind, u64>,
}
impl ResourceLimits {
pub fn new(limits: impl IntoIterator<Item = (LimitKind, u64)>) -> Self {
Self {
limits: limits.into_iter().collect(),
}
}
pub fn unlimited() -> Self {
Self::new(LimitKind::ALL.into_iter().map(|kind| (kind, u64::MAX)))
}
pub fn limit(&self, kind: &LimitKind) -> u64 {
*self.limits.get(kind).unwrap_or(&0)
}
}
impl Default for ResourceLimits {
fn default() -> Self {
Self::unlimited()
}
}
impl LimitKind {
pub const ALL: [Self; 8] = [
Self::ApiCall,
Self::Spawn,
Self::LogBytes,
Self::MetadataBytes,
Self::DebugReadBytes,
Self::UiEvent,
Self::RendezvousAttempt,
Self::ArtifactDownloadBytes,
];
}
pub const TASK_JOIN_TIMEOUT_SECONDS_ENV: &str = "CLUSTERFLUX_TASK_JOIN_TIMEOUT_SECONDS";
pub const DEFAULT_TASK_JOIN_TIMEOUT_SECONDS: u64 = 24 * 60 * 60;
pub fn task_join_timeout() -> Duration {
std::env::var(TASK_JOIN_TIMEOUT_SECONDS_ENV)
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0)
.map(Duration::from_secs)
.unwrap_or_else(|| Duration::from_secs(DEFAULT_TASK_JOIN_TIMEOUT_SECONDS))
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum TaskJoinError {
#[error(
"timed out after {waited_seconds} seconds waiting for task {task}; the child task was left running"
)]
Timeout {
task: crate::TaskInstanceId,
waited_seconds: u64,
},
#[error("task join for {task} was cancelled; the child task was left running")]
Cancelled { task: crate::TaskInstanceId },
}
impl TaskJoinError {
pub fn timeout(task: crate::TaskInstanceId, waited: Duration) -> Self {
Self::Timeout {
task,
waited_seconds: waited.as_secs(),
}
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceMeter {
used: BTreeMap<LimitKind, u64>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum LimitError {
#[error(
"resource limit exceeded for {kind:?}: requested {requested}, used {used}, limit {limit}"
)]
Exceeded {
kind: LimitKind,
requested: u64,
used: u64,
limit: u64,
},
#[error("task argument is too large: {size} bytes exceeds {limit} bytes")]
LargeTaskArgument { size: u64, limit: u64 },
}
impl ResourceMeter {
pub fn can_charge(
&self,
limits: &ResourceLimits,
kind: LimitKind,
amount: u64,
) -> Result<(), LimitError> {
let used = self.used.get(&kind).copied().unwrap_or(0);
let limit = limits.limit(&kind);
if used.saturating_add(amount) > limit {
return Err(LimitError::Exceeded {
kind,
requested: amount,
used,
limit,
});
}
Ok(())
}
pub fn charge(
&mut self,
limits: &ResourceLimits,
kind: LimitKind,
amount: u64,
) -> Result<(), LimitError> {
self.can_charge(limits, kind, amount)?;
let used = self.used.get(&kind).copied().unwrap_or(0);
self.used.insert(kind, used + amount);
Ok(())
}
pub fn used(&self, kind: &LimitKind) -> u64 {
self.used.get(kind).copied().unwrap_or(0)
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogRecord {
pub task: TaskInstanceId,
pub bytes: Vec<u8>,
pub truncated: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct LogBuffer {
max_bytes: usize,
used_bytes: usize,
records: Vec<LogRecord>,
backpressured: bool,
}
impl LogBuffer {
pub fn new(max_bytes: usize) -> Self {
Self {
max_bytes,
used_bytes: 0,
records: Vec::new(),
backpressured: false,
}
}
pub fn push(&mut self, task: TaskInstanceId, bytes: impl AsRef<[u8]>) {
let bytes = bytes.as_ref();
let remaining = self.max_bytes.saturating_sub(self.used_bytes);
let truncated = bytes.len() > remaining;
let stored = bytes[..bytes.len().min(remaining)].to_vec();
self.used_bytes += stored.len();
if truncated {
self.backpressured = true;
}
self.records.push(LogRecord {
task,
bytes: stored,
truncated,
});
}
pub fn records(&self) -> &[LogRecord] {
&self.records
}
pub fn backpressured(&self) -> bool {
self.backpressured
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum LargeArgumentPolicy {
Allow,
Warn,
Reject,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskArgumentBudget {
pub max_inline_bytes: u64,
pub policy: LargeArgumentPolicy,
}
impl TaskArgumentBudget {
pub fn validate(&self, size: u64) -> Result<Option<String>, LimitError> {
if size <= self.max_inline_bytes {
return Ok(None);
}
match self.policy {
LargeArgumentPolicy::Allow => Ok(None),
LargeArgumentPolicy::Warn => Ok(Some(format!(
"task argument is {size} bytes; prefer SourceSnapshot, Blob, Artifact, or VFS handles"
))),
LargeArgumentPolicy::Reject => Err(LimitError::LargeTaskArgument {
size,
limit: self.max_inline_bytes,
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resource_meter_rejects_usage_before_work_starts() {
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::Spawn, 1)]),
};
let mut meter = ResourceMeter::default();
meter.charge(&limits, LimitKind::Spawn, 1).unwrap();
let error = meter.charge(&limits, LimitKind::Spawn, 1).unwrap_err();
assert!(matches!(error, LimitError::Exceeded { .. }));
}
#[test]
fn resource_meter_can_check_limits_without_consuming() {
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 4)]),
};
let mut meter = ResourceMeter::default();
meter
.can_charge(&limits, LimitKind::ArtifactDownloadBytes, 4)
.unwrap();
assert_eq!(meter.used(&LimitKind::ArtifactDownloadBytes), 0);
meter
.charge(&limits, LimitKind::ArtifactDownloadBytes, 3)
.unwrap();
assert!(matches!(
meter.can_charge(&limits, LimitKind::ArtifactDownloadBytes, 2),
Err(LimitError::Exceeded { .. })
));
}
#[test]
fn log_buffer_caps_backpressures_and_keeps_task_association() {
let mut logs = LogBuffer::new(4);
logs.push(TaskInstanceId::from("task-a"), b"abcdef");
assert!(logs.backpressured());
assert_eq!(logs.records()[0].task, TaskInstanceId::from("task-a"));
assert_eq!(logs.records()[0].bytes, b"abcd");
assert!(logs.records()[0].truncated);
}
#[test]
fn large_task_arguments_are_rejected_or_warned() {
let reject = TaskArgumentBudget {
max_inline_bytes: 4,
policy: LargeArgumentPolicy::Reject,
};
assert!(reject.validate(5).is_err());
let warn = TaskArgumentBudget {
max_inline_bytes: 4,
policy: LargeArgumentPolicy::Warn,
};
assert!(warn.validate(5).unwrap().unwrap().contains("Artifact"));
}
}

377
crates/clusterflux-core/src/operator_panel.rs Normal file
View file

@ -0,0 +1,377 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
ArtifactId, DownloadAction, DownloadError, ProcessId, ProjectId, TaskInstanceId, TenantId,
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PanelWidgetKind {
Text {
value: String,
},
Progress {
current: u64,
total: u64,
},
Button {
action: String,
},
Toggle {
value: bool,
},
Select {
options: Vec<String>,
selected: String,
},
ArtifactDownload {
artifact: ArtifactId,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelWidget {
pub id: String,
pub label: String,
pub kind: PanelWidgetKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ControlPlaneAction {
RestartTask(TaskInstanceId),
CancelProcess,
DebugProcess,
DownloadArtifact(ArtifactId),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelState {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub widgets: BTreeMap<String, PanelWidget>,
pub program_ui_events_enabled: bool,
pub control_plane_actions: Vec<ControlPlaneAction>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PanelEventKind {
ButtonClicked,
ToggleChanged(bool),
SelectChanged(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PanelEvent {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub widget_id: String,
pub kind: PanelEventKind,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RateLimit {
pub max_events: u64,
pub used_events: u64,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum PanelError {
#[error("custom HTML or JavaScript is not supported in operator panels")]
CustomContentDenied,
#[error(
"operator panel widget `{0}` is not allowed to collect secrets or OAuth-like credentials"
)]
CredentialCollectionDenied(String),
#[error("panel event scope does not match tenant/project/process")]
ScopeMismatch,
#[error("program UI events are disabled while debug process is stopped")]
ProgramEventsDisabled,
#[error("panel event rate limit exceeded")]
RateLimited,
#[error("unknown panel widget `{0}`")]
UnknownWidget(String),
#[error("artifact download action is unavailable: {0}")]
DownloadUnavailable(String),
}
impl PanelState {
pub fn new(tenant: TenantId, project: ProjectId, process: ProcessId) -> Self {
Self {
tenant,
project,
process,
widgets: BTreeMap::new(),
program_ui_events_enabled: true,
control_plane_actions: Vec::new(),
}
}
pub fn add_widget(&mut self, widget: PanelWidget) -> Result<(), PanelError> {
validate_widget(&widget)?;
self.widgets.insert(widget.id.clone(), widget);
Ok(())
}
pub fn add_download_widget_from_action(
&mut self,
widget_id: impl Into<String>,
label: impl Into<String>,
action: Result<DownloadAction, DownloadError>,
) -> Result<(), PanelError> {
let action = action.map_err(|err| PanelError::DownloadUnavailable(err.to_string()))?;
let artifact = action.artifact;
self.add_widget(PanelWidget {
id: widget_id.into(),
label: label.into(),
kind: PanelWidgetKind::ArtifactDownload {
artifact: artifact.clone(),
},
})?;
self.control_plane_actions
.push(ControlPlaneAction::DownloadArtifact(artifact));
Ok(())
}
pub fn reject_custom_content(_html_or_js: &str) -> Result<(), PanelError> {
Err(PanelError::CustomContentDenied)
}
pub fn freeze_program_ui_events(&mut self) {
self.program_ui_events_enabled = false;
}
pub fn set_control_plane_actions(&mut self, actions: Vec<ControlPlaneAction>) {
self.control_plane_actions = actions;
}
pub fn accept_event(
&self,
event: &PanelEvent,
limit: &mut RateLimit,
) -> Result<(), PanelError> {
if !self.program_ui_events_enabled {
return Err(PanelError::ProgramEventsDisabled);
}
if self.tenant != event.tenant
|| self.project != event.project
|| self.process != event.process
{
return Err(PanelError::ScopeMismatch);
}
if !self.widgets.contains_key(&event.widget_id) {
return Err(PanelError::UnknownWidget(event.widget_id.clone()));
}
if limit.used_events >= limit.max_events {
return Err(PanelError::RateLimited);
}
limit.used_events += 1;
Ok(())
}
pub fn control_plane_actions_available(&self) -> &[ControlPlaneAction] {
&self.control_plane_actions
}
}
fn validate_widget(widget: &PanelWidget) -> Result<(), PanelError> {
let mut checked_text = vec![widget.id.as_str(), widget.label.as_str()];
match &widget.kind {
PanelWidgetKind::Button { action } => checked_text.push(action),
PanelWidgetKind::Select { options, selected } => {
checked_text.push(selected);
checked_text.extend(options.iter().map(String::as_str));
}
PanelWidgetKind::Text { .. }
| PanelWidgetKind::Progress { .. }
| PanelWidgetKind::Toggle { .. }
| PanelWidgetKind::ArtifactDownload { .. } => {}
}
let combined = checked_text.join(" ").to_ascii_lowercase();
if combined.contains("password")
|| combined.contains("token")
|| combined.contains("oauth")
|| combined.contains("secret")
{
return Err(PanelError::CredentialCollectionDenied(widget.id.clone()));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn panel() -> PanelState {
PanelState::new(
TenantId::from("tenant"),
ProjectId::from("project"),
ProcessId::from("process"),
)
}
#[test]
fn panel_uses_typed_widgets_and_rejects_custom_content() {
let mut panel = panel();
panel
.add_widget(PanelWidget {
id: "progress".to_owned(),
label: "Build".to_owned(),
kind: PanelWidgetKind::Progress {
current: 1,
total: 2,
},
})
.unwrap();
assert!(PanelState::reject_custom_content("<script>alert(1)</script>").is_err());
assert!(panel.widgets.contains_key("progress"));
}
#[test]
fn panel_rejects_password_or_oauth_collection_widgets() {
let mut panel = panel();
let error = panel
.add_widget(PanelWidget {
id: "oauth_token".to_owned(),
label: "OAuth Token".to_owned(),
kind: PanelWidgetKind::Text {
value: String::new(),
},
})
.unwrap_err();
assert!(matches!(error, PanelError::CredentialCollectionDenied(_)));
}
#[test]
fn panel_rejects_credential_collection_in_interactive_fields() {
let mut panel = panel();
let button_error = panel
.add_widget(PanelWidget {
id: "continue".to_owned(),
label: "Continue".to_owned(),
kind: PanelWidgetKind::Button {
action: "collect-secret".to_owned(),
},
})
.unwrap_err();
assert!(matches!(
button_error,
PanelError::CredentialCollectionDenied(_)
));
let select_error = panel
.add_widget(PanelWidget {
id: "auth-mode".to_owned(),
label: "Auth Mode".to_owned(),
kind: PanelWidgetKind::Select {
options: vec!["password".to_owned(), "public key".to_owned()],
selected: "public key".to_owned(),
},
})
.unwrap_err();
assert!(matches!(
select_error,
PanelError::CredentialCollectionDenied(_)
));
}
#[test]
fn panel_events_are_scoped_and_rate_limited() {
let mut panel = panel();
panel
.add_widget(PanelWidget {
id: "restart".to_owned(),
label: "Restart".to_owned(),
kind: PanelWidgetKind::Button {
action: "restart".to_owned(),
},
})
.unwrap();
let event = PanelEvent {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
widget_id: "restart".to_owned(),
kind: PanelEventKind::ButtonClicked,
};
let mut limit = RateLimit {
max_events: 1,
used_events: 0,
};
panel.accept_event(&event, &mut limit).unwrap();
assert_eq!(
panel.accept_event(&event, &mut limit),
Err(PanelError::RateLimited)
);
}
#[test]
fn stopped_debug_process_keeps_control_plane_actions_available() {
let mut panel = panel();
panel.freeze_program_ui_events();
panel.set_control_plane_actions(vec![
ControlPlaneAction::RestartTask(TaskInstanceId::from("task")),
ControlPlaneAction::DownloadArtifact(ArtifactId::from("artifact")),
]);
let event = PanelEvent {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: ProcessId::from("process"),
widget_id: "missing".to_owned(),
kind: PanelEventKind::ButtonClicked,
};
let mut limit = RateLimit {
max_events: 1,
used_events: 0,
};
assert_eq!(
panel.accept_event(&event, &mut limit),
Err(PanelError::ProgramEventsDisabled)
);
assert_eq!(panel.control_plane_actions_available().len(), 2);
}
#[test]
fn download_widget_is_only_created_from_available_action() {
let mut panel = panel();
let action = Ok(DownloadAction {
artifact: ArtifactId::from("artifact"),
source: crate::StorageLocation::RetainedNode(crate::NodeId::from("node")),
scoped_token_subject: "tenant/project/process/artifact".to_owned(),
});
panel
.add_download_widget_from_action("download-artifact", "Download", action)
.unwrap();
assert!(matches!(
panel.widgets["download-artifact"].kind,
PanelWidgetKind::ArtifactDownload { .. }
));
assert!(matches!(
panel.control_plane_actions_available()[0],
ControlPlaneAction::DownloadArtifact(_)
));
let before = panel.widgets.len();
let error = panel
.add_download_widget_from_action(
"missing-download",
"Download",
Err(DownloadError::Unavailable),
)
.unwrap_err();
assert_eq!(panel.widgets.len(), before);
assert!(matches!(error, PanelError::DownloadUnavailable(_)));
}
}

134
crates/clusterflux-core/src/policy.rs Normal file
View file

@ -0,0 +1,134 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use crate::{Action, AuthContext, Capability, Scope};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum PolicyReason {
Allowed,
MissingCapability(Capability),
HostedNativeComputeDenied,
HostedContainerDenied,
QuotaExceeded(String),
Unauthorized(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Decision {
pub allowed: bool,
pub reason: PolicyReason,
}
impl Decision {
pub fn allow() -> Self {
Self {
allowed: true,
reason: PolicyReason::Allowed,
}
}
pub fn deny(reason: PolicyReason) -> Self {
Self {
allowed: false,
reason,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResourceRequest {
pub action: Action,
pub required_capabilities: BTreeSet<Capability>,
pub hosted_control_plane: bool,
}
pub trait CapabilityPolicy {
fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision;
}
pub trait ServicePolicy: CapabilityPolicy + Send + Sync {}
impl<T> ServicePolicy for T where T: CapabilityPolicy + Send + Sync {}
#[derive(Clone, Debug, Default)]
pub struct LocalTrustedPolicy;
impl CapabilityPolicy for LocalTrustedPolicy {
fn decide(&self, context: &AuthContext, scope: &Scope, request: &ResourceRequest) -> Decision {
let authz = crate::auth::same_tenant_project(context, scope);
if !authz.allowed {
return Decision::deny(PolicyReason::Unauthorized(authz.reason));
}
if request.hosted_control_plane && request.action == Action::RunNativeCommand {
return Decision::deny(PolicyReason::HostedNativeComputeDenied);
}
if request.hosted_control_plane && request.action == Action::RunContainer {
return Decision::deny(PolicyReason::HostedContainerDenied);
}
Decision::allow()
}
}
#[cfg(test)]
mod tests {
use crate::{Actor, ProjectId, TenantId, UserId};
use super::*;
#[test]
fn public_policy_interface_denies_hosted_native_compute() {
let policy = LocalTrustedPolicy;
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::User(UserId::from("user")),
};
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: None,
task: None,
node: None,
artifact: None,
};
let request = ResourceRequest {
action: Action::RunNativeCommand,
required_capabilities: BTreeSet::new(),
hosted_control_plane: true,
};
let decision = policy.decide(&context, &scope, &request);
assert!(!decision.allowed);
assert_eq!(decision.reason, PolicyReason::HostedNativeComputeDenied);
}
#[test]
fn local_trusted_policy_allows_owner_controlled_native_capability_request() {
let policy = LocalTrustedPolicy;
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
actor: Actor::User(UserId::from("owner")),
};
let scope = Scope {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
process: None,
task: None,
node: None,
artifact: None,
};
let request = ResourceRequest {
action: Action::RunNativeCommand,
required_capabilities: BTreeSet::from([Capability::Command]),
hosted_control_plane: false,
};
let decision = policy.decide(&context, &scope, &request);
assert!(decision.allowed);
assert_eq!(decision.reason, PolicyReason::Allowed);
}
}

303
crates/clusterflux-core/src/project.rs Normal file
View file

@ -0,0 +1,303 @@
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use syn::{punctuated::Punctuated, Expr, Item, Lit, Meta, Token};
use thiserror::Error;
use crate::{discover_environments, environment::EnvironmentError, EnvironmentResource};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Entrypoint {
pub name: String,
pub function: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ProjectModel {
pub root: PathBuf,
pub environments: Vec<EnvironmentResource>,
pub entrypoints: BTreeMap<String, Entrypoint>,
pub default_entrypoint: String,
pub required_config_file: Option<PathBuf>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ProjectModelError {
#[error("environment discovery failed: {0}")]
Environment(String),
#[error("Clusterflux entrypoint discovery failed: {0}")]
EntrypointDiscovery(String),
#[error(
"no Clusterflux entrypoint is declared; add `#[clusterflux::main]` to a function under src/"
)]
NoEntrypoints,
#[error("unknown Clusterflux entrypoint `{name}`; available entrypoints: {available:?}")]
UnknownEntrypoint {
name: String,
available: Vec<String>,
},
}
impl ProjectModel {
pub fn discover_without_config(root: &Path) -> Result<Self, ProjectModelError> {
let environments = discover_environments(root).map_err(|err| {
ProjectModelError::Environment(match err {
EnvironmentError::Read { path, source } => {
format!("failed to read {}: {source}", path.display())
}
})
})?;
let entrypoints = discover_entrypoints(root)?;
let default_entrypoint = if entrypoints.contains_key("build") {
"build".to_owned()
} else {
entrypoints.keys().next().cloned().unwrap_or_default()
};
Ok(Self {
root: root.to_path_buf(),
environments,
entrypoints,
default_entrypoint,
required_config_file: None,
})
}
pub fn select_entrypoint(&self, name: Option<&str>) -> Result<&Entrypoint, ProjectModelError> {
if self.entrypoints.is_empty() {
return Err(ProjectModelError::NoEntrypoints);
}
let name = name.unwrap_or(&self.default_entrypoint);
self.entrypoints
.get(name)
.ok_or_else(|| ProjectModelError::UnknownEntrypoint {
name: name.to_owned(),
available: self.entrypoints.keys().cloned().collect(),
})
}
}
fn discover_entrypoints(root: &Path) -> Result<BTreeMap<String, Entrypoint>, ProjectModelError> {
let source_root = root.join("src");
if !source_root.is_dir() {
return Ok(BTreeMap::new());
}
let mut source_files = Vec::new();
collect_rust_sources(&source_root, &mut source_files)?;
source_files.sort();
let mut entrypoints = BTreeMap::new();
for path in source_files {
let source = fs::read_to_string(&path).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to read {}: {error}",
path.display()
))
})?;
let syntax = syn::parse_file(&source).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to parse {}: {error}",
path.display()
))
})?;
collect_entrypoint_items(&syntax.items, &path, &mut entrypoints)?;
}
Ok(entrypoints)
}
fn collect_rust_sources(
directory: &Path,
files: &mut Vec<PathBuf>,
) -> Result<(), ProjectModelError> {
let entries = fs::read_dir(directory).map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to read {}: {error}",
directory.display()
))
})?;
for entry in entries {
let entry = entry.map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to inspect {}: {error}",
directory.display()
))
})?;
let path = entry.path();
let file_type = entry.file_type().map_err(|error| {
ProjectModelError::EntrypointDiscovery(format!(
"failed to inspect {}: {error}",
path.display()
))
})?;
if file_type.is_dir() {
collect_rust_sources(&path, files)?;
} else if file_type.is_file() && path.extension().is_some_and(|ext| ext == "rs") {
files.push(path);
}
}
Ok(())
}
fn collect_entrypoint_items(
items: &[Item],
path: &Path,
entrypoints: &mut BTreeMap<String, Entrypoint>,
) -> Result<(), ProjectModelError> {
for item in items {
match item {
Item::Fn(function) => {
let Some(attribute) = function.attrs.iter().find(|attribute| {
let segments = attribute
.path()
.segments
.iter()
.map(|segment| segment.ident.to_string())
.collect::<Vec<_>>();
segments.as_slice() == ["clusterflux", "main"]
}) else {
continue;
};
let function_name = function.sig.ident.to_string();
let default_name = function_name
.strip_suffix("_main")
.unwrap_or(&function_name);
let name = entrypoint_name(attribute, default_name);
let entrypoint = Entrypoint {
name: name.clone(),
function: function_name,
};
if let Some(existing) = entrypoints.insert(name.clone(), entrypoint) {
return Err(ProjectModelError::EntrypointDiscovery(format!(
"duplicate entrypoint `{name}` in {}; it was already declared by `{}`",
path.display(),
existing.function
)));
}
}
Item::Mod(module) => {
if let Some((_, nested)) = &module.content {
collect_entrypoint_items(nested, path, entrypoints)?;
}
}
_ => {}
}
}
Ok(())
}
fn entrypoint_name(attribute: &syn::Attribute, default: &str) -> String {
let Meta::List(_) = &attribute.meta else {
return default.to_owned();
};
let Ok(arguments) = attribute.parse_args_with(Punctuated::<Meta, Token![,]>::parse_terminated)
else {
return default.to_owned();
};
arguments
.into_iter()
.find_map(|meta| {
let Meta::NameValue(name_value) = meta else {
return None;
};
if !name_value.path.is_ident("name") {
return None;
}
let Expr::Lit(expression) = name_value.value else {
return None;
};
let Lit::Str(value) = expression.lit else {
return None;
};
Some(value.value())
})
.unwrap_or_else(|| default.to_owned())
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn project_works_without_hand_written_configuration_file() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("envs/linux")).unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(
temp.path().join("envs/linux/Containerfile"),
"FROM alpine\n",
)
.unwrap();
fs::write(
temp.path().join("src/main.rs"),
"#[clusterflux::main]\npub fn build_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert_eq!(model.required_config_file, None);
assert_eq!(model.environments[0].name, "linux");
assert_eq!(model.select_entrypoint(None).unwrap().name, "build");
}
#[test]
fn project_can_define_multiple_default_entrypoints() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src/nested")).unwrap();
fs::write(
temp.path().join("src/lib.rs"),
"#[clusterflux::main(name = \"check\")]\npub fn test_main() {}\n",
)
.unwrap();
fs::write(
temp.path().join("src/nested/release.rs"),
"#[clusterflux::main]\npub fn release_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert_eq!(
model.select_entrypoint(Some("check")).unwrap().function,
"test_main"
);
assert_eq!(
model.select_entrypoint(Some("release")).unwrap().function,
"release_main"
);
}
#[test]
fn unknown_entrypoint_lists_available_choices() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(
temp.path().join("src/main.rs"),
"#[clusterflux::main]\npub fn build_main() {}\n",
)
.unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
let error = model.select_entrypoint(Some("deploy")).unwrap_err();
assert!(matches!(error, ProjectModelError::UnknownEntrypoint { .. }));
}
#[test]
fn project_without_declared_entrypoint_does_not_invent_product_surfaces() {
let temp = tempfile::tempdir().unwrap();
fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write(temp.path().join("src/main.rs"), "fn main() {}\n").unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap();
assert!(model.entrypoints.is_empty());
assert_eq!(model.default_entrypoint, "");
assert_eq!(
model.select_entrypoint(None).unwrap_err(),
ProjectModelError::NoEntrypoints
);
}
}

472
crates/clusterflux-core/src/scheduler.rs Normal file
View file

@ -0,0 +1,472 @@
use std::collections::{BTreeMap, BTreeSet};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
ArtifactId, Capability, Digest, EnvironmentRequirements, NodeCapabilities, NodeId, ProjectId,
TenantId,
};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeDescriptor {
pub id: NodeId,
pub tenant: TenantId,
pub project: ProjectId,
pub capabilities: NodeCapabilities,
pub cached_environments: BTreeSet<Digest>,
pub dependency_caches: BTreeSet<Digest>,
pub source_snapshots: BTreeSet<Digest>,
pub artifact_locations: BTreeSet<ArtifactId>,
pub direct_connectivity: bool,
pub online: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct PlacementRequest {
pub tenant: TenantId,
pub project: ProjectId,
pub environment: Option<EnvironmentRequirements>,
pub environment_digest: Option<Digest>,
#[serde(default)]
pub environment_cache_required: bool,
pub required_capabilities: BTreeSet<Capability>,
pub dependency_cache: Option<Digest>,
pub source_snapshot: Option<Digest>,
pub required_artifacts: BTreeSet<ArtifactId>,
pub quota_available: bool,
pub policy_allowed: bool,
pub prefer_node: Option<NodeId>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Placement {
pub node: NodeId,
pub score: i64,
pub reasons: Vec<String>,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
#[error("no capable node for placement: {message}")]
pub struct PlacementError {
pub message: String,
}
pub trait Scheduler {
fn place(
&self,
nodes: &[NodeDescriptor],
request: &PlacementRequest,
) -> Result<Placement, PlacementError>;
}
#[derive(Clone, Debug, Default)]
pub struct DefaultScheduler;
impl Scheduler for DefaultScheduler {
fn place(
&self,
nodes: &[NodeDescriptor],
request: &PlacementRequest,
) -> Result<Placement, PlacementError> {
let mut scored = Vec::new();
let mut rejection_counts = BTreeMap::<String, usize>::new();
for node in nodes {
match compatibility(node, request) {
Ok(mut placement) => {
locality_score(node, request, &mut placement);
scored.push(placement);
}
Err(reasons) => {
for reason in reasons {
*rejection_counts.entry(reason).or_default() += 1;
}
}
}
}
scored
.into_iter()
.max_by_key(|placement| placement.score)
.ok_or_else(|| PlacementError {
message: rejection_counts
.into_iter()
.map(|(reason, count)| format!("{reason} ({count} node(s))"))
.collect::<Vec<_>>()
.join("; "),
})
}
}
fn compatibility(
node: &NodeDescriptor,
request: &PlacementRequest,
) -> Result<Placement, Vec<String>> {
let mut reasons = Vec::new();
if !node.online {
reasons.push("node offline".to_owned());
}
if node.tenant != request.tenant {
reasons.push("tenant mismatch".to_owned());
}
if node.project != request.project {
reasons.push("project mismatch".to_owned());
}
if !request.quota_available {
reasons.push("quota unavailable for placement".to_owned());
}
if !request.policy_allowed {
reasons.push("policy denied placement".to_owned());
}
for capability in &request.required_capabilities {
if !node.capabilities.capabilities.contains(capability) {
reasons.push(format!("missing capability {capability:?}"));
}
}
if let Some(environment) = &request.environment {
if let Some(required_os) = &environment.os {
if &node.capabilities.os != required_os {
reasons.push(format!("environment requires os {required_os:?}"));
}
}
if let Some(required_arch) = &environment.arch {
if &node.capabilities.arch != required_arch {
reasons.push(format!("environment requires arch {required_arch}"));
}
}
for capability in &environment.capabilities {
if !node.capabilities.capabilities.contains(capability) {
reasons.push(format!("environment requires capability {capability:?}"));
}
}
}
if request.environment_cache_required {
match request.environment_digest.as_ref() {
Some(digest) if !node.cached_environments.contains(digest) => {
reasons.push(format!(
"required named environment cache {digest} is unavailable"
));
}
None => reasons.push("required named environment cache digest is missing".to_owned()),
Some(_) => {}
}
}
let source_transfer_required = request
.source_snapshot
.as_ref()
.is_some_and(|digest| !node.source_snapshots.contains(digest));
if source_transfer_required && !node.direct_connectivity {
reasons.push("source snapshot unavailable and direct connectivity unavailable".to_owned());
}
let missing_artifacts = request
.required_artifacts
.iter()
.filter(|artifact| !node.artifact_locations.contains(*artifact))
.count();
if missing_artifacts > 0 && !node.direct_connectivity {
reasons.push(format!(
"{missing_artifacts} required artifact(s) unavailable and direct connectivity unavailable"
));
}
if reasons.is_empty() {
Ok(Placement {
node: node.id.clone(),
score: 0,
reasons: Vec::new(),
})
} else {
Err(reasons)
}
}
fn locality_score(node: &NodeDescriptor, request: &PlacementRequest, placement: &mut Placement) {
if request.prefer_node.as_ref() == Some(&node.id) {
placement.score += 100;
placement.reasons.push("preferred node".to_owned());
}
if request
.environment_digest
.as_ref()
.is_some_and(|digest| node.cached_environments.contains(digest))
{
placement.score += 50;
placement.reasons.push("warm environment cache".to_owned());
}
if request
.source_snapshot
.as_ref()
.is_some_and(|digest| node.source_snapshots.contains(digest))
{
placement.score += 40;
placement
.reasons
.push("source snapshot already local".to_owned());
}
if request
.dependency_cache
.as_ref()
.is_some_and(|digest| node.dependency_caches.contains(digest))
{
placement.score += 30;
placement.reasons.push("warm dependency cache".to_owned());
}
let artifact_hits = request
.required_artifacts
.iter()
.filter(|artifact| node.artifact_locations.contains(*artifact))
.count() as i64;
if artifact_hits > 0 {
placement.score += 10 * artifact_hits;
placement.reasons.push(format!(
"{artifact_hits} required artifact(s) already local"
));
}
}
#[cfg(test)]
mod tests {
use crate::{EnvironmentBackend, Os};
use super::*;
fn node(id: &str, cached_source: bool) -> NodeDescriptor {
let source = Digest::sha256("source");
NodeDescriptor {
id: NodeId::from(id),
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
capabilities: NodeCapabilities {
os: Os::Linux,
arch: "x86_64".to_owned(),
capabilities: BTreeSet::from([
Capability::Command,
Capability::Containers,
Capability::RootlessPodman,
]),
environment_backends: BTreeSet::from([EnvironmentBackend::Container]),
source_providers: BTreeSet::from(["filesystem".to_owned()]),
},
cached_environments: BTreeSet::from([Digest::sha256("env")]),
dependency_caches: if cached_source {
BTreeSet::from([Digest::sha256("deps")])
} else {
BTreeSet::new()
},
source_snapshots: if cached_source {
BTreeSet::from([source])
} else {
BTreeSet::new()
},
artifact_locations: BTreeSet::new(),
direct_connectivity: true,
online: true,
}
}
#[test]
fn scheduler_prefers_warm_source_and_environment() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::linux_container()),
environment_digest: Some(Digest::sha256("env")),
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: Some(Digest::sha256("deps")),
source_snapshot: Some(Digest::sha256("source")),
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let placement = DefaultScheduler
.place(&[node("cold", false), node("warm", true)], &request)
.unwrap();
assert_eq!(placement.node, NodeId::from("warm"));
assert!(placement
.reasons
.iter()
.any(|reason| reason.contains("source")));
assert!(placement
.reasons
.iter()
.any(|reason| reason.contains("dependency")));
}
#[test]
fn scheduler_requires_requested_named_environment_cache() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: Some(Digest::sha256("missing-environment")),
environment_cache_required: true,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("uncached", false)], &request)
.unwrap_err();
assert!(error.message.contains("named environment cache"));
}
#[test]
fn scheduler_failure_names_missing_constraint() {
let mut request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
request.required_capabilities.insert(Capability::Command);
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("WindowsCommandDev"));
}
#[test]
fn scheduler_failure_names_environment_constraint() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::windows_command_dev()),
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("environment requires os Windows"));
assert!(error
.message
.contains("environment requires capability WindowsCommandDev"));
}
#[test]
fn scheduler_requires_direct_connectivity_when_transfer_is_needed() {
let mut disconnected = node("disconnected", false);
disconnected.direct_connectivity = false;
let mut local = node("local", true);
local.direct_connectivity = false;
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: Some(Digest::sha256("source")),
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let placement = DefaultScheduler
.place(&[disconnected, local], &request)
.unwrap();
assert_eq!(placement.node, NodeId::from("local"));
let mut disconnected = node("disconnected", false);
disconnected.direct_connectivity = false;
let error = DefaultScheduler
.place(&[disconnected], &request)
.unwrap_err();
assert!(error
.message
.contains("source snapshot unavailable and direct connectivity unavailable"));
}
#[test]
fn scheduler_failure_names_required_artifact_transfer_constraint() {
let mut disconnected = node("disconnected", true);
disconnected.direct_connectivity = false;
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::from([ArtifactId::from("cache")]),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[disconnected], &request)
.unwrap_err();
assert!(error
.message
.contains("1 required artifact(s) unavailable and direct connectivity unavailable"));
}
#[test]
fn scheduler_failure_names_quota_and_policy_constraints() {
let mut request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: false,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("quota unavailable for placement"));
request.quota_available = true;
request.policy_allowed = false;
let error = DefaultScheduler
.place(&[node("linux", false)], &request)
.unwrap_err();
assert!(error.message.contains("policy denied placement"));
}
}

324
crates/clusterflux-core/src/source.rs Normal file
View file

@ -0,0 +1,324 @@
use std::collections::BTreeSet;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Capability, Digest, ProjectId, TenantId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum SourceProviderKind {
Filesystem,
Git,
Custom(String),
}
impl SourceProviderKind {
pub fn provider_id(&self) -> &str {
match self {
SourceProviderKind::Filesystem => "filesystem",
SourceProviderKind::Git => "git",
SourceProviderKind::Custom(provider) => provider,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub enum SourceTransferMode {
RequiredContent,
ExplicitSnapshotChunks,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceTransferPolicy {
pub local_source_bytes_remain_node_local: bool,
pub coordinator_receives_source_bytes_by_default: bool,
pub default_full_repo_tarball: bool,
pub allowed_remote_transfer: BTreeSet<SourceTransferMode>,
}
impl SourceTransferPolicy {
pub fn local_first_snapshot_chunks() -> Self {
Self {
local_source_bytes_remain_node_local: true,
coordinator_receives_source_bytes_by_default: false,
default_full_repo_tarball: false,
allowed_remote_transfer: BTreeSet::from([
SourceTransferMode::RequiredContent,
SourceTransferMode::ExplicitSnapshotChunks,
]),
}
}
}
impl Default for SourceTransferPolicy {
fn default() -> Self {
Self::local_first_snapshot_chunks()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourceProviderManifest {
pub kind: SourceProviderKind,
pub digest: Digest,
pub description: String,
#[serde(default)]
pub coordinator_requires_checkout_access: bool,
#[serde(default)]
pub transfer_policy: SourceTransferPolicy,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum SourceManifestError {
#[error("source provider manifest digest is not a valid sha256 digest: {0}")]
InvalidDigest(String),
#[error("custom source provider id `{0}` is invalid")]
InvalidProviderId(String),
#[error("source provider manifest description must be non-empty")]
EmptyDescription,
#[error("source provider manifest description is too long")]
DescriptionTooLong,
#[error("source provider manifest description contains control characters")]
DescriptionControlCharacter,
#[error("source provider manifest would require coordinator checkout access")]
CoordinatorCheckoutAccess,
#[error("source provider manifest would send source bytes to the coordinator by default")]
CoordinatorReceivesSourceBytes,
#[error("source provider manifest would default to a full-repo tarball")]
DefaultFullRepoTarball,
#[error("source provider manifest has no allowed remote transfer mode")]
MissingRemoteTransferMode,
}
pub trait SourceProviderModule {
fn kind(&self) -> SourceProviderKind;
fn manifest(&self) -> SourceProviderManifest;
}
impl SourceProviderManifest {
pub fn local_first(kind: SourceProviderKind, description: impl Into<String>) -> Self {
let transfer_policy = SourceTransferPolicy::local_first_snapshot_chunks();
let digest = Self::digest_for(&kind, false, &transfer_policy);
Self {
kind,
digest,
description: description.into(),
coordinator_requires_checkout_access: false,
transfer_policy,
}
}
pub fn validate_public_mvp(&self) -> Result<(), SourceManifestError> {
self.validate_shape()?;
if self.coordinator_requires_checkout_access {
return Err(SourceManifestError::CoordinatorCheckoutAccess);
}
if self
.transfer_policy
.coordinator_receives_source_bytes_by_default
{
return Err(SourceManifestError::CoordinatorReceivesSourceBytes);
}
if self.transfer_policy.default_full_repo_tarball {
return Err(SourceManifestError::DefaultFullRepoTarball);
}
if self.transfer_policy.allowed_remote_transfer.is_empty() {
return Err(SourceManifestError::MissingRemoteTransferMode);
}
Ok(())
}
fn validate_shape(&self) -> Result<(), SourceManifestError> {
if !self.digest.is_valid_sha256() {
return Err(SourceManifestError::InvalidDigest(
self.digest.as_str().to_owned(),
));
}
if let SourceProviderKind::Custom(provider) = &self.kind {
if !valid_provider_id(provider) {
return Err(SourceManifestError::InvalidProviderId(provider.clone()));
}
}
if self.description.trim().is_empty() {
return Err(SourceManifestError::EmptyDescription);
}
if self.description.len() > 256 {
return Err(SourceManifestError::DescriptionTooLong);
}
if self.description.chars().any(char::is_control) {
return Err(SourceManifestError::DescriptionControlCharacter);
}
Ok(())
}
fn digest_for(
kind: &SourceProviderKind,
coordinator_requires_checkout_access: bool,
transfer_policy: &SourceTransferPolicy,
) -> Digest {
let mut modes = transfer_policy
.allowed_remote_transfer
.iter()
.map(|mode| format!("{mode:?}"))
.collect::<Vec<_>>();
modes.sort();
let mut parts = vec![
b"source-provider-manifest:v2".to_vec(),
kind.provider_id().as_bytes().to_vec(),
coordinator_requires_checkout_access
.to_string()
.into_bytes(),
transfer_policy
.local_source_bytes_remain_node_local
.to_string()
.into_bytes(),
transfer_policy
.coordinator_receives_source_bytes_by_default
.to_string()
.into_bytes(),
transfer_policy
.default_full_repo_tarball
.to_string()
.into_bytes(),
];
parts.extend(modes.into_iter().map(String::into_bytes));
Digest::from_parts(parts)
}
}
fn valid_provider_id(provider: &str) -> bool {
!provider.is_empty()
&& provider.len() <= 64
&& provider
.bytes()
.all(|byte| matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.'))
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SourcePreparation {
pub tenant: TenantId,
pub project: ProjectId,
pub provider: SourceProviderKind,
pub required_capabilities: BTreeSet<Capability>,
pub coordinator_requires_checkout_access: bool,
}
impl SourcePreparation {
pub fn node_task(tenant: TenantId, project: ProjectId, provider: SourceProviderKind) -> Self {
let capability = match provider {
SourceProviderKind::Filesystem => Capability::SourceFilesystem,
SourceProviderKind::Git => Capability::SourceGit,
SourceProviderKind::Custom(_) => Capability::SourceFilesystem,
};
Self {
tenant,
project,
provider,
required_capabilities: BTreeSet::from([capability]),
coordinator_requires_checkout_access: false,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn source_preparation_can_be_scheduled_as_node_task() {
let prep = SourcePreparation::node_task(
TenantId::from("tenant"),
ProjectId::from("project"),
SourceProviderKind::Git,
);
assert!(!prep.coordinator_requires_checkout_access);
assert!(prep.required_capabilities.contains(&Capability::SourceGit));
}
#[test]
fn local_first_source_manifest_rejects_bulk_coordinator_paths() {
let manifest = SourceProviderManifest::local_first(
SourceProviderKind::Git,
"node-side Git snapshot provider",
);
assert!(manifest.validate_public_mvp().is_ok());
assert!(!manifest.coordinator_requires_checkout_access);
assert!(
!manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default
);
assert!(!manifest.transfer_policy.default_full_repo_tarball);
assert!(manifest
.transfer_policy
.allowed_remote_transfer
.contains(&SourceTransferMode::ExplicitSnapshotChunks));
}
#[test]
fn source_manifest_validation_treats_manifest_as_hostile_input() {
let mut manifest = SourceProviderManifest::local_first(
SourceProviderKind::Custom("gitlab-lfs".to_owned()),
"custom provider",
);
assert!(manifest.validate_public_mvp().is_ok());
manifest.kind = SourceProviderKind::Custom("../checkout".to_owned());
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::InvalidProviderId(
"../checkout".to_owned()
))
);
manifest.kind = SourceProviderKind::Git;
manifest.digest = Digest::sha256("valid");
manifest.coordinator_requires_checkout_access = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::CoordinatorCheckoutAccess)
);
manifest.coordinator_requires_checkout_access = false;
manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::CoordinatorReceivesSourceBytes)
);
manifest
.transfer_policy
.coordinator_receives_source_bytes_by_default = false;
manifest.transfer_policy.default_full_repo_tarball = true;
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::DefaultFullRepoTarball)
);
}
#[test]
fn source_manifest_rejects_malformed_digest_from_json() {
let mut manifest = SourceProviderManifest::local_first(
SourceProviderKind::Filesystem,
"filesystem provider",
);
let value = serde_json::to_value(&manifest).unwrap();
let mut object = value.as_object().unwrap().clone();
object.insert(
"digest".to_owned(),
serde_json::Value::String("sha256:not-a-real-digest".to_owned()),
);
manifest = serde_json::from_value(serde_json::Value::Object(object)).unwrap();
assert_eq!(
manifest.validate_public_mvp(),
Err(SourceManifestError::InvalidDigest(
"sha256:not-a-real-digest".to_owned()
))
);
}
}

243
crates/clusterflux-core/src/transport.rs Normal file
View file

@ -0,0 +1,243 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{ArtifactId, Digest, NodeId, ProcessId, ProjectId, TenantId};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum TransportKind {
NativeQuic,
}
pub trait Transport {
fn kind(&self) -> TransportKind;
fn authenticated_direct_connections(&self) -> bool;
}
#[derive(Clone, Debug, Default)]
pub struct NativeQuicTransport;
impl Transport for NativeQuicTransport {
fn kind(&self) -> TransportKind {
TransportKind::NativeQuic
}
fn authenticated_direct_connections(&self) -> bool {
true
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DataPlaneScope {
pub tenant: TenantId,
pub project: ProjectId,
pub process: ProcessId,
pub object: DataPlaneObject,
pub authorization_subject: String,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum DataPlaneObject {
Artifact(ArtifactId),
Blob(Digest),
SourceSnapshot(Digest),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct NodeEndpoint {
pub node: NodeId,
pub advertised_addr: String,
pub public_key_fingerprint: Digest,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RendezvousRequest {
pub scope: DataPlaneScope,
pub source: NodeEndpoint,
pub destination: NodeEndpoint,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DirectBulkTransferPlan {
pub transport: TransportKind,
pub scope: DataPlaneScope,
pub source: NodeEndpoint,
pub destination: NodeEndpoint,
pub authorization_digest: Digest,
pub coordinator_assisted_rendezvous: bool,
pub coordinator_bulk_relay_allowed: bool,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum TransportError {
#[error(
"direct node-to-node connectivity is unavailable for scoped data-plane transfer: {reason}; coordinator bulk relay is disabled"
)]
DirectConnectivityUnavailable { reason: String },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum BulkTransferDecision {
DirectAuthenticated { scope: DataPlaneScope },
FailClear { message: String },
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
#[error("bulk relay through coordinator is not allowed by default")]
pub struct BulkRelayDenied;
impl NativeQuicTransport {
pub fn plan_authenticated_direct_bulk_transfer(
&self,
request: RendezvousRequest,
direct_connectivity: bool,
failure_reason: impl Into<String>,
) -> Result<DirectBulkTransferPlan, TransportError> {
if !direct_connectivity {
return Err(TransportError::DirectConnectivityUnavailable {
reason: failure_reason.into(),
});
}
let authorization_digest = data_plane_authorization_digest(&request);
Ok(DirectBulkTransferPlan {
transport: self.kind(),
scope: request.scope,
source: request.source,
destination: request.destination,
authorization_digest,
coordinator_assisted_rendezvous: true,
coordinator_bulk_relay_allowed: false,
})
}
}
pub fn direct_bulk_transfer_or_error(
scope: DataPlaneScope,
direct_connectivity: bool,
) -> BulkTransferDecision {
if direct_connectivity {
BulkTransferDecision::DirectAuthenticated { scope }
} else {
BulkTransferDecision::FailClear {
message:
"direct node-to-node connectivity is unavailable; coordinator bulk relay is disabled"
.to_owned(),
}
}
}
fn data_plane_authorization_digest(request: &RendezvousRequest) -> Digest {
let object = match &request.scope.object {
DataPlaneObject::Artifact(artifact) => format!("artifact:{artifact}"),
DataPlaneObject::Blob(digest) => format!("blob:{}", digest.as_str()),
DataPlaneObject::SourceSnapshot(digest) => format!("source:{}", digest.as_str()),
};
Digest::from_parts([
b"dataplane-auth:v1".as_slice(),
request.scope.tenant.as_str().as_bytes(),
request.scope.project.as_str().as_bytes(),
request.scope.process.as_str().as_bytes(),
object.as_bytes(),
request.scope.authorization_subject.as_bytes(),
request.source.node.as_str().as_bytes(),
request.source.public_key_fingerprint.as_str().as_bytes(),
request.destination.node.as_str().as_bytes(),
request
.destination
.public_key_fingerprint
.as_str()
.as_bytes(),
])
}
#[cfg(test)]
mod tests {
use super::*;
fn endpoint(name: &str) -> NodeEndpoint {
NodeEndpoint {
node: NodeId::from(name),
advertised_addr: format!("{name}.mesh.invalid:4433"),
public_key_fingerprint: Digest::sha256(format!("{name}-public-key")),
}
}
fn scope(project: &str) -> DataPlaneScope {
DataPlaneScope {
tenant: TenantId::from("tenant"),
project: ProjectId::from(project),
process: ProcessId::from("process"),
object: DataPlaneObject::Artifact(ArtifactId::from("artifact")),
authorization_subject: "node-a-to-node-b".to_owned(),
}
}
#[test]
fn failed_direct_transfer_does_not_silently_relay() {
let decision = direct_bulk_transfer_or_error(scope("project"), false);
assert!(matches!(decision, BulkTransferDecision::FailClear { .. }));
}
#[test]
fn native_quic_rendezvous_plan_is_scoped_and_disallows_coordinator_bulk_relay() {
let transport = NativeQuicTransport;
let request = RendezvousRequest {
scope: scope("project"),
source: endpoint("node-a"),
destination: endpoint("node-b"),
};
let plan = transport
.plan_authenticated_direct_bulk_transfer(request.clone(), true, "")
.unwrap();
let changed_scope_plan = transport
.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope("other-project"),
..request
},
true,
"",
)
.unwrap();
assert_eq!(plan.transport, TransportKind::NativeQuic);
assert_eq!(plan.scope.tenant, TenantId::from("tenant"));
assert_eq!(plan.scope.project, ProjectId::from("project"));
assert_eq!(plan.scope.process, ProcessId::from("process"));
assert_eq!(
plan.scope.object,
DataPlaneObject::Artifact(ArtifactId::from("artifact"))
);
assert_eq!(plan.source.node, NodeId::from("node-a"));
assert_eq!(plan.destination.node, NodeId::from("node-b"));
assert!(plan.coordinator_assisted_rendezvous);
assert!(!plan.coordinator_bulk_relay_allowed);
assert_ne!(
plan.authorization_digest,
changed_scope_plan.authorization_digest
);
}
#[test]
fn failed_direct_rendezvous_reports_clear_error_instead_of_relaying() {
let error = NativeQuicTransport
.plan_authenticated_direct_bulk_transfer(
RendezvousRequest {
scope: scope("project"),
source: endpoint("node-a"),
destination: endpoint("node-b"),
},
false,
"nat traversal failed",
)
.unwrap_err();
assert!(error.to_string().contains("nat traversal failed"));
assert!(error
.to_string()
.contains("coordinator bulk relay is disabled"));
}
}

241
crates/clusterflux-core/src/vfs.rs Normal file
View file

@ -0,0 +1,241 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{Digest, NodeId, TaskInstanceId};
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct VfsPath(String);
impl VfsPath {
pub fn new(path: impl Into<String>) -> Result<Self, VfsError> {
let path = path.into();
if !path.starts_with("/vfs/") {
return Err(VfsError::InvalidPath(path));
}
Ok(Self(path))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsObject {
pub path: VfsPath,
pub digest: Digest,
pub size: u64,
pub producer: TaskInstanceId,
pub node: NodeId,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct VfsManifest {
pub epoch: u64,
pub producer: TaskInstanceId,
pub node: NodeId,
pub objects: BTreeMap<VfsPath, VfsObject>,
pub large_bytes_uploaded: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum SyncPolicy {
MetadataOnly,
ExplicitNode(NodeId),
ExplicitStore(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum VfsSyncDecision {
NoBytesMoved,
MoveBytesToNode(NodeId),
MoveBytesToStore(String),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReuseDecision {
SameNodeZeroCopy,
NeedsTransfer { from: NodeId, to: NodeId },
Unavailable,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum VfsError {
#[error("VFS path must start with /vfs/: {0}")]
InvalidPath(String),
#[error("path is not visible in the published VFS manifest: {0}")]
NotVisible(String),
}
#[derive(Clone, Debug)]
pub struct VfsOverlay {
task: TaskInstanceId,
node: NodeId,
epoch: u64,
pending: BTreeMap<VfsPath, VfsObject>,
published: BTreeMap<VfsPath, VfsObject>,
}
impl VfsOverlay {
pub fn new(task: TaskInstanceId, node: NodeId) -> Self {
Self {
task,
node,
epoch: 0,
pending: BTreeMap::new(),
published: BTreeMap::new(),
}
}
pub fn write(&mut self, path: VfsPath, digest: Digest, size: u64) -> VfsObject {
let object = VfsObject {
path: path.clone(),
digest,
size,
producer: self.task.clone(),
node: self.node.clone(),
};
self.pending.insert(path, object.clone());
object
}
pub fn flush(&mut self) -> VfsManifest {
self.epoch += 1;
self.published.append(&mut self.pending);
VfsManifest {
epoch: self.epoch,
producer: self.task.clone(),
node: self.node.clone(),
objects: self.published.clone(),
large_bytes_uploaded: false,
}
}
pub fn sync(&self, policy: SyncPolicy) -> VfsSyncDecision {
match policy {
SyncPolicy::MetadataOnly => VfsSyncDecision::NoBytesMoved,
SyncPolicy::ExplicitNode(node) => VfsSyncDecision::MoveBytesToNode(node),
SyncPolicy::ExplicitStore(store) => VfsSyncDecision::MoveBytesToStore(store),
}
}
pub fn read_published<'a>(
manifest: &'a VfsManifest,
path: &VfsPath,
) -> Result<&'a VfsObject, VfsError> {
manifest
.objects
.get(path)
.ok_or_else(|| VfsError::NotVisible(path.as_str().to_owned()))
}
pub fn reuse_for_consumer(
manifest: &VfsManifest,
path: &VfsPath,
consumer_node: &NodeId,
) -> ReuseDecision {
let Some(object) = manifest.objects.get(path) else {
return ReuseDecision::Unavailable;
};
if &object.node == consumer_node {
ReuseDecision::SameNodeZeroCopy
} else {
ReuseDecision::NeedsTransfer {
from: object.node.clone(),
to: consumer_node.clone(),
}
}
}
pub fn discard_unflushed(&mut self) {
self.pending.clear();
}
pub fn pending_len(&self) -> usize {
self.pending.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn path() -> VfsPath {
VfsPath::new("/vfs/artifacts/app").unwrap()
}
#[test]
fn flush_publishes_manifest_without_large_byte_upload() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let manifest = overlay.flush();
assert_eq!(manifest.epoch, 1);
assert!(!manifest.large_bytes_uploaded);
assert!(manifest.objects.contains_key(&path()));
}
#[test]
fn downstream_task_can_read_after_flush_but_not_before() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let empty = VfsManifest {
epoch: 0,
producer: TaskInstanceId::from("task"),
node: NodeId::from("node-a"),
objects: BTreeMap::new(),
large_bytes_uploaded: false,
};
assert!(VfsOverlay::read_published(&empty, &path()).is_err());
let manifest = overlay.flush();
assert!(VfsOverlay::read_published(&manifest, &path()).is_ok());
}
#[test]
fn sync_is_explicit_and_policy_driven() {
let overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
assert_eq!(
overlay.sync(SyncPolicy::MetadataOnly),
VfsSyncDecision::NoBytesMoved
);
assert_eq!(
overlay.sync(SyncPolicy::ExplicitStore("s3://bucket/app".to_owned())),
VfsSyncDecision::MoveBytesToStore("s3://bucket/app".to_owned())
);
}
#[test]
fn same_node_reuse_avoids_transfer() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
let manifest = overlay.flush();
assert_eq!(
VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-a")),
ReuseDecision::SameNodeZeroCopy
);
assert_eq!(
VfsOverlay::reuse_for_consumer(&manifest, &path(), &NodeId::from("node-b")),
ReuseDecision::NeedsTransfer {
from: NodeId::from("node-a"),
to: NodeId::from("node-b")
}
);
}
#[test]
fn unflushed_task_local_changes_can_be_discarded() {
let mut overlay = VfsOverlay::new(TaskInstanceId::from("task"), NodeId::from("node-a"));
overlay.write(path(), Digest::sha256("binary"), 6);
overlay.discard_unflushed();
assert_eq!(overlay.pending_len(), 0);
}
}

114
crates/clusterflux-core/src/wire.rs Normal file
View file

@ -0,0 +1,114 @@
use serde_json::{json, Value};
pub const COORDINATOR_PROTOCOL_VERSION: u64 = 1;
pub const COORDINATOR_WIRE_REQUEST_TYPE: &str = "coordinator_request";
pub fn coordinator_wire_request(request_id: impl Into<String>, payload: Value) -> Value {
let operation = coordinator_payload_operation(&payload);
let authentication = coordinator_authentication_metadata(&payload);
json!({
"type": COORDINATOR_WIRE_REQUEST_TYPE,
"protocol_version": COORDINATOR_PROTOCOL_VERSION,
"request_id": request_id.into(),
"operation": operation,
"authentication": authentication,
"payload": payload,
})
}
pub fn coordinator_payload_operation(payload: &Value) -> String {
payload
.get("type")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_owned()
}
pub fn coordinator_authentication_metadata(payload: &Value) -> Value {
let operation = coordinator_payload_operation(payload);
match operation.as_str() {
"authenticated" => json!({
"kind": "cli_session",
"session": true,
"request_operation": payload
.get("request")
.map(coordinator_payload_operation)
.unwrap_or_else(|| "unknown".to_owned()),
}),
"signed_node" => json!({
"kind": "node_signature",
"node": payload.get("node").and_then(Value::as_str),
}),
"node_heartbeat" if payload.get("node_signature").is_some() => json!({
"kind": "node_signature",
"node": payload.get("node").and_then(Value::as_str),
}),
"start_process" | "launch_task" if payload.get("agent_signature").is_some() => json!({
"kind": "agent_signature",
"agent": payload.get("actor_agent").and_then(Value::as_str),
"fingerprint": payload.get("agent_public_key_fingerprint").and_then(Value::as_str),
}),
"admin_status" | "suspend_tenant" if payload.get("admin_proof").is_some() => json!({
"kind": "admin_proof",
"actor": payload.get("actor_user").and_then(Value::as_str),
"nonce": payload.get("admin_nonce").and_then(Value::as_str),
"issued_at_epoch_seconds": payload.get("issued_at_epoch_seconds").and_then(Value::as_u64),
}),
"exchange_node_enrollment_grant" => json!({
"kind": "node_enrollment_grant",
"node": payload.get("node").and_then(Value::as_str),
}),
_ => json!({
"kind": "none",
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn coordinator_wire_request_wraps_payload_without_exposing_secret_metadata() {
let envelope = coordinator_wire_request(
"cli-1",
json!({
"type": "authenticated",
"session_secret": "secret-value",
"request": { "type": "list_projects" },
}),
);
assert_eq!(envelope["type"], COORDINATOR_WIRE_REQUEST_TYPE);
assert_eq!(envelope["protocol_version"], COORDINATOR_PROTOCOL_VERSION);
assert_eq!(envelope["request_id"], "cli-1");
assert_eq!(envelope["operation"], "authenticated");
assert_eq!(envelope["authentication"]["kind"], "cli_session");
assert_eq!(
envelope["authentication"]["request_operation"],
"list_projects"
);
assert_eq!(envelope["authentication"].get("session_secret"), None);
assert_eq!(envelope["payload"]["session_secret"], "secret-value");
}
#[test]
fn coordinator_wire_request_describes_signature_metadata() {
let envelope = coordinator_wire_request(
"node-1",
json!({
"type": "signed_node",
"node": "node-a",
"node_signature": {
"nonce": "nonce",
"issued_at_epoch_seconds": 1,
"signature": "ed25519:sig"
},
"request": { "type": "poll_task_assignment", "node": "node-a" },
}),
);
assert_eq!(envelope["authentication"]["kind"], "node_signature");
assert_eq!(envelope["authentication"]["node"], "node-a");
}
}