Compare commits

..

2 commits

Author SHA1 Message Date
Clusterflux release dry run
da9ed82251 Publish Clusterflux dryrun a43e907efd9d 2026-07-15 10:13:02 +02:00
Clusterflux release dry run
6f52bb46cd Public dry run dryrun-a43e907efd9d
Source commit: a43e907efd9d1561c23fe73499478e881f868355

Public tree identity: sha256:453dea30195485dd8939f575a69b39f4bb7acd84c4df23f8aa29b55d044f2673
2026-07-15 01:54:51 +02:00
226 changed files with 7672 additions and 10306 deletions

4
.gitignore vendored
View file

@ -1,6 +1,6 @@
/target/ /target/
/.disasmer/ /.clusterflux/
**/.disasmer/ **/.clusterflux/
/vscode-extension/node_modules/ /vscode-extension/node_modules/
/private/*/Cargo.lock /private/*/Cargo.lock
/private/*/target/ /private/*/target/

View file

@ -0,0 +1,22 @@
{
"kind": "clusterflux-filtered-public-tree",
"source_commit": "a43e907efd9d1561c23fe73499478e881f868355",
"release_name": "dryrun-a43e907efd9d",
"filtered_out": [
"private/**",
"internal/**",
"experiments/**",
".git",
"target",
"git-ignored source paths",
"root/*.md except README.md",
"**/.clusterflux/**",
".forgejo/**"
],
"public_export": {
"host_neutral": true,
"include_forgejo_workflows": false
},
"forgejo_host": "git.michelpaulissen.com",
"default_hosted_coordinator_endpoint": "https://clusterflux.michelpaulissen.com"
}

523
Cargo.lock generated
View file

@ -197,15 +197,15 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.12.0" version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.65" version = "1.2.67"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
dependencies = [ dependencies = [
"find-msvc-tools", "find-msvc-tools",
"shlex", "shlex",
@ -274,6 +274,134 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "clusterflux-cli"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"clap",
"clusterflux-control",
"clusterflux-core",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"wasmparser 0.245.1",
]
[[package]]
name = "clusterflux-control"
version = "0.1.0"
dependencies = [
"serde_json",
"thiserror 1.0.69",
"ureq",
]
[[package]]
name = "clusterflux-coordinator"
version = "0.1.0"
dependencies = [
"base64",
"clusterflux-core",
"clusterflux-wasm-runtime",
"postgres",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 1.0.69",
"wasmparser 0.245.1",
]
[[package]]
name = "clusterflux-core"
version = "0.1.0"
dependencies = [
"base64",
"ed25519-dalek",
"getrandom 0.3.4",
"hex",
"serde",
"serde_json",
"sha2 0.10.9",
"syn",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "clusterflux-dap"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"clusterflux-control",
"clusterflux-core",
"clusterflux-node",
"serde_json",
"tempfile",
]
[[package]]
name = "clusterflux-macros"
version = "0.1.0"
dependencies = [
"hex",
"proc-macro2",
"quote",
"serde_json",
"sha2 0.10.9",
"syn",
]
[[package]]
name = "clusterflux-node"
version = "0.1.0"
dependencies = [
"base64",
"clusterflux-control",
"clusterflux-core",
"clusterflux-wasm-runtime",
"libc",
"quinn",
"rcgen",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 1.0.69",
"tokio",
"wasmparser 0.245.1",
"wat",
]
[[package]]
name = "clusterflux-sdk"
version = "0.1.0"
dependencies = [
"base64",
"clusterflux-control",
"clusterflux-core",
"clusterflux-macros",
"futures-executor",
"serde",
"serde_json",
]
[[package]]
name = "clusterflux-wasm-runtime"
version = "0.1.0"
dependencies = [
"clusterflux-core",
"serde",
"serde_json",
"thiserror 1.0.69",
"tokio",
"wasmtime",
]
[[package]] [[package]]
name = "cmov" name = "cmov"
version = "0.5.4" version = "0.5.4"
@ -589,133 +717,6 @@ dependencies = [
"ctutils", "ctutils",
] ]
[[package]]
name = "disasmer-cli"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"clap",
"disasmer-control",
"disasmer-core",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"wasmparser 0.245.1",
]
[[package]]
name = "disasmer-control"
version = "0.1.0"
dependencies = [
"serde_json",
"thiserror 1.0.69",
"ureq",
]
[[package]]
name = "disasmer-coordinator"
version = "0.1.0"
dependencies = [
"base64",
"disasmer-core",
"disasmer-wasm-runtime",
"postgres",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 1.0.69",
"wasmparser 0.245.1",
]
[[package]]
name = "disasmer-core"
version = "0.1.0"
dependencies = [
"base64",
"ed25519-dalek",
"getrandom 0.3.4",
"hex",
"serde",
"serde_json",
"sha2 0.10.9",
"syn",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "disasmer-dap"
version = "0.1.0"
dependencies = [
"anyhow",
"base64",
"disasmer-control",
"disasmer-core",
"disasmer-node",
"serde_json",
"tempfile",
]
[[package]]
name = "disasmer-macros"
version = "0.1.0"
dependencies = [
"hex",
"proc-macro2",
"quote",
"serde_json",
"sha2 0.10.9",
"syn",
]
[[package]]
name = "disasmer-node"
version = "0.1.0"
dependencies = [
"base64",
"disasmer-control",
"disasmer-core",
"disasmer-wasm-runtime",
"libc",
"quinn",
"rcgen",
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 1.0.69",
"tokio",
"wasmparser 0.245.1",
]
[[package]]
name = "disasmer-sdk"
version = "0.1.0"
dependencies = [
"base64",
"disasmer-control",
"disasmer-core",
"disasmer-macros",
"futures-executor",
"serde",
"serde_json",
]
[[package]]
name = "disasmer-wasm-runtime"
version = "0.1.0"
dependencies = [
"disasmer-core",
"serde",
"serde_json",
"thiserror 1.0.69",
"tokio",
"wasmtime",
]
[[package]] [[package]]
name = "displaydoc" name = "displaydoc"
version = "0.2.6" version = "0.2.6"
@ -912,11 +913,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"r-efi 5.3.0", "r-efi 5.3.0",
"wasip2", "wasip2",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -926,9 +925,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"r-efi 6.0.0", "r-efi 6.0.0",
"rand_core 0.10.1", "rand_core 0.10.1",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -1147,7 +1148,7 @@ dependencies = [
name = "launch-build-demo" name = "launch-build-demo"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"disasmer-sdk", "clusterflux-sdk",
"futures-executor", "futures-executor",
"serde", "serde",
"serde_json", "serde_json",
@ -1179,9 +1180,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]] [[package]]
name = "libredox" name = "libredox"
version = "0.1.17" version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652"
dependencies = [ dependencies = [
"libc", "libc",
] ]
@ -1240,9 +1241,9 @@ dependencies = [
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.8.2" version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]] [[package]]
name = "memfd" name = "memfd"
@ -1261,9 +1262,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]] [[package]]
name = "mio" name = "mio"
version = "1.2.1" version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [ dependencies = [
"libc", "libc",
"wasi 0.11.1+wasi-snapshot-preview1", "wasi 0.11.1+wasi-snapshot-preview1",
@ -1282,9 +1283,9 @@ dependencies = [
[[package]] [[package]]
name = "num-bigint" name = "num-bigint"
version = "0.4.6" version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
dependencies = [ dependencies = [
"num-integer", "num-integer",
"num-traits", "num-traits",
@ -1478,7 +1479,7 @@ dependencies = [
"hmac", "hmac",
"md-5", "md-5",
"memchr", "memchr",
"rand 0.10.1", "rand",
"sha2 0.11.0", "sha2 0.11.0",
"stringprep", "stringprep",
] ]
@ -1511,15 +1512,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]] [[package]]
name = "proc-macro2" name = "proc-macro2"
version = "1.0.106" version = "1.0.106"
@ -1574,14 +1566,15 @@ dependencies = [
[[package]] [[package]]
name = "quinn-proto" name = "quinn-proto"
version = "0.11.15" version = "0.11.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
dependencies = [ dependencies = [
"bytes", "bytes",
"getrandom 0.3.4", "getrandom 0.4.3",
"lru-slab", "lru-slab",
"rand 0.9.4", "rand",
"rand_pcg",
"ring", "ring",
"rustc-hash", "rustc-hash",
"rustls", "rustls",
@ -1595,16 +1588,16 @@ dependencies = [
[[package]] [[package]]
name = "quinn-udp" name = "quinn-udp"
version = "0.5.14" version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
dependencies = [ dependencies = [
"cfg_aliases", "cfg_aliases",
"libc", "libc",
"once_cell", "once_cell",
"socket2", "socket2",
"tracing", "tracing",
"windows-sys 0.60.2", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@ -1630,35 +1623,15 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.9.4" version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
dependencies = [
"rand_chacha",
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
dependencies = [ dependencies = [
"chacha20", "chacha20",
"getrandom 0.4.3", "getrandom 0.4.3",
"rand_core 0.10.1", "rand_core 0.10.1",
] ]
[[package]]
name = "rand_chacha"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.5",
]
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.6.4" version = "0.6.4"
@ -1668,21 +1641,21 @@ dependencies = [
"getrandom 0.2.17", "getrandom 0.2.17",
] ]
[[package]]
name = "rand_core"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.10.1" version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "rand_pcg"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
dependencies = [
"rand_core 0.10.1",
]
[[package]] [[package]]
name = "rcgen" name = "rcgen"
version = "0.14.8" version = "0.14.8"
@ -1736,9 +1709,9 @@ dependencies = [
[[package]] [[package]]
name = "rustc-hash" name = "rustc-hash"
version = "2.1.2" version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
@ -1773,9 +1746,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.23.41" version = "0.23.42"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [ dependencies = [
"log", "log",
"once_cell", "once_cell",
@ -1788,9 +1761,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls-pki-types" name = "rustls-pki-types"
version = "1.14.1" version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
dependencies = [ dependencies = [
"web-time", "web-time",
"zeroize", "zeroize",
@ -1809,9 +1782,9 @@ dependencies = [
[[package]] [[package]]
name = "rustversion" name = "rustversion"
version = "1.0.22" version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
@ -1928,9 +1901,9 @@ dependencies = [
[[package]] [[package]]
name = "socket2" name = "socket2"
version = "0.6.4" version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.61.2", "windows-sys 0.61.2",
@ -2067,9 +2040,9 @@ dependencies = [
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.51" version = "0.3.53"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
dependencies = [ dependencies = [
"deranged", "deranged",
"num-conv", "num-conv",
@ -2087,9 +2060,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]] [[package]]
name = "time-macros" name = "time-macros"
version = "0.2.30" version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
dependencies = [ dependencies = [
"num-conv", "num-conv",
"time-core", "time-core",
@ -2107,9 +2080,9 @@ dependencies = [
[[package]] [[package]]
name = "tinyvec" name = "tinyvec"
version = "1.11.0" version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f"
dependencies = [ dependencies = [
"tinyvec_macros", "tinyvec_macros",
] ]
@ -2165,7 +2138,7 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
"postgres-protocol", "postgres-protocol",
"postgres-types", "postgres-types",
"rand 0.10.1", "rand",
"socket2", "socket2",
"tokio", "tokio",
"tokio-util", "tokio-util",
@ -2390,12 +2363,12 @@ dependencies = [
[[package]] [[package]]
name = "wasm-encoder" name = "wasm-encoder"
version = "0.252.0" version = "0.253.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8185ae345fa5687c054626ff9a50e7089797a343d9904d1dc9820eb4c4d3196f" checksum = "59972d6cd272259de647b7c1f1912e45e289c75ffd4be04e10695507cd7e1b59"
dependencies = [ dependencies = [
"leb128fmt", "leb128fmt",
"wasmparser 0.252.0", "wasmparser 0.253.0",
] ]
[[package]] [[package]]
@ -2413,9 +2386,9 @@ dependencies = [
[[package]] [[package]]
name = "wasmparser" name = "wasmparser"
version = "0.252.0" version = "0.253.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3eb099dcadcde5be9eef55e3a337128efd4e44b4c93122487e4d2e4e1c6627c" checksum = "19db11f87d2486580e1e8b6f494c54df7e0566b87d0b599db843c24019667339"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"indexmap", "indexmap",
@ -2634,22 +2607,22 @@ dependencies = [
[[package]] [[package]]
name = "wast" name = "wast"
version = "252.0.0" version = "253.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942a3449d6a593fccc111a6241c8df52bda168af30e40bf9580d4394d7374c65" checksum = "d3264542f8965c5d84fb1085d924bfba9a6314bb228eff13a2de14d7627664d0"
dependencies = [ dependencies = [
"bumpalo", "bumpalo",
"leb128fmt", "leb128fmt",
"memchr", "memchr",
"unicode-width", "unicode-width",
"wasm-encoder 0.252.0", "wasm-encoder 0.253.0",
] ]
[[package]] [[package]]
name = "wat" name = "wat"
version = "1.252.0" version = "1.253.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c72a4ba7088f7bac94cf516e49882bdf97068904a563768cf249efc839ec42cb" checksum = "4bfc5ce906144200c972ec617470aa35bd847472e170b26dde3e80541c674055"
dependencies = [ dependencies = [
"wast", "wast",
] ]
@ -2726,16 +2699,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [ dependencies = [
"windows-targets 0.52.6", "windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
] ]
[[package]] [[package]]
@ -2753,31 +2717,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [ dependencies = [
"windows_aarch64_gnullvm 0.52.6", "windows_aarch64_gnullvm",
"windows_aarch64_msvc 0.52.6", "windows_aarch64_msvc",
"windows_i686_gnu 0.52.6", "windows_i686_gnu",
"windows_i686_gnullvm 0.52.6", "windows_i686_gnullvm",
"windows_i686_msvc 0.52.6", "windows_i686_msvc",
"windows_x86_64_gnu 0.52.6", "windows_x86_64_gnu",
"windows_x86_64_gnullvm 0.52.6", "windows_x86_64_gnullvm",
"windows_x86_64_msvc 0.52.6", "windows_x86_64_msvc",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
] ]
[[package]] [[package]]
@ -2786,96 +2733,48 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]] [[package]]
name = "windows_aarch64_msvc" name = "windows_aarch64_msvc"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]] [[package]]
name = "windows_i686_gnu" name = "windows_i686_gnu"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]] [[package]]
name = "windows_i686_gnullvm" name = "windows_i686_gnullvm"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]] [[package]]
name = "windows_i686_msvc" name = "windows_i686_msvc"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]] [[package]]
name = "windows_x86_64_gnu" name = "windows_x86_64_gnu"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]] [[package]]
name = "windows_x86_64_gnullvm" name = "windows_x86_64_gnullvm"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]] [[package]]
name = "windows_x86_64_msvc" name = "windows_x86_64_msvc"
version = "0.52.6" version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]] [[package]]
name = "wit-bindgen" name = "wit-bindgen"
version = "0.57.1" version = "0.57.1"
@ -2958,26 +2857,6 @@ dependencies = [
"synstructure", "synstructure",
] ]
[[package]]
name = "zerocopy"
version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "zerofrom" name = "zerofrom"
version = "0.1.8" version = "0.1.8"
@ -3040,6 +2919,6 @@ dependencies = [
[[package]] [[package]]
name = "zmij" name = "zmij"
version = "1.0.21" version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

View file

@ -1,22 +1,22 @@
[workspace] [workspace]
resolver = "2" resolver = "2"
members = [ members = [
"crates/disasmer-cli", "crates/clusterflux-cli",
"crates/disasmer-control", "crates/clusterflux-control",
"crates/disasmer-coordinator", "crates/clusterflux-coordinator",
"crates/disasmer-core", "crates/clusterflux-core",
"crates/disasmer-dap", "crates/clusterflux-dap",
"crates/disasmer-macros", "crates/clusterflux-macros",
"crates/disasmer-node", "crates/clusterflux-node",
"crates/disasmer-sdk", "crates/clusterflux-sdk",
"crates/disasmer-wasm-runtime", "crates/clusterflux-wasm-runtime",
"examples/launch-build-demo", "examples/launch-build-demo",
] ]
[workspace.package] [workspace.package]
edition = "2021" edition = "2021"
license = "Apache-2.0 OR MIT" license = "Apache-2.0 OR MIT"
repository = "https://git.michelpaulissen.com/michel/disasmer" repository = "https://git.michelpaulissen.com/michel/clusterflux"
[workspace.dependencies] [workspace.dependencies]
anyhow = "1.0" anyhow = "1.0"

View file

@ -1,21 +0,0 @@
{
"kind": "disasmer-filtered-public-tree",
"source_commit": "20b59dc46b72103c2f8a516c692b5cc3d54fab19",
"release_name": "dryrun-20b59dc46b72",
"filtered_out": [
"private/**",
"experiments/**",
".git",
"target",
"git-ignored source paths",
"root/*.md except README.md and SECURITY.md",
"**/.disasmer/**",
".forgejo/**"
],
"public_export": {
"host_neutral": true,
"include_forgejo_workflows": false
},
"forgejo_host": "git.michelpaulissen.com",
"default_hosted_coordinator_endpoint": "https://disasmer.michelpaulissen.com"
}

View file

@ -1,6 +1,6 @@
MIT License MIT License
Copyright (c) 2026 Disasmer contributors Copyright (c) 2026 Clusterflux contributors
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

501
README.md
View file

@ -1,421 +1,80 @@
# Disasmer # Clusterflux
Disasmer is a distributed Wasm runtime for source-debuggable, local-first execution. The flagship workflow is a build written as Rust source code: one virtual process, many virtual threads/tasks, ordinary debugger controls, attached user nodes, and explicit artifact handling. Clusterflux runs a Rust-defined workflow as one distributed virtual process. A
coordinator hosts the async main, while attached nodes execute Wasm tasks,
The MVP shape is: rootless containers, and native commands. Tasks exchange canonical values and
typed handles instead of sharing host memory.
```text
examples/launch-build-demo/envs/linux/Containerfile Start with [Getting started](docs/getting-started.md). It takes you through
examples/launch-build-demo/src/build.rs authentication, project setup, node enrollment, a run, debugging, task restart,
``` and artifact download.
```rust ## What you get
use disasmer::env;
- One virtual process with distinct task instances and restart attempts.
let linux = env!("linux"); - Bundle-declared environments resolved by digest.
``` - Native work only on nodes you attach.
- Metadata-first artifacts whose bytes remain on retaining nodes by default.
`envs/<name>/Containerfile` or `envs/<name>/Dockerfile` defines an environment named `<name>`. Source code references that environment by logical name with `env!("name")`, not by runner label or machine name. - VS Code debugging backed by coordinator task and attempt snapshots.
- Full and partial Debug Epochs with explicit consistency status.
Bundle metadata is inspectable before launch: - Human Authentik sessions plus scoped public-key identities for agents and nodes.
- A public coordinator, node runtime, CLI, SDK, and DAP adapter for self-hosting.
```bash
disasmer bundle inspect --project examples/launch-build-demo ## Install from this checkout
```
~~~bash
The inspection output includes discovered environments, selected input digests, default source-provider choices, and the bundle identity. Container images are not embedded by default. cargo install --path crates/clusterflux-cli --bin clusterflux
cargo install --path crates/clusterflux-node --bin clusterflux-node
The development example lives in `examples/launch-build-demo`. Its coordinator cargo install --path crates/clusterflux-coordinator --bin clusterflux-coordinator
main prepares a node-local source snapshot, compiles a real static executable cargo install --path crates/clusterflux-dap --bin clusterflux-debug-dap
with `cc` in the declared rootless-Podman environment, flushes the output file, ~~~
and passes its artifact handle to a downstream deterministic packaging task.
Rootless Podman is required on Linux nodes that build or run a declared
Canonical public documentation covers the [architecture](docs/architecture.md), Containerfile environment. Install VS Code when you want the graphical debug
[security model](docs/security.md), [task ABI](docs/task-abi.md), workflow.
[artifact semantics](docs/artifacts.md), [debugging](docs/debugging.md), and
[self-hosting](docs/self-hosting.md). ## First run
## Quickstart For the hosted service:
Prerequisites for the local MVP path are a Rust toolchain, Node.js for smoke ~~~bash
scripts, rootless Podman for Linux environment materialization, and VS Code when clusterflux login --browser
debugging through the extension. clusterflux project init --new-project my-project --name "My Project" --yes
clusterflux node enroll --project-id my-project --json
Build the public workspace and install the local command binaries: ~~~
```bash Use the returned short-lived enrollment grant once with "clusterflux node
cargo build --workspace attach", then start "clusterflux-node --worker" from the project directory. See
cargo install --path crates/disasmer-cli --bin disasmer [Nodes](docs/nodes.md) for the complete sequence.
cargo install --path crates/disasmer-node --bin disasmer-node
cargo install --path crates/disasmer-coordinator --bin disasmer-coordinator Choose an entrypoint and run it:
cargo install --path crates/disasmer-dap --bin disasmer-debug-dap
``` ~~~bash
clusterflux bundle inspect --project examples/launch-build-demo
Install or run the VS Code extension from this checkout: clusterflux run --project examples/launch-build-demo build
~~~
```bash
code --extensionDevelopmentPath "$(pwd)/vscode-extension" examples/launch-build-demo Inspect the result:
```
~~~bash
For a persistent local install, copy or symlink `vscode-extension` into the VS clusterflux process status
Code user extension directory under a versioned folder such as clusterflux task list
`disasmer.disasmer-vscode-0.1.0`, then restart VS Code. clusterflux logs
clusterflux artifact list
Inspect and run the flagship project: clusterflux artifact download app.txt --to ./app.txt
~~~
```bash
disasmer bundle inspect --project examples/launch-build-demo To run your own coordinator instead, follow [Self-hosting](docs/self-hosting.md).
disasmer run --local --project examples/launch-build-demo build The hosted website is not required for self-hosted projects.
```
## Documentation
For explicit process-boundary inspection, run the local services yourself:
- [Getting started](docs/getting-started.md)
```bash - [Architecture](docs/architecture.md)
disasmer-coordinator --listen 127.0.0.1:7999 --allow-local-trusted-loopback - [Nodes](docs/nodes.md)
disasmer node attach --coordinator 127.0.0.1:7999 - [Environments](docs/environments.md)
disasmer run --local --coordinator 127.0.0.1:7999 --project examples/launch-build-demo build - [Artifacts](docs/artifacts.md)
``` - [Debugging](docs/debugging.md)
- [Task ABI](docs/task-abi.md)
`disasmer run --local` starts a loopback coordinator and user-attached local - [Self-hosting](docs/self-hosting.md)
node for the run when no coordinator address is supplied. `disasmer run [entry]` - [Security](docs/security.md)
selects an entrypoint such as `build`; `--project` overrides the project
directory. When the CLI has a hosted login and no local override is selected,
`disasmer run` uses the hosted coordinator. Use `--local` or
`--coordinator <host:port>` to force local coordinator mode. The
`--allow-local-trusted-loopback` switch is an explicit single-machine development
compatibility mode: it is rejected on non-loopback listeners and must not be
used for a shared or remotely reachable coordinator.
The MVP keeps bundles inline in the existing 1 MiB control frame. The CLI
therefore supports raw Wasm modules up to approximately 696 KiB (712704 bytes)
after reserving space for the authenticated request and task metadata. It builds,
resolves, validates, and checks this boundary before creating the virtual
process. Larger bundles fail without leaving an active process; a larger-bundle
transport is explicitly post-MVP.
In VS Code, open `examples/launch-build-demo`, start the `Disasmer: Launch
Virtual Process` configuration or press F5, inspect the Disasmer nodes,
processes, logs, artifacts, and inspector views, and use the ordinary debugger
controls for breakpoints, continue, pause, and restart. Pause is
participant-acknowledged; source stepping still fails explicitly instead of
simulating success. Restarting a terminal task rebuilds the bundle: a compatible
edit launches only that task under a fresh instance ID while preserving its
validated arguments, handles, environment, and clean VFS boundary. An active
task cannot be silently replaced, and an ABI-incompatible edit clearly requires
a whole-process restart. Artifact download
behavior is exercised by `node scripts/artifact-download-smoke.js`; final export
is explicit and should go through an attached receiver node or user-provided
storage integration rather than hidden coordinator storage.
Cleanup for the local quickstart is ordinary process and artifact cleanup: stop
the coordinator process, stop attached node processes, remove any temporary
artifacts under `target/acceptance/`, and remove the local VS Code extension copy
or symlink if one was installed.
## First-Run Diagnostics
Use these messages as first checks before debugging infrastructure:
- Missing nodes: attach a node with `disasmer node attach --coordinator <host:port>` or check the Disasmer Nodes view.
- Missing environments: add `envs/<name>/Containerfile` or `envs/<name>/Dockerfile`; the VS Code extension highlights unknown `env!("name")` references.
- Quota limits: hosted/community denials are returned before dispatch with a specific community tier reason.
- Unavailable artifacts: downloads fail before showing a link when retention, size, authorization, quota, or connectivity makes streaming impossible.
- Auth failures: browser login uses the device/browser flow, while agents and nodes use public-key identity and scoped enrollment grants.
- Failed debug freezes: all-stop reports the participant that could not freeze instead of claiming success.
- Source-provider capability gaps: source preparation stays pending until a node reports `SourceGit` or `SourceFilesystem`.
## Repository Shape
The public workspace contains the open-source contract layer:
```text
crates/disasmer-core shared identities, policy traits, scheduling, VFS/artifacts
crates/disasmer-coordinator local coordinator state model
crates/disasmer-dap Debug Adapter Protocol adapter for VS Code/debug clients
crates/disasmer-macros #[disasmer::main] and #[disasmer::task]
crates/disasmer-node node backend interfaces and local node runtime
crates/disasmer-cli CLI command surface
crates/disasmer-sdk user-facing Rust SDK
scripts/verify-public-split.sh
```
Hosted-only policy code lives under `private/**`. The public split is verified by copying the repo without `private/**` and running the public workspace tests.
Deployment names and authority names describe different things. The
**standalone Core coordinator** is the open-source self-hosted server. The
**Hosted coordinator** is the managed deployment that adds hosted policy around
Core. Within either deployment, **Client**, **Node**, **Identity**, and
**Operator** name authority lanes. “Public” and “private” describe source or
release visibility; they do not grant authority.
## Hosted Coordinator
Public CLI binaries default to the hosted coordinator at
`https://disasmer.michelpaulissen.com` unless local/self-hosted mode is
selected. The CLI asks the hosted coordinator to create a state-, nonce-, and
PKCE-bound login transaction, opens the returned Authentik authorization URL,
and polls with an opaque transaction secret. Authentik returns to the hosted
`/auth/callback`; provider codes and identity claims never pass through the CLI.
The hosted website for the MVP is deliberately barebones HTML with no CSS;
layout and UX polish are later work.
The hosted coordinator combines private hosted policy code with the open-source
Core coordinator. Client traffic uses authenticated user sessions, Identity
traffic handles browser/OIDC login, Node traffic is signed by enrolled workers,
and privileged Operator actions require a separate private credential. It does
not give community users arbitrary hosted native commands or hosted containers.
Real work runs on attached nodes.
Self-hosted users do not need the hosted website. They can run the public
coordinator and public node runtime from this repository, attach their own
nodes, and use the CLI/API for normal project, process, log, artifact, debug,
quota, and admin operations.
The standalone Core coordinator uses strict Client authority by default. A
self-hosted operator supplies one scoped bootstrap session through protected
service configuration:
```bash
DISASMER_SELF_HOSTED_SESSION_SECRET="$SELF_HOSTED_SESSION_SECRET" \
DISASMER_SELF_HOSTED_TENANT=my-team \
DISASMER_SELF_HOSTED_PROJECT=my-project \
DISASMER_SELF_HOSTED_USER=me \
disasmer-coordinator --listen 127.0.0.1:7999
```
From the project directory, connect the CLI without placing the secret in a
process argument:
```bash
printf '%s\n' "$SELF_HOSTED_SESSION_SECRET" | \
disasmer auth connect-self-hosted \
--coordinator 127.0.0.1:7999 \
--tenant my-team --project-id my-project --user me \
--session-secret-stdin
```
The CLI verifies the scoped session before writing `.disasmer/session.json`; on
Unix the file is mode `0600`. Enrollment-grant creation and ordinary Client
operations then use this authenticated session. Signed Node and Agent requests
remain separate authority lanes.
The native Core transport is plaintext and therefore refuses non-loopback
listeners. For administration from another machine, create an authenticated SSH
tunnel to the coordinator loopback port and connect the CLI to the tunnel's
local `127.0.0.1` endpoint. Do not expose port 7999 directly.
Release dry-run mechanics, Forgejo publication, filtered repository preparation,
deployment evidence, and public-release e2e commands are source-side release
operations, not the product quickstart. They live in
`public_release_dryrun.md`.
## Coordinator State
The coordinator separates durable project and identity state from live runtime state. Tenants, users, projects, node identities, credentials, source-provider configuration, durable service policy records, and explicit project permissions can be stored in Postgres. Active virtual processes, live virtual threads, scheduler state, debug epochs, ephemeral VFS manifests, and transient artifact locations remain in coordinator memory for the MVP and are not represented in the Postgres schema.
Process lifecycle is independent of task count. `disasmer process list` and
`disasmer process status` read the live virtual-process registry directly.
`disasmer process cancel --yes` requests cooperative cancellation and leaves the
process visible as `cancelling` until its runtime exits. `disasmer process abort
--yes` is the explicit forced operation: it terminates the coordinator-side
process identity, signals active node work to stop, and immediately releases the
single-process slot. VS Code uses the same live registry for its sidebar and
offers Attach, Restart, Cancel, or Abort when F5 finds an existing process.
The local workspace and the hosted **Coordinator Project** are distinct. A
Coordinator Project is the identity, authorization, quota, and one-active-process
boundary; a local workspace is the source/bundle launch target. When another
workspace launch is occupying the same Coordinator Project, VS Code avoids
blindly attaching with the wrong source and offers different-project actions.
## Local-First Builds
Disasmer is designed so local source checkouts and large outputs stay node-local unless user code or policy explicitly moves bytes.
`flush()` publishes metadata and visibility information. It makes produced artifacts visible to downstream tasks without implying durable coordinator storage.
`sync()` is explicit. It may move bytes to another node or user-provided storage according to code or configured policy.
User-provided storage/export integrations are ordinary project code or external
commands, such as publishing through a CLI. Disasmer records metadata and
coordinates capability, scheduling, and transfer decisions; it does not provide
or manage an explicit artifact-store feature as part of the MVP.
Artifacts are best-effort retained on nodes by default. If unsynced node-local bytes are garbage collected or the retaining node is lost, the artifact becomes unavailable instead of silently recovering from coordinator storage.
Artifact downloads are created only when current retention, size, authorization, and community tier accounting allow it. Download links are scoped to the tenant, project, process, artifact, actor, and policy context, expire after a bounded TTL, can be revoked, and streaming usage is charged before and during transfer.
On Linux nodes, Containerfile and Dockerfile environments are materialized with rootless user Podman. The node runtime has a concrete runner path for `podman build` followed by `podman run`; tests use a recording runner, while real attached nodes can use the `std::process::Command` runner. A local build from an attached local checkout uses a node-local bind mount into the selected environment; the coordinator does not create a full-repo tarball or route compiler file reads. Command output is associated with the virtual thread that started it and can be staged into the VFS artifact namespace. `node scripts/wasmtime-node-smoke.js` builds the launch example for `wasm32-unknown-unknown`, runs its exported task through the node Wasmtime runtime, and verifies a Wasm task can invoke a native command through the versioned `disasmer.command_run_v1` host capability without moving command execution into the hosted coordinator.
## Nodes And Trust
Users attach their own nodes for real work. The hosted coordinator is a control plane for identity, rendezvous, scheduling metadata, debug sessions, logs, artifact metadata, and operator state. The community tier does not provide arbitrary hosted native commands or hosted containers.
Agent and worker automation should use enrolled public-key identity. User OAuth or browser session tokens are not task credentials and should not be passed to nodes.
```bash
disasmer login
disasmer login --browser
disasmer agent enroll --public-key <agent-public-key>
DISASMER_AGENT_PRIVATE_KEY=<agent-private-key> disasmer run --non-interactive build
disasmer run --local --non-interactive build
disasmer node attach --enrollment-grant <grant> --public-key <node-public-key>
disasmer-node --coordinator https://disasmer.michelpaulissen.com --tenant <tenant> --project-id <project> --node <node-id> --public-key <node-public-key> --enrollment-grant <grant> --worker --emit-ready
```
`disasmer login` uses a short-lived human device flow and does not ask users to paste long-lived secrets. `disasmer login --browser` opens the server-provided authorization URL, polls the hosted transaction, and stores only normalized CLI-session metadata in `.disasmer/session.json`; provider authorization codes, OAuth tokens, and identity claims are not accepted or persisted by the CLI. `disasmer login --browser --plan` is the diagnostic JSON login plan path. `--non-interactive` never opens a browser; `disasmer login --browser --non-interactive` and an unauthenticated implicit hosted `disasmer run --non-interactive` fail with an authentication-category report and next actions instead of prompting or guessing. Node attach exchanges a short-lived enrollment grant for a scoped long-lived node identity and then exits; a long-lived `disasmer-node --worker` process must be running for coordinator-side launches and DAP live-services debugging to place real command/container work on that node.
Hosted agent public keys are project-scoped records: a signed-in user can
register, list, rotate, and revoke an agent key without giving the agent browser
or OAuth credentials. An enrolled agent can run CLI commands non-interactively
with `DISASMER_AGENT_PRIVATE_KEY`; the CLI derives the registered public key,
signs workflow requests, and records the public-key fingerprint in its run plan
instead of starting a browser flow. `DISASMER_AGENT_PUBLIC_KEY` may identify or
cross-check the registered key, but cannot authenticate without the private key.
`disasmer node attach` auto-detects OS, architecture, command/container,
VFS/artifact, source-provider, and environment capabilities. `--cap <name>` is
available as an override for unusual local setups, not as required normal
configuration.
For local process-boundary testing, the public workspace includes a TCP JSON-line coordinator service and a node runtime binary:
```bash
cargo run -p disasmer-coordinator -- --listen 127.0.0.1:0
cargo run -p disasmer-node -- --coordinator <host:port> --worker \
--project-root examples/launch-build-demo
```
`scripts/real-flagship-harness.js` starts those as separate long-lived processes. The real `build` Wasm entrypoint spawns its named Wasm tasks through the coordinator; the Linux task selects `env!("linux")`, runs its command through rootless Podman against the node-local checkout, publishes content-addressed bytes, and reports task and artifact metadata. The download and export smokes reuse that workflow and then retrieve the retained bytes through an authorized reverse stream. `scripts/wasmtime-assignment-smoke.js` separately proves cooperative cancellation observed by task code and forced abort of uncooperative Wasm/native work.
For self-hosted local clouds, trusted teams, or VPN deployments, run the public
coordinator on the chosen listen address and attach team-owned nodes with their
public keys. This path uses the open-source coordinator, scheduler, node
heartbeat, task metadata, retained-node downloads, and direct node-to-node export
planning without private hosted OIDC or community tier policy modules.
`node scripts/self-hosted-coordinator-smoke.js` exercises that public path with
two trusted Linux nodes.
## Debugging
The VS Code extension contributes the normal `disasmer` debug type, refreshes bundle metadata before launch, and starts the DAP adapter. Bundle probe ownership comes from actual `#[disasmer::main]` and `#[disasmer::task]` declarations and their declared names, not function-name heuristics. In the verified local-services path, an executing Wasm probe reports a breakpoint hit, every active participant acknowledges the freeze, and only then does DAP emit an all-stop `stopped` event. Runtime child tasks appear as dynamic virtual threads. Continue requests a real coordinator resume. Stack, task-argument, live task-handle, native-command status, and output data come from node acknowledgements; values the runtime did not report are labeled unavailable instead of being inferred or fabricated. Each dispatched task captures a clean entry checkpoint containing its TaskSpec, bundle, ABI, arguments, environment digest, VFS epoch, and required-artifact manifest. After a terminal failure, `restartFrame` rebuilds the current source and relaunches only a boundary-compatible task with a fresh instance ID; an ABI-incompatible edit requires a whole virtual-process restart without mutating the existing process. Active-task restart and arbitrary source stepping fail explicitly. Disasmer-specific side views expose nodes, virtual processes, logs, artifacts, and inspector state alongside the normal debugger UI.
Live-services launch now uses the same bundle builder, Wasm entrypoint TaskSpec,
probe plan, worker placement, Debug Epoch observation, continuation, and restart
code as local-services; only ownership of the already-attached worker differs.
That path remains labeled integration-verified until the final revision-bound
hosted E2E is recorded. F5 and DAP acceptance therefore continue to exercise
local services as well as the strict hosted deployment rather than treating
deployment configuration alone as live proof.
The built-in operator panel preview is rendered by the coordinator from live
process, task, log, and artifact metadata. It uses typed widgets only, keeps
program UI events scoped and rate-limited, and keeps control-plane actions such
as debug, cancel, restart, and artifact download available when program UI events
are disabled.
## Windows
Windows node support uses the same node protocol and capability model as Linux. MVP Windows command execution is user-attached development execution, labeled `windows-command-dev`. Production-grade managed Windows sandboxing is behind an explicit backend stub until it is implemented and validated independently. When the acceptance report shows `windows_validation: "not-run"`, Windows support is best-effort and unvalidated for that release; the public smoke only verifies protocol, placement, metadata, and download behavior for a Windows-capable node identity.
Real Windows validation runs through the manual Forgejo workflow
`.forgejo/workflows/windows-validation.yml` when the intermittent Windows runner is
online. That workflow records `windows_validation: "forgejo-windows-runner"` in
the acceptance report, runs `disasmer node attach` against a coordinator, starts a
Windows node process, executes a simple `windows-command-dev` command, publishes
artifact metadata, checks Windows placement, and verifies the debugger can show a
Windows virtual thread.
## Source Providers
Git is an optional source-provider module included by default. The VFS core depends on source-provider manifests and snapshot handles, not on Git internals. Non-Git source providers can implement the public source-provider interface and run snapshot preparation as node work instead of requiring coordinator filesystem access.
Source preparation is scheduled as node work. The coordinator can return a
pending source-preparation status while waiting for any capable node, then assign
the work after a node reports `SourceGit` or `SourceFilesystem` capability; it
does not need checkout or provider access itself.
Source-provider manifests are validated as public input and must declare a
local-first transfer policy: local source bytes stay node-local, the coordinator
does not receive source bytes by default, and remote preparation uses required
content or explicit snapshot chunks rather than a full-repo tarball.
## Verification
Run the public workspace checks:
```bash
scripts/acceptance-public.sh
```
Run the CLI-first non-e2e gate before any final public-release e2e attempt:
```bash
scripts/acceptance-cli-first.sh
```
This gate composes the CLI/API, local-services, self-hosted, hosted-compat,
debugger, artifact, safety, and public dry-run contract checks without invoking
`public-release-dryrun-e2e.js` or the final evidence verifier. Leave
`DISASMER_PUBLIC_RELEASE_DRYRUN_E2E` and `DISASMER_PUBLIC_RELEASE_DRYRUN_FINAL`
unset until the CLI-first criteria are otherwise implemented to pass.
Or run the individual public checks:
```bash
cargo test --workspace
node scripts/acceptance-report-smoke.js
node scripts/public-private-boundary-smoke.js
node scripts/release-blocker-smoke.js
node scripts/docs-smoke.js
node scripts/public-release-dryrun-contract-smoke.js
node scripts/wasmtime-node-smoke.js
node scripts/podman-backend-smoke.js
node scripts/vscode-extension-smoke.js
node scripts/vscode-f5-smoke.js
node scripts/node-attach-smoke.js
node scripts/wasmtime-assignment-smoke.js
node scripts/cli-local-run-smoke.js
node scripts/artifact-download-smoke.js
node scripts/artifact-export-smoke.js
node scripts/operator-panel-smoke.js
node scripts/source-preparation-smoke.js
node scripts/scheduler-placement-smoke.js
node scripts/windows-best-effort-smoke.js
node scripts/quic-smoke.js
node scripts/flagship-demo-smoke.js
node scripts/dap-smoke.js
node scripts/prepare-public-release-dryrun.js
scripts/verify-public-split.sh
```
When the Forgejo Windows runner is online, run the manual `Windows validation`
workflow as the release Windows gate. The local equivalent on a Windows machine is:
```powershell
$env:DISASMER_WINDOWS_VALIDATION = "forgejo-windows-runner"
node scripts/acceptance-report.js windows
cargo fmt --all --check
cargo test -p disasmer-node windows_backend_is_labeled_user_attached_dev_execution
node scripts/windows-runner-smoke.js
```
Run private hosted policy checks separately:
```bash
scripts/acceptance-private.sh
```
The private hosted gate includes `hosted-client-compat-smoke.js`, which starts
the hosted dry-run service locally and verifies that the released Client CLI and
signed Node runtime can attach, launch work through coordinator task assignment,
and publish debug/log/artifact metadata through the hosted Client protocol
boundary.
Both acceptance scripts write an environment report under `target/acceptance/`
with the commit SHA, toolchain versions, OS/kernel, Podman/Postgres discovery,
browser/VS Code harness metadata, and Windows-validation status. The report
records Podman as `available` only when rootless Podman is discoverable; if it is
missing or not rootless, Linux Podman backend behavior is marked `incomplete`
with the reason and the Podman backend smoke blocks acceptance with the same
incomplete reason.

View file

@ -1,26 +0,0 @@
# Security policy
Disasmer is pre-release software that executes untrusted Wasm and can delegate
explicit host capabilities to attached nodes. Security reports are welcome even
when the affected behavior has not shipped in a numbered release.
## Supported versions
Until the first public release, only the current `main` branch is supported.
After releases begin, this table will identify supported release lines.
## Reporting a vulnerability
Do not open a public issue for a suspected vulnerability. Email
`ops@michelpaulissen.com` with:
- the affected commit or release;
- the relevant deployment mode (hosted, self-hosted, or attached node);
- reproduction steps or a minimal proof of concept;
- the security boundary or tenant scope that was crossed; and
- any known mitigations.
Please avoid accessing other users' data, disrupting hosted services, or
retaining secrets while testing. We will acknowledge receipt, coordinate a fix
and disclosure, and credit reporters who want to be named. No response-time or
bounty commitment is made before the first public release.

View file

@ -1,20 +1,20 @@
[package] [package]
name = "disasmer-cli" name = "clusterflux-cli"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
license.workspace = true license.workspace = true
repository.workspace = true repository.workspace = true
[[bin]] [[bin]]
name = "disasmer" name = "clusterflux"
path = "src/main.rs" path = "src/main.rs"
[dependencies] [dependencies]
anyhow.workspace = true anyhow.workspace = true
base64.workspace = true base64.workspace = true
clap.workspace = true clap.workspace = true
disasmer-core = { path = "../disasmer-core" } clusterflux-core = { path = "../clusterflux-core" }
disasmer-control = { path = "../disasmer-control" } clusterflux-control = { path = "../clusterflux-control" }
serde.workspace = true serde.workspace = true
serde_json.workspace = true serde_json.workspace = true
sha2.workspace = true sha2.workspace = true

View file

@ -1,7 +1,7 @@
use std::path::PathBuf; use std::path::PathBuf;
use anyhow::Result; use anyhow::Result;
use disasmer_core::admin_request_proof; use clusterflux_core::admin_request_proof;
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::client::JsonLineSession; use crate::client::JsonLineSession;
@ -94,31 +94,31 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
.unwrap_or(json!(false)), .unwrap_or(json!(false)),
"project_init": project_init, "project_init": project_init,
"admin_surfaces": { "admin_surfaces": {
"coordinator": "disasmer-coordinator", "coordinator": "clusterflux-coordinator",
"project": "disasmer project init/status/list/select", "project": "clusterflux project init/status/list/select",
"node": "disasmer node enroll/list/status/revoke", "node": "clusterflux node enroll/list/status/revoke",
"process": "disasmer run/process status/process restart/process cancel", "process": "clusterflux run/process status/process restart/process cancel",
"logs": "disasmer logs", "logs": "clusterflux logs",
"artifacts": "disasmer artifact list/download/export", "artifacts": "clusterflux artifact list/download/export",
"quota": "disasmer quota status", "quota": "clusterflux quota status",
"policy": "disasmer admin status/suspend-tenant", "policy": "clusterflux admin status/suspend-tenant",
}, },
"bootstrap_sequence": [ "bootstrap_sequence": [
{ {
"step": "start_self_hosted_coordinator", "step": "start_self_hosted_coordinator",
"command": "disasmer-coordinator --listen 127.0.0.1:0", "command": "clusterflux-coordinator --listen 127.0.0.1:0",
"private_website_required": false, "private_website_required": false,
}, },
{ {
"step": "create_or_link_project", "step": "create_or_link_project",
"command": "disasmer project init --yes", "command": "clusterflux project init --yes",
"completed": true, "completed": true,
"private_website_required": false, "private_website_required": false,
}, },
{ {
"step": "create_node_enrollment_grant", "step": "create_node_enrollment_grant",
"command": format!( "command": format!(
"disasmer node enroll --coordinator {coordinator} --tenant {} --project-id {}", "clusterflux node enroll --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project tenant, project
), ),
"private_website_required": false, "private_website_required": false,
@ -126,7 +126,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
{ {
"step": "attach_worker_node", "step": "attach_worker_node",
"command": format!( "command": format!(
"disasmer node attach --coordinator {coordinator} --tenant {} --project-id {} --worker", "clusterflux node attach --coordinator {coordinator} --tenant {} --project-id {} --worker",
tenant, project tenant, project
), ),
"private_website_required": false, "private_website_required": false,
@ -134,7 +134,7 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
{ {
"step": "run_process", "step": "run_process",
"command": format!( "command": format!(
"disasmer run --coordinator {coordinator} --tenant {} --project-id {}", "clusterflux run --coordinator {coordinator} --tenant {} --project-id {}",
tenant, project tenant, project
), ),
"private_website_required": false, "private_website_required": false,
@ -142,17 +142,17 @@ pub(crate) fn admin_bootstrap_report(args: AdminBootstrapArgs, cwd: PathBuf) ->
{ {
"step": "inspect_status_logs_artifacts", "step": "inspect_status_logs_artifacts",
"commands": [ "commands": [
"disasmer process status", "clusterflux process status",
"disasmer task list", "clusterflux task list",
"disasmer logs", "clusterflux logs",
"disasmer artifact list", "clusterflux artifact list",
"disasmer quota status", "clusterflux quota status",
], ],
"private_website_required": false, "private_website_required": false,
}, },
{ {
"step": "revoke_access", "step": "revoke_access",
"command": "disasmer admin revoke-node --node <node-id> --yes", "command": "clusterflux admin revoke-node --node <node-id> --yes",
"private_website_required": false, "private_website_required": false,
} }
], ],
@ -173,7 +173,7 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
"actor_user": args.scope.user, "actor_user": args.scope.user,
"target_tenant": tenant, "target_tenant": tenant,
}), }),
"disasmer admin suspend-tenant --yes".to_owned(), "clusterflux admin suspend-tenant --yes".to_owned(),
)); ));
} }
if let Some(coordinator) = &args.scope.coordinator { if let Some(coordinator) = &args.scope.coordinator {
@ -226,11 +226,11 @@ pub(crate) fn admin_suspend_tenant_report(args: AdminSuspendTenantArgs) -> Resul
fn admin_token_for_request(explicit: Option<&str>) -> Result<String> { fn admin_token_for_request(explicit: Option<&str>) -> Result<String> {
explicit explicit
.map(str::to_owned) .map(str::to_owned)
.or_else(|| std::env::var("DISASMER_ADMIN_TOKEN").ok()) .or_else(|| std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok())
.filter(|token| !token.trim().is_empty()) .filter(|token| !token.trim().is_empty())
.ok_or_else(|| { .ok_or_else(|| {
anyhow::anyhow!( anyhow::anyhow!(
"admin command requires --admin-token or DISASMER_ADMIN_TOKEN for coordinator requests" "admin command requires --admin-token or CLUSTERFLUX_ADMIN_TOKEN for coordinator requests"
) )
}) })
} }

View file

@ -1,4 +1,4 @@
use disasmer_core::Digest; use clusterflux_core::Digest;
use serde::Serialize; use serde::Serialize;
use crate::AgentEnrollArgs; use crate::AgentEnrollArgs;

View file

@ -294,7 +294,7 @@ fn download_link_to_path(
.pointer("/link/scoped_token_digest") .pointer("/link/scoped_token_digest")
.cloned() .cloned()
.context("artifact download link did not include a scoped token digest")?; .context("artifact download link did not include a scoped token digest")?;
let expected_digest: disasmer_core::Digest = serde_json::from_value( let expected_digest: clusterflux_core::Digest = serde_json::from_value(
link_response link_response
.pointer("/link/artifact_digest") .pointer("/link/artifact_digest")
.cloned() .cloned()
@ -326,7 +326,7 @@ fn download_link_to_path(
std::fs::create_dir_all(parent) std::fs::create_dir_all(parent)
.with_context(|| format!("failed to create {}", parent.display()))?; .with_context(|| format!("failed to create {}", parent.display()))?;
let mut temporary = tempfile::Builder::new() let mut temporary = tempfile::Builder::new()
.prefix(".disasmer-download-") .prefix(".clusterflux-download-")
.tempfile_in(parent) .tempfile_in(parent)
.with_context(|| { .with_context(|| {
format!( format!(
@ -433,8 +433,8 @@ fn download_link_to_path(
})); }));
} }
let actual_digest_hex = format!("{:x}", hasher.finalize()); let actual_digest_hex = format!("{:x}", hasher.finalize());
let actual_digest = let actual_digest = clusterflux_core::Digest::from_sha256_hex(&actual_digest_hex)
disasmer_core::Digest::from_sha256_hex(&actual_digest_hex).map_err(anyhow::Error::msg)?; .map_err(anyhow::Error::msg)?;
if received_bytes != expected_size || actual_digest != expected_digest { if received_bytes != expected_size || actual_digest != expected_digest {
return Ok(json!({ return Ok(json!({
"status": "download_integrity_failed", "status": "download_integrity_failed",

View file

@ -268,7 +268,7 @@ fn coordinator_auth_status_summary(
"signup_failure_details_exposed": false, "signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message), "machine_error": cli_error_summary(&message),
"error": message, "error": message,
"next_actions": ["disasmer doctor", "check coordinator status"], "next_actions": ["clusterflux doctor", "check coordinator status"],
"coordinator_session_requests": 0, "coordinator_session_requests": 0,
}); });
} }
@ -301,7 +301,7 @@ fn coordinator_auth_status_summary(
"signup_failure_details_exposed": false, "signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message), "machine_error": cli_error_summary(&message),
"error": message, "error": message,
"next_actions": ["disasmer login --browser"], "next_actions": ["clusterflux login --browser"],
"coordinator_session_requests": 0, "coordinator_session_requests": 0,
}); });
} }
@ -321,7 +321,7 @@ fn coordinator_auth_status_summary(
"signup_failure_details_exposed": false, "signup_failure_details_exposed": false,
"machine_error": cli_error_summary(&message), "machine_error": cli_error_summary(&message),
"error": message, "error": message,
"next_actions": ["disasmer doctor", "check coordinator status"], "next_actions": ["clusterflux doctor", "check coordinator status"],
"coordinator_session_requests": session.requests(), "coordinator_session_requests": session.requests(),
}); });
} }
@ -343,7 +343,7 @@ fn coordinator_auth_status_summary(
"signup_failure_details_exposed": false, "signup_failure_details_exposed": false,
"machine_error": cli_error_summary(message), "machine_error": cli_error_summary(message),
"coordinator_response_type": "error", "coordinator_response_type": "error",
"next_actions": ["disasmer doctor", "disasmer login --browser"], "next_actions": ["clusterflux doctor", "clusterflux login --browser"],
"coordinator_session_requests": coordinator_session_requests, "coordinator_session_requests": coordinator_session_requests,
}); });
} }
@ -416,8 +416,8 @@ pub(crate) fn non_interactive_browser_login_report(args: &LoginArgs) -> Value {
"browser login requires an interactive browser, but non-interactive mode is enabled"; "browser login requires an interactive browser, but non-interactive mode is enabled";
let next_actions = vec![ let next_actions = vec![
"rerun without --non-interactive to open the browser", "rerun without --non-interactive to open the browser",
"disasmer login --browser --plan", "clusterflux login --browser --plan",
"use DISASMER_AGENT_PRIVATE_KEY for automation", "use CLUSTERFLUX_AGENT_PRIVATE_KEY for automation",
]; ];
json!({ json!({
"command": "login", "command": "login",
@ -460,11 +460,11 @@ pub(crate) fn auth_state_value(cwd: &Path) -> Result<Value> {
})) }))
} }
CliSession::HumanSession => { CliSession::HumanSession => {
let expires_at = std::env::var("DISASMER_TOKEN_EXPIRES_AT").ok(); let expires_at = std::env::var("CLUSTERFLUX_TOKEN_EXPIRES_AT").ok();
Ok(json!({ Ok(json!({
"kind": "human", "kind": "human",
"authenticated": true, "authenticated": true,
"source": "DISASMER_TOKEN", "source": "CLUSTERFLUX_TOKEN",
"provider_tokens_exposed_to_nodes": false, "provider_tokens_exposed_to_nodes": false,
"expires_at": expires_at, "expires_at": expires_at,
"token_expiry_posture": if expires_at.is_some() { "expires_at" } else { "unknown_env_token" }, "token_expiry_posture": if expires_at.is_some() { "expires_at" } else { "unknown_env_token" },
@ -479,7 +479,7 @@ pub(crate) fn auth_state_value(cwd: &Path) -> Result<Value> {
"kind": "agent_public_key", "kind": "agent_public_key",
"authenticated": true, "authenticated": true,
"agent": agent, "agent": agent,
"source": "DISASMER_AGENT_PRIVATE_KEY", "source": "CLUSTERFLUX_AGENT_PRIVATE_KEY",
"public_key_fingerprint": public_key_fingerprint, "public_key_fingerprint": public_key_fingerprint,
"browser_interaction_required": browser_interaction_required, "browser_interaction_required": browser_interaction_required,
"token_expiry_posture": "not_applicable_public_key", "token_expiry_posture": "not_applicable_public_key",
@ -667,7 +667,7 @@ pub(crate) fn execute_interactive_browser_login(args: LoginArgs) -> Result<Login
}), }),
}; };
eprintln!("Opening Disasmer browser login: {authorization_url}"); eprintln!("Opening Clusterflux browser login: {authorization_url}");
eprintln!("Waiting for the hosted login callback to complete."); eprintln!("Waiting for the hosted login callback to complete.");
open_browser(&authorization_url)?; open_browser(&authorization_url)?;
@ -717,7 +717,7 @@ pub(crate) fn print_browser_login_success(report: &LoginCompletionReport) {
} }
fn open_browser(url: &str) -> Result<()> { fn open_browser(url: &str) -> Result<()> {
let mut command = if let Some(command) = std::env::var_os("DISASMER_BROWSER_OPEN_COMMAND") { let mut command = if let Some(command) = std::env::var_os("CLUSTERFLUX_BROWSER_OPEN_COMMAND") {
Command::new(command) Command::new(command)
} else { } else {
platform_browser_command() platform_browser_command()
@ -751,7 +751,7 @@ fn platform_browser_command() -> Command {
} }
fn browser_login_timeout() -> Duration { fn browser_login_timeout() -> Duration {
let seconds = std::env::var("DISASMER_BROWSER_LOGIN_TIMEOUT_SECONDS") let seconds = std::env::var("CLUSTERFLUX_BROWSER_LOGIN_TIMEOUT_SECONDS")
.ok() .ok()
.and_then(|value| value.parse::<u64>().ok()) .and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0) .filter(|seconds| *seconds > 0)

View file

@ -62,7 +62,7 @@ pub(crate) fn session_scope_mismatch_status(fields: &[&str]) -> Value {
"account_state_known": false, "account_state_known": false,
"private_moderation_details_exposed": false, "private_moderation_details_exposed": false,
"signup_failure_details_exposed": false, "signup_failure_details_exposed": false,
"next_actions": ["disasmer login --browser"], "next_actions": ["clusterflux login --browser"],
}) })
} }

View file

@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
use std::process::Command; use std::process::Command;
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use disasmer_core::Digest; use clusterflux_core::Digest;
use serde_json::{json, Value}; use serde_json::{json, Value};
use wasmparser::{Parser, Payload}; use wasmparser::{Parser, Payload};
@ -47,24 +47,24 @@ pub(crate) fn build_report(args: BuildArgs, cwd: PathBuf) -> Result<Value> {
let environment_manifest = serde_json::to_vec(&inspection.metadata.environments)?; let environment_manifest = serde_json::to_vec(&inspection.metadata.environments)?;
append_custom_section( append_custom_section(
&mut wasm.bytes, &mut wasm.bytes,
"disasmer.environments", "clusterflux.environments",
&environment_manifest, &environment_manifest,
); );
let bundle_digest = Digest::sha256(&wasm.bytes); let bundle_digest = Digest::sha256(&wasm.bytes);
let task_descriptors = descriptor_records(&wasm.bytes, "disasmer.tasks")?; let task_descriptors = descriptor_records(&wasm.bytes, "clusterflux.tasks")?;
let entrypoint_descriptors = descriptor_records(&wasm.bytes, "disasmer.entrypoints")?; let entrypoint_descriptors = descriptor_records(&wasm.bytes, "clusterflux.entrypoints")?;
if task_descriptors.is_empty() { if task_descriptors.is_empty() {
bail!( bail!(
"compiled Wasm module contains no #[disasmer::task] descriptors; annotate at least one exported task" "compiled Wasm module contains no #[clusterflux::task] descriptors; annotate at least one exported task"
); );
} }
if entrypoint_descriptors.is_empty() { if entrypoint_descriptors.is_empty() {
bail!( bail!(
"compiled Wasm module contains no #[disasmer::main] descriptors; annotate at least one entrypoint" "compiled Wasm module contains no #[clusterflux::main] descriptors; annotate at least one entrypoint"
); );
} }
let output = args.output.unwrap_or_else(|| { let output = args.output.unwrap_or_else(|| {
inspection.project.join(".disasmer/build").join( inspection.project.join(".clusterflux/build").join(
bundle_digest bundle_digest
.as_str() .as_str()
.trim_start_matches("sha256:") .trim_start_matches("sha256:")
@ -171,7 +171,7 @@ fn compile_project_wasm(project: &Path) -> Result<CompiledWasm> {
.is_some_and(|types| types.iter().any(|kind| kind == "cdylib")) .is_some_and(|types| types.iter().any(|kind| kind == "cdylib"))
}) })
}) })
.context("Disasmer project library must include crate-type = [\"cdylib\"]")?; .context("Clusterflux project library must include crate-type = [\"cdylib\"]")?;
let target_name = target["name"] let target_name = target["name"]
.as_str() .as_str()
.context("cargo target name missing")?; .context("cargo target name missing")?;
@ -272,7 +272,7 @@ fn write_bundle(
)?; )?;
write_json(&debug_path, &json!(inspection.metadata.debug_metadata))?; write_json(&debug_path, &json!(inspection.metadata.debug_metadata))?;
let manifest = json!({ let manifest = json!({
"kind": "disasmer-bundle", "kind": "clusterflux-bundle",
"format_version": 1, "format_version": 1,
"package": wasm.package, "package": wasm.package,
"target": wasm.target, "target": wasm.target,

View file

@ -2,7 +2,7 @@ use std::collections::BTreeSet;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use disasmer_core::{ use clusterflux_core::{
diagnose_environment_references, discover_source_debug_probes, BundleDebugProbe, diagnose_environment_references, discover_source_debug_probes, BundleDebugProbe,
BundleIdentityInputs, BundleMetadata, Digest, EnvironmentReference, ProjectModel, BundleIdentityInputs, BundleMetadata, Digest, EnvironmentReference, ProjectModel,
SelectedInput, SourceProviderKind, SourceProviderManifest, SelectedInput, SourceProviderKind, SourceProviderManifest,
@ -246,7 +246,7 @@ fn source_provider_manifest(kind: &SourceProviderKind) -> SourceProviderManifest
fn environment_diagnostics_for_inputs( fn environment_diagnostics_for_inputs(
project: &Path, project: &Path,
selected_inputs: &[SelectedInput], selected_inputs: &[SelectedInput],
environments: &[disasmer_core::EnvironmentResource], environments: &[clusterflux_core::EnvironmentResource],
) -> Result<Vec<EnvironmentDiagnosticReport>> { ) -> Result<Vec<EnvironmentDiagnosticReport>> {
let mut diagnostics = Vec::new(); let mut diagnostics = Vec::new();
for input in selected_inputs { for input in selected_inputs {
@ -287,7 +287,7 @@ fn discover_debug_probes(
fn pre_schedule_diagnostics( fn pre_schedule_diagnostics(
source_provider_statuses: &[SourceProviderStatus], source_provider_statuses: &[SourceProviderStatus],
environment_diagnostics: &[EnvironmentDiagnosticReport], environment_diagnostics: &[EnvironmentDiagnosticReport],
environments: &[disasmer_core::EnvironmentResource], environments: &[clusterflux_core::EnvironmentResource],
) -> Vec<CliDiagnostic> { ) -> Vec<CliDiagnostic> {
let mut diagnostics = Vec::new(); let mut diagnostics = Vec::new();
diagnostics.extend( diagnostics.extend(
@ -301,7 +301,7 @@ fn pre_schedule_diagnostics(
next_actions: vec![ next_actions: vec![
"create the missing envs/<name>/Containerfile or envs/<name>/Dockerfile" "create the missing envs/<name>/Containerfile or envs/<name>/Dockerfile"
.to_owned(), .to_owned(),
"rerun disasmer inspect".to_owned(), "rerun clusterflux inspect".to_owned(),
], ],
}), }),
); );
@ -326,7 +326,7 @@ fn pre_schedule_diagnostics(
), ),
next_actions: vec![ next_actions: vec![
"choose an available source provider with --source-provider".to_owned(), "choose an available source provider with --source-provider".to_owned(),
"rerun disasmer inspect --json".to_owned(), "rerun clusterflux inspect --json".to_owned(),
], ],
}), }),
); );

View file

@ -1,6 +1,6 @@
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use disasmer_control::{endpoint_identity, endpoint_is_loopback, ControlSession}; use clusterflux_control::{endpoint_identity, endpoint_is_loopback, ControlSession};
use disasmer_core::coordinator_wire_request; use clusterflux_core::coordinator_wire_request;
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::config::StoredCliSession; use crate::config::StoredCliSession;
@ -75,7 +75,7 @@ pub(crate) fn authenticated_or_local_trusted_request(
Ok(local_trusted_request) Ok(local_trusted_request)
} else { } else {
anyhow::bail!( anyhow::bail!(
"no authenticated CLI session matches coordinator {coordinator}; run `disasmer login --browser` from the current project" "no authenticated CLI session matches coordinator {coordinator}; run `clusterflux login --browser` from the current project"
) )
} }
} }

View file

@ -5,7 +5,8 @@ use serde::{Deserialize, Serialize};
use crate::CliScopeArgs; use crate::CliScopeArgs;
pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str = "https://disasmer.michelpaulissen.com"; pub(crate) const DEFAULT_HOSTED_COORDINATOR_ENDPOINT: &str =
"https://clusterflux.michelpaulissen.com";
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct ProjectConfig { pub(crate) struct ProjectConfig {
@ -37,11 +38,11 @@ pub(crate) fn default_hosted_coordinator_endpoint() -> String {
} }
pub(crate) fn project_config_file(project: &Path) -> PathBuf { pub(crate) fn project_config_file(project: &Path) -> PathBuf {
project.join(".disasmer").join("project.json") project.join(".clusterflux").join("project.json")
} }
pub(crate) fn session_config_file(project: &Path) -> PathBuf { pub(crate) fn session_config_file(project: &Path) -> PathBuf {
project.join(".disasmer").join("session.json") project.join(".clusterflux").join("session.json")
} }
pub(crate) fn read_project_config(project: &Path) -> Result<Option<ProjectConfig>> { pub(crate) fn read_project_config(project: &Path) -> Result<Option<ProjectConfig>> {

View file

@ -23,9 +23,9 @@ pub(crate) fn exec_dap(args: DapArgs) -> Result<()> {
let status = Command::new(dap_binary_path()?) let status = Command::new(dap_binary_path()?)
.args(args.args) .args(args.args)
.status() .status()
.context("failed to launch disasmer-debug-dap")?; .context("failed to launch clusterflux-debug-dap")?;
if !status.success() { if !status.success() {
anyhow::bail!("disasmer-debug-dap exited with {status}"); anyhow::bail!("clusterflux-debug-dap exited with {status}");
} }
Ok(()) Ok(())
} }

View file

@ -2,8 +2,8 @@ use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use disasmer_control::ControlSession; use clusterflux_control::ControlSession;
use disasmer_core::{coordinator_wire_request, Capability, NodeCapabilities}; use clusterflux_core::{coordinator_wire_request, Capability, NodeCapabilities};
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::auth::auth_state_value; use crate::auth::auth_state_value;
@ -25,9 +25,9 @@ pub(crate) fn doctor_report(args: DoctorArgs, cwd: PathBuf) -> Result<Value> {
"cargo": command_available("cargo"), "cargo": command_available("cargo"),
"git": command_available("git"), "git": command_available("git"),
"podman": command_available("podman"), "podman": command_available("podman"),
"disasmer-node": command_available("disasmer-node") || sibling_binary("disasmer-node").is_some(), "clusterflux-node": command_available("clusterflux-node") || sibling_binary("clusterflux-node").is_some(),
"disasmer-coordinator": command_available("disasmer-coordinator") || sibling_binary("disasmer-coordinator").is_some(), "clusterflux-coordinator": command_available("clusterflux-coordinator") || sibling_binary("clusterflux-coordinator").is_some(),
"disasmer-debug-dap": command_available("disasmer-debug-dap") || sibling_binary("disasmer-debug-dap").is_some(), "clusterflux-debug-dap": command_available("clusterflux-debug-dap") || sibling_binary("clusterflux-debug-dap").is_some(),
}); });
let node_readiness = NodeCapabilities::detect_current(); let node_readiness = NodeCapabilities::detect_current();
let node_readiness_summary = node_readiness_summary(&node_readiness, &dependencies); let node_readiness_summary = node_readiness_summary(&node_readiness, &dependencies);
@ -42,10 +42,10 @@ pub(crate) fn doctor_report(args: DoctorArgs, cwd: PathBuf) -> Result<Value> {
"node_readiness": node_readiness, "node_readiness": node_readiness,
"node_readiness_summary": node_readiness_summary, "node_readiness_summary": node_readiness_summary,
"next_actions": [ "next_actions": [
"disasmer login --browser", "clusterflux login --browser",
"disasmer project init", "clusterflux project init",
"disasmer node attach", "clusterflux node attach",
"disasmer run" "clusterflux run"
] ]
})) }))
} }
@ -60,7 +60,7 @@ fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value)
.contains(&Capability::SourceFilesystem); .contains(&Capability::SourceFilesystem);
let has_container_backend = capabilities let has_container_backend = capabilities
.environment_backends .environment_backends
.contains(&disasmer_core::EnvironmentBackend::Container); .contains(&clusterflux_core::EnvironmentBackend::Container);
let podman_available = dependencies let podman_available = dependencies
.get("podman") .get("podman")
.and_then(Value::as_bool) .and_then(Value::as_bool)
@ -69,8 +69,8 @@ fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value)
.get("git") .get("git")
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false); .unwrap_or(false);
let disasmer_node_available = dependencies let clusterflux_node_available = dependencies
.get("disasmer-node") .get("clusterflux-node")
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false); .unwrap_or(false);
@ -81,8 +81,8 @@ fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value)
if capabilities.source_providers.contains("git") && !git_available { if capabilities.source_providers.contains("git") && !git_available {
missing.push("git"); missing.push("git");
} }
if !disasmer_node_available { if !clusterflux_node_available {
missing.push("disasmer-node"); missing.push("clusterflux-node");
} }
let basic_runtime_ready = true; let basic_runtime_ready = true;
@ -96,14 +96,14 @@ fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value)
}; };
let next_actions = if status == "ready_to_attach" { let next_actions = if status == "ready_to_attach" {
vec![ vec![
"disasmer node enroll", "clusterflux node enroll",
"disasmer node attach", "clusterflux node attach",
"disasmer-node --worker", "clusterflux-node --worker",
] ]
} else { } else {
vec![ vec![
"install missing local dependencies", "install missing local dependencies",
"rerun disasmer doctor", "rerun clusterflux doctor",
] ]
}; };
@ -118,7 +118,7 @@ fn node_readiness_summary(capabilities: &NodeCapabilities, dependencies: &Value)
"container_backend_reported": has_container_backend, "container_backend_reported": has_container_backend,
"podman_binary_available": podman_available, "podman_binary_available": podman_available,
"git_binary_available": git_available, "git_binary_available": git_available,
"node_binary_available": disasmer_node_available, "node_binary_available": clusterflux_node_available,
"missing_local_dependencies": missing, "missing_local_dependencies": missing,
"source_providers": capabilities.source_providers.iter().collect::<Vec<_>>(), "source_providers": capabilities.source_providers.iter().collect::<Vec<_>>(),
"next_actions": next_actions, "next_actions": next_actions,
@ -130,7 +130,7 @@ fn coordinator_reachability(coordinator: Option<&str>) -> Value {
return json!({ return json!({
"checked": false, "checked": false,
"status": "not_configured", "status": "not_configured",
"next_action": "run disasmer login --browser or pass --coordinator" "next_action": "run clusterflux login --browser or pass --coordinator"
}); });
}; };

View file

@ -210,45 +210,48 @@ fn cli_error_retryable_after_user_action(category: &str) -> bool {
fn cli_error_next_actions(category: &str) -> Vec<&'static str> { fn cli_error_next_actions(category: &str) -> Vec<&'static str> {
match category { match category {
"authentication" => vec!["disasmer login --browser", "disasmer auth status"], "authentication" => vec!["clusterflux login --browser", "clusterflux auth status"],
"authorization" => vec![ "authorization" => vec![
"disasmer auth status", "clusterflux auth status",
"check tenant/project selection", "check tenant/project selection",
"ask an admin to grant access", "ask an admin to grant access",
], ],
"quota" => vec![ "quota" => vec![
"disasmer quota status", "clusterflux quota status",
"reduce concurrent work or wait for usage to fall", "reduce concurrent work or wait for usage to fall",
], ],
"policy" => vec![ "policy" => vec![
"disasmer doctor", "clusterflux doctor",
"check coordinator policy for this action", "check coordinator policy for this action",
], ],
"capability" => vec![ "capability" => vec![
"disasmer node list", "clusterflux node list",
"attach a node with the required capabilities", "attach a node with the required capabilities",
"check tenant/project on the attached node", "check tenant/project on the attached node",
], ],
"connectivity" => vec![ "connectivity" => vec![
"disasmer doctor", "clusterflux doctor",
"check the coordinator endpoint and network reachability", "check the coordinator endpoint and network reachability",
], ],
"environment" => vec![ "environment" => vec![
"disasmer inspect", "clusterflux inspect",
"check envs/<name>/Containerfile or envs/<name>/Dockerfile", "check envs/<name>/Containerfile or envs/<name>/Dockerfile",
], ],
"program" => vec!["disasmer logs", "fix the program or task command and rerun"], "program" => vec![
"clusterflux logs",
"fix the program or task command and rerun",
],
"active_process" => vec![ "active_process" => vec![
"disasmer process list", "clusterflux process list",
"disasmer process status", "clusterflux process status",
"disasmer debug attach", "clusterflux debug attach",
"disasmer process restart --yes", "clusterflux process restart --yes",
"disasmer process cancel --yes", "clusterflux process cancel --yes",
"disasmer process abort --yes", "clusterflux process abort --yes",
"use another Coordinator Project", "use another Coordinator Project",
], ],
_ => vec![ _ => vec![
"disasmer doctor", "clusterflux doctor",
"rerun with --json for machine-readable details", "rerun with --json for machine-readable details",
], ],
} }

View file

@ -1,5 +1,5 @@
use anyhow::Result; use anyhow::Result;
use disasmer_core::Digest; use clusterflux_core::Digest;
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::client::{authenticated_or_local_trusted_request, JsonLineSession}; use crate::client::{authenticated_or_local_trusted_request, JsonLineSession};
@ -182,7 +182,7 @@ pub(crate) fn key_revoke_report_with_session(
"user": args.scope.user, "user": args.scope.user,
"agent": args.agent, "agent": args.agent,
}), }),
format!("disasmer key revoke --agent {} --yes", args.agent), format!("clusterflux key revoke --agent {} --yes", args.agent),
)); ));
} }
let coordinator = effective_coordinator(&args.scope.coordinator, stored_session); let coordinator = effective_coordinator(&args.scope.coordinator, stored_session);

View file

@ -23,7 +23,7 @@ pub(crate) fn logout_report(args: AuthLogoutArgs, cwd: PathBuf, command: &str) -
"session_file": session_file, "session_file": session_file,
"node_credentials_untouched": true, "node_credentials_untouched": true,
}), }),
format!("disasmer {command} --yes"), format!("clusterflux {command} --yes"),
)); ));
} }
let stored_session = read_cli_session(&cwd).ok().flatten(); let stored_session = read_cli_session(&cwd).ok().flatten();
@ -73,7 +73,7 @@ fn revoke_stored_cli_session_if_possible(stored_session: Option<&StoredCliSessio
"coordinator": coordinator, "coordinator": coordinator,
"error": message, "error": message,
"machine_error": cli_error_summary(&message), "machine_error": cli_error_summary(&message),
"next_actions": ["disasmer login --browser"], "next_actions": ["clusterflux login --browser"],
}); });
} }
}; };
@ -95,7 +95,7 @@ fn revoke_stored_cli_session_if_possible(stored_session: Option<&StoredCliSessio
"error": message, "error": message,
"machine_error": cli_error_summary(&message), "machine_error": cli_error_summary(&message),
"coordinator_session_requests": coordinator_session.requests(), "coordinator_session_requests": coordinator_session.requests(),
"next_actions": ["disasmer login --browser"], "next_actions": ["clusterflux login --browser"],
}); });
} }
}; };
@ -113,7 +113,7 @@ fn revoke_stored_cli_session_if_possible(stored_session: Option<&StoredCliSessio
"coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("unknown"), "coordinator_response_type": response.get("type").and_then(Value::as_str).unwrap_or("unknown"),
"machine_error": cli_error_summary(message), "machine_error": cli_error_summary(message),
"coordinator_session_requests": coordinator_session.requests(), "coordinator_session_requests": coordinator_session.requests(),
"next_actions": ["disasmer login --browser"], "next_actions": ["clusterflux login --browser"],
}); });
} }
json!({ json!({

View file

@ -8,7 +8,7 @@ use std::path::PathBuf;
use clap::{Args, Parser, Subcommand}; use clap::{Args, Parser, Subcommand};
#[cfg(test)] #[cfg(test)]
use disasmer_core::{Capability, Digest, ProjectModel, SourceProviderKind}; use clusterflux_core::{Capability, Digest, ProjectModel, SourceProviderKind};
#[cfg(test)] #[cfg(test)]
use serde_json::json; use serde_json::json;
use serde_json::Value; use serde_json::Value;
@ -126,17 +126,17 @@ use tools::dap_binary_path;
#[derive(Clone, Debug, Parser)] #[derive(Clone, Debug, Parser)]
#[command( #[command(
name = "disasmer", name = "clusterflux",
version, version,
arg_required_else_help = true, arg_required_else_help = true,
about = "Disasmer distributed Wasm runtime CLI.", about = "Clusterflux distributed Wasm runtime CLI.",
after_help = "Primary workflow: after_help = "Primary workflow:
1. disasmer login --browser 1. clusterflux login --browser
2. disasmer project init 2. clusterflux project init
3. disasmer node enroll; disasmer node attach; disasmer-node --worker 3. clusterflux node enroll; clusterflux node attach; clusterflux-node --worker
4. disasmer run [entry] --project <path> 4. clusterflux run [entry] --project <path>
5. Debug with VS Code \"Disasmer: Launch Virtual Process\" or disasmer dap 5. Debug with VS Code \"Clusterflux: Launch Virtual Process\" or clusterflux dap
6. Inspect with disasmer process list, disasmer process status, task list, logs, and artifact list 6. Inspect with clusterflux process list, clusterflux process status, task list, logs, and artifact list
7. Request cooperative shutdown with process cancel; force termination with process abort 7. Request cooperative shutdown with process cancel; force termination with process abort
Use --json on primary commands for scriptable output. Hosted account creation happens in the browser login flow." Use --json on primary commands for scriptable output. Hosted account creation happens in the browser login flow."
@ -322,7 +322,7 @@ struct ProjectInitArgs {
scope: CliScopeArgs, scope: CliScopeArgs,
#[arg(long, default_value = "project")] #[arg(long, default_value = "project")]
new_project: String, new_project: String,
#[arg(long, default_value = "Disasmer Project")] #[arg(long, default_value = "Clusterflux Project")]
name: String, name: String,
#[arg(long)] #[arg(long)]
yes: bool, yes: bool,
@ -635,7 +635,7 @@ struct AdminStatusArgs {
struct AdminBootstrapArgs { struct AdminBootstrapArgs {
#[command(flatten)] #[command(flatten)]
scope: CliScopeArgs, scope: CliScopeArgs,
#[arg(long, default_value = "Disasmer Project")] #[arg(long, default_value = "Clusterflux Project")]
name: String, name: String,
#[arg(long)] #[arg(long)]
yes: bool, yes: bool,

View file

@ -2,7 +2,7 @@ use std::collections::BTreeMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use disasmer_core::{ use clusterflux_core::{
generate_ed25519_private_key, node_ed25519_public_key_from_private_key, sign_node_request, generate_ed25519_private_key, node_ed25519_public_key_from_private_key, sign_node_request,
signed_request_payload_digest, Capability, Digest, EnvironmentBackend, NodeCapabilities, signed_request_payload_digest, Capability, Digest, EnvironmentBackend, NodeCapabilities,
NodeId, NodeId,
@ -60,7 +60,7 @@ pub(crate) struct CapabilityGrantDisclosure {
#[derive(Clone, Debug, PartialEq, Eq, Serialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub(crate) struct NodeAttachDetectionEvidence { pub(crate) struct NodeAttachDetectionEvidence {
pub(crate) auto_detected: bool, pub(crate) auto_detected: bool,
pub(crate) os: disasmer_core::Os, pub(crate) os: clusterflux_core::Os,
pub(crate) arch: String, pub(crate) arch: String,
pub(crate) command_backend: String, pub(crate) command_backend: String,
pub(crate) command_backend_available: bool, pub(crate) command_backend_available: bool,
@ -287,7 +287,7 @@ pub(crate) fn node_revoke_report(args: NodeRevokeArgs, cwd: PathBuf) -> Result<V
"user": args.scope.user, "user": args.scope.user,
"node": args.node, "node": args.node,
}), }),
format!("disasmer node revoke --node {} --yes", args.node), format!("clusterflux node revoke --node {} --yes", args.node),
)); ));
} }
let stored_session = read_cli_session(&cwd)?; let stored_session = read_cli_session(&cwd)?;
@ -581,7 +581,7 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
.unwrap_or(derived_public_key.clone()); .unwrap_or(derived_public_key.clone());
if public_key != derived_public_key { if public_key != derived_public_key {
anyhow::bail!( anyhow::bail!(
"node attach --public-key must match DISASMER_NODE_PRIVATE_KEY or the stored local node credential" "node attach --public-key must match CLUSTERFLUX_NODE_PRIVATE_KEY or the stored local node credential"
); );
} }
let mut plan = attach_plan(args); let mut plan = attach_plan(args);
@ -662,7 +662,7 @@ pub(crate) fn execute_node_attach(args: AttachArgs) -> Result<NodeAttachReport>
} }
fn node_private_key_for_attach(node: &str) -> Result<String> { fn node_private_key_for_attach(node: &str) -> Result<String> {
if let Ok(private_key) = std::env::var("DISASMER_NODE_PRIVATE_KEY") { if let Ok(private_key) = std::env::var("CLUSTERFLUX_NODE_PRIVATE_KEY") {
return Ok(private_key); return Ok(private_key);
} }
load_or_create_local_node_credential(&std::env::current_dir()?, node) load_or_create_local_node_credential(&std::env::current_dir()?, node)
@ -698,7 +698,7 @@ pub(crate) fn load_or_create_local_node_credential(project: &Path, node: &str) -
let public_key = let public_key =
node_ed25519_public_key_from_private_key(&private_key).map_err(anyhow::Error::msg)?; node_ed25519_public_key_from_private_key(&private_key).map_err(anyhow::Error::msg)?;
let credential = StoredNodeCredential { let credential = StoredNodeCredential {
kind: "disasmer_node_credential".to_owned(), kind: "clusterflux_node_credential".to_owned(),
node: node.to_owned(), node: node.to_owned(),
private_key: private_key.clone(), private_key: private_key.clone(),
public_key, public_key,
@ -774,7 +774,7 @@ pub(crate) fn local_node_credential_file(project: &Path, node: &str) -> PathBuf
let digest = Digest::sha256(node); let digest = Digest::sha256(node);
let file_stem = digest.as_str().trim_start_matches("sha256:"); let file_stem = digest.as_str().trim_start_matches("sha256:");
project project
.join(".disasmer") .join(".clusterflux")
.join("nodes") .join("nodes")
.join(format!("{file_stem}.json")) .join(format!("{file_stem}.json"))
} }
@ -811,7 +811,7 @@ fn signed_node_request_json(
} }
pub(crate) fn default_node_id() -> String { pub(crate) fn default_node_id() -> String {
std::env::var("DISASMER_NODE_ID") std::env::var("CLUSTERFLUX_NODE_ID")
.or_else(|_| std::env::var("HOSTNAME")) .or_else(|_| std::env::var("HOSTNAME"))
.or_else(|_| std::env::var("COMPUTERNAME")) .or_else(|_| std::env::var("COMPUTERNAME"))
.unwrap_or_else(|_| "node-local".to_owned()) .unwrap_or_else(|_| "node-local".to_owned())

View file

@ -21,25 +21,25 @@ pub(crate) fn human_report(value: &Value) -> String {
let title = value let title = value
.get("command") .get("command")
.and_then(Value::as_str) .and_then(Value::as_str)
.map(|command| format!("Disasmer {command}")) .map(|command| format!("Clusterflux {command}"))
.or_else(|| { .or_else(|| {
if value.get("human_flow").is_some() { if value.get("human_flow").is_some() {
Some("Disasmer login".to_owned()) Some("Clusterflux login".to_owned())
} else if value.get("metadata").is_some() } else if value.get("metadata").is_some()
&& value.get("source_provider_manifest").is_some() && value.get("source_provider_manifest").is_some()
{ {
Some("Disasmer bundle inspect".to_owned()) Some("Clusterflux bundle inspect".to_owned())
} else if value.get("entry").is_some() && value.get("session").is_some() { } else if value.get("entry").is_some() && value.get("session").is_some() {
Some("Disasmer run".to_owned()) Some("Clusterflux run".to_owned())
} else if value.get("capabilities").is_some() && value.get("node").is_some() { } else if value.get("capabilities").is_some() && value.get("node").is_some() {
Some("Disasmer node attach".to_owned()) Some("Clusterflux node attach".to_owned())
} else if value.get("public_key_fingerprint").is_some() { } else if value.get("public_key_fingerprint").is_some() {
Some("Disasmer agent enroll".to_owned()) Some("Clusterflux agent enroll".to_owned())
} else { } else {
None None
} }
}) })
.unwrap_or_else(|| "Disasmer report".to_owned()); .unwrap_or_else(|| "Clusterflux report".to_owned());
lines.push(title); lines.push(title);
push_string_field(&mut lines, value, "status", "status"); push_string_field(&mut lines, value, "status", "status");

View file

@ -157,7 +157,10 @@ pub(crate) fn process_restart_report_with_session(
"project": args.scope.project, "project": args.scope.project,
"process": args.process, "process": args.process,
}), }),
format!("disasmer process restart --process {} --yes", args.process), format!(
"clusterflux process restart --process {} --yes",
args.process
),
)); ));
} }
if let Some(coordinator) = &args.scope.coordinator { if let Some(coordinator) = &args.scope.coordinator {
@ -223,7 +226,10 @@ pub(crate) fn process_cancel_report_with_session(
"node": args.node, "node": args.node,
"task": args.task, "task": args.task,
}), }),
format!("disasmer process cancel --process {} --yes", args.process), format!(
"clusterflux process cancel --process {} --yes",
args.process
),
)); ));
} }
if let Some(coordinator) = &args.scope.coordinator { if let Some(coordinator) = &args.scope.coordinator {
@ -287,7 +293,7 @@ pub(crate) fn process_abort_report_with_session(
"project": args.scope.project, "project": args.scope.project,
"process": args.process, "process": args.process,
}), }),
format!("disasmer process abort --process {} --yes", args.process), format!("clusterflux process abort --process {} --yes", args.process),
)); ));
} }
let Some(coordinator) = &args.scope.coordinator else { let Some(coordinator) = &args.scope.coordinator else {

View file

@ -101,7 +101,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
"current_directory_link": { "current_directory_link": {
"cwd": cwd, "cwd": cwd,
"config_file": config_file, "config_file": config_file,
"config_format": "disasmer_project_config_v1", "config_format": "clusterflux_project_config_v1",
"links_current_directory": true, "links_current_directory": true,
"writes_current_directory_only": true, "writes_current_directory_only": true,
"private_website_required": false, "private_website_required": false,
@ -113,7 +113,7 @@ pub(crate) fn project_init_report(args: ProjectInitArgs, cwd: PathBuf) -> Result
"coordinator": config.coordinator.clone(), "coordinator": config.coordinator.clone(),
"project_name": args.name.clone(), "project_name": args.name.clone(),
"default_project_id_used": args.new_project == "project", "default_project_id_used": args.new_project == "project",
"default_project_name_used": args.name == "Disasmer Project", "default_project_name_used": args.name == "Clusterflux Project",
"browser_interaction_required": false, "browser_interaction_required": false,
"private_website_required": false, "private_website_required": false,
}, },
@ -147,7 +147,7 @@ pub(crate) fn project_status_report(args: ProjectStatusArgs, cwd: PathBuf) -> Re
|| session.user != config.user) || session.user != config.user)
{ {
anyhow::bail!( anyhow::bail!(
"stored CLI session is for {}/{}/{} but this workspace is configured for {}/{}/{}; run `disasmer login --browser` from this workspace", "stored CLI session is for {}/{}/{} but this workspace is configured for {}/{}/{}; run `clusterflux login --browser` from this workspace",
session.tenant, session.tenant,
session.project, session.project,
session.user, session.user,

View file

@ -4,10 +4,10 @@ use std::time::{Duration, Instant};
use anyhow::{Context, Result}; use anyhow::{Context, Result};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_control::MAX_CONTROL_FRAME_BYTES; use clusterflux_control::MAX_CONTROL_FRAME_BYTES;
use disasmer_core::{ use clusterflux_core::{
agent_ed25519_public_key_from_private_key, sign_agent_workflow_request, agent_ed25519_public_key_from_private_key, agent_workflow_request_scope_from_payload,
signed_request_payload_digest, AgentWorkflowScope, Digest, sign_agent_workflow_request, signed_request_payload_digest, Digest,
}; };
use serde::Serialize; use serde::Serialize;
use serde_json::{json, Value}; use serde_json::{json, Value};
@ -122,8 +122,8 @@ fn non_interactive_run_requires_auth_report(args: RunArgs, cwd: PathBuf) -> Valu
let entry = args.entry.unwrap_or_else(|| "build".to_owned()); let entry = args.entry.unwrap_or_else(|| "build".to_owned());
let message = "non-interactive run requires an authenticated human or agent session unless --local or --coordinator is explicit"; let message = "non-interactive run requires an authenticated human or agent session unless --local or --coordinator is explicit";
let next_actions = vec![ let next_actions = vec![
"disasmer login --browser", "clusterflux login --browser",
"set DISASMER_AGENT_PRIVATE_KEY for automation", "set CLUSTERFLUX_AGENT_PRIVATE_KEY for automation",
"pass --local to run against local services", "pass --local to run against local services",
"pass --coordinator for an explicit self-hosted coordinator", "pass --coordinator for an explicit self-hosted coordinator",
]; ];
@ -191,7 +191,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
&& !crate::client::is_loopback_coordinator(&coordinator) && !crate::client::is_loopback_coordinator(&coordinator)
{ {
anyhow::bail!( anyhow::bail!(
"no authenticated CLI session matches coordinator {coordinator}; run `disasmer login --browser` from the current project" "no authenticated CLI session matches coordinator {coordinator}; run `clusterflux login --browser` from the current project"
); );
} }
let process = "vp-current".to_owned(); let process = "vp-current".to_owned();
@ -209,7 +209,17 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
&user, &user,
human_session_secret.as_deref(), human_session_secret.as_deref(),
); );
let response = session.request_allow_error(request)?; let response = request_process_start_with_rollback(
&mut session,
request,
&coordinator,
&plan.session,
&user,
human_session_secret.as_deref(),
&tenant,
&project,
&process,
)?;
let run_start = run_start_summary(&response); let run_start = run_start_summary(&response);
let status = run_start let status = run_start
.get("status") .get("status")
@ -239,6 +249,7 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
"required_artifacts": [], "required_artifacts": [],
"args": [], "args": [],
"vfs_epoch": response.get("epoch").and_then(Value::as_u64).unwrap_or_default(), "vfs_epoch": response.get("epoch").and_then(Value::as_u64).unwrap_or_default(),
"failure_policy": "fail_fast",
"bundle_digest": bundle.digest, "bundle_digest": bundle.digest,
}); });
let mut launch_task_request = json!({ let mut launch_task_request = json!({
@ -256,23 +267,37 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
&user, &user,
human_session_secret.as_deref(), human_session_secret.as_deref(),
); );
let launch = session.request_allow_error(launch_task_request)?; let launch = session
if !matches!( .request_allow_error(launch_task_request)
launch.get("type").and_then(Value::as_str), .and_then(|response| {
Some("main_launched") if response.get("type").and_then(Value::as_str) == Some("main_launched") {
) { Ok(response)
rollback_failed_process_launch( } else {
&mut session, anyhow::bail!("coordinator main launch was not acknowledged: {response}")
&plan.session, }
&user, });
human_session_secret.as_deref(), match launch {
&tenant, Ok(launch) => launch,
&project, Err(launch_error) => {
&process, let rollback = ProcessLaunchRollback {
&launch, cli_session: &plan.session,
)?; fallback_user: &user,
human_session_secret: human_session_secret.as_deref(),
tenant: &tenant,
project: &project,
process: &process,
};
let cleanup = rollback_failed_process_launch_reconnecting(
&coordinator,
&rollback,
&json!({ "error": launch_error.to_string() }),
);
if let Err(cleanup_error) = cleanup {
anyhow::bail!("{launch_error}; process cleanup also failed: {cleanup_error}");
}
return Err(launch_error);
}
} }
launch
} else { } else {
Value::Null Value::Null
}; };
@ -304,12 +329,62 @@ fn coordinator_run_report(plan: RunPlan) -> Result<Value> {
})) }))
} }
#[allow(clippy::too_many_arguments)]
fn request_process_start_with_rollback(
session: &mut JsonLineSession,
request: Value,
coordinator: &str,
cli_session: &CliSession,
fallback_user: &str,
human_session_secret: Option<&str>,
tenant: &str,
project: &str,
process: &str,
) -> Result<Value> {
let rollback = ProcessLaunchRollback {
cli_session,
fallback_user,
human_session_secret,
tenant,
project,
process,
};
match session.request_allow_error(request) {
Ok(response) => Ok(response),
Err(start_error) => {
let cleanup = rollback_failed_process_launch_reconnecting(
coordinator,
&rollback,
&json!({ "error": start_error.to_string(), "phase": "start_process" }),
);
if let Err(cleanup_error) = cleanup {
let cleanup_message = cleanup_error.to_string();
if !cleanup_message.contains("process abort requires an active virtual process") {
anyhow::bail!(
"{start_error}; ambiguous process-start cleanup also failed: {cleanup_error}"
);
}
}
Err(start_error)
}
}
}
struct ProcessLaunchRollback<'a> {
cli_session: &'a CliSession,
fallback_user: &'a str,
human_session_secret: Option<&'a str>,
tenant: &'a str,
project: &'a str,
process: &'a str,
}
fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> { fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> {
if module_size_bytes <= MAX_INLINE_WASM_MODULE_BYTES { if module_size_bytes <= MAX_INLINE_WASM_MODULE_BYTES {
return Ok(()); return Ok(());
} }
anyhow::bail!( anyhow::bail!(
"built Wasm module is {module_size_bytes} bytes, but the current {}-byte inline control frame supports at most about {} KiB ({} raw bytes); reduce the MVP bundle's dependencies or release optimization footprint. Larger bundle transport is post-MVP. No virtual process was created", "built Wasm module is {module_size_bytes} bytes, but the current {}-byte inline control frame supports at most about {} KiB ({} raw bytes); reduce the bundle dependency or optimization footprint. Larger bundles require an out-of-band transport. No virtual process was created",
MAX_CONTROL_FRAME_BYTES, MAX_CONTROL_FRAME_BYTES,
MAX_INLINE_WASM_MODULE_BYTES / 1024, MAX_INLINE_WASM_MODULE_BYTES / 1024,
MAX_INLINE_WASM_MODULE_BYTES, MAX_INLINE_WASM_MODULE_BYTES,
@ -318,34 +393,40 @@ fn validate_inline_bundle_size(module_size_bytes: usize) -> Result<()> {
fn rollback_failed_process_launch( fn rollback_failed_process_launch(
session: &mut JsonLineSession, session: &mut JsonLineSession,
cli_session: &CliSession, context: &ProcessLaunchRollback<'_>,
fallback_user: &str,
human_session_secret: Option<&str>,
tenant: &str,
project: &str,
process: &str,
launch_response: &Value, launch_response: &Value,
) -> Result<()> { ) -> Result<()> {
let rollback = authenticated_human_or_local_trusted_workflow( let rollback = authenticated_human_or_local_trusted_workflow(
json!({ json!({
"type": "abort_process", "type": "abort_process",
"tenant": tenant, "tenant": context.tenant,
"project": project, "project": context.project,
"process": process, "process": context.process,
}), }),
cli_session, context.cli_session,
fallback_user, context.fallback_user,
human_session_secret, context.human_session_secret,
); );
let rollback_response = session.request_allow_error(rollback)?; let rollback_response = session.request_allow_error(rollback)?;
if rollback_response.get("type").and_then(Value::as_str) != Some("process_aborted") { if rollback_response.get("type").and_then(Value::as_str) != Some("process_aborted") {
anyhow::bail!( anyhow::bail!(
"coordinator main launch failed ({launch_response}) and rollback was not acknowledged ({rollback_response}); inspect virtual process {process} before retrying" "coordinator main launch failed ({launch_response}) and rollback was not acknowledged ({rollback_response}); inspect virtual process {} before retrying",
context.process
); );
} }
Ok(()) Ok(())
} }
fn rollback_failed_process_launch_reconnecting(
coordinator: &str,
context: &ProcessLaunchRollback<'_>,
launch_response: &Value,
) -> Result<()> {
let mut cleanup_session = JsonLineSession::connect(coordinator)
.context("open a fresh coordinator connection for failed-launch cleanup")?;
rollback_failed_process_launch(&mut cleanup_session, context, launch_response)
}
fn build_bundle_for_run(project: &Path, entry: &str) -> Result<RunBundle> { fn build_bundle_for_run(project: &Path, entry: &str) -> Result<RunBundle> {
let build_report = crate::build::build_report( let build_report = crate::build::build_report(
BuildArgs { BuildArgs {
@ -362,7 +443,7 @@ fn build_bundle_for_run(project: &Path, entry: &str) -> Result<RunBundle> {
.get("diagnostics") .get("diagnostics")
.cloned() .cloned()
.unwrap_or(Value::Null); .unwrap_or(Value::Null);
anyhow::bail!("Disasmer bundle build was blocked before run: {diagnostics}"); anyhow::bail!("Clusterflux bundle build was blocked before run: {diagnostics}");
} }
let artifact = build_report let artifact = build_report
.get("bundle_artifact") .get("bundle_artifact")
@ -403,11 +484,11 @@ fn build_bundle_for_run(project: &Path, entry: &str) -> Result<RunBundle> {
.filter_map(|descriptor| descriptor.get("name").and_then(Value::as_str)) .filter_map(|descriptor| descriptor.get("name").and_then(Value::as_str))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
format!( format!(
"bundle has no Disasmer entrypoint `{entry}`; available entrypoints: {available:?}" "bundle has no Clusterflux entrypoint `{entry}`; available entrypoints: {available:?}"
) )
})?; })?;
if descriptor.get("abi_version").and_then(Value::as_u64) != Some(1) { if descriptor.get("abi_version").and_then(Value::as_u64) != Some(1) {
anyhow::bail!("entrypoint `{entry}` does not use supported Disasmer ABI version 1"); anyhow::bail!("entrypoint `{entry}` does not use supported Clusterflux ABI version 1");
} }
let entry_export = descriptor let entry_export = descriptor
.get("export") .get("export")
@ -439,7 +520,7 @@ fn human_run_session_secret(
if !matches!(plan.session, CliSession::HumanSession) { if !matches!(plan.session, CliSession::HumanSession) {
return Ok(None); return Ok(None);
} }
if let Ok(token) = std::env::var("DISASMER_TOKEN") { if let Ok(token) = std::env::var("CLUSTERFLUX_TOKEN") {
if !token.trim().is_empty() { if !token.trim().is_empty() {
return Ok(Some(token)); return Ok(Some(token));
} }
@ -505,9 +586,9 @@ pub(crate) fn run_start_summary(response: &Value) -> Value {
"restart": false, "restart": false,
"single_active_process_boundary": true, "single_active_process_boundary": true,
"next_actions": [ "next_actions": [
"disasmer process status", "clusterflux process status",
"disasmer logs", "clusterflux logs",
"disasmer process cancel" "clusterflux process cancel"
], ],
}); });
} }
@ -543,16 +624,16 @@ pub(crate) fn run_start_summary(response: &Value) -> Value {
"safe_failure": true, "safe_failure": true,
"next_actions": if active_conflict { "next_actions": if active_conflict {
json!([ json!([
"disasmer process list", "clusterflux process list",
"disasmer process status", "clusterflux process status",
"disasmer debug attach", "clusterflux debug attach",
"disasmer process restart --yes", "clusterflux process restart --yes",
"disasmer process cancel --yes", "clusterflux process cancel --yes",
"disasmer process abort --yes", "clusterflux process abort --yes",
"use another Coordinator Project" "use another Coordinator Project"
]) ])
} else { } else {
json!(["disasmer doctor", "check coordinator status"]) json!(["clusterflux doctor", "check coordinator status"])
}, },
}) })
} }
@ -566,16 +647,16 @@ pub(crate) fn should_execute_local_node(plan: &RunPlan) -> bool {
} }
fn execute_local_node_run(plan: RunPlan) -> Result<RunExecutionReport> { fn execute_local_node_run(plan: RunPlan) -> Result<RunExecutionReport> {
let environments = disasmer_core::discover_environments(&plan.project)?; let environments = clusterflux_core::discover_environments(&plan.project)?;
let detected = disasmer_core::NodeCapabilities::detect_current(); let detected = clusterflux_core::NodeCapabilities::detect_current();
if environments.iter().any(|environment| { if environments.iter().any(|environment| {
environment environment
.requirements .requirements
.capabilities .capabilities
.contains(&disasmer_core::Capability::RootlessPodman) .contains(&clusterflux_core::Capability::RootlessPodman)
}) && !detected }) && !detected
.capabilities .capabilities
.contains(&disasmer_core::Capability::RootlessPodman) .contains(&clusterflux_core::Capability::RootlessPodman)
{ {
anyhow::bail!( anyhow::bail!(
"local project declares a Linux container environment, but rootless Podman is not available; configure rootless Podman or attach a capable user node" "local project declares a Linux container environment, but rootless Podman is not available; configure rootless Podman or attach a capable user node"
@ -589,7 +670,7 @@ fn execute_local_node_run(plan: RunPlan) -> Result<RunExecutionReport> {
let coordinator_address = local_coordinator.address.clone(); let coordinator_address = local_coordinator.address.clone();
let mut coordinator_plan = plan.clone(); let mut coordinator_plan = plan.clone();
coordinator_plan.coordinator = coordinator_plan.coordinator =
CoordinatorSelection::LocalOverride(format!("disasmer+tcp://{coordinator_address}")); CoordinatorSelection::LocalOverride(format!("clusterflux+tcp://{coordinator_address}"));
let run = coordinator_run_report(coordinator_plan)?; let run = coordinator_run_report(coordinator_plan)?;
let launch_type = run.pointer("/task_launch/type").and_then(Value::as_str); let launch_type = run.pointer("/task_launch/type").and_then(Value::as_str);
if launch_type != Some("main_launched") { if launch_type != Some("main_launched") {
@ -722,13 +803,13 @@ fn wait_for_local_main_completion(coordinator: &str, process: &str, task: &str)
pub(crate) fn session_from_env() -> Result<CliSession> { pub(crate) fn session_from_env() -> Result<CliSession> {
if let Some(session) = agent_session_from_keys( if let Some(session) = agent_session_from_keys(
std::env::var("DISASMER_AGENT_ID").unwrap_or_else(|_| "agent".to_owned()), std::env::var("CLUSTERFLUX_AGENT_ID").unwrap_or_else(|_| "agent".to_owned()),
std::env::var("DISASMER_AGENT_PUBLIC_KEY").ok(), std::env::var("CLUSTERFLUX_AGENT_PUBLIC_KEY").ok(),
std::env::var("DISASMER_AGENT_PRIVATE_KEY").ok(), std::env::var("CLUSTERFLUX_AGENT_PRIVATE_KEY").ok(),
)? { )? {
return Ok(session); return Ok(session);
} }
if std::env::var_os("DISASMER_TOKEN").is_some() { if std::env::var_os("CLUSTERFLUX_TOKEN").is_some() {
return Ok(CliSession::HumanSession); return Ok(CliSession::HumanSession);
} }
Ok(CliSession::Anonymous) Ok(CliSession::Anonymous)
@ -742,11 +823,11 @@ pub(crate) fn agent_session_from_keys(
if let Some(private_key) = private_key { if let Some(private_key) = private_key {
let derived_public_key = agent_ed25519_public_key_from_private_key(&private_key) let derived_public_key = agent_ed25519_public_key_from_private_key(&private_key)
.map_err(anyhow::Error::msg) .map_err(anyhow::Error::msg)
.context("DISASMER_AGENT_PRIVATE_KEY is not a valid Ed25519 private key")?; .context("CLUSTERFLUX_AGENT_PRIVATE_KEY is not a valid Ed25519 private key")?;
if let Some(configured_public_key) = configured_public_key.as_deref() { if let Some(configured_public_key) = configured_public_key.as_deref() {
if configured_public_key != derived_public_key { if configured_public_key != derived_public_key {
anyhow::bail!( anyhow::bail!(
"DISASMER_AGENT_PUBLIC_KEY does not match DISASMER_AGENT_PRIVATE_KEY" "CLUSTERFLUX_AGENT_PUBLIC_KEY does not match CLUSTERFLUX_AGENT_PRIVATE_KEY"
); );
} }
} }
@ -760,7 +841,7 @@ pub(crate) fn agent_session_from_keys(
} }
if configured_public_key.is_some() { if configured_public_key.is_some() {
anyhow::bail!( anyhow::bail!(
"DISASMER_AGENT_PUBLIC_KEY identifies a registered key but cannot authenticate; set DISASMER_AGENT_PRIVATE_KEY to prove key possession" "CLUSTERFLUX_AGENT_PUBLIC_KEY identifies a registered key but cannot authenticate; set CLUSTERFLUX_AGENT_PRIVATE_KEY to prove key possession"
); );
} }
Ok(None) Ok(None)
@ -809,32 +890,15 @@ fn agent_signature_for_request(
request: &serde_json::Map<String, Value>, request: &serde_json::Map<String, Value>,
agent: &str, agent: &str,
private_key: Option<&str>, private_key: Option<&str>,
) -> Option<disasmer_core::AgentSignedRequest> { ) -> Option<clusterflux_core::AgentSignedRequest> {
let private_key = private_key?; let private_key = private_key?;
let request_kind = request.get("type")?.as_str()?; let payload = Value::Object(request.clone());
let tenant = request.get("tenant")?.as_str()?; let scope = agent_workflow_request_scope_from_payload(&payload).ok()?;
let project = request.get("project")?.as_str()?; let agent = clusterflux_core::AgentId::from(agent);
let task_spec = request.get("task_spec"); let payload_digest = signed_request_payload_digest(&payload);
let process = request
.get("process")
.or_else(|| task_spec.and_then(|spec| spec.get("process")))?
.as_str()?;
let task = request
.get("task")
.or_else(|| task_spec.and_then(|spec| spec.get("task")))
.and_then(Value::as_str)
.map(disasmer_core::TaskInstanceId::from);
let payload_digest = signed_request_payload_digest(&Value::Object(request.clone()));
sign_agent_workflow_request( sign_agent_workflow_request(
private_key, private_key,
AgentWorkflowScope { scope.for_agent(&agent),
tenant: &disasmer_core::TenantId::from(tenant),
project: &disasmer_core::ProjectId::from(project),
agent: &disasmer_core::AgentId::from(agent),
request_kind,
process: &disasmer_core::ProcessId::from(process),
task: task.as_ref(),
},
&payload_digest, &payload_digest,
crate::tools::command_nonce("agent-signature"), crate::tools::command_nonce("agent-signature"),
crate::tools::unix_timestamp_seconds(), crate::tools::unix_timestamp_seconds(),
@ -853,14 +917,14 @@ mod transactional_launch_tests {
fn inline_module_limit_is_derived_from_the_control_frame() { fn inline_module_limit_is_derived_from_the_control_frame() {
assert_eq!(MAX_CONTROL_FRAME_BYTES, 1024 * 1024); assert_eq!(MAX_CONTROL_FRAME_BYTES, 1024 * 1024);
assert_eq!(MAX_INLINE_WASM_MODULE_BYTES % 3, 0); assert_eq!(MAX_INLINE_WASM_MODULE_BYTES % 3, 0);
assert!(MAX_INLINE_WASM_MODULE_BYTES >= 690 * 1024); let inline_limit = std::hint::black_box(MAX_INLINE_WASM_MODULE_BYTES);
assert!(MAX_INLINE_WASM_MODULE_BYTES <= 700 * 1024); assert!((690 * 1024..=700 * 1024).contains(&inline_limit));
validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES).unwrap(); validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES).unwrap();
let error = validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES + 1) let error = validate_inline_bundle_size(MAX_INLINE_WASM_MODULE_BYTES + 1)
.unwrap_err() .unwrap_err()
.to_string(); .to_string();
assert!(error.contains("No virtual process was created")); assert!(error.contains("No virtual process was created"));
assert!(error.contains("post-MVP")); assert!(error.contains("out-of-band transport"));
} }
#[test] #[test]
@ -927,17 +991,76 @@ mod transactional_launch_tests {
.unwrap(); .unwrap();
}); });
let mut session = JsonLineSession::connect(&address.to_string()).unwrap(); let mut session = JsonLineSession::connect(&address.to_string()).unwrap();
let rollback = ProcessLaunchRollback {
cli_session: &CliSession::Anonymous,
fallback_user: "user-a",
human_session_secret: None,
tenant: "tenant-a",
project: "project-a",
process: "vp-current",
};
rollback_failed_process_launch( rollback_failed_process_launch(
&mut session, &mut session,
&rollback,
&json!({"type": "error", "message": "main launch failed"}),
)
.unwrap();
server.join().unwrap();
}
#[test]
fn dropped_start_response_reconnects_and_aborts_ambiguous_process() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap().to_string();
let server = std::thread::spawn(move || {
let (stream, _) = listener.accept().unwrap();
let mut start_line = String::new();
std::io::BufReader::new(stream.try_clone().unwrap())
.read_line(&mut start_line)
.unwrap();
assert!(start_line.contains("\"type\":\"start_process\""));
drop(stream);
let (mut cleanup_stream, _) = listener.accept().unwrap();
let mut cleanup_line = String::new();
std::io::BufReader::new(cleanup_stream.try_clone().unwrap())
.read_line(&mut cleanup_line)
.unwrap();
assert!(cleanup_line.contains("\"type\":\"abort_process\""));
writeln!(
cleanup_stream,
"{}",
json!({
"type": "process_aborted",
"process": "vp-current",
"aborted_tasks": [],
"affected_nodes": []
})
)
.unwrap();
});
let mut session = JsonLineSession::connect(&address).unwrap();
let error = request_process_start_with_rollback(
&mut session,
json!({
"type": "start_process",
"tenant": "tenant-a",
"project": "project-a",
"actor_user": "user-a",
"process": "vp-current",
"restart": false
}),
&address,
&CliSession::Anonymous, &CliSession::Anonymous,
"user-a", "user-a",
None, None,
"tenant-a", "tenant-a",
"project-a", "project-a",
"vp-current", "vp-current",
&json!({"type": "error", "message": "main launch failed"}),
) )
.unwrap(); .unwrap_err()
.to_string();
assert!(error.contains("closed") || error.contains("response"));
server.join().unwrap(); server.join().unwrap();
} }
} }

View file

@ -126,12 +126,12 @@ fn read_coordinator_ready_address(child: &mut Child) -> Result<String> {
} }
fn node_command() -> Result<Command> { fn node_command() -> Result<Command> {
if let Some(path) = std::env::var_os("DISASMER_NODE_BIN") { if let Some(path) = std::env::var_os("CLUSTERFLUX_NODE_BIN") {
return Ok(Command::new(path)); return Ok(Command::new(path));
} }
let mut sibling = std::env::current_exe().context("cannot locate current executable")?; let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
sibling.set_file_name(format!("disasmer-node{}", std::env::consts::EXE_SUFFIX)); sibling.set_file_name(format!("clusterflux-node{}", std::env::consts::EXE_SUFFIX));
if sibling.is_file() { if sibling.is_file() {
return Ok(Command::new(sibling)); return Ok(Command::new(sibling));
} }
@ -141,22 +141,22 @@ fn node_command() -> Result<Command> {
"run", "run",
"-q", "-q",
"-p", "-p",
"disasmer-node", "clusterflux-node",
"--bin", "--bin",
"disasmer-node", "clusterflux-node",
"--", "--",
]); ]);
Ok(command) Ok(command)
} }
fn coordinator_command() -> Result<Command> { fn coordinator_command() -> Result<Command> {
if let Some(path) = std::env::var_os("DISASMER_COORDINATOR_BIN") { if let Some(path) = std::env::var_os("CLUSTERFLUX_COORDINATOR_BIN") {
return Ok(Command::new(path)); return Ok(Command::new(path));
} }
let mut sibling = std::env::current_exe().context("cannot locate current executable")?; let mut sibling = std::env::current_exe().context("cannot locate current executable")?;
sibling.set_file_name(format!( sibling.set_file_name(format!(
"disasmer-coordinator{}", "clusterflux-coordinator{}",
std::env::consts::EXE_SUFFIX std::env::consts::EXE_SUFFIX
)); ));
if sibling.is_file() { if sibling.is_file() {
@ -168,9 +168,9 @@ fn coordinator_command() -> Result<Command> {
"run", "run",
"-q", "-q",
"-p", "-p",
"disasmer-coordinator", "clusterflux-coordinator",
"--bin", "--bin",
"disasmer-coordinator", "clusterflux-coordinator",
"--", "--",
]); ]);
Ok(command) Ok(command)

View file

@ -54,7 +54,7 @@ pub(crate) fn task_restart_report_with_session(
"task": args.task, "task": args.task,
}), }),
format!( format!(
"disasmer task restart {} --process {} --yes", "clusterflux task restart {} --process {} --yes",
args.task, args.process args.task, args.process
), ),
)); ));

View file

@ -12,7 +12,7 @@ fn parse(args: &[&str]) -> Cli {
fn write_runnable_wasm_project(project: &Path) { fn write_runnable_wasm_project(project: &Path) {
let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); let cli_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
let sdk = cli_manifest_dir.parent().unwrap().join("disasmer-sdk"); let sdk = cli_manifest_dir.parent().unwrap().join("clusterflux-sdk");
let shared_target = cli_manifest_dir let shared_target = cli_manifest_dir
.parent() .parent()
.and_then(Path::parent) .and_then(Path::parent)
@ -23,7 +23,7 @@ fn write_runnable_wasm_project(project: &Path) {
fs::write( fs::write(
project.join("Cargo.toml"), project.join("Cargo.toml"),
format!( format!(
"[package]\nname = \"cli-run-fixture\"\nversion = \"0.0.0\"\nedition = \"2021\"\n\n[workspace]\n\n[lib]\ncrate-type = [\"cdylib\"]\n\n[dependencies]\ndisasmer = {{ package = \"disasmer-sdk\", path = {} }}\n", "[package]\nname = \"cli-run-fixture\"\nversion = \"0.0.0\"\nedition = \"2021\"\n\n[workspace]\n\n[lib]\ncrate-type = [\"cdylib\"]\n\n[dependencies]\nclusterflux = {{ package = \"clusterflux-sdk\", path = {} }}\n",
serde_json::to_string(&sdk.to_string_lossy()).unwrap() serde_json::to_string(&sdk.to_string_lossy()).unwrap()
), ),
) )
@ -39,14 +39,14 @@ fn write_runnable_wasm_project(project: &Path) {
fs::write( fs::write(
project.join("src/lib.rs"), project.join("src/lib.rs"),
r#" r#"
#[disasmer::task] #[clusterflux::task]
#[unsafe(no_mangle)] #[unsafe(no_mangle)]
pub extern "C" fn task_add_one(value: i32) -> i32 { value + 1 } pub extern "C" fn task_add_one(value: i32) -> i32 { value + 1 }
#[disasmer::main] #[clusterflux::main]
pub fn build_main() -> i32 { 7 } pub fn build_main() -> i32 { 7 }
#[disasmer::main(name = "test")] #[clusterflux::main(name = "test")]
pub fn test_main() -> i32 { 8 } pub fn test_main() -> i32 { 8 }
"#, "#,
) )
@ -62,7 +62,7 @@ fn cli_error_classifier_distinguishes_mvp_failure_categories() {
20, 20,
), ),
( (
"CLI session credential has expired; run disasmer login --browser again", "CLI session credential has expired; run clusterflux login --browser again",
"authentication", "authentication",
20, 20,
), ),
@ -163,7 +163,7 @@ fn command_report_exit_code_marks_command_failures_only() {
#[test] #[test]
fn top_level_version_is_available() { fn top_level_version_is_available() {
let command = Cli::command(); let command = Cli::command();
assert_eq!(command.get_name(), "disasmer"); assert_eq!(command.get_name(), "clusterflux");
assert!(command.get_version().is_some()); assert!(command.get_version().is_some());
} }
@ -171,7 +171,7 @@ fn top_level_version_is_available() {
fn run_defaults_to_current_project_and_build_entry() { fn run_defaults_to_current_project_and_build_entry() {
let Cli { let Cli {
command: Commands::Run(args), command: Commands::Run(args),
} = parse(&["disasmer", "run"]) } = parse(&["clusterflux", "run"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -188,7 +188,7 @@ fn run_defaults_to_current_project_and_build_entry() {
fn non_interactive_run_without_session_requires_explicit_auth_or_local() { fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
let Cli { let Cli {
command: Commands::Run(args), command: Commands::Run(args),
} = parse(&["disasmer", "run", "--non-interactive", "--json"]) } = parse(&["clusterflux", "run", "--non-interactive", "--json"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -208,8 +208,8 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
.iter() .iter()
.filter_map(Value::as_str) .filter_map(Value::as_str)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert!(next_actions.contains(&"disasmer login --browser")); assert!(next_actions.contains(&"clusterflux login --browser"));
assert!(next_actions.contains(&"set DISASMER_AGENT_PRIVATE_KEY for automation")); assert!(next_actions.contains(&"set CLUSTERFLUX_AGENT_PRIVATE_KEY for automation"));
assert!(next_actions.contains(&"pass --local to run against local services")); assert!(next_actions.contains(&"pass --local to run against local services"));
} }
@ -217,7 +217,7 @@ fn non_interactive_run_without_session_requires_explicit_auth_or_local() {
fn run_project_and_named_entry_are_respected() { fn run_project_and_named_entry_are_respected() {
let Cli { let Cli {
command: Commands::Run(args), command: Commands::Run(args),
} = parse(&["disasmer", "run", "test", "--project", "/other"]) } = parse(&["clusterflux", "run", "test", "--project", "/other"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -239,7 +239,7 @@ fn node_attach_detects_and_accepts_capability_overrides() {
command: Commands::Node { command: Commands::Node {
command: NodeCommands::Attach(args), command: NodeCommands::Attach(args),
}, },
} = parse(&["disasmer", "node", "attach", "--cap", "quic-direct"]) } = parse(&["clusterflux", "node", "attach", "--cap", "quic-direct"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -280,7 +280,7 @@ fn node_attach_discloses_dangerous_capability_grants() {
command: NodeCommands::Attach(args), command: NodeCommands::Attach(args),
}, },
} = parse(&[ } = parse(&[
"disasmer", "clusterflux",
"node", "node",
"attach", "attach",
"--cap", "--cap",
@ -371,7 +371,8 @@ fn agent_environment_auth_requires_matching_private_key_possession() {
assert!(public_only.to_string().contains("cannot authenticate")); assert!(public_only.to_string().contains("cannot authenticate"));
let private_key = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="; let private_key = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
let public_key = disasmer_core::agent_ed25519_public_key_from_private_key(private_key).unwrap(); let public_key =
clusterflux_core::agent_ed25519_public_key_from_private_key(private_key).unwrap();
let session = agent_session_from_keys( let session = agent_session_from_keys(
"agent-ci".to_owned(), "agent-ci".to_owned(),
Some(public_key.clone()), Some(public_key.clone()),
@ -415,7 +416,8 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
.unwrap(); .unwrap();
let private_key = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="; let private_key = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
let public_key = disasmer_core::agent_ed25519_public_key_from_private_key(private_key).unwrap(); let public_key =
clusterflux_core::agent_ed25519_public_key_from_private_key(private_key).unwrap();
let fingerprint = Digest::sha256(&public_key); let fingerprint = Digest::sha256(&public_key);
let expected_fingerprint = fingerprint.to_string(); let expected_fingerprint = fingerprint.to_string();
let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let listener = TcpListener::bind("127.0.0.1:0").unwrap();
@ -460,10 +462,14 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
launch["payload"]["task_spec"]["task_instance"], launch["payload"]["task_spec"]["task_instance"],
"ti:vp-current:main" "ti:vp-current:main"
); );
assert_eq!(
launch["payload"]["task_spec"]["failure_policy"],
"fail_fast"
);
assert!(launch_line.contains(r#""required_capabilities":[]"#)); assert!(launch_line.contains(r#""required_capabilities":[]"#));
assert!(launch_line.contains(r#""wasm_module_base64":""#)); assert!(launch_line.contains(r#""wasm_module_base64":""#));
assert!(launch_line.contains(r#""kind":"coordinator_node_wasm""#)); assert!(launch_line.contains(r#""kind":"coordinator_node_wasm""#));
assert!(launch_line.contains(r#""export":"disasmer_entry_v1_"#)); assert!(launch_line.contains(r#""export":"clusterflux_entry_v1_"#));
assert!(!launch_line.contains(r#""command":""#)); assert!(!launch_line.contains(r#""command":""#));
assert!(launch_line.contains(r#""actor_agent":"agent-ci""#)); assert!(launch_line.contains(r#""actor_agent":"agent-ci""#));
assert!(launch_line.contains(&format!( assert!(launch_line.contains(&format!(
@ -493,7 +499,7 @@ fn run_with_agent_public_key_sends_attributable_workflow_actor() {
RunArgs { RunArgs {
entry: Some("build".to_owned()), entry: Some("build".to_owned()),
project: Some(temp.path().to_path_buf()), project: Some(temp.path().to_path_buf()),
coordinator: Some(format!("disasmer+tcp://{addr}")), coordinator: Some(format!("clusterflux+tcp://{addr}")),
local: false, local: false,
non_interactive: true, non_interactive: true,
json: false, json: false,
@ -611,7 +617,7 @@ fn run_with_human_session_derives_workflow_scope_from_authenticated_envelope() {
fn run_local_flag_overrides_logged_in_hosted_selection() { fn run_local_flag_overrides_logged_in_hosted_selection() {
let Cli { let Cli {
command: Commands::Run(args), command: Commands::Run(args),
} = parse(&["disasmer", "run", "--local"]) } = parse(&["clusterflux", "run", "--local"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -677,7 +683,7 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
assert!(launch_line.contains(r#""required_capabilities":[]"#)); assert!(launch_line.contains(r#""required_capabilities":[]"#));
assert!(launch_line.contains(r#""wasm_module_base64":""#)); assert!(launch_line.contains(r#""wasm_module_base64":""#));
assert!(launch_line.contains(r#""kind":"coordinator_node_wasm""#)); assert!(launch_line.contains(r#""kind":"coordinator_node_wasm""#));
assert!(launch_line.contains(r#""export":"disasmer_entry_v1_"#)); assert!(launch_line.contains(r#""export":"clusterflux_entry_v1_"#));
assert!(!launch_line.contains(r#""command":""#)); assert!(!launch_line.contains(r#""command":""#));
assert!(!launch_line.contains("--manifest-path")); assert!(!launch_line.contains("--manifest-path"));
stream stream
@ -702,7 +708,7 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
RunArgs { RunArgs {
entry: Some("build".to_owned()), entry: Some("build".to_owned()),
project: Some(temp.path().to_path_buf()), project: Some(temp.path().to_path_buf()),
coordinator: Some(format!("disasmer+tcp://{addr}")), coordinator: Some(format!("clusterflux+tcp://{addr}")),
local: false, local: false,
non_interactive: false, non_interactive: false,
json: false, json: false,
@ -715,7 +721,7 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
RunArgs { RunArgs {
entry: Some("test".to_owned()), entry: Some("test".to_owned()),
project: Some(temp.path().to_path_buf()), project: Some(temp.path().to_path_buf()),
coordinator: Some(format!("disasmer+tcp://{addr}")), coordinator: Some(format!("clusterflux+tcp://{addr}")),
local: false, local: false,
non_interactive: false, non_interactive: false,
json: false, json: false,
@ -757,7 +763,7 @@ fn run_contacts_configured_coordinator_and_reports_active_process_conflicts() {
.as_array() .as_array()
.unwrap() .unwrap()
.iter() .iter()
.any(|action| action == "disasmer process restart --yes")); .any(|action| action == "clusterflux process restart --yes"));
} }
#[test] #[test]
@ -781,14 +787,14 @@ fn run_rejection_reports_machine_readable_error_category() {
.as_array() .as_array()
.unwrap() .unwrap()
.iter() .iter()
.any(|action| action == "disasmer quota status")); .any(|action| action == "clusterflux quota status"));
} }
#[test] #[test]
fn local_only_run_executes_ephemeral_local_services() { fn local_only_run_executes_ephemeral_local_services() {
let Cli { let Cli {
command: Commands::Run(args), command: Commands::Run(args),
} = parse(&["disasmer", "run", "--local"]) } = parse(&["clusterflux", "run", "--local"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -801,7 +807,7 @@ fn local_only_run_executes_ephemeral_local_services() {
fn login_defaults_to_the_server_owned_browser_flow() { fn login_defaults_to_the_server_owned_browser_flow() {
let Cli { let Cli {
command: Commands::Login(args), command: Commands::Login(args),
} = parse(&["disasmer", "login"]) } = parse(&["clusterflux", "login"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -822,7 +828,7 @@ fn login_defaults_to_the_server_owned_browser_flow() {
fn browser_login_flow_is_available_for_humans() { fn browser_login_flow_is_available_for_humans() {
let Cli { let Cli {
command: Commands::Login(args), command: Commands::Login(args),
} = parse(&["disasmer", "login", "--browser"]) } = parse(&["clusterflux", "login", "--browser"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -853,7 +859,7 @@ fn login_inherits_the_current_project_scope() {
.unwrap(); .unwrap();
let Cli { let Cli {
command: Commands::Login(args), command: Commands::Login(args),
} = parse(&["disasmer", "login", "--browser"]) } = parse(&["clusterflux", "login", "--browser"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -899,7 +905,7 @@ fn auth_status_reports_a_session_that_does_not_match_the_current_project() {
command: Commands::Auth { command: Commands::Auth {
command: AuthCommands::Status(args), command: AuthCommands::Status(args),
}, },
} = parse(&["disasmer", "auth", "status"]) } = parse(&["clusterflux", "auth", "status"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -921,7 +927,7 @@ fn auth_status_reports_a_session_that_does_not_match_the_current_project() {
fn browser_login_non_interactive_fails_before_opening_browser() { fn browser_login_non_interactive_fails_before_opening_browser() {
let Cli { let Cli {
command: Commands::Login(args), command: Commands::Login(args),
} = parse(&["disasmer", "login", "--browser", "--non-interactive"]) } = parse(&["clusterflux", "login", "--browser", "--non-interactive"])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -942,7 +948,7 @@ fn browser_login_non_interactive_fails_before_opening_browser() {
.filter_map(Value::as_str) .filter_map(Value::as_str)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert!(next_actions.contains(&"rerun without --non-interactive to open the browser")); assert!(next_actions.contains(&"rerun without --non-interactive to open the browser"));
assert!(next_actions.contains(&"use DISASMER_AGENT_PRIVATE_KEY for automation")); assert!(next_actions.contains(&"use CLUSTERFLUX_AGENT_PRIVATE_KEY for automation"));
} }
#[test] #[test]
@ -973,7 +979,7 @@ fn stored_browser_login_session_omits_provider_token_values() {
"project": "project-live", "project": "project-live",
"user": "user-live", "user": "user-live",
"cli_session_credential_kind": "CliDeviceSession", "cli_session_credential_kind": "CliDeviceSession",
"cli_session_secret": "disasmer-cli-session-secret", "cli_session_secret": "clusterflux-cli-session-secret",
"expires_at": "2026-07-04T00:00:00Z", "expires_at": "2026-07-04T00:00:00Z",
"access_token": "provider-secret", "access_token": "provider-secret",
"id_token": "provider-id-token", "id_token": "provider-id-token",
@ -993,7 +999,7 @@ fn stored_browser_login_session_omits_provider_token_values() {
assert_eq!(stored.user, "user-live"); assert_eq!(stored.user, "user-live");
assert_eq!( assert_eq!(
stored.session_secret.as_deref(), stored.session_secret.as_deref(),
Some("disasmer-cli-session-secret") Some("clusterflux-cli-session-secret")
); );
assert_eq!(stored.token_expiry_posture, "expires_at"); assert_eq!(stored.token_expiry_posture, "expires_at");
assert!(stored.provider_tokens_exposed_to_cli); assert!(stored.provider_tokens_exposed_to_cli);
@ -1014,7 +1020,7 @@ fn stored_browser_login_session_accepts_hosted_epoch_expiry() {
"project": "project-live", "project": "project-live",
"user": "user-live", "user": "user-live",
"cli_session_credential_kind": "CliDeviceSession", "cli_session_credential_kind": "CliDeviceSession",
"cli_session_secret": "disasmer-cli-session-secret", "cli_session_secret": "clusterflux-cli-session-secret",
"expires_at_epoch_seconds": 1_800_000_000_u64, "expires_at_epoch_seconds": 1_800_000_000_u64,
"provider_tokens_sent_to_nodes": false "provider_tokens_sent_to_nodes": false
} }
@ -1034,7 +1040,13 @@ fn agent_enroll_uses_public_key_without_browser_each_run() {
command: Commands::Agent { command: Commands::Agent {
command: AgentCommands::Enroll(args), command: AgentCommands::Enroll(args),
}, },
} = parse(&["disasmer", "agent", "enroll", "--public-key", "agent-key"]) } = parse(&[
"clusterflux",
"agent",
"enroll",
"--public-key",
"agent-key",
])
else { else {
panic!("wrong command"); panic!("wrong command");
}; };
@ -1063,7 +1075,7 @@ fn bundle_inspect_discovers_environments_selected_inputs_and_source_providers()
fs::write(temp.path().join("Cargo.toml"), "[package]\nname='demo'\n").unwrap(); fs::write(temp.path().join("Cargo.toml"), "[package]\nname='demo'\n").unwrap();
fs::write( fs::write(
temp.path().join("src/main.rs"), temp.path().join("src/main.rs"),
"fn main() {}\n\n#[disasmer::task]\nfn compile_linux() {}\n\n#[disasmer::main]\nfn build_main() {}\n", "fn main() {}\n\n#[clusterflux::task]\nfn compile_linux() {}\n\n#[clusterflux::main]\nfn build_main() {}\n",
) )
.unwrap(); .unwrap();
@ -1072,7 +1084,7 @@ fn bundle_inspect_discovers_environments_selected_inputs_and_source_providers()
command: BundleCommands::Inspect(args), command: BundleCommands::Inspect(args),
}, },
} = parse(&[ } = parse(&[
"disasmer", "clusterflux",
"bundle", "bundle",
"inspect", "inspect",
"--project", "--project",
@ -1273,7 +1285,7 @@ fn bundle_inspect_reports_missing_environment_references_before_schedule() {
); );
assert!(inspection.environment_diagnostics[0] assert!(inspection.environment_diagnostics[0]
.message .message
.contains("missing Disasmer environment `linux`")); .contains("missing Clusterflux environment `linux`"));
assert!(inspection assert!(inspection
.pre_schedule_diagnostics .pre_schedule_diagnostics
.iter() .iter()
@ -1492,7 +1504,7 @@ fn node_attach_can_exchange_enrollment_grant() {
command: NodeCommands::Attach(args), command: NodeCommands::Attach(args),
}, },
} = parse(&[ } = parse(&[
"disasmer", "clusterflux",
"node", "node",
"attach", "attach",
"--enrollment-grant", "--enrollment-grant",
@ -1519,7 +1531,7 @@ fn node_attach_enrollment_uses_default_public_key_when_not_explicit() {
command: NodeCommands::Attach(args), command: NodeCommands::Attach(args),
}, },
} = parse(&[ } = parse(&[
"disasmer", "clusterflux",
"node", "node",
"attach", "attach",
"--node", "--node",
@ -1553,16 +1565,16 @@ fn node_attach_local_credential_is_durable_and_project_scoped() {
assert!(credential_file.exists()); assert!(credential_file.exists());
assert!( assert!(
credential_file credential_file
.strip_prefix(temp.path().join(".disasmer").join("nodes")) .strip_prefix(temp.path().join(".clusterflux").join("nodes"))
.unwrap() .unwrap()
.components() .components()
.count() .count()
== 1 == 1
); );
let public_key = disasmer_core::node_ed25519_public_key_from_private_key(&first).unwrap(); let public_key = clusterflux_core::node_ed25519_public_key_from_private_key(&first).unwrap();
let bytes = fs::read(credential_file).unwrap(); let bytes = fs::read(credential_file).unwrap();
let stored: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); let stored: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(stored["kind"], "disasmer_node_credential"); assert_eq!(stored["kind"], "clusterflux_node_credential");
assert_eq!(stored["node"], node); assert_eq!(stored["node"], node);
assert_eq!(stored["public_key"], public_key); assert_eq!(stored["public_key"], public_key);
assert_eq!(stored["credential_scope"], "local_project_node_identity"); assert_eq!(stored["credential_scope"], "local_project_node_identity");
@ -1575,7 +1587,7 @@ fn node_attach_local_credential_is_durable_and_project_scoped() {
.permissions() .permissions()
.mode() .mode()
& 0o777; & 0o777;
let directory_mode = fs::metadata(temp.path().join(".disasmer").join("nodes")) let directory_mode = fs::metadata(temp.path().join(".clusterflux").join("nodes"))
.unwrap() .unwrap()
.permissions() .permissions()
.mode() .mode()
@ -1606,16 +1618,17 @@ fn node_attach_refuses_a_symlink_credential_target() {
fn hosted_coordinator_remains_a_real_https_control_endpoint() { fn hosted_coordinator_remains_a_real_https_control_endpoint() {
assert_eq!( assert_eq!(
control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(), control_endpoint_identity(DEFAULT_HOSTED_COORDINATOR_ENDPOINT).unwrap(),
"https://disasmer.michelpaulissen.com/api/v1/control" "https://clusterflux.michelpaulissen.com/api/v1/control"
); );
assert_eq!( assert_eq!(
control_endpoint_identity("https://disasmer.michelpaulissen.com/api/v1/control").unwrap(), control_endpoint_identity("https://clusterflux.michelpaulissen.com/api/v1/control")
"https://disasmer.michelpaulissen.com/api/v1/control" .unwrap(),
"https://clusterflux.michelpaulissen.com/api/v1/control"
); );
assert!(control_endpoint_identity("http://operator.example.test").is_err()); assert!(control_endpoint_identity("http://operator.example.test").is_err());
assert_eq!( assert_eq!(
control_endpoint_identity("127.0.0.1:7999").unwrap(), control_endpoint_identity("127.0.0.1:7999").unwrap(),
"disasmer+tcp://127.0.0.1:7999" "clusterflux+tcp://127.0.0.1:7999"
); );
} }
@ -1723,7 +1736,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
let mut line = String::new(); let mut line = String::new();
reader.read_line(&mut line).unwrap(); reader.read_line(&mut line).unwrap();
assert!(line.contains(r#""type":"authenticated""#)); assert!(line.contains(r#""type":"authenticated""#));
assert!(line.contains(r#""session_secret":"disasmer-cli-session-secret""#)); assert!(line.contains(r#""session_secret":"clusterflux-cli-session-secret""#));
assert!(line.contains(r#""type":"auth_status""#)); assert!(line.contains(r#""type":"auth_status""#));
assert!(!line.contains(r#""actor_user":"user-session""#)); assert!(!line.contains(r#""actor_user":"user-session""#));
stream stream
@ -1742,7 +1755,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
project: "project-session".to_owned(), project: "project-session".to_owned(),
user: "user-session".to_owned(), user: "user-session".to_owned(),
cli_session_credential_kind: "CliDeviceSession".to_owned(), cli_session_credential_kind: "CliDeviceSession".to_owned(),
session_secret: Some("disasmer-cli-session-secret".to_owned()), session_secret: Some("clusterflux-cli-session-secret".to_owned()),
token_expiry_posture: "expires_at".to_owned(), token_expiry_posture: "expires_at".to_owned(),
expires_at: Some("2026-07-04T00:00:00Z".to_owned()), expires_at: Some("2026-07-04T00:00:00Z".to_owned()),
provider_tokens_exposed_to_cli: false, provider_tokens_exposed_to_cli: false,
@ -1799,7 +1812,7 @@ fn auth_status_reads_stored_cli_session_without_provider_tokens() {
#[test] #[test]
fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() { fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
for message in [ for message in [
"unauthorized coordinator action: CLI session credential has expired; run disasmer login --browser again", "unauthorized coordinator action: CLI session credential has expired; run clusterflux login --browser again",
"unauthorized coordinator action: CLI session credential has been revoked", "unauthorized coordinator action: CLI session credential has been revoked",
] { ] {
let temp = tempfile::tempdir().unwrap(); let temp = tempfile::tempdir().unwrap();
@ -1867,7 +1880,7 @@ fn auth_status_reports_expired_or_revoked_cli_session_as_login_required() {
.as_array() .as_array()
.unwrap() .unwrap()
.iter() .iter()
.any(|action| action == "disasmer login --browser") .any(|action| action == "clusterflux login --browser")
); );
} }
} }
@ -2068,13 +2081,13 @@ fn auth_status_reports_disabled_deleted_and_manual_review_safely() {
#[test] #[test]
fn cli_first_mvp_command_surface_parses() { fn cli_first_mvp_command_surface_parses() {
for args in [ for args in [
&["disasmer", "doctor"][..], &["clusterflux", "doctor"][..],
&["disasmer", "auth", "status"], &["clusterflux", "auth", "status"],
&["disasmer", "logout", "--yes"], &["clusterflux", "logout", "--yes"],
&["disasmer", "auth", "logout", "--yes"], &["clusterflux", "auth", "logout", "--yes"],
&["disasmer", "login", "--browser", "--non-interactive"], &["clusterflux", "login", "--browser", "--non-interactive"],
&[ &[
"disasmer", "clusterflux",
"key", "key",
"add", "add",
"--agent", "--agent",
@ -2082,47 +2095,52 @@ fn cli_first_mvp_command_surface_parses() {
"--public-key", "--public-key",
"key", "key",
], ],
&["disasmer", "key", "list"], &["clusterflux", "key", "list"],
&["disasmer", "key", "revoke", "--agent", "agent", "--yes"], &["clusterflux", "key", "revoke", "--agent", "agent", "--yes"],
&["disasmer", "project", "init", "--yes"], &["clusterflux", "project", "init", "--yes"],
&["disasmer", "project", "status"], &["clusterflux", "project", "status"],
&["disasmer", "project", "list"], &["clusterflux", "project", "list"],
&["disasmer", "project", "select", "project"], &["clusterflux", "project", "select", "project"],
&["disasmer", "inspect"], &["clusterflux", "inspect"],
&["disasmer", "build"], &["clusterflux", "build"],
&["disasmer", "run", "--non-interactive"], &["clusterflux", "run", "--non-interactive"],
&["disasmer", "node", "enroll"], &["clusterflux", "node", "enroll"],
&["disasmer", "node", "list"], &["clusterflux", "node", "list"],
&["disasmer", "node", "status"], &["clusterflux", "node", "status"],
&["disasmer", "node", "revoke", "--node", "node", "--yes"], &["clusterflux", "node", "revoke", "--node", "node", "--yes"],
&["disasmer", "process", "list"], &["clusterflux", "process", "list"],
&["disasmer", "process", "status"], &["clusterflux", "process", "status"],
&["disasmer", "process", "restart", "--yes"], &["clusterflux", "process", "restart", "--yes"],
&["disasmer", "process", "cancel", "--yes"], &["clusterflux", "process", "cancel", "--yes"],
&["disasmer", "process", "abort", "--yes"], &["clusterflux", "process", "abort", "--yes"],
&["disasmer", "task", "list"], &["clusterflux", "task", "list"],
&["disasmer", "task", "restart", "compile-linux", "--yes"], &["clusterflux", "task", "restart", "compile-linux", "--yes"],
&["disasmer", "logs"], &["clusterflux", "logs"],
&["disasmer", "artifact", "list"], &["clusterflux", "artifact", "list"],
&["disasmer", "artifact", "download", "artifact"], &["clusterflux", "artifact", "download", "artifact"],
&[ &[
"disasmer", "artifact", "export", "artifact", "--to", "/tmp/out", "clusterflux",
"artifact",
"export",
"artifact",
"--to",
"/tmp/out",
], ],
&["disasmer", "dap", "--plan"], &["clusterflux", "dap", "--plan"],
&["disasmer", "debug", "attach"], &["clusterflux", "debug", "attach"],
&["disasmer", "quota", "status"], &["clusterflux", "quota", "status"],
&["disasmer", "admin", "status"], &["clusterflux", "admin", "status"],
&["disasmer", "admin", "bootstrap", "--yes"], &["clusterflux", "admin", "bootstrap", "--yes"],
&[ &[
"disasmer", "clusterflux",
"admin", "admin",
"revoke-node", "revoke-node",
"--node", "--node",
"node", "node",
"--yes", "--yes",
], ],
&["disasmer", "admin", "stop-process", "--yes"], &["clusterflux", "admin", "stop-process", "--yes"],
&["disasmer", "admin", "suspend-tenant", "--yes"], &["clusterflux", "admin", "suspend-tenant", "--yes"],
] { ] {
let _ = parse(args); let _ = parse(args);
} }
@ -2131,9 +2149,9 @@ fn cli_first_mvp_command_surface_parses() {
#[test] #[test]
fn cli_has_no_direct_hosted_account_creation_command() { fn cli_has_no_direct_hosted_account_creation_command() {
for args in [ for args in [
&["disasmer", "signup"][..], &["clusterflux", "signup"][..],
&["disasmer", "account", "create"], &["clusterflux", "account", "create"],
&["disasmer", "login", "--create-account"], &["clusterflux", "login", "--create-account"],
] { ] {
let error = Cli::try_parse_from(args).unwrap_err().to_string(); let error = Cli::try_parse_from(args).unwrap_err().to_string();
assert!( assert!(
@ -2145,7 +2163,7 @@ fn cli_has_no_direct_hosted_account_creation_command() {
let mut command = Cli::command(); let mut command = Cli::command();
let help = command.render_help().to_string(); let help = command.render_help().to_string();
assert!(help.contains("Hosted account creation happens in the browser login flow.")); assert!(help.contains("Hosted account creation happens in the browser login flow."));
assert!(!help.contains("disasmer signup")); assert!(!help.contains("clusterflux signup"));
assert!(!help.contains("account create")); assert!(!help.contains("account create"));
} }
@ -2181,7 +2199,7 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
); );
assert_eq!( assert_eq!(
report["admin_surfaces"]["node"], report["admin_surfaces"]["node"],
"disasmer node enroll/list/status/revoke" "clusterflux node enroll/list/status/revoke"
); );
let steps = report["bootstrap_sequence"].as_array().unwrap(); let steps = report["bootstrap_sequence"].as_array().unwrap();
for expected in [ for expected in [
@ -2208,12 +2226,12 @@ fn admin_bootstrap_reports_self_hosted_cli_only_path() {
.get("command") .get("command")
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("") .unwrap_or("")
.contains("disasmer node enroll"))); .contains("clusterflux node enroll")));
assert!(steps.iter().any(|step| step assert!(steps.iter().any(|step| step
.get("command") .get("command")
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("") .unwrap_or("")
.contains("disasmer admin revoke-node"))); .contains("clusterflux admin revoke-node")));
} }
#[test] #[test]
@ -2223,15 +2241,15 @@ fn top_level_help_exposes_primary_workflow_without_auth() {
for expected in [ for expected in [
"Primary workflow:", "Primary workflow:",
"disasmer login --browser", "clusterflux login --browser",
"disasmer project init", "clusterflux project init",
"disasmer node enroll", "clusterflux node enroll",
"disasmer node attach", "clusterflux node attach",
"disasmer-node --worker", "clusterflux-node --worker",
"disasmer run [entry] --project <path>", "clusterflux run [entry] --project <path>",
"Disasmer: Launch Virtual Process", "Clusterflux: Launch Virtual Process",
"disasmer dap", "clusterflux dap",
"disasmer process status", "clusterflux process status",
"task list", "task list",
"logs", "logs",
"artifact list", "artifact list",
@ -2437,19 +2455,19 @@ fn mutating_commands_require_yes_before_side_effects() {
#[test] #[test]
fn cli_first_json_mode_parses_for_primary_commands() { fn cli_first_json_mode_parses_for_primary_commands() {
for args in [ for args in [
&["disasmer", "doctor", "--json"][..], &["clusterflux", "doctor", "--json"][..],
&["disasmer", "login", "--json"], &["clusterflux", "login", "--json"],
&[ &[
"disasmer", "clusterflux",
"login", "login",
"--browser", "--browser",
"--non-interactive", "--non-interactive",
"--json", "--json",
], ],
&["disasmer", "logout", "--yes", "--json"], &["clusterflux", "logout", "--yes", "--json"],
&["disasmer", "auth", "status", "--json"], &["clusterflux", "auth", "status", "--json"],
&[ &[
"disasmer", "clusterflux",
"agent", "agent",
"enroll", "enroll",
"--public-key", "--public-key",
@ -2457,7 +2475,7 @@ fn cli_first_json_mode_parses_for_primary_commands() {
"--json", "--json",
], ],
&[ &[
"disasmer", "clusterflux",
"key", "key",
"add", "add",
"--agent", "--agent",
@ -2466,26 +2484,32 @@ fn cli_first_json_mode_parses_for_primary_commands() {
"key", "key",
"--json", "--json",
], ],
&["disasmer", "project", "init", "--yes", "--json"], &["clusterflux", "project", "init", "--yes", "--json"],
&["disasmer", "inspect", "--json"], &["clusterflux", "inspect", "--json"],
&["disasmer", "build", "--json"], &["clusterflux", "build", "--json"],
&["disasmer", "bundle", "inspect", "--json"], &["clusterflux", "bundle", "inspect", "--json"],
&["disasmer", "run", "--json"], &["clusterflux", "run", "--json"],
&["disasmer", "run", "--non-interactive", "--json"], &["clusterflux", "run", "--non-interactive", "--json"],
&["disasmer", "node", "attach", "--json"], &["clusterflux", "node", "attach", "--json"],
&["disasmer", "node", "enroll", "--json"], &["clusterflux", "node", "enroll", "--json"],
&["disasmer", "process", "status", "--json"], &["clusterflux", "process", "status", "--json"],
&["disasmer", "task", "list", "--json"], &["clusterflux", "task", "list", "--json"],
&["disasmer", "logs", "--json"], &["clusterflux", "logs", "--json"],
&["disasmer", "artifact", "list", "--json"], &["clusterflux", "artifact", "list", "--json"],
&["disasmer", "artifact", "download", "artifact", "--json"], &["clusterflux", "artifact", "download", "artifact", "--json"],
&[ &[
"disasmer", "artifact", "export", "artifact", "--to", "/tmp/out", "--json", "clusterflux",
"artifact",
"export",
"artifact",
"--to",
"/tmp/out",
"--json",
], ],
&["disasmer", "dap", "--plan", "--json"], &["clusterflux", "dap", "--plan", "--json"],
&["disasmer", "debug", "attach", "--json"], &["clusterflux", "debug", "attach", "--json"],
&["disasmer", "quota", "status", "--json"], &["clusterflux", "quota", "status", "--json"],
&["disasmer", "admin", "status", "--json"], &["clusterflux", "admin", "status", "--json"],
] { ] {
let _ = parse(args); let _ = parse(args);
} }
@ -2780,7 +2804,7 @@ fn admin_status_and_suspend_use_public_coordinator_api() {
let issued_at_epoch_seconds = payload["issued_at_epoch_seconds"].as_u64().unwrap(); let issued_at_epoch_seconds = payload["issued_at_epoch_seconds"].as_u64().unwrap();
assert_eq!( assert_eq!(
payload["admin_proof"], payload["admin_proof"],
disasmer_core::admin_request_proof( clusterflux_core::admin_request_proof(
"admin-token", "admin-token",
expected, expected,
tenant, tenant,
@ -2881,7 +2905,7 @@ fn debug_attach_reports_public_authorization() {
}, },
process: "vp".to_owned(), process: "vp".to_owned(),
}, },
"/tmp/disasmer-debug-dap-test".to_owned(), "/tmp/clusterflux-debug-dap-test".to_owned(),
) )
.unwrap(); .unwrap();
server.join().unwrap(); server.join().unwrap();
@ -3053,7 +3077,7 @@ fn user_control_commands_use_authenticated_envelope_with_stored_cli_session() {
scope: coordinator_scope, scope: coordinator_scope,
process: "vp".to_owned(), process: "vp".to_owned(),
}, },
"/tmp/disasmer-debug-dap-test".to_owned(), "/tmp/clusterflux-debug-dap-test".to_owned(),
Some(&session), Some(&session),
) )
.unwrap(); .unwrap();
@ -3066,15 +3090,15 @@ fn human_report_is_text_not_json() {
"command": "doctor", "command": "doctor",
"status": "ok", "status": "ok",
"coordinator": "127.0.0.1:9443", "coordinator": "127.0.0.1:9443",
"next_actions": ["disasmer login --browser", "disasmer project init"], "next_actions": ["clusterflux login --browser", "clusterflux project init"],
}); });
let human = human_report(&report); let human = human_report(&report);
assert!(!human.trim_start().starts_with('{')); assert!(!human.trim_start().starts_with('{'));
assert!(human.contains("Disasmer doctor")); assert!(human.contains("Clusterflux doctor"));
assert!(human.contains("status: ok")); assert!(human.contains("status: ok"));
assert!(human.contains("coordinator: 127.0.0.1:9443")); assert!(human.contains("coordinator: 127.0.0.1:9443"));
assert!(human.contains("disasmer login --browser")); assert!(human.contains("clusterflux login --browser"));
} }
#[test] #[test]
@ -3102,7 +3126,7 @@ fn project_init_select_and_status_use_local_project_config() {
assert_eq!(init["project_config_written"], true); assert_eq!(init["project_config_written"], true);
assert_eq!( assert_eq!(
init["current_directory_link"]["config_format"], init["current_directory_link"]["config_format"],
"disasmer_project_config_v1" "clusterflux_project_config_v1"
); );
assert_eq!( assert_eq!(
init["current_directory_link"]["links_current_directory"], init["current_directory_link"]["links_current_directory"],
@ -4403,7 +4427,7 @@ fn build_command_reuses_bundle_inspection_without_full_repo_upload() {
assert_eq!(report["content_addressed"], true); assert_eq!(report["content_addressed"], true);
assert_eq!(report["contains_full_repository_upload"], false); assert_eq!(report["contains_full_repository_upload"], false);
assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 9); assert_eq!(report["bundle_artifact"]["task_descriptor_count"], 9);
assert_eq!(report["bundle_artifact"]["entrypoint_count"], 3); assert_eq!(report["bundle_artifact"]["entrypoint_count"], 4);
assert!(output.join("module.wasm").is_file()); assert!(output.join("module.wasm").is_file());
assert!(output.join("manifest.json").is_file()); assert!(output.join("manifest.json").is_file());
assert!(output.join("task-descriptors.json").is_file()); assert!(output.join("task-descriptors.json").is_file());

View file

@ -39,25 +39,25 @@ pub(crate) fn sibling_binary(name: &str) -> Option<PathBuf> {
} }
pub(crate) fn dap_binary_path() -> Result<PathBuf> { pub(crate) fn dap_binary_path() -> Result<PathBuf> {
if let Some(path) = std::env::var_os("DISASMER_DAP_BIN") { if let Some(path) = std::env::var_os("CLUSTERFLUX_DAP_BIN") {
return Ok(PathBuf::from(path)); return Ok(PathBuf::from(path));
} }
if let Some(path) = sibling_binary("disasmer-debug-dap") { if let Some(path) = sibling_binary("clusterflux-debug-dap") {
return Ok(path); return Ok(path);
} }
let release = PathBuf::from("target/release").join(format!( let release = PathBuf::from("target/release").join(format!(
"disasmer-debug-dap{}", "clusterflux-debug-dap{}",
std::env::consts::EXE_SUFFIX std::env::consts::EXE_SUFFIX
)); ));
if release.is_file() { if release.is_file() {
return Ok(release); return Ok(release);
} }
let debug = PathBuf::from("target/debug").join(format!( let debug = PathBuf::from("target/debug").join(format!(
"disasmer-debug-dap{}", "clusterflux-debug-dap{}",
std::env::consts::EXE_SUFFIX std::env::consts::EXE_SUFFIX
)); ));
if debug.is_file() { if debug.is_file() {
return Ok(debug); return Ok(debug);
} }
anyhow::bail!("could not locate disasmer-debug-dap; set DISASMER_DAP_BIN") anyhow::bail!("could not locate clusterflux-debug-dap; set CLUSTERFLUX_DAP_BIN")
} }

View file

@ -1,5 +1,5 @@
[package] [package]
name = "disasmer-control" name = "clusterflux-control"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
license.workspace = true license.workspace = true

View file

@ -84,7 +84,9 @@ impl ControlSession {
}); });
} }
let loopback_address = endpoint.strip_prefix("disasmer+tcp://").unwrap_or(endpoint); let loopback_address = endpoint
.strip_prefix("clusterflux+tcp://")
.unwrap_or(endpoint);
if !endpoint_is_loopback(loopback_address) { if !endpoint_is_loopback(loopback_address) {
return Err(ControlTransportError::InsecureRemote(endpoint.to_owned())); return Err(ControlTransportError::InsecureRemote(endpoint.to_owned()));
} }
@ -185,9 +187,11 @@ pub fn endpoint_identity(endpoint: &str) -> Result<String, ControlTransportError
} }
return control_api_url(endpoint); return control_api_url(endpoint);
} }
let loopback_address = endpoint.strip_prefix("disasmer+tcp://").unwrap_or(endpoint); let loopback_address = endpoint
.strip_prefix("clusterflux+tcp://")
.unwrap_or(endpoint);
if endpoint_is_loopback(loopback_address) { if endpoint_is_loopback(loopback_address) {
return Ok(format!("disasmer+tcp://{loopback_address}")); return Ok(format!("clusterflux+tcp://{loopback_address}"));
} }
Err(ControlTransportError::InsecureRemote(endpoint.to_owned())) Err(ControlTransportError::InsecureRemote(endpoint.to_owned()))
} }
@ -197,7 +201,7 @@ pub fn endpoint_is_loopback(endpoint: &str) -> bool {
.trim() .trim()
.strip_prefix("https://") .strip_prefix("https://")
.or_else(|| endpoint.trim().strip_prefix("http://")) .or_else(|| endpoint.trim().strip_prefix("http://"))
.or_else(|| endpoint.trim().strip_prefix("disasmer+tcp://")) .or_else(|| endpoint.trim().strip_prefix("clusterflux+tcp://"))
.unwrap_or(endpoint.trim()) .unwrap_or(endpoint.trim())
.split('/') .split('/')
.next() .next()
@ -236,19 +240,19 @@ mod tests {
#[test] #[test]
fn hosted_endpoints_are_real_https_api_urls() { fn hosted_endpoints_are_real_https_api_urls() {
assert_eq!( assert_eq!(
control_api_url("https://disasmer.example").unwrap(), control_api_url("https://clusterflux.example").unwrap(),
"https://disasmer.example/api/v1/control" "https://clusterflux.example/api/v1/control"
); );
assert_eq!( assert_eq!(
endpoint_identity("https://disasmer.example/api/v1/control").unwrap(), endpoint_identity("https://clusterflux.example/api/v1/control").unwrap(),
"https://disasmer.example/api/v1/control" "https://clusterflux.example/api/v1/control"
); );
} }
#[test] #[test]
fn plaintext_transport_is_restricted_to_loopback() { fn plaintext_transport_is_restricted_to_loopback() {
assert!(endpoint_is_loopback("127.0.0.1:7999")); assert!(endpoint_is_loopback("127.0.0.1:7999"));
assert!(endpoint_is_loopback("disasmer+tcp://127.0.0.1:7999")); assert!(endpoint_is_loopback("clusterflux+tcp://127.0.0.1:7999"));
assert!(endpoint_is_loopback("http://[::1]:7999")); assert!(endpoint_is_loopback("http://[::1]:7999"));
assert!(matches!( assert!(matches!(
ControlSession::connect("http://example.com:7999"), ControlSession::connect("http://example.com:7999"),

View file

@ -1,5 +1,5 @@
[package] [package]
name = "disasmer-coordinator" name = "clusterflux-coordinator"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
license.workspace = true license.workspace = true
@ -7,8 +7,8 @@ repository.workspace = true
[dependencies] [dependencies]
base64.workspace = true base64.workspace = true
disasmer-core = { path = "../disasmer-core" } clusterflux-core = { path = "../clusterflux-core" }
disasmer-wasm-runtime = { path = "../disasmer-wasm-runtime" } clusterflux-wasm-runtime = { path = "../clusterflux-wasm-runtime" }
postgres.workspace = true postgres.workspace = true
serde.workspace = true serde.workspace = true
serde_json.workspace = true serde_json.workspace = true

View file

@ -1,4 +1,4 @@
use disasmer_core::{ use clusterflux_core::{
verify_agent_workflow_signature, Actor, AgentId, AgentSignedRequest, AgentWorkflowScope, verify_agent_workflow_signature, Actor, AgentId, AgentSignedRequest, AgentWorkflowScope,
AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId, AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId,
}; };

View file

@ -1,6 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use disasmer_core::{ use clusterflux_core::{
AgentId, CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId, AgentId, CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
}; };
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};

View file

@ -1,6 +1,6 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use disasmer_core::{ use clusterflux_core::{
Actor, AuthContext, Authorization, CredentialKind, Digest, EnrollmentError, EnrollmentGrant, Actor, AuthContext, Authorization, CredentialKind, Digest, EnrollmentError, EnrollmentGrant,
NodeCredential, NodeId, ProcessId, ProjectId, SourceProviderKind, TenantId, UserId, NodeCredential, NodeId, ProcessId, ProjectId, SourceProviderKind, TenantId, UserId,
}; };
@ -21,10 +21,12 @@ pub use postgres_store::{
PostgresDurableStore, PostgresStoreError, PostgresTable, POSTGRES_DURABLE_TABLES, PostgresDurableStore, PostgresStoreError, PostgresTable, POSTGRES_DURABLE_TABLES,
}; };
pub use service::{ pub use service::{
CoordinatorAdmission, CoordinatorRequest, CoordinatorResponse, CoordinatorService, ArtifactRelayConfiguration, ArtifactRelayError, ArtifactRelayUsage, CoordinatorAdmission,
CoordinatorServiceError, DebugAcknowledgementState, DebugAuditEvent, CoordinatorMainRuntimeConfiguration, CoordinatorRequest, CoordinatorResponse,
CoordinatorService, CoordinatorServiceError, DebugAcknowledgementState, DebugAuditEvent,
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
TaskAssignment, TaskCancellationTarget, TaskCompletionEvent, TaskExecutor, TaskTerminalState, TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskTerminalState,
}; };
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
@ -288,7 +290,7 @@ impl Coordinator {
Vec::new() Vec::new()
} else { } else {
vec![ vec![
"disasmer auth status --json".to_owned(), "clusterflux auth status --json".to_owned(),
"contact the hosted operator or use a self-hosted coordinator".to_owned(), "contact the hosted operator or use a self-hosted coordinator".to_owned(),
] ]
}; };
@ -554,7 +556,7 @@ impl Coordinator {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use disasmer_core::AgentId; use clusterflux_core::AgentId;
use super::*; use super::*;

View file

@ -1,12 +1,12 @@
use std::io::Write; use std::io::Write;
use disasmer_coordinator::{service::bind_listener, CoordinatorService}; use clusterflux_coordinator::{service::bind_listener, CoordinatorService};
use disasmer_core::{ProjectId, TenantId, UserId}; use clusterflux_core::{ProjectId, TenantId, UserId};
use serde_json::json; use serde_json::json;
fn main() -> Result<(), Box<dyn std::error::Error>> { fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut listen = "127.0.0.1:0".to_owned(); let mut listen = "127.0.0.1:0".to_owned();
let mut allow_local_trusted = std::env::var("DISASMER_ALLOW_LOCAL_TRUSTED_LOOPBACK") let mut allow_local_trusted = std::env::var("CLUSTERFLUX_ALLOW_LOCAL_TRUSTED_LOOPBACK")
.ok() .ok()
.as_deref() .as_deref()
== Some("1"); == Some("1");
@ -22,21 +22,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
let (listener, addr) = bind_listener(&listen)?; let (listener, addr) = bind_listener(&listen)?;
let database_url = std::env::var("DATABASE_URL").ok(); let database_url = std::env::var("DATABASE_URL").ok();
let mut service = CoordinatorService::new_with_database_url(1, database_url.as_deref())?; let mut service = CoordinatorService::new_with_database_url(1, database_url.as_deref())?;
let self_hosted_session_secret = std::env::var("DISASMER_SELF_HOSTED_SESSION_SECRET") let self_hosted_session_secret = std::env::var("CLUSTERFLUX_SELF_HOSTED_SESSION_SECRET")
.ok() .ok()
.filter(|secret| !secret.trim().is_empty()); .filter(|secret| !secret.trim().is_empty());
if let Some(session_secret) = self_hosted_session_secret.as_deref() { if let Some(session_secret) = self_hosted_session_secret.as_deref() {
service.issue_cli_session( service.issue_cli_session(
TenantId::new( TenantId::new(
std::env::var("DISASMER_SELF_HOSTED_TENANT") std::env::var("CLUSTERFLUX_SELF_HOSTED_TENANT")
.unwrap_or_else(|_| "tenant".to_owned()), .unwrap_or_else(|_| "tenant".to_owned()),
), ),
ProjectId::new( ProjectId::new(
std::env::var("DISASMER_SELF_HOSTED_PROJECT") std::env::var("CLUSTERFLUX_SELF_HOSTED_PROJECT")
.unwrap_or_else(|_| "project".to_owned()), .unwrap_or_else(|_| "project".to_owned()),
), ),
UserId::new( UserId::new(
std::env::var("DISASMER_SELF_HOSTED_USER").unwrap_or_else(|_| "user".to_owned()), std::env::var("CLUSTERFLUX_SELF_HOSTED_USER").unwrap_or_else(|_| "user".to_owned()),
), ),
session_secret, session_secret,
None, None,

View file

@ -17,52 +17,52 @@ pub struct PostgresTable {
pub const POSTGRES_DURABLE_TABLES: &[PostgresTable] = &[ pub const POSTGRES_DURABLE_TABLES: &[PostgresTable] = &[
PostgresTable { PostgresTable {
name: "disasmer_tenants", name: "clusterflux_tenants",
durable_record: "tenants", durable_record: "tenants",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_users", name: "clusterflux_users",
durable_record: "users", durable_record: "users",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_projects", name: "clusterflux_projects",
durable_record: "projects", durable_record: "projects",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_node_identities", name: "clusterflux_node_identities",
durable_record: "node identities", durable_record: "node identities",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_credentials", name: "clusterflux_credentials",
durable_record: "credentials", durable_record: "credentials",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_cli_sessions", name: "clusterflux_cli_sessions",
durable_record: "CLI sessions", durable_record: "CLI sessions",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_agent_public_keys", name: "clusterflux_agent_public_keys",
durable_record: "agent public keys", durable_record: "agent public keys",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_source_provider_configs", name: "clusterflux_source_provider_configs",
durable_record: "source-provider configuration", durable_record: "source-provider configuration",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_service_policy_records", name: "clusterflux_service_policy_records",
durable_record: "durable service policy records", durable_record: "durable service policy records",
restart_surviving: true, restart_surviving: true,
}, },
PostgresTable { PostgresTable {
name: "disasmer_project_permissions", name: "clusterflux_project_permissions",
durable_record: "explicit project permissions", durable_record: "explicit project permissions",
restart_surviving: true, restart_surviving: true,
}, },
@ -134,39 +134,39 @@ impl FallibleDurableStore for PostgresDurableStore {
let mut state = DurableState::default(); let mut state = DurableState::default();
for record in self.query_records::<TenantRecord>( for record in self.query_records::<TenantRecord>(
"SELECT record FROM disasmer_tenants ORDER BY tenant_id", "SELECT record FROM clusterflux_tenants ORDER BY tenant_id",
)? { )? {
state.tenants.insert(record.id.clone(), record); state.tenants.insert(record.id.clone(), record);
} }
for record in for record in self
self.query_records::<UserRecord>("SELECT record FROM disasmer_users ORDER BY user_id")? .query_records::<UserRecord>("SELECT record FROM clusterflux_users ORDER BY user_id")?
{ {
state.users.insert(record.id.clone(), record); state.users.insert(record.id.clone(), record);
} }
for record in self.query_records::<ProjectRecord>( for record in self.query_records::<ProjectRecord>(
"SELECT record FROM disasmer_projects ORDER BY project_id", "SELECT record FROM clusterflux_projects ORDER BY project_id",
)? { )? {
state.projects.insert(record.id.clone(), record); state.projects.insert(record.id.clone(), record);
} }
for record in self.query_records::<NodeIdentityRecord>( for record in self.query_records::<NodeIdentityRecord>(
"SELECT record FROM disasmer_node_identities ORDER BY node_id", "SELECT record FROM clusterflux_node_identities ORDER BY node_id",
)? { )? {
state.node_identities.insert(record.id.clone(), record); state.node_identities.insert(record.id.clone(), record);
} }
for record in self.query_records::<CredentialRecord>( for record in self.query_records::<CredentialRecord>(
"SELECT record FROM disasmer_credentials ORDER BY subject", "SELECT record FROM clusterflux_credentials ORDER BY subject",
)? { )? {
state.credentials.insert(record.subject.clone(), record); state.credentials.insert(record.subject.clone(), record);
} }
for record in self.query_records::<CliSessionRecord>( for record in self.query_records::<CliSessionRecord>(
"SELECT record FROM disasmer_cli_sessions ORDER BY session_digest", "SELECT record FROM clusterflux_cli_sessions ORDER BY session_digest",
)? { )? {
state state
.cli_sessions .cli_sessions
.insert(record.session_digest.clone(), record); .insert(record.session_digest.clone(), record);
} }
for record in self.query_records::<AgentPublicKeyRecord>( for record in self.query_records::<AgentPublicKeyRecord>(
"SELECT record FROM disasmer_agent_public_keys ORDER BY tenant_id, project_id, agent_id", "SELECT record FROM clusterflux_agent_public_keys ORDER BY tenant_id, project_id, agent_id",
)? { )? {
state.agent_public_keys.insert( state.agent_public_keys.insert(
( (
@ -178,7 +178,7 @@ impl FallibleDurableStore for PostgresDurableStore {
); );
} }
for record in self.query_records::<SourceProviderConfigRecord>( for record in self.query_records::<SourceProviderConfigRecord>(
"SELECT record FROM disasmer_source_provider_configs ORDER BY tenant_id, project_id, provider_key", "SELECT record FROM clusterflux_source_provider_configs ORDER BY tenant_id, project_id, provider_key",
)? { )? {
let provider_key = format!("{:?}", record.provider); let provider_key = format!("{:?}", record.provider);
state.source_provider_configs.insert( state.source_provider_configs.insert(
@ -187,14 +187,14 @@ impl FallibleDurableStore for PostgresDurableStore {
); );
} }
for record in self.query_records::<ServicePolicyRecord>( for record in self.query_records::<ServicePolicyRecord>(
"SELECT record FROM disasmer_service_policy_records ORDER BY tenant_id, name", "SELECT record FROM clusterflux_service_policy_records ORDER BY tenant_id, name",
)? { )? {
state state
.service_policy_records .service_policy_records
.insert((record.tenant.clone(), record.name.clone()), record); .insert((record.tenant.clone(), record.name.clone()), record);
} }
for record in self.query_records::<ProjectPermissionRecord>( for record in self.query_records::<ProjectPermissionRecord>(
"SELECT record FROM disasmer_project_permissions ORDER BY tenant_id, project_id, user_id", "SELECT record FROM clusterflux_project_permissions ORDER BY tenant_id, project_id, user_id",
)? { )? {
state.project_permissions.insert( state.project_permissions.insert(
( (
@ -213,44 +213,44 @@ impl FallibleDurableStore for PostgresDurableStore {
let mut tx = self.client.transaction()?; let mut tx = self.client.transaction()?;
tx.batch_execute( tx.batch_execute(
" "
DELETE FROM disasmer_project_permissions; DELETE FROM clusterflux_project_permissions;
DELETE FROM disasmer_service_policy_records; DELETE FROM clusterflux_service_policy_records;
DELETE FROM disasmer_source_provider_configs; DELETE FROM clusterflux_source_provider_configs;
DELETE FROM disasmer_agent_public_keys; DELETE FROM clusterflux_agent_public_keys;
DELETE FROM disasmer_cli_sessions; DELETE FROM clusterflux_cli_sessions;
DELETE FROM disasmer_credentials; DELETE FROM clusterflux_credentials;
DELETE FROM disasmer_node_identities; DELETE FROM clusterflux_node_identities;
DELETE FROM disasmer_projects; DELETE FROM clusterflux_projects;
DELETE FROM disasmer_users; DELETE FROM clusterflux_users;
DELETE FROM disasmer_tenants; DELETE FROM clusterflux_tenants;
", ",
)?; )?;
for record in state.tenants.values() { for record in state.tenants.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_tenants (tenant_id, record) VALUES ($1, $2)", "INSERT INTO clusterflux_tenants (tenant_id, record) VALUES ($1, $2)",
&[&record.id.as_str(), &value], &[&record.id.as_str(), &value],
)?; )?;
} }
for record in state.users.values() { for record in state.users.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_users (user_id, tenant_id, record) VALUES ($1, $2, $3)", "INSERT INTO clusterflux_users (user_id, tenant_id, record) VALUES ($1, $2, $3)",
&[&record.id.as_str(), &record.tenant.as_str(), &value], &[&record.id.as_str(), &record.tenant.as_str(), &value],
)?; )?;
} }
for record in state.projects.values() { for record in state.projects.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_projects (project_id, tenant_id, record) VALUES ($1, $2, $3)", "INSERT INTO clusterflux_projects (project_id, tenant_id, record) VALUES ($1, $2, $3)",
&[&record.id.as_str(), &record.tenant.as_str(), &value], &[&record.id.as_str(), &record.tenant.as_str(), &value],
)?; )?;
} }
for record in state.node_identities.values() { for record in state.node_identities.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_node_identities (node_id, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)", "INSERT INTO clusterflux_node_identities (node_id, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
&[ &[
&record.id.as_str(), &record.id.as_str(),
&record.tenant.as_str(), &record.tenant.as_str(),
@ -263,14 +263,14 @@ impl FallibleDurableStore for PostgresDurableStore {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
let project_id = record.project.as_ref().map(|project| project.as_str()); let project_id = record.project.as_ref().map(|project| project.as_str());
tx.execute( tx.execute(
"INSERT INTO disasmer_credentials (subject, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)", "INSERT INTO clusterflux_credentials (subject, tenant_id, project_id, record) VALUES ($1, $2, $3, $4)",
&[&record.subject.as_str(), &record.tenant.as_str(), &project_id, &value], &[&record.subject.as_str(), &record.tenant.as_str(), &project_id, &value],
)?; )?;
} }
for record in state.cli_sessions.values() { for record in state.cli_sessions.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_cli_sessions (session_digest, tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4, $5)", "INSERT INTO clusterflux_cli_sessions (session_digest, tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4, $5)",
&[ &[
&record.session_digest.as_str(), &record.session_digest.as_str(),
&record.tenant.as_str(), &record.tenant.as_str(),
@ -283,7 +283,7 @@ impl FallibleDurableStore for PostgresDurableStore {
for record in state.agent_public_keys.values() { for record in state.agent_public_keys.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_agent_public_keys (tenant_id, project_id, user_id, agent_id, record) VALUES ($1, $2, $3, $4, $5)", "INSERT INTO clusterflux_agent_public_keys (tenant_id, project_id, user_id, agent_id, record) VALUES ($1, $2, $3, $4, $5)",
&[ &[
&record.tenant.as_str(), &record.tenant.as_str(),
&record.project.as_str(), &record.project.as_str(),
@ -296,7 +296,7 @@ impl FallibleDurableStore for PostgresDurableStore {
for ((_, _, provider_key), record) in &state.source_provider_configs { for ((_, _, provider_key), record) in &state.source_provider_configs {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_source_provider_configs (tenant_id, project_id, provider_key, record) VALUES ($1, $2, $3, $4)", "INSERT INTO clusterflux_source_provider_configs (tenant_id, project_id, provider_key, record) VALUES ($1, $2, $3, $4)",
&[ &[
&record.tenant.as_str(), &record.tenant.as_str(),
&record.project.as_str(), &record.project.as_str(),
@ -308,14 +308,14 @@ impl FallibleDurableStore for PostgresDurableStore {
for record in state.service_policy_records.values() { for record in state.service_policy_records.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_service_policy_records (tenant_id, name, record) VALUES ($1, $2, $3)", "INSERT INTO clusterflux_service_policy_records (tenant_id, name, record) VALUES ($1, $2, $3)",
&[&record.tenant.as_str(), &record.name.as_str(), &value], &[&record.tenant.as_str(), &record.name.as_str(), &value],
)?; )?;
} }
for record in state.project_permissions.values() { for record in state.project_permissions.values() {
let value = Self::record_value(record)?; let value = Self::record_value(record)?;
tx.execute( tx.execute(
"INSERT INTO disasmer_project_permissions (tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4)", "INSERT INTO clusterflux_project_permissions (tenant_id, project_id, user_id, record) VALUES ($1, $2, $3, $4)",
&[ &[
&record.tenant.as_str(), &record.tenant.as_str(),
&record.project.as_str(), &record.project.as_str(),
@ -331,73 +331,73 @@ impl FallibleDurableStore for PostgresDurableStore {
} }
const POSTGRES_SCHEMA_SQL: &str = r#" const POSTGRES_SCHEMA_SQL: &str = r#"
CREATE TABLE IF NOT EXISTS disasmer_tenants ( CREATE TABLE IF NOT EXISTS clusterflux_tenants (
tenant_id TEXT PRIMARY KEY, tenant_id TEXT PRIMARY KEY,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_users ( CREATE TABLE IF NOT EXISTS clusterflux_users (
user_id TEXT PRIMARY KEY, user_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_projects ( CREATE TABLE IF NOT EXISTS clusterflux_projects (
project_id TEXT PRIMARY KEY, project_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_node_identities ( CREATE TABLE IF NOT EXISTS clusterflux_node_identities (
node_id TEXT PRIMARY KEY, node_id TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_credentials ( CREATE TABLE IF NOT EXISTS clusterflux_credentials (
subject TEXT PRIMARY KEY, subject TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_cli_sessions ( CREATE TABLE IF NOT EXISTS clusterflux_cli_sessions (
session_digest TEXT PRIMARY KEY, session_digest TEXT PRIMARY KEY,
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES disasmer_users(user_id) ON DELETE CASCADE, user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
record JSONB NOT NULL record JSONB NOT NULL
); );
CREATE TABLE IF NOT EXISTS disasmer_agent_public_keys ( CREATE TABLE IF NOT EXISTS clusterflux_agent_public_keys (
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES disasmer_users(user_id) ON DELETE CASCADE, user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
agent_id TEXT NOT NULL, agent_id TEXT NOT NULL,
record JSONB NOT NULL, record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, agent_id) PRIMARY KEY (tenant_id, project_id, agent_id)
); );
CREATE TABLE IF NOT EXISTS disasmer_source_provider_configs ( CREATE TABLE IF NOT EXISTS clusterflux_source_provider_configs (
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
provider_key TEXT NOT NULL, provider_key TEXT NOT NULL,
record JSONB NOT NULL, record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, provider_key) PRIMARY KEY (tenant_id, project_id, provider_key)
); );
CREATE TABLE IF NOT EXISTS disasmer_service_policy_records ( CREATE TABLE IF NOT EXISTS clusterflux_service_policy_records (
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
name TEXT NOT NULL, name TEXT NOT NULL,
record JSONB NOT NULL, record JSONB NOT NULL,
PRIMARY KEY (tenant_id, name) PRIMARY KEY (tenant_id, name)
); );
CREATE TABLE IF NOT EXISTS disasmer_project_permissions ( CREATE TABLE IF NOT EXISTS clusterflux_project_permissions (
tenant_id TEXT NOT NULL REFERENCES disasmer_tenants(tenant_id) ON DELETE CASCADE, tenant_id TEXT NOT NULL REFERENCES clusterflux_tenants(tenant_id) ON DELETE CASCADE,
project_id TEXT NOT NULL REFERENCES disasmer_projects(project_id) ON DELETE CASCADE, project_id TEXT NOT NULL REFERENCES clusterflux_projects(project_id) ON DELETE CASCADE,
user_id TEXT NOT NULL REFERENCES disasmer_users(user_id) ON DELETE CASCADE, user_id TEXT NOT NULL REFERENCES clusterflux_users(user_id) ON DELETE CASCADE,
record JSONB NOT NULL, record JSONB NOT NULL,
PRIMARY KEY (tenant_id, project_id, user_id) PRIMARY KEY (tenant_id, project_id, user_id)
); );
@ -405,7 +405,7 @@ CREATE TABLE IF NOT EXISTS disasmer_project_permissions (
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use disasmer_core::{ use clusterflux_core::{
CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId, CredentialKind, Digest, NodeId, ProjectId, SourceProviderKind, TenantId, UserId,
}; };
@ -420,16 +420,16 @@ mod tests {
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(names.len(), 10); assert_eq!(names.len(), 10);
assert!(names.contains(&"disasmer_tenants")); assert!(names.contains(&"clusterflux_tenants"));
assert!(names.contains(&"disasmer_users")); assert!(names.contains(&"clusterflux_users"));
assert!(names.contains(&"disasmer_projects")); assert!(names.contains(&"clusterflux_projects"));
assert!(names.contains(&"disasmer_node_identities")); assert!(names.contains(&"clusterflux_node_identities"));
assert!(names.contains(&"disasmer_credentials")); assert!(names.contains(&"clusterflux_credentials"));
assert!(names.contains(&"disasmer_cli_sessions")); assert!(names.contains(&"clusterflux_cli_sessions"));
assert!(names.contains(&"disasmer_agent_public_keys")); assert!(names.contains(&"clusterflux_agent_public_keys"));
assert!(names.contains(&"disasmer_source_provider_configs")); assert!(names.contains(&"clusterflux_source_provider_configs"));
assert!(names.contains(&"disasmer_service_policy_records")); assert!(names.contains(&"clusterflux_service_policy_records"));
assert!(names.contains(&"disasmer_project_permissions")); assert!(names.contains(&"clusterflux_project_permissions"));
assert!(PostgresDurableStore::durable_tables() assert!(PostgresDurableStore::durable_tables()
.iter() .iter()
.all(|table| table.restart_surviving)); .all(|table| table.restart_surviving));
@ -494,7 +494,7 @@ mod tests {
first.start_process( first.start_process(
TenantId::from("tenant"), TenantId::from("tenant"),
ProjectId::from("project"), ProjectId::from("project"),
disasmer_core::ProcessId::from("process"), clusterflux_core::ProcessId::from("process"),
); );
first.try_persist(&mut store).unwrap(); first.try_persist(&mut store).unwrap();
@ -507,7 +507,7 @@ mod tests {
#[test] #[test]
fn postgres_round_trip_runs_when_dsn_is_configured() { fn postgres_round_trip_runs_when_dsn_is_configured() {
let Ok(dsn) = std::env::var("DISASMER_TEST_POSTGRES") else { let Ok(dsn) = std::env::var("CLUSTERFLUX_TEST_POSTGRES") else {
return; return;
}; };

View file

@ -6,7 +6,7 @@
use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use disasmer_core::{ use clusterflux_core::{
Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError, Actor, AgentId, ArtifactRegistry, CapabilityReportError, CredentialKind, Digest, LimitError,
NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId, NativeQuicTransport, NodeDescriptor, NodeId, PanelState, Placement, ProcessId, ProjectId,
RateLimit, TenantId, TransportError, UserId, RateLimit, TenantId, TransportError, UserId,
@ -31,6 +31,7 @@ mod process_launch;
mod processes; mod processes;
mod protocol; mod protocol;
mod quota; mod quota;
mod relay;
mod routing; mod routing;
mod signed_nodes; mod signed_nodes;
mod tcp; mod tcp;
@ -45,10 +46,12 @@ pub use protocol::{
ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest, ArtifactTransferAssignment, AuthenticatedCoordinatorRequest, CoordinatorRequest,
CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent, CoordinatorResponse, DebugAcknowledgementState, DebugAuditEvent,
DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus, DebugParticipantAcknowledgement, SourcePreparationDisposition, SourcePreparationStatus,
TaskAssignment, TaskCancellationTarget, TaskCompletionEvent, TaskExecutor, TaskAssignment, TaskAttemptSnapshot, TaskAttemptState, TaskCancellationTarget,
TaskReplacementBundle, TaskTerminalState, VirtualProcessStatus, WorkflowActor, TaskCompletionEvent, TaskExecutor, TaskFailureResolution, TaskReplacementBundle,
TaskTerminalState, VirtualProcessStatus, WorkflowActor,
}; };
pub use quota::CoordinatorQuotaConfiguration; pub use quota::CoordinatorQuotaConfiguration;
pub use relay::{ArtifactRelayConfiguration, ArtifactRelayError, ArtifactRelayUsage};
pub use tcp::{bind_listener, ClientAuthorityMode}; pub use tcp::{bind_listener, ClientAuthorityMode};
pub use wire_protocol::CoordinatorWireRequest; pub use wire_protocol::CoordinatorWireRequest;
@ -61,12 +64,36 @@ const MAX_ENROLLMENT_GRANTS_PER_PROJECT: usize = 64;
const MAX_TASK_EVENTS_PER_PROCESS: usize = 128; const MAX_TASK_EVENTS_PER_PROCESS: usize = 128;
const MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS: usize = 256; const MAX_DEBUG_AUDIT_EVENTS_PER_PROCESS: usize = 256;
const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128; const MAX_RESTART_CHECKPOINTS_PER_PROCESS: usize = 128;
const MAX_TASK_EVENTS_TOTAL: usize = 8_192;
const MAX_DEBUG_AUDIT_EVENTS_TOTAL: usize = 8_192;
const MAX_RESTART_CHECKPOINTS_TOTAL: usize = 4_096;
const MAX_TASK_ATTEMPT_HISTORIES: usize = 4_096;
const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256; const MAX_IN_FLIGHT_TASKS_PER_PROCESS: usize = 256;
const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024; const MAX_NODE_REPORTED_OBJECTS_PER_KIND: usize = 1_024;
const DEFAULT_NODE_STALE_AFTER_SECONDS: u64 = 30;
fn bounded_ttl(requested: u64, maximum: u64) -> u64 { fn bounded_ttl(requested: u64, maximum: u64) -> u64 {
requested.clamp(1, maximum) requested.clamp(1, maximum)
} }
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorMainRuntimeConfiguration {
pub fuel_units_per_second: u64,
pub fuel_burst_seconds: u64,
pub memory_bytes: usize,
pub nested_join_timeout_ms: u64,
}
impl Default for CoordinatorMainRuntimeConfiguration {
fn default() -> Self {
Self {
fuel_units_per_second: 10_000_000,
fuel_burst_seconds: 60,
memory_bytes: 256 * 1024 * 1024,
nested_join_timeout_ms: 24 * 60 * 60 * 1_000,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorAdmission { pub struct CoordinatorAdmission {
pub workflow_placement_allowed: bool, pub workflow_placement_allowed: bool,
@ -95,15 +122,15 @@ pub enum CoordinatorServiceError {
#[error("coordinator request failed: {0}")] #[error("coordinator request failed: {0}")]
Coordinator(#[from] CoordinatorError), Coordinator(#[from] CoordinatorError),
#[error("artifact download request failed: {0}")] #[error("artifact download request failed: {0}")]
Download(#[from] disasmer_core::DownloadError), Download(#[from] clusterflux_core::DownloadError),
#[error("scheduler placement failed: {0}")] #[error("scheduler placement failed: {0}")]
Scheduler(#[from] disasmer_core::PlacementError), Scheduler(#[from] clusterflux_core::PlacementError),
#[error("transport request failed: {0}")] #[error("transport request failed: {0}")]
Transport(#[from] TransportError), Transport(#[from] TransportError),
#[error("resource limit failed: {0}")] #[error("resource limit failed: {0}")]
Resource(#[from] LimitError), Resource(#[from] LimitError),
#[error("operator panel request failed: {0}")] #[error("operator panel request failed: {0}")]
Panel(#[from] disasmer_core::PanelError), Panel(#[from] clusterflux_core::PanelError),
#[error("invalid node capability report: {0}")] #[error("invalid node capability report: {0}")]
CapabilityReport(#[from] CapabilityReportError), CapabilityReport(#[from] CapabilityReportError),
#[error("invalid VFS artifact path reported by node: {0}")] #[error("invalid VFS artifact path reported by node: {0}")]
@ -118,8 +145,12 @@ pub struct CoordinatorService {
coordinator: Coordinator, coordinator: Coordinator,
store: RuntimeDurableStore, store: RuntimeDurableStore,
node_descriptors: BTreeMap<NodeId, NodeDescriptor>, node_descriptors: BTreeMap<NodeId, NodeDescriptor>,
enrollment_grants: BTreeMap<EnrollmentGrantKey, disasmer_core::EnrollmentGrant>, node_last_seen_epoch_seconds: BTreeMap<NodeId, u64>,
node_stale_after_seconds: u64,
debug_freeze_timeout: std::time::Duration,
enrollment_grants: BTreeMap<EnrollmentGrantKey, clusterflux_core::EnrollmentGrant>,
task_events: VecDeque<TaskCompletionEvent>, task_events: VecDeque<TaskCompletionEvent>,
process_scope_history: VecDeque<ProcessControlKey>,
debug_audit_events: VecDeque<DebugAuditEvent>, debug_audit_events: VecDeque<DebugAuditEvent>,
debug_epochs: BTreeMap<ProcessControlKey, u64>, debug_epochs: BTreeMap<ProcessControlKey, u64>,
debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>, debug_epoch_runtime: BTreeMap<ProcessControlKey, debug::DebugEpochRuntime>,
@ -128,6 +159,8 @@ pub struct CoordinatorService {
task_assignments: BTreeMap<TaskAssignmentKey, VecDeque<TaskAssignment>>, task_assignments: BTreeMap<TaskAssignmentKey, VecDeque<TaskAssignment>>,
task_restart_checkpoints: BTreeMap<TaskRestartKey, processes::TaskRestartCheckpoint>, task_restart_checkpoints: BTreeMap<TaskRestartKey, processes::TaskRestartCheckpoint>,
task_restart_checkpoint_order: VecDeque<TaskRestartKey>, task_restart_checkpoint_order: VecDeque<TaskRestartKey>,
task_attempts: BTreeMap<TaskRestartKey, Vec<protocol::TaskAttemptSnapshot>>,
restart_launches: BTreeSet<TaskRestartKey>,
main_runtime: main_runtime::CoordinatorMainRuntime, main_runtime: main_runtime::CoordinatorMainRuntime,
pending_task_launches: VecDeque<processes::PendingTaskLaunch>, pending_task_launches: VecDeque<processes::PendingTaskLaunch>,
task_placements: BTreeMap<TaskControlKey, Placement>, task_placements: BTreeMap<TaskControlKey, Placement>,
@ -144,6 +177,7 @@ pub struct CoordinatorService {
artifact_registry: ArtifactRegistry, artifact_registry: ArtifactRegistry,
artifact_reverse_transfers: BTreeMap<String, artifacts::ArtifactReverseTransfer>, artifact_reverse_transfers: BTreeMap<String, artifacts::ArtifactReverseTransfer>,
artifact_transfer_by_token: BTreeMap<Digest, String>, artifact_transfer_by_token: BTreeMap<Digest, String>,
artifact_relay: relay::ArtifactRelayLedger,
transport: NativeQuicTransport, transport: NativeQuicTransport,
quota: quota::CoordinatorQuota, quota: quota::CoordinatorQuota,
admission: CoordinatorAdmission, admission: CoordinatorAdmission,
@ -154,6 +188,27 @@ pub struct CoordinatorService {
} }
impl CoordinatorService { impl CoordinatorService {
pub fn configure_artifact_relay(
&mut self,
configuration: ArtifactRelayConfiguration,
) -> Result<(), ArtifactRelayError> {
self.artifact_relay.configure(configuration)
}
pub fn artifact_relay_usage(&self) -> ArtifactRelayUsage {
self.artifact_relay.usage()
}
pub fn set_debug_freeze_timeout(&mut self, timeout: std::time::Duration) {
self.debug_freeze_timeout = timeout.max(std::time::Duration::from_millis(1));
}
pub fn configure_coordinator_main_runtime(
&mut self,
configuration: CoordinatorMainRuntimeConfiguration,
) -> Result<(), CoordinatorServiceError> {
self.main_runtime.configure(configuration)
}
pub(super) fn authorize_node_for_process_or_termination( pub(super) fn authorize_node_for_process_or_termination(
&self, &self,
node: &NodeId, node: &NodeId,
@ -186,7 +241,7 @@ impl CoordinatorService {
pub fn new(coordinator_epoch: u64) -> Self { pub fn new(coordinator_epoch: u64) -> Self {
Self::new_with_optional_admin_token_and_admission( Self::new_with_optional_admin_token_and_admission(
coordinator_epoch, coordinator_epoch,
std::env::var("DISASMER_ADMIN_TOKEN").ok(), std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
CoordinatorAdmission::default(), CoordinatorAdmission::default(),
) )
} }
@ -202,7 +257,7 @@ impl CoordinatorService {
pub fn new_with_admission(coordinator_epoch: u64, admission: CoordinatorAdmission) -> Self { pub fn new_with_admission(coordinator_epoch: u64, admission: CoordinatorAdmission) -> Self {
Self::new_with_optional_admin_token_and_admission( Self::new_with_optional_admin_token_and_admission(
coordinator_epoch, coordinator_epoch,
std::env::var("DISASMER_ADMIN_TOKEN").ok(), std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
admission, admission,
) )
} }
@ -228,7 +283,7 @@ impl CoordinatorService {
) -> Result<Self, CoordinatorServiceError> { ) -> Result<Self, CoordinatorServiceError> {
Self::try_new_with_optional_admin_token_admission_and_database_url( Self::try_new_with_optional_admin_token_admission_and_database_url(
coordinator_epoch, coordinator_epoch,
std::env::var("DISASMER_ADMIN_TOKEN").ok(), std::env::var("CLUSTERFLUX_ADMIN_TOKEN").ok(),
CoordinatorAdmission::default(), CoordinatorAdmission::default(),
database_url, database_url,
CoordinatorQuotaConfiguration::default(), CoordinatorQuotaConfiguration::default(),
@ -281,8 +336,16 @@ impl CoordinatorService {
coordinator, coordinator,
store, store,
node_descriptors: BTreeMap::new(), node_descriptors: BTreeMap::new(),
node_last_seen_epoch_seconds: BTreeMap::new(),
node_stale_after_seconds: std::env::var("CLUSTERFLUX_NODE_STALE_AFTER_SECONDS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.filter(|seconds| *seconds > 0)
.unwrap_or(DEFAULT_NODE_STALE_AFTER_SECONDS),
debug_freeze_timeout: std::time::Duration::from_secs(5),
enrollment_grants: BTreeMap::new(), enrollment_grants: BTreeMap::new(),
task_events: VecDeque::new(), task_events: VecDeque::new(),
process_scope_history: VecDeque::new(),
debug_audit_events: VecDeque::new(), debug_audit_events: VecDeque::new(),
debug_epochs: BTreeMap::new(), debug_epochs: BTreeMap::new(),
debug_epoch_runtime: BTreeMap::new(), debug_epoch_runtime: BTreeMap::new(),
@ -291,6 +354,8 @@ impl CoordinatorService {
task_assignments: BTreeMap::new(), task_assignments: BTreeMap::new(),
task_restart_checkpoints: BTreeMap::new(), task_restart_checkpoints: BTreeMap::new(),
task_restart_checkpoint_order: VecDeque::new(), task_restart_checkpoint_order: VecDeque::new(),
task_attempts: BTreeMap::new(),
restart_launches: BTreeSet::new(),
main_runtime: main_runtime::CoordinatorMainRuntime::default(), main_runtime: main_runtime::CoordinatorMainRuntime::default(),
pending_task_launches: VecDeque::new(), pending_task_launches: VecDeque::new(),
task_placements: BTreeMap::new(), task_placements: BTreeMap::new(),
@ -307,6 +372,7 @@ impl CoordinatorService {
artifact_registry: ArtifactRegistry::default(), artifact_registry: ArtifactRegistry::default(),
artifact_reverse_transfers: BTreeMap::new(), artifact_reverse_transfers: BTreeMap::new(),
artifact_transfer_by_token: BTreeMap::new(), artifact_transfer_by_token: BTreeMap::new(),
artifact_relay: relay::ArtifactRelayLedger::default(),
transport: NativeQuicTransport, transport: NativeQuicTransport,
quota: quota::CoordinatorQuota::new(quota_configuration), quota: quota::CoordinatorQuota::new(quota_configuration),
admission, admission,

View file

@ -1,6 +1,6 @@
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use disasmer_core::{ use clusterflux_core::{
admin_request_proof_from_token_digest, CredentialKind, Digest, TenantId, UserId, admin_request_proof_from_token_digest, CredentialKind, Digest, TenantId, UserId,
}; };

View file

@ -1,7 +1,7 @@
use std::io::{Read, Seek, SeekFrom, Write}; use std::io::{Read, Seek, SeekFrom, Write};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{ use clusterflux_core::{
generate_opaque_token, Actor, ArtifactId, AuthContext, DataPlaneObject, DataPlaneScope, Digest, generate_opaque_token, Actor, ArtifactId, AuthContext, DataPlaneObject, DataPlaneScope, Digest,
DownloadPolicy, NodeEndpoint, NodeId, ProjectId, RendezvousRequest, ResourceLimits, DownloadPolicy, NodeEndpoint, NodeId, ProjectId, RendezvousRequest, ResourceLimits,
ResourceMeter, StorageLocation, TenantId, UserId, ResourceMeter, StorageLocation, TenantId, UserId,
@ -10,13 +10,13 @@ use sha2::{Digest as _, Sha256};
use crate::CoordinatorError; use crate::CoordinatorError;
use super::relay::RelayFinishReason;
use super::{ use super::{
bounded_ttl, ArtifactTransferAssignment, CoordinatorResponse, CoordinatorService, bounded_ttl, ArtifactTransferAssignment, CoordinatorResponse, CoordinatorService,
CoordinatorServiceError, CoordinatorServiceError,
}; };
pub(super) const MAX_ARTIFACT_REVERSE_CHUNK_BYTES: u64 = 256 * 1024; pub(super) const MAX_ARTIFACT_REVERSE_CHUNK_BYTES: u64 = 256 * 1024;
const MAX_CONCURRENT_ARTIFACT_REVERSE_TRANSFERS: usize = 4;
#[derive(Debug)] #[derive(Debug)]
pub(super) struct ArtifactReverseTransfer { pub(super) struct ArtifactReverseTransfer {
@ -57,6 +57,7 @@ impl CoordinatorService {
.artifact_registry .artifact_registry
.downloadable_size(&context, &artifact, &policy)?; .downloadable_size(&context, &artifact, &policy)?;
let now_epoch_seconds = self.current_epoch_seconds()?; let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay.expire(now_epoch_seconds);
self.quota.can_charge_download( self.quota.can_charge_download(
&context.tenant, &context.tenant,
&context.project, &context.project,
@ -69,14 +70,45 @@ impl CoordinatorService {
ttl_seconds, ttl_seconds,
self.admission.max_artifact_download_ttl_seconds, self.admission.max_artifact_download_ttl_seconds,
); );
let link = self.artifact_registry.create_download_link( let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let account = match &context.actor {
Actor::User(user) => user.clone(),
Actor::Agent(_) | Actor::Node(_) | Actor::Task(_) => {
return Err(CoordinatorServiceError::Protocol(
"artifact relay download requires a user account".to_owned(),
))
}
};
self.artifact_relay
.reserve(
token_nonce.clone(),
context.tenant.clone(),
context.project.clone(),
account,
downloadable_size,
MAX_ARTIFACT_REVERSE_CHUNK_BYTES,
expires_at_epoch_seconds,
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let link = match self.artifact_registry.create_download_link(
&context, &context,
&artifact, &artifact,
&policy, &policy,
&token_nonce, &token_nonce,
now_epoch_seconds, now_epoch_seconds,
ttl_seconds, ttl_seconds,
)?; ) {
Ok(link) => link,
Err(error) => {
self.artifact_relay
.finish(&token_nonce, RelayFinishReason::Cancelled);
return Err(error.into());
}
};
self.artifact_relay
.rekey(&token_nonce, link.scoped_token_digest.as_str().to_owned())
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
Ok(CoordinatorResponse::ArtifactDownloadLink { link }) Ok(CoordinatorResponse::ArtifactDownloadLink { link })
} }
@ -94,6 +126,7 @@ impl CoordinatorService {
let artifact = ArtifactId::new(artifact); let artifact = ArtifactId::new(artifact);
let policy = DownloadPolicy { max_bytes }; let policy = DownloadPolicy { max_bytes };
let now_epoch_seconds = self.current_epoch_seconds()?; let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay.expire(now_epoch_seconds);
self.artifact_registry self.artifact_registry
.expire_download_links(now_epoch_seconds); .expire_download_links(now_epoch_seconds);
let downloadable_size = self let downloadable_size = self
@ -102,7 +135,7 @@ impl CoordinatorService {
let validation_limits = ResourceLimits::unlimited(); let validation_limits = ResourceLimits::unlimited();
let mut validation_meter = ResourceMeter::default(); let mut validation_meter = ResourceMeter::default();
let mut stream = self.artifact_registry.open_download_stream( let mut stream = self.artifact_registry.open_download_stream(
disasmer_core::DownloadStreamRequest { clusterflux_core::DownloadStreamRequest {
context: &context, context: &context,
artifact: &artifact, artifact: &artifact,
policy: &policy, policy: &policy,
@ -123,6 +156,8 @@ impl CoordinatorService {
{ {
self.artifact_reverse_transfers.remove(&transfer_id); self.artifact_reverse_transfers.remove(&transfer_id);
self.artifact_transfer_by_token.remove(&token_digest); self.artifact_transfer_by_token.remove(&token_digest);
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Failed);
return Err(CoordinatorServiceError::Protocol(format!( return Err(CoordinatorServiceError::Protocol(format!(
"retaining node could not stream artifact: {message}" "retaining node could not stream artifact: {message}"
))); )));
@ -141,9 +176,16 @@ impl CoordinatorService {
|| transfer.artifact != artifact || transfer.artifact != artifact
|| transfer.token_digest != token_digest || transfer.token_digest != token_digest
{ {
return Err(disasmer_core::DownloadError::InvalidToken.into()); return Err(clusterflux_core::DownloadError::InvalidToken.into());
} }
if transfer.received_bytes != transfer.expected_size_bytes { if transfer.received_bytes != transfer.expected_size_bytes {
self.artifact_relay
.charge_egress(
token_digest.as_str(),
self.artifact_relay.framing_overhead_bytes(),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
return Ok(CoordinatorResponse::ArtifactDownloadStream { return Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link, link: stream.link,
streamed_bytes: 0, streamed_bytes: 0,
@ -205,12 +247,23 @@ impl CoordinatorService {
streamed_bytes, streamed_bytes,
now_epoch_seconds, now_epoch_seconds,
)?; )?;
let content_base64 = BASE64_STANDARD.encode(content);
self.artifact_relay
.charge_egress(
token_digest.as_str(),
(content_base64.len() as u64)
.saturating_add(self.artifact_relay.framing_overhead_bytes()),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
if let Some(transfer) = self.artifact_reverse_transfers.get_mut(&transfer_id) { if let Some(transfer) = self.artifact_reverse_transfers.get_mut(&transfer_id) {
transfer.delivered_offset = end; transfer.delivered_offset = end;
} }
if complete { if complete {
self.artifact_reverse_transfers.remove(&transfer_id); self.artifact_reverse_transfers.remove(&transfer_id);
self.artifact_transfer_by_token.remove(&token_digest); self.artifact_transfer_by_token.remove(&token_digest);
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Completed);
} }
return Ok(CoordinatorResponse::ArtifactDownloadStream { return Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link, link: stream.link,
@ -219,27 +272,22 @@ impl CoordinatorService {
content_bytes_available: true, content_bytes_available: true,
content_offset: Some(content_offset), content_offset: Some(content_offset),
content_eof: complete, content_eof: complete,
content_base64: Some(BASE64_STANDARD.encode(content)), content_base64: Some(content_base64),
content_source: Some("retaining_node_reverse_stream".to_owned()), content_source: Some("retaining_node_reverse_stream".to_owned()),
}); });
} }
if self.artifact_reverse_transfers.len() >= MAX_CONCURRENT_ARTIFACT_REVERSE_TRANSFERS {
return Err(CoordinatorServiceError::Protocol(format!(
"artifact reverse transfer concurrency limit of {MAX_CONCURRENT_ARTIFACT_REVERSE_TRANSFERS} reached"
)));
}
let StorageLocation::RetainedNode(source_node) = &stream.link.source else { let StorageLocation::RetainedNode(source_node) = &stream.link.source else {
return Err(disasmer_core::DownloadError::Unavailable.into()); return Err(clusterflux_core::DownloadError::Unavailable.into());
}; };
let metadata = self let metadata = self
.artifact_registry .artifact_registry
.metadata(&artifact) .metadata(&artifact)
.ok_or(disasmer_core::DownloadError::NotFound)?; .ok_or(clusterflux_core::DownloadError::NotFound)?;
let transfer_id = generate_opaque_token("artifact_transfer") let transfer_id = generate_opaque_token("artifact_transfer")
.map_err(CoordinatorServiceError::Protocol)?; .map_err(CoordinatorServiceError::Protocol)?;
let spool = tempfile::Builder::new() let spool = tempfile::Builder::new()
.prefix("disasmer-artifact-transfer-") .prefix("clusterflux-artifact-transfer-")
.tempfile() .tempfile()
.map_err(|error| { .map_err(|error| {
CoordinatorServiceError::Protocol(format!( CoordinatorServiceError::Protocol(format!(
@ -266,6 +314,13 @@ impl CoordinatorService {
.insert(transfer_id.clone(), transfer); .insert(transfer_id.clone(), transfer);
self.artifact_transfer_by_token self.artifact_transfer_by_token
.insert(token_digest, transfer_id); .insert(token_digest, transfer_id);
self.artifact_relay
.charge_egress(
stream.link.scoped_token_digest.as_str(),
self.artifact_relay.framing_overhead_bytes(),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
Ok(CoordinatorResponse::ArtifactDownloadStream { Ok(CoordinatorResponse::ArtifactDownloadStream {
link: stream.link, link: stream.link,
streamed_bytes: 0, streamed_bytes: 0,
@ -302,6 +357,8 @@ impl CoordinatorService {
if let Some(transfer_id) = self.artifact_transfer_by_token.remove(&token_digest) { if let Some(transfer_id) = self.artifact_transfer_by_token.remove(&token_digest) {
self.artifact_reverse_transfers.remove(&transfer_id); self.artifact_reverse_transfers.remove(&transfer_id);
} }
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Cancelled);
Ok(CoordinatorResponse::ArtifactDownloadLinkRevoked { link }) Ok(CoordinatorResponse::ArtifactDownloadLinkRevoked { link })
} }
@ -326,12 +383,12 @@ impl CoordinatorService {
}, },
)?; )?;
let StorageLocation::RetainedNode(source_node) = action.source else { let StorageLocation::RetainedNode(source_node) = action.source else {
return Err(disasmer_core::DownloadError::Unavailable.into()); return Err(clusterflux_core::DownloadError::Unavailable.into());
}; };
let metadata = self let metadata = self
.artifact_registry .artifact_registry
.metadata(&artifact) .metadata(&artifact)
.ok_or(disasmer_core::DownloadError::NotFound)?; .ok_or(clusterflux_core::DownloadError::NotFound)?;
let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?; let source = self.export_endpoint(&source_node, &context.tenant, &context.project)?;
let destination = let destination =
self.export_endpoint(&receiver_node, &context.tenant, &context.project)?; self.export_endpoint(&receiver_node, &context.tenant, &context.project)?;
@ -407,6 +464,33 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let node = NodeId::new(node); let node = NodeId::new(node);
self.authorize_artifact_transfer_node(&tenant, &project, &node)?; self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
let token_digest = {
let transfer = self
.artifact_reverse_transfers
.get(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"unknown artifact reverse transfer".to_owned(),
)
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer is outside the signed node scope".to_owned(),
)
.into());
}
transfer.token_digest.clone()
};
let now_epoch_seconds = self.current_epoch_seconds()?;
let ingress_wire_bytes = (content_base64.len() as u64)
.saturating_add(self.artifact_relay.framing_overhead_bytes());
self.artifact_relay
.charge_ingress(token_digest.as_str(), ingress_wire_bytes, now_epoch_seconds)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let transfer = self let transfer = self
.artifact_reverse_transfers .artifact_reverse_transfers
.get_mut(&transfer_id) .get_mut(&transfer_id)
@ -511,6 +595,35 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let node = NodeId::new(node); let node = NodeId::new(node);
self.authorize_artifact_transfer_node(&tenant, &project, &node)?; self.authorize_artifact_transfer_node(&tenant, &project, &node)?;
let token_digest = {
let transfer = self
.artifact_reverse_transfers
.get(&transfer_id)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"unknown artifact reverse transfer".to_owned(),
)
})?;
if transfer.tenant != tenant
|| transfer.project != project
|| transfer.source_node != node
|| transfer.artifact.as_str() != artifact
{
return Err(CoordinatorError::Unauthorized(
"artifact reverse transfer failure is outside the signed node scope".to_owned(),
)
.into());
}
transfer.token_digest.clone()
};
let now_epoch_seconds = self.current_epoch_seconds()?;
self.artifact_relay
.charge_ingress(
token_digest.as_str(),
(message.len() as u64).saturating_add(self.artifact_relay.framing_overhead_bytes()),
now_epoch_seconds,
)
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
let transfer = self let transfer = self
.artifact_reverse_transfers .artifact_reverse_transfers
.get_mut(&transfer_id) .get_mut(&transfer_id)
@ -533,6 +646,8 @@ impl CoordinatorService {
} else { } else {
message.chars().take(1024).collect() message.chars().take(1024).collect()
}); });
self.artifact_relay
.finish(token_digest.as_str(), RelayFinishReason::Failed);
Ok(CoordinatorResponse::ArtifactTransferFailed { transfer_id }) Ok(CoordinatorResponse::ArtifactTransferFailed { transfer_id })
} }
@ -566,7 +681,10 @@ impl CoordinatorService {
for (id, token) in expired { for (id, token) in expired {
self.artifact_reverse_transfers.remove(&id); self.artifact_reverse_transfers.remove(&id);
self.artifact_transfer_by_token.remove(&token); self.artifact_transfer_by_token.remove(&token);
self.artifact_relay
.finish(token.as_str(), RelayFinishReason::Expired);
} }
self.artifact_relay.expire(now_epoch_seconds);
} }
fn export_endpoint( fn export_endpoint(
@ -586,7 +704,7 @@ impl CoordinatorService {
.into()); .into());
} }
let descriptor = self.node_descriptors.get(node).ok_or_else(|| { let descriptor = self.node_descriptors.get(node).ok_or_else(|| {
disasmer_core::DownloadError::DirectConnectivityUnavailable(format!( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} has not reported export connectivity" "node {node} has not reported export connectivity"
)) ))
})?; })?;
@ -596,9 +714,9 @@ impl CoordinatorService {
) )
.into()); .into());
} }
if !descriptor.online { if !self.node_is_live(node) {
return Err( return Err(
disasmer_core::DownloadError::DirectConnectivityUnavailable(format!( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"node {node} is offline for artifact export" "node {node} is offline for artifact export"
)) ))
.into(), .into(),
@ -606,7 +724,7 @@ impl CoordinatorService {
} }
if !descriptor.direct_connectivity { if !descriptor.direct_connectivity {
return Err( return Err(
disasmer_core::DownloadError::DirectConnectivityUnavailable(format!( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"direct connectivity unavailable to node {node} for artifact export" "direct connectivity unavailable to node {node} for artifact export"
)) ))
.into(), .into(),
@ -622,19 +740,21 @@ impl CoordinatorService {
fn ensure_download_source_connectivity( fn ensure_download_source_connectivity(
&self, &self,
source: &StorageLocation, source: &StorageLocation,
) -> Result<(), disasmer_core::DownloadError> { ) -> Result<(), clusterflux_core::DownloadError> {
let StorageLocation::RetainedNode(node) = source else { let StorageLocation::RetainedNode(node) = source else {
return Ok(()); return Ok(());
}; };
let descriptor = self.node_descriptors.get(node).ok_or_else(|| { let _descriptor = self.node_descriptors.get(node).ok_or_else(|| {
disasmer_core::DownloadError::DirectConnectivityUnavailable(format!( clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
"retaining node {node} has not reported online status for artifact download" "retaining node {node} has not reported online status for artifact download"
)) ))
})?; })?;
if !descriptor.online { if !self.node_is_live(node) {
return Err(disasmer_core::DownloadError::DirectConnectivityUnavailable( return Err(
format!("retaining node {node} is offline for artifact download"), clusterflux_core::DownloadError::DirectConnectivityUnavailable(format!(
)); "retaining node {node} is offline for artifact download"
)),
);
} }
Ok(()) Ok(())
} }

View file

@ -1,4 +1,4 @@
use disasmer_core::{AuthContext, UserId}; use clusterflux_core::{AuthContext, UserId};
use super::{ use super::{
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService, AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,

View file

@ -1,4 +1,4 @@
use disasmer_core::{Actor, AuthContext, UserId}; use clusterflux_core::{Actor, AuthContext, UserId};
use crate::CoordinatorError; use crate::CoordinatorError;
@ -26,6 +26,7 @@ pub(super) enum PublicUserOperation {
ListProcesses, ListProcesses,
QuotaStatus, QuotaStatus,
RestartTask, RestartTask,
ResolveTaskFailure,
DebugAttach, DebugAttach,
SetDebugBreakpoints, SetDebugBreakpoints,
InspectDebugBreakpoints, InspectDebugBreakpoints,
@ -33,6 +34,7 @@ pub(super) enum PublicUserOperation {
ResumeDebugEpoch, ResumeDebugEpoch,
InspectDebugEpoch, InspectDebugEpoch,
ListTaskEvents, ListTaskEvents,
ListTaskSnapshots,
JoinTask, JoinTask,
CreateArtifactDownloadLink, CreateArtifactDownloadLink,
OpenArtifactDownloadStream, OpenArtifactDownloadStream,
@ -63,6 +65,7 @@ impl PublicUserOperation {
Self::ListProcesses => "list_processes", Self::ListProcesses => "list_processes",
Self::QuotaStatus => "quota_status", Self::QuotaStatus => "quota_status",
Self::RestartTask => "restart_task", Self::RestartTask => "restart_task",
Self::ResolveTaskFailure => "resolve_task_failure",
Self::DebugAttach => "debug_attach", Self::DebugAttach => "debug_attach",
Self::SetDebugBreakpoints => "set_debug_breakpoints", Self::SetDebugBreakpoints => "set_debug_breakpoints",
Self::InspectDebugBreakpoints => "inspect_debug_breakpoints", Self::InspectDebugBreakpoints => "inspect_debug_breakpoints",
@ -70,6 +73,7 @@ impl PublicUserOperation {
Self::ResumeDebugEpoch => "resume_debug_epoch", Self::ResumeDebugEpoch => "resume_debug_epoch",
Self::InspectDebugEpoch => "inspect_debug_epoch", Self::InspectDebugEpoch => "inspect_debug_epoch",
Self::ListTaskEvents => "list_task_events", Self::ListTaskEvents => "list_task_events",
Self::ListTaskSnapshots => "list_task_snapshots",
Self::JoinTask => "join_task", Self::JoinTask => "join_task",
Self::CreateArtifactDownloadLink => "create_artifact_download_link", Self::CreateArtifactDownloadLink => "create_artifact_download_link",
Self::OpenArtifactDownloadStream => "open_artifact_download_stream", Self::OpenArtifactDownloadStream => "open_artifact_download_stream",
@ -112,6 +116,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses, AuthenticatedCoordinatorRequest::ListProcesses => Self::ListProcesses,
AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus, AuthenticatedCoordinatorRequest::QuotaStatus => Self::QuotaStatus,
AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask, AuthenticatedCoordinatorRequest::RestartTask { .. } => Self::RestartTask,
AuthenticatedCoordinatorRequest::ResolveTaskFailure { .. } => Self::ResolveTaskFailure,
AuthenticatedCoordinatorRequest::DebugAttach { .. } => Self::DebugAttach, AuthenticatedCoordinatorRequest::DebugAttach { .. } => Self::DebugAttach,
AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } => { AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } => {
Self::SetDebugBreakpoints Self::SetDebugBreakpoints
@ -123,6 +128,7 @@ impl From<&AuthenticatedCoordinatorRequest> for PublicUserOperation {
AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. } => Self::ResumeDebugEpoch, AuthenticatedCoordinatorRequest::ResumeDebugEpoch { .. } => Self::ResumeDebugEpoch,
AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch, AuthenticatedCoordinatorRequest::InspectDebugEpoch { .. } => Self::InspectDebugEpoch,
AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents, AuthenticatedCoordinatorRequest::ListTaskEvents { .. } => Self::ListTaskEvents,
AuthenticatedCoordinatorRequest::ListTaskSnapshots { .. } => Self::ListTaskSnapshots,
AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask, AuthenticatedCoordinatorRequest::JoinTask { .. } => Self::JoinTask,
AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => { AuthenticatedCoordinatorRequest::CreateArtifactDownloadLink { .. } => {
Self::CreateArtifactDownloadLink Self::CreateArtifactDownloadLink
@ -166,7 +172,7 @@ pub(super) fn authorize_authenticated_user_operation(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use disasmer_core::{AgentId, ProjectId, TenantId}; use clusterflux_core::{AgentId, ProjectId, TenantId};
use super::*; use super::*;

View file

@ -1,6 +1,7 @@
use std::collections::{BTreeMap, BTreeSet}; use std::collections::{BTreeMap, BTreeSet};
use std::time::Instant;
use disasmer_core::{ use clusterflux_core::{
Actor, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId, Actor, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, UserId,
WasmHostDebugProbeResult, WASM_TASK_ABI_VERSION, WasmHostDebugProbeResult, WASM_TASK_ABI_VERSION,
}; };
@ -28,6 +29,7 @@ pub(super) struct DebugEpochRuntime {
pub(super) command: String, pub(super) command: String,
pub(super) expected: BTreeSet<TaskControlKey>, pub(super) expected: BTreeSet<TaskControlKey>,
pub(super) acknowledgements: BTreeMap<TaskControlKey, DebugParticipantAcknowledgement>, pub(super) acknowledgements: BTreeMap<TaskControlKey, DebugParticipantAcknowledgement>,
pub(super) deadline: Instant,
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
@ -51,7 +53,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -90,7 +92,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -147,7 +149,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -405,6 +407,14 @@ impl CoordinatorService {
) )
.into()); .into());
} }
let attempt_key = task_restart_key(&tenant_id, &project_id, &process_id, &task_id);
if let Some(attempt) = self
.task_attempts
.get_mut(&attempt_key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
{
attempt.probe_symbol = Some(probe_symbol.clone());
}
let process_key = process_control_key(&tenant_id, &project_id, &process_id); let process_key = process_control_key(&tenant_id, &project_id, &process_id);
let Some(plan) = self.debug_breakpoints.get(&process_key).cloned() else { let Some(plan) = self.debug_breakpoints.get(&process_key).cloned() else {
return Ok(CoordinatorResponse::DebugProbeHit { return Ok(CoordinatorResponse::DebugProbeHit {
@ -536,7 +546,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -586,15 +596,32 @@ impl CoordinatorService {
&& acknowledgements && acknowledgements
.iter() .iter()
.all(|ack| ack.state == DebugAcknowledgementState::Frozen); .all(|ack| ack.state == DebugAcknowledgementState::Frozen);
let frozen_count = acknowledgements
.iter()
.filter(|ack| ack.state == DebugAcknowledgementState::Frozen)
.count();
let freeze_deadline_elapsed =
runtime.command == "freeze" && Instant::now() >= runtime.deadline;
let partially_frozen = runtime.command == "freeze"
&& freeze_deadline_elapsed
&& frozen_count > 0
&& !fully_frozen;
let fully_resumed = runtime.command == "resume" let fully_resumed = runtime.command == "resume"
&& all_acknowledged && all_acknowledged
&& acknowledgements && acknowledgements
.iter() .iter()
.all(|ack| ack.state == DebugAcknowledgementState::Running); .all(|ack| ack.state == DebugAcknowledgementState::Running);
let missing = runtime
.expected
.iter()
.filter(|key| !runtime.acknowledgements.contains_key(*key))
.cloned()
.collect::<Vec<_>>();
let failed = acknowledgements let failed = acknowledgements
.iter() .iter()
.any(|ack| ack.state == DebugAcknowledgementState::Failed); .any(|ack| ack.state == DebugAcknowledgementState::Failed)
let failure_messages = acknowledgements || (freeze_deadline_elapsed && !missing.is_empty());
let mut failure_messages = acknowledgements
.iter() .iter()
.filter(|ack| ack.state == DebugAcknowledgementState::Failed) .filter(|ack| ack.state == DebugAcknowledgementState::Failed)
.map(|ack| { .map(|ack| {
@ -602,7 +629,15 @@ impl CoordinatorService {
.clone() .clone()
.unwrap_or_else(|| format!("task {} failed debug control", ack.task)) .unwrap_or_else(|| format!("task {} failed debug control", ack.task))
}) })
.collect(); .collect::<Vec<_>>();
if freeze_deadline_elapsed {
failure_messages.extend(missing.iter().map(|(_, _, _, node, task)| {
format!(
"task {task} on node {node} did not acknowledge frozen state within {} ms",
self.debug_freeze_timeout.as_millis()
)
}));
}
let expected_tasks = runtime let expected_tasks = runtime
.expected .expected
.into_iter() .into_iter()
@ -620,6 +655,7 @@ impl CoordinatorService {
expected_tasks, expected_tasks,
acknowledgements, acknowledgements,
fully_frozen, fully_frozen,
partially_frozen,
fully_resumed, fully_resumed,
failed, failed,
failure_messages, failure_messages,
@ -664,7 +700,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -715,11 +751,18 @@ impl CoordinatorService {
"cannot resume debug epoch {expected} for {process}: participant state is unavailable" "cannot resume debug epoch {expected} for {process}: participant state is unavailable"
)) ))
})?; })?;
if runtime.command != "freeze" let has_frozen = runtime
|| !runtime_all_in_state(runtime, DebugAcknowledgementState::Frozen) .acknowledgements
{ .values()
.any(|ack| ack.state == DebugAcknowledgementState::Frozen);
let settled = runtime
.expected
.iter()
.all(|key| runtime.acknowledgements.contains_key(key))
|| Instant::now() >= runtime.deadline;
if runtime.command != "freeze" || !has_frozen || !settled {
return Err(CoordinatorServiceError::Protocol(format!( return Err(CoordinatorServiceError::Protocol(format!(
"cannot resume debug epoch {expected} for {process}: all expected participants have not acknowledged frozen state" "cannot resume debug epoch {expected} for {process}: no settled frozen participant set is available"
))); )));
} }
current current
@ -740,6 +783,21 @@ impl CoordinatorService {
next next
} }
}; };
let resumable = if command == "resume" {
self.debug_epoch_runtime
.get(&process_key)
.map(|runtime| {
runtime
.acknowledgements
.iter()
.filter(|(_, ack)| ack.state == DebugAcknowledgementState::Frozen)
.map(|(key, _)| key.clone())
.collect::<BTreeSet<_>>()
})
.unwrap_or_default()
} else {
BTreeSet::new()
};
let mut affected_tasks = self let mut affected_tasks = self
.enqueue_debug_command_for_active_tasks(&tenant, &project, &process, epoch, command); .enqueue_debug_command_for_active_tasks(&tenant, &project, &process, epoch, command);
let mut expected = self let mut expected = self
@ -770,6 +828,17 @@ impl CoordinatorService {
task: control.task_instance.clone(), task: control.task_instance.clone(),
}); });
} }
if command == "resume" {
expected.retain(|key| resumable.contains(key));
affected_tasks.retain(|target| {
resumable
.iter()
.any(|(_, _, _, node, task)| node == &target.node && task == &target.task)
});
self.debug_commands.retain(|key, pending| {
pending.epoch != epoch || pending.command != "resume" || resumable.contains(key)
});
}
self.debug_epoch_runtime.insert( self.debug_epoch_runtime.insert(
process_key.clone(), process_key.clone(),
DebugEpochRuntime { DebugEpochRuntime {
@ -777,6 +846,7 @@ impl CoordinatorService {
command: command.to_owned(), command: command.to_owned(),
expected, expected,
acknowledgements: BTreeMap::new(), acknowledgements: BTreeMap::new(),
deadline: Instant::now() + self.debug_freeze_timeout,
}, },
); );
if let Some(control) = self if let Some(control) = self
@ -787,7 +857,13 @@ impl CoordinatorService {
{ {
if command == "freeze" { if command == "freeze" {
control.debug.request_freeze(epoch); control.debug.request_freeze(epoch);
} else { } else if resumable.contains(&task_control_key(
&tenant,
&project,
&process,
&NodeId::from("coordinator-main"),
&control.task_instance,
)) {
control.debug.request_resume(epoch); control.debug.request_resume(epoch);
} }
} }
@ -876,14 +952,13 @@ impl CoordinatorService {
task: control.task_instance.clone(), task: control.task_instance.clone(),
epoch, epoch,
state, state,
stack_frames: vec![control stack_frames: control
.stopped_probe_symbol .stopped_probe_symbol
.as_deref() .as_deref()
.and_then(|symbol| symbol.strip_prefix("disasmer.probe.")) .and_then(|symbol| symbol.strip_prefix("clusterflux.probe."))
.map(|function| format!("{function}::wasm")) .map(|function| format!("{function}::wasm"))
.unwrap_or_else(|| { .into_iter()
format!("coordinator_main::wasm ({})", control.task_definition) .collect(),
})],
local_values: Vec::new(), local_values: Vec::new(),
task_args: Vec::new(), task_args: Vec::new(),
handles, handles,
@ -989,6 +1064,9 @@ impl CoordinatorService {
}; };
self.debug_audit_events.remove(index); self.debug_audit_events.remove(index);
} }
while self.debug_audit_events.len() >= super::MAX_DEBUG_AUDIT_EVENTS_TOTAL {
self.debug_audit_events.pop_front();
}
self.debug_audit_events.push_back(event.clone()); self.debug_audit_events.push_back(event.clone());
Ok(event) Ok(event)
} }

View file

@ -1,7 +1,7 @@
use std::collections::BTreeSet; use std::collections::BTreeSet;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{ use clusterflux_core::{
Actor, Capability, CredentialKind, Digest, EnvironmentResource, ProcessId, ProjectId, Actor, Capability, CredentialKind, Digest, EnvironmentResource, ProcessId, ProjectId,
RestartDecision, RestartPolicy, RestartRequest, TaskDispatch, TaskInstanceId, TenantId, UserId, RestartDecision, RestartPolicy, RestartRequest, TaskDispatch, TaskInstanceId, TenantId, UserId,
WasmExportAbi, WasmExportAbi,
@ -10,9 +10,10 @@ use disasmer_core::{
use crate::{CoordinatorError, CoordinatorServiceError}; use crate::{CoordinatorError, CoordinatorServiceError};
use super::keys::task_restart_key; use super::keys::task_restart_key;
use super::protocol::{TaskAttemptState, TaskFailureResolution};
use super::{ use super::{
AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService, AuthenticatedCoordinatorRequest, CoordinatorRequest, CoordinatorResponse, CoordinatorService,
TaskReplacementBundle, WorkflowActor, TaskReplacementBundle, TaskTerminalState, WorkflowActor,
}; };
impl CoordinatorService { impl CoordinatorService {
@ -158,7 +159,7 @@ impl CoordinatorService {
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let process = ProcessId::new(process); let process = ProcessId::new(process);
let task = TaskInstanceId::new(task); let task = TaskInstanceId::new(task);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -211,6 +212,7 @@ impl CoordinatorService {
let checkpoint = self.task_restart_checkpoints.get(&checkpoint_key).cloned(); let checkpoint = self.task_restart_checkpoints.get(&checkpoint_key).cloned();
let mut accepted = false; let mut accepted = false;
let mut restarted_task_instance = None; let mut restarted_task_instance = None;
let mut restarted_attempt_id = None;
let mut clean_boundary_available = false; let mut clean_boundary_available = false;
let mut requires_whole_process_restart = true; let mut requires_whole_process_restart = true;
let message = if active_main { let message = if active_main {
@ -272,14 +274,16 @@ impl CoordinatorService {
RestartDecision::RestartTask { from_vfs_epoch, .. } => { RestartDecision::RestartTask { from_vfs_epoch, .. } => {
clean_boundary_available = true; clean_boundary_available = true;
let mut assignment = checkpoint.assignment; let mut assignment = checkpoint.assignment;
let new_task_instance = disasmer_core::TaskInstanceId::new( assignment.task = task.clone();
disasmer_core::generate_opaque_token("ti") assignment.task_spec.task_instance = task.clone();
.map_err(CoordinatorServiceError::Protocol)?, let next_attempt = self
.task_attempts
.get(&checkpoint_key)
.map_or(1, |attempts| attempts.len() + 1);
assignment.artifact_path = format!(
"/vfs/artifacts/{}-attempt-{next_attempt}-result.json",
task.as_str()
); );
assignment.task = new_task_instance.clone();
assignment.task_spec.task_instance = new_task_instance.clone();
assignment.artifact_path =
format!("/vfs/artifacts/{}-result.json", new_task_instance.as_str());
if let Some(replacement) = replacement { if let Some(replacement) = replacement {
assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm { assignment.task_spec.dispatch = TaskDispatch::CoordinatorNodeWasm {
export: Some(replacement.export), export: Some(replacement.export),
@ -296,6 +300,9 @@ impl CoordinatorService {
Some(replacement.bundle_digest.clone()); Some(replacement.bundle_digest.clone());
assignment.wasm_module_base64 = replacement.wasm_module_base64; assignment.wasm_module_base64 = replacement.wasm_module_base64;
} }
let restart_task_spec = assignment.task_spec.clone();
let restart_artifact_path = assignment.artifact_path.clone();
self.restart_launches.insert(checkpoint_key.clone());
let launch = self.handle_launch_task_with_actor( let launch = self.handle_launch_task_with_actor(
tenant.clone(), tenant.clone(),
project.clone(), project.clone(),
@ -312,18 +319,34 @@ impl CoordinatorService {
true, true,
assignment.artifact_path, assignment.artifact_path,
assignment.wasm_module_base64, assignment.wasm_module_base64,
)?; );
self.restart_launches.remove(&checkpoint_key);
let launch = launch?;
let queued = matches!(launch, CoordinatorResponse::TaskQueued { .. });
accepted = matches!( accepted = matches!(
launch, &launch,
CoordinatorResponse::TaskLaunched { .. } CoordinatorResponse::TaskLaunched { .. }
| CoordinatorResponse::TaskQueued { .. } | CoordinatorResponse::TaskQueued { .. }
); );
if accepted { if accepted {
restarted_task_instance = Some(new_task_instance.clone()); restarted_task_instance = Some(task.clone());
restarted_attempt_id = self
.task_attempts
.get(&checkpoint_key)
.and_then(|attempts| attempts.last())
.map(|attempt| attempt.attempt_id.clone());
if restarted_attempt_id.is_none() {
restarted_attempt_id = Some(self.begin_task_attempt(
&restart_task_spec,
None,
Some(&restart_artifact_path),
queued,
)?);
}
} }
requires_whole_process_restart = !accepted; requires_whole_process_restart = !accepted;
format!( format!(
"selected task restarted as new instance {new_task_instance} from clean VFS entry boundary epoch {from_vfs_epoch}; unflushed task changes were discarded" "selected logical task {task} restarted as a new attempt from clean VFS entry boundary epoch {from_vfs_epoch}; unflushed task changes were discarded"
) )
} }
RestartDecision::RestartWholeVirtualProcess { message } => message, RestartDecision::RestartWholeVirtualProcess { message } => message,
@ -346,6 +369,7 @@ impl CoordinatorService {
process, process,
task, task,
restarted_task_instance, restarted_task_instance,
restarted_attempt_id,
actor, actor,
accepted, accepted,
clean_boundary_available, clean_boundary_available,
@ -358,6 +382,87 @@ impl CoordinatorService {
audit_event, audit_event,
}) })
} }
pub(super) fn handle_resolve_task_failure(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
resolution: TaskFailureResolution,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let actor = UserId::new(actor_user);
let process = ProcessId::new(process);
let task = TaskInstanceId::new(task);
let context = clusterflux_core::AuthContext {
tenant: tenant.clone(),
project: project.clone(),
actor: Actor::User(actor),
};
let authorization = self.coordinator.authorize_debug_attach(&context, &process);
if !authorization.allowed {
return Err(CoordinatorError::Unauthorized(format!(
"task failure resolution denied: {}",
authorization.reason
))
.into());
}
let key = task_restart_key(&tenant, &project, &process, &task);
let attempt = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
.filter(|attempt| attempt.state == TaskAttemptState::FailedAwaitingAction)
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"task is not failed awaiting operator action".to_owned(),
)
})?;
attempt.state = match resolution {
TaskFailureResolution::AcceptFailure => TaskAttemptState::Failed,
TaskFailureResolution::Cancel => TaskAttemptState::Cancelled,
};
attempt.command_state = Some(
match resolution {
TaskFailureResolution::AcceptFailure => "failure_accepted",
TaskFailureResolution::Cancel => "cancelled",
}
.to_owned(),
);
let attempt_id = attempt.attempt_id.clone();
let mut event = self
.task_events
.iter()
.rev()
.find(|event| {
event.tenant == tenant
&& event.project == project
&& event.process == process
&& event.task == task
&& event.attempt_id.as_deref() == Some(attempt_id.as_str())
})
.cloned()
.ok_or_else(|| {
CoordinatorServiceError::Protocol(
"failed attempt terminal event is unavailable".to_owned(),
)
})?;
if resolution == TaskFailureResolution::Cancel {
event.terminal_state = TaskTerminalState::Cancelled;
event.stderr_tail = "operator cancelled task after failure".to_owned();
}
self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event);
Ok(CoordinatorResponse::TaskFailureResolved {
process,
task,
attempt_id,
resolution,
})
}
} }
struct ValidatedTaskReplacement { struct ValidatedTaskReplacement {
@ -372,7 +477,7 @@ struct ValidatedTaskReplacement {
fn validate_task_replacement( fn validate_task_replacement(
replacement: &TaskReplacementBundle, replacement: &TaskReplacementBundle,
task_definition: &disasmer_core::TaskDefinitionId, task_definition: &clusterflux_core::TaskDefinitionId,
environment_id: Option<&str>, environment_id: Option<&str>,
) -> Result<ValidatedTaskReplacement, CoordinatorServiceError> { ) -> Result<ValidatedTaskReplacement, CoordinatorServiceError> {
let module = BASE64_STANDARD let module = BASE64_STANDARD
@ -398,7 +503,7 @@ fn validate_task_replacement(
if descriptor if descriptor
.get("abi_version") .get("abi_version")
.and_then(serde_json::Value::as_u64) .and_then(serde_json::Value::as_u64)
!= Some(disasmer_core::WASM_TASK_ABI_VERSION as u64) != Some(clusterflux_core::WASM_TASK_ABI_VERSION as u64)
{ {
return Err(CoordinatorServiceError::Protocol(format!( return Err(CoordinatorServiceError::Protocol(format!(
"replacement task `{task_definition}` uses an unsupported task ABI" "replacement task `{task_definition}` uses an unsupported task ABI"
@ -455,7 +560,7 @@ fn validate_task_replacement(
required_capabilities.extend(environment.requirements.capabilities.iter().cloned()); required_capabilities.extend(environment.requirements.capabilities.iter().cloned());
} }
let environment_digest = environment.as_ref().map_or_else( let environment_digest = environment.as_ref().map_or_else(
|| Digest::sha256("disasmer.environment.unconstrained.v1"), || Digest::sha256("clusterflux.environment.unconstrained.v1"),
|environment| environment.digest.clone(), |environment| environment.digest.clone(),
); );
Ok(ValidatedTaskReplacement { Ok(ValidatedTaskReplacement {

View file

@ -1,4 +1,6 @@
use disasmer_core::{ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath}; use clusterflux_core::{
ArtifactId, NodeId, ProcessId, ProjectId, TaskInstanceId, TenantId, VfsPath,
};
pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId); pub(super) type TaskControlKey = (TenantId, ProjectId, ProcessId, NodeId, TaskInstanceId);
pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId); pub(super) type TaskRestartKey = (TenantId, ProjectId, ProcessId, TaskInstanceId);

View file

@ -1,11 +1,12 @@
use disasmer_core::{ use clusterflux_core::{
ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue, ArtifactFlush, ArtifactId, Digest, NodeId, ProcessId, ProjectId, TaskBoundaryValue,
TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath, TaskInstanceId, TaskJoinResult, TaskJoinState, TenantId, UserId, VfsPath,
}; };
use crate::CoordinatorError; use crate::CoordinatorError;
use super::keys::{process_control_key, task_control_key}; use super::keys::{process_control_key, task_control_key, task_restart_key};
use super::protocol::TaskAttemptState;
use super::{ use super::{
artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError, artifact_id_from_path, CoordinatorResponse, CoordinatorService, CoordinatorServiceError,
TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES, TaskCompletionEvent, TaskTerminalState, MAX_TASK_LOG_TAIL_BYTES,
@ -160,6 +161,7 @@ impl CoordinatorService {
executor: super::TaskExecutor::Node, executor: super::TaskExecutor::Node,
task_definition: checkpoint.assignment.task_spec.task_definition.clone(), task_definition: checkpoint.assignment.task_spec.task_definition.clone(),
task, task,
attempt_id: None,
placement: None, placement: None,
terminal_state: terminal_state terminal_state: terminal_state
.unwrap_or_else(|| TaskTerminalState::from_status_code(status_code)), .unwrap_or_else(|| TaskTerminalState::from_status_code(status_code)),
@ -215,16 +217,25 @@ impl CoordinatorService {
self.task_aborts.remove(&task_key); self.task_aborts.remove(&task_key);
self.debug_commands.remove(&task_key); self.debug_commands.remove(&task_key);
self.active_tasks.remove(&task_key); self.active_tasks.remove(&task_key);
if !self let no_active_tasks =
.active_tasks !self
.iter() .active_tasks
.any(|(task_tenant, task_project, task_process, _, _)| { .iter()
task_tenant == &event.tenant .any(|(task_tenant, task_project, task_process, _, _)| {
&& task_project == &event.project task_tenant == &event.tenant
&& task_process == &event.process && task_project == &event.project
}) && task_process == &event.process
{ });
if no_active_tasks {
self.process_aborts.remove(&process_key); self.process_aborts.remove(&process_key);
if self.process_cancellations.remove(&process_key)
&& !self.main_runtime.controls.contains_key(&process_key)
{
self.coordinator
.abort_process(&event.tenant, &event.project, &event.process)?;
self.clear_debug_state_for_process(&event.tenant, &event.project, &event.process);
self.clear_operator_panel_state(&event.tenant, &event.project, &event.process);
}
} }
if process_was_aborted { if process_was_aborted {
let checkpoint_key = super::keys::task_restart_key( let checkpoint_key = super::keys::task_restart_key(
@ -237,8 +248,11 @@ impl CoordinatorService {
self.task_restart_checkpoint_order self.task_restart_checkpoint_order
.retain(|retained| retained != &checkpoint_key); .retain(|retained| retained != &checkpoint_key);
} }
let awaiting_operator = self.finish_task_attempt(&mut event);
self.record_task_completion_event(event.clone()); self.record_task_completion_event(event.clone());
self.notify_coordinator_main_waiters(&event); if !awaiting_operator {
self.notify_coordinator_main_waiters(&event);
}
Ok(CoordinatorResponse::TaskRecorded { Ok(CoordinatorResponse::TaskRecorded {
process: event.process, process: event.process,
task: event.task, task: event.task,
@ -275,6 +289,33 @@ impl CoordinatorService {
Ok(CoordinatorResponse::TaskEvents { events }) Ok(CoordinatorResponse::TaskEvents { events })
} }
pub(super) fn handle_list_task_snapshots(
&mut self,
tenant: String,
project: String,
actor_user: String,
process: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant);
let project = ProjectId::new(project);
let _actor = UserId::new(actor_user);
let process = ProcessId::new(process);
self.authorize_task_event_process_scope(&tenant, &project, &process)?;
let snapshots = self
.task_attempts
.iter()
.filter(
|((attempt_tenant, attempt_project, attempt_process, _), _)| {
attempt_tenant == &tenant
&& attempt_project == &project
&& attempt_process == &process
},
)
.flat_map(|(_, attempts)| attempts.iter().cloned())
.collect();
Ok(CoordinatorResponse::TaskSnapshots { snapshots })
}
fn authorize_task_event_process_scope( fn authorize_task_event_process_scope(
&self, &self,
tenant: &TenantId, tenant: &TenantId,
@ -287,13 +328,25 @@ impl CoordinatorService {
.is_some(); .is_some();
let historical_in_scope = self.task_events.iter().any(|event| { let historical_in_scope = self.task_events.iter().any(|event| {
event.tenant == *tenant && event.project == *project && event.process == *process event.tenant == *tenant && event.project == *project && event.process == *process
}); }) || self.process_scope_history.iter().any(
|(historical_tenant, historical_project, historical_process)| {
historical_tenant == tenant
&& historical_project == project
&& historical_process == process
},
);
let process_exists_outside_scope = self let process_exists_outside_scope = self
.coordinator .coordinator
.active_process_exists_outside_scope(tenant, project, process) .active_process_exists_outside_scope(tenant, project, process)
|| self.task_events.iter().any(|event| { || self.task_events.iter().any(|event| {
event.process == *process && (event.tenant != *tenant || event.project != *project) event.process == *process && (event.tenant != *tenant || event.project != *project)
}); })
|| self.process_scope_history.iter().any(
|(historical_tenant, historical_project, historical_process)| {
historical_process == process
&& (historical_tenant != tenant || historical_project != project)
},
);
if !active_in_scope && !historical_in_scope && process_exists_outside_scope { if !active_in_scope && !historical_in_scope && process_exists_outside_scope {
return Err(CoordinatorError::Unauthorized( return Err(CoordinatorError::Unauthorized(
"task event access is outside the virtual process tenant/project scope".to_owned(), "task event access is outside the virtual process tenant/project scope".to_owned(),
@ -362,6 +415,31 @@ impl CoordinatorService {
process: ProcessId, process: ProcessId,
task: TaskInstanceId, task: TaskInstanceId,
) -> TaskJoinResult { ) -> TaskJoinResult {
let attempt_key = task_restart_key(&tenant, &project, &process, &task);
if self
.task_attempts
.get(&attempt_key)
.is_some_and(|attempts| {
attempts
.iter()
.rev()
.find(|attempt| attempt.current)
.is_some_and(|attempt| {
matches!(
attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
})
{
return TaskJoinResult::pending(
process,
task,
"logical task is still running or awaiting operator action",
);
}
let event = self.task_events.iter().rev().find(|event| { let event = self.task_events.iter().rev().find(|event| {
event.tenant == tenant event.tenant == tenant
&& event.project == project && event.project == project
@ -396,6 +474,17 @@ impl CoordinatorService {
pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) { pub(super) fn record_task_completion_event(&mut self, mut event: TaskCompletionEvent) {
event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated); event.stdout_tail = bounded_log_tail(event.stdout_tail, &mut event.stdout_truncated);
event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated); event.stderr_tail = bounded_log_tail(event.stderr_tail, &mut event.stderr_truncated);
let process_scope = (
event.tenant.clone(),
event.project.clone(),
event.process.clone(),
);
self.process_scope_history
.retain(|retained| retained != &process_scope);
while self.process_scope_history.len() >= super::MAX_TASK_EVENTS_TOTAL {
self.process_scope_history.pop_front();
}
self.process_scope_history.push_back(process_scope);
while self while self
.task_events .task_events
.iter() .iter()
@ -416,9 +505,46 @@ impl CoordinatorService {
}; };
self.task_events.remove(index); self.task_events.remove(index);
} }
while self.task_events.len() >= super::MAX_TASK_EVENTS_TOTAL {
self.task_events.pop_front();
}
self.task_events.push_back(event); self.task_events.push_back(event);
} }
fn finish_task_attempt(&mut self, event: &mut TaskCompletionEvent) -> bool {
let key = task_restart_key(&event.tenant, &event.project, &event.process, &event.task);
let Some(attempt) = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
else {
return false;
};
event.attempt_id = Some(attempt.attempt_id.clone());
attempt.status_code = event.status_code;
attempt.artifact_path = event.artifact_path.clone();
attempt.artifact_digest = event.artifact_digest.clone();
attempt.artifact_size_bytes = event.artifact_size_bytes;
attempt.error = (!event.stderr_tail.trim().is_empty()).then(|| event.stderr_tail.clone());
let awaiting_operator = event.terminal_state == TaskTerminalState::Failed
&& attempt.failure_policy == clusterflux_core::TaskFailurePolicy::AwaitOperator;
attempt.state = if awaiting_operator {
TaskAttemptState::FailedAwaitingAction
} else {
match event.terminal_state {
TaskTerminalState::Completed => TaskAttemptState::Completed,
TaskTerminalState::Failed => TaskAttemptState::Failed,
TaskTerminalState::Cancelled => TaskAttemptState::Cancelled,
}
};
attempt.command_state = Some(if awaiting_operator {
"failed_awaiting_action".to_owned()
} else {
format!("{:?}", event.terminal_state).to_ascii_lowercase()
});
awaiting_operator
}
fn flush_artifact_metadata( fn flush_artifact_metadata(
&mut self, &mut self,
flush: ArtifactFlush, flush: ArtifactFlush,

View file

@ -1,10 +1,11 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{self, Receiver, Sender, SyncSender, TryRecvError}; use std::sync::mpsc::{self, Receiver, Sender, SyncSender};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::time::Duration;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{ use clusterflux_core::{
Capability, CredentialKind, Digest, EnvironmentResource, NodeId, ProcessId, ProjectId, Capability, CredentialKind, Digest, EnvironmentResource, NodeId, ProcessId, ProjectId,
TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec, TaskBoundaryValue, TaskDefinitionId, TaskDispatch, TaskInstanceId, TaskJoinState, TaskSpec,
TenantId, WasmExportAbi, WasmHostCommandRequest, WasmHostCommandResult, TenantId, WasmExportAbi, WasmHostCommandRequest, WasmHostCommandResult,
@ -13,12 +14,16 @@ use disasmer_core::{
WasmHostTaskHandle, WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest, WasmHostTaskHandle, WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation, WasmTaskOutcome, WasmTaskResult, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation, WasmTaskOutcome, WasmTaskResult,
}; };
use disasmer_wasm_runtime::{WasmDebugControl, WasmTaskHost, WasmtimeTaskRuntime}; use clusterflux_wasm_runtime::{
WasmDebugControl, WasmTaskHost, WasmtimeRuntimeLimits, WasmtimeTaskRuntime,
};
use wasmparser::{Parser, Payload}; use wasmparser::{Parser, Payload};
use crate::{CoordinatorError, CoordinatorServiceError}; use crate::{CoordinatorError, CoordinatorServiceError};
use super::keys::{process_control_key, task_restart_key, ProcessControlKey, TaskRestartKey}; use super::keys::{
process_control_key, task_control_key, task_restart_key, ProcessControlKey, TaskRestartKey,
};
use super::{ use super::{
CoordinatorResponse, CoordinatorService, TaskCompletionEvent, TaskExecutor, TaskTerminalState, CoordinatorResponse, CoordinatorService, TaskCompletionEvent, TaskExecutor, TaskTerminalState,
WorkflowActor, WorkflowActor,
@ -39,7 +44,7 @@ enum MainCommand {
StartTask { StartTask {
scope: MainScope, scope: MainScope,
handle_id: u64, handle_id: u64,
task_spec: TaskSpec, task_spec: Box<TaskSpec>,
wasm_module_base64: String, wasm_module_base64: String,
response: SyncSender<Result<WasmHostTaskHandle, String>>, response: SyncSender<Result<WasmHostTaskHandle, String>>,
}, },
@ -76,6 +81,8 @@ pub(super) struct CoordinatorMainRuntime {
pub(super) controls: BTreeMap<ProcessControlKey, CoordinatorMainControl>, pub(super) controls: BTreeMap<ProcessControlKey, CoordinatorMainControl>,
join_waiters: BTreeMap<TaskRestartKey, Vec<SyncSender<Result<WasmHostTaskJoinResult, String>>>>, join_waiters: BTreeMap<TaskRestartKey, Vec<SyncSender<Result<WasmHostTaskJoinResult, String>>>>,
next_launch_id: u64, next_launch_id: u64,
runtime_limits: WasmtimeRuntimeLimits,
nested_join_timeout: Duration,
} }
impl Default for CoordinatorMainRuntime { impl Default for CoordinatorMainRuntime {
@ -87,11 +94,35 @@ impl Default for CoordinatorMainRuntime {
controls: BTreeMap::new(), controls: BTreeMap::new(),
join_waiters: BTreeMap::new(), join_waiters: BTreeMap::new(),
next_launch_id: 1, next_launch_id: 1,
runtime_limits: WasmtimeRuntimeLimits::default(),
nested_join_timeout: Duration::from_secs(24 * 60 * 60),
} }
} }
} }
impl CoordinatorMainRuntime { impl CoordinatorMainRuntime {
pub(super) fn configure(
&mut self,
configuration: super::CoordinatorMainRuntimeConfiguration,
) -> Result<(), CoordinatorServiceError> {
if configuration.nested_join_timeout_ms == 0 {
return Err(CoordinatorServiceError::Protocol(
"coordinator main nested join timeout must be positive".to_owned(),
));
}
let limits = WasmtimeRuntimeLimits {
fuel_units_per_second: configuration.fuel_units_per_second,
fuel_burst_seconds: configuration.fuel_burst_seconds,
memory_bytes: configuration.memory_bytes,
};
limits
.validate()
.map_err(|error| CoordinatorServiceError::Protocol(error.to_string()))?;
self.runtime_limits = limits;
self.nested_join_timeout = Duration::from_millis(configuration.nested_join_timeout_ms);
Ok(())
}
pub(super) fn is_waiting_for_task( pub(super) fn is_waiting_for_task(
&self, &self,
tenant: &TenantId, tenant: &TenantId,
@ -107,11 +138,8 @@ impl CoordinatorMainRuntime {
fn drain_commands(&self) -> Vec<MainCommand> { fn drain_commands(&self) -> Vec<MainCommand> {
let mut commands = Vec::new(); let mut commands = Vec::new();
loop { while let Ok(command) = self.receiver.try_recv() {
match self.receiver.try_recv() { commands.push(command);
Ok(command) => commands.push(command),
Err(TryRecvError::Empty | TryRecvError::Disconnected) => break,
}
} }
commands commands
} }
@ -157,8 +185,10 @@ impl CoordinatorMainRuntime {
scope.task_instance.clone(), scope.task_instance.clone(),
Vec::new(), Vec::new(),
); );
let runtime_limits = self.runtime_limits.clone();
let nested_join_timeout = self.nested_join_timeout;
std::thread::Builder::new() std::thread::Builder::new()
.name(format!("disasmer-main-{}", scope.process)) .name(format!("clusterflux-main-{}", scope.process))
.spawn(move || { .spawn(move || {
let host = CoordinatorMainHost { let host = CoordinatorMainHost {
scope: scope.clone(), scope: scope.clone(),
@ -171,8 +201,9 @@ impl CoordinatorMainRuntime {
wasm_module_base64, wasm_module_base64,
next_handle_id: 1, next_handle_id: 1,
handles, handles,
nested_join_timeout,
}; };
let result = WasmtimeTaskRuntime::new() let result = WasmtimeTaskRuntime::new_with_limits(runtime_limits)
.and_then(|runtime| { .and_then(|runtime| {
runtime.run_task_export_verified_with_task_host( runtime.run_task_export_verified_with_task_host(
&module, &module,
@ -244,6 +275,7 @@ struct CoordinatorMainHost {
wasm_module_base64: String, wasm_module_base64: String,
next_handle_id: u64, next_handle_id: u64,
handles: Arc<Mutex<HashMap<u64, TaskSpec>>>, handles: Arc<Mutex<HashMap<u64, TaskSpec>>>,
nested_join_timeout: Duration,
} }
impl WasmTaskHost for CoordinatorMainHost { impl WasmTaskHost for CoordinatorMainHost {
@ -364,6 +396,7 @@ impl WasmTaskHost for CoordinatorMainHost {
required_artifacts, required_artifacts,
args: request.args, args: request.args,
vfs_epoch: self.scope.epoch, vfs_epoch: self.scope.epoch,
failure_policy: request.failure_policy,
bundle_digest: Some(self.bundle_digest.clone()), bundle_digest: Some(self.bundle_digest.clone()),
}; };
let (response, receiver) = mpsc::sync_channel(1); let (response, receiver) = mpsc::sync_channel(1);
@ -371,7 +404,7 @@ impl WasmTaskHost for CoordinatorMainHost {
.send(MainCommand::StartTask { .send(MainCommand::StartTask {
scope: self.scope.clone(), scope: self.scope.clone(),
handle_id, handle_id,
task_spec: task_spec.clone(), task_spec: Box::new(task_spec.clone()),
wasm_module_base64: self.wasm_module_base64.clone(), wasm_module_base64: self.wasm_module_base64.clone(),
response, response,
}) })
@ -406,9 +439,18 @@ impl WasmTaskHost for CoordinatorMainHost {
response, response,
}) })
.map_err(|_| "coordinator main command channel closed".to_owned())?; .map_err(|_| "coordinator main command channel closed".to_owned())?;
let joined = receiver let joined =
.recv() receiver
.map_err(|_| "coordinator main task-join response channel closed".to_owned())?; .recv_timeout(self.nested_join_timeout)
.map_err(|error| match error {
mpsc::RecvTimeoutError::Timeout => format!(
"coordinator main nested task join timed out after {} ms",
self.nested_join_timeout.as_millis()
),
mpsc::RecvTimeoutError::Disconnected => {
"coordinator main task-join response channel closed".to_owned()
}
})?;
self.handles self.handles
.lock() .lock()
.map_err(|_| "coordinator main handle registry is unavailable".to_owned())? .map_err(|_| "coordinator main handle registry is unavailable".to_owned())?
@ -429,7 +471,7 @@ impl WasmTaskHost for CoordinatorMainHost {
) -> Result<WasmHostTaskControlResult, String> { ) -> Result<WasmHostTaskControlResult, String> {
request.validate()?; request.validate()?;
Ok(WasmHostTaskControlResult { Ok(WasmHostTaskControlResult {
abi_version: disasmer_core::WASM_TASK_ABI_VERSION, abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
cancellation_requested: self.abort.load(Ordering::Acquire), cancellation_requested: self.abort.load(Ordering::Acquire),
}) })
} }
@ -473,7 +515,7 @@ impl CoordinatorService {
actor_user: Option<String>, actor_user: Option<String>,
actor_agent: Option<String>, actor_agent: Option<String>,
agent_public_key_fingerprint: Option<Digest>, agent_public_key_fingerprint: Option<Digest>,
agent_signature: Option<disasmer_core::AgentSignedRequest>, agent_signature: Option<clusterflux_core::AgentSignedRequest>,
request_payload_digest: Option<&Digest>, request_payload_digest: Option<&Digest>,
task_spec: TaskSpec, task_spec: TaskSpec,
wasm_module_base64: String, wasm_module_base64: String,
@ -602,6 +644,7 @@ impl CoordinatorService {
wasm_module_base64, wasm_module_base64,
response, response,
} => { } => {
let task_spec = *task_spec;
if !self.main_runtime.is_current_scope(&scope) { if !self.main_runtime.is_current_scope(&scope) {
let _ = response.send(Err( let _ = response.send(Err(
"coordinator main process incarnation was replaced".to_owned(), "coordinator main process incarnation was replaced".to_owned(),
@ -630,7 +673,7 @@ impl CoordinatorService {
.and_then(|launch| match launch { .and_then(|launch| match launch {
CoordinatorResponse::TaskLaunched { .. } CoordinatorResponse::TaskLaunched { .. }
| CoordinatorResponse::TaskQueued { .. } => Ok(WasmHostTaskHandle { | CoordinatorResponse::TaskQueued { .. } => Ok(WasmHostTaskHandle {
abi_version: disasmer_core::WASM_TASK_ABI_VERSION, abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
handle_id, handle_id,
task_spec, task_spec,
}), }),
@ -666,7 +709,7 @@ impl CoordinatorService {
"completed child task omitted its boundary result".to_owned() "completed child task omitted its boundary result".to_owned()
}) })
.map(|result| WasmHostTaskJoinResult { .map(|result| WasmHostTaskJoinResult {
abi_version: disasmer_core::WASM_TASK_ABI_VERSION, abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
task_instance, task_instance,
result, result,
}); });
@ -732,7 +775,7 @@ impl CoordinatorService {
.clone() .clone()
.ok_or_else(|| "completed child task omitted its boundary result".to_owned()) .ok_or_else(|| "completed child task omitted its boundary result".to_owned())
.map(|result| WasmHostTaskJoinResult { .map(|result| WasmHostTaskJoinResult {
abi_version: disasmer_core::WASM_TASK_ABI_VERSION, abi_version: clusterflux_core::WASM_TASK_ABI_VERSION,
task_instance: event.task.clone(), task_instance: event.task.clone(),
result, result,
}), }),
@ -776,6 +819,7 @@ impl CoordinatorService {
executor: TaskExecutor::CoordinatorMain, executor: TaskExecutor::CoordinatorMain,
task_definition: scope.task_definition, task_definition: scope.task_definition,
task: scope.task_instance, task: scope.task_instance,
attempt_id: None,
placement: None, placement: None,
terminal_state, terminal_state,
status_code: if error.is_empty() { Some(0) } else { None }, status_code: if error.is_empty() { Some(0) } else { None },
@ -802,111 +846,28 @@ impl CoordinatorService {
handles.clear(); handles.clear();
} }
} }
// Completion keeps the virtual process and its final debug handshake let process_key = process_control_key(&scope.tenant, &scope.project, &scope.process);
// inspectable. In particular, the DAP client may still be waiting for for (task_tenant, task_project, task_process, node, task) in self.active_tasks.clone() {
// every participant's resume acknowledgement when the main Wasm task if task_tenant == scope.tenant
// exits. A new process incarnation or an explicit abort clears this && task_project == scope.project
// ephemeral state. && task_process == scope.process
} {
} self.task_aborts.insert(task_control_key(
&task_tenant,
#[cfg(test)] &task_project,
mod tests { &task_process,
use super::*; &node,
&task,
#[test] ));
fn completed_main_keeps_process_and_debug_handshake_visible_until_explicit_abort() { }
let mut service = CoordinatorService::new(7); }
let tenant = TenantId::from("tenant"); self.process_aborts.insert(process_key.clone());
let project = ProjectId::from("project"); let _ = self
let process = ProcessId::from("vp-current");
let main_task = TaskInstanceId::from("ti:vp-current:main");
let scope = MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main_task.clone(),
epoch: 7,
launch_id: 1,
};
service
.coordinator .coordinator
.start_process(tenant.clone(), project.clone(), process.clone()); .abort_process(&scope.tenant, &scope.project, &scope.process);
let process_key = process_control_key(&tenant, &project, &process); self.main_runtime.controls.remove(&process_key);
service.main_runtime.controls.insert( self.clear_debug_state_for_process(&scope.tenant, &scope.project, &scope.process);
process_key.clone(), self.clear_operator_panel_state(&scope.tenant, &scope.project, &scope.process);
CoordinatorMainControl {
task_definition: scope.task_definition.clone(),
task_instance: main_task.clone(),
abort: Arc::new(AtomicBool::new(false)),
debug: Arc::new(WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: Arc::new(Mutex::new(HashMap::new())),
launch_id: 1,
},
);
service.debug_epochs.insert(process_key.clone(), 2);
service.debug_epoch_runtime.insert(
process_key.clone(),
super::super::debug::DebugEpochRuntime {
epoch: 2,
command: "resume".to_owned(),
expected: BTreeSet::new(),
acknowledgements: BTreeMap::new(),
},
);
service.record_coordinator_main_completion(
scope,
Ok(WasmTaskResult::completed(
main_task,
TaskBoundaryValue::SmallJson(serde_json::Value::Null),
)),
);
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_some());
let CoordinatorResponse::ProcessStatuses { processes, .. } = service
.handle_list_processes(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
"user".to_owned(),
)
.unwrap()
else {
panic!("expected process statuses");
};
assert_eq!(processes.len(), 1);
assert_eq!(processes[0].state, "completed");
assert_eq!(processes[0].main_state.as_deref(), Some("completed"));
assert_eq!(service.debug_epochs.get(&process_key), Some(&2));
assert_eq!(
service
.debug_epoch_runtime
.get(&process_key)
.map(|runtime| runtime.command.as_str()),
Some("resume")
);
service
.handle_abort_process(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
"user".to_owned(),
process.as_str().to_owned(),
)
.unwrap();
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
} }
} }
@ -921,7 +882,7 @@ pub(super) fn task_descriptors(
else { else {
continue; continue;
}; };
if section.name() != "disasmer.tasks" { if section.name() != "clusterflux.tasks" {
continue; continue;
} }
for record in section for record in section
@ -955,7 +916,7 @@ pub(super) fn bundle_environments(
else { else {
continue; continue;
}; };
if section.name() != "disasmer.environments" { if section.name() != "clusterflux.environments" {
continue; continue;
} }
let environments: Vec<EnvironmentResource> = serde_json::from_slice(section.data()) let environments: Vec<EnvironmentResource> = serde_json::from_slice(section.data())
@ -997,3 +958,82 @@ pub(super) fn capability_from_descriptor(capability: &str) -> Result<Capability,
other => Err(format!("unknown task capability `{other}`")), other => Err(format!("unknown task capability `{other}`")),
} }
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn completed_main_releases_process_slot_and_debug_state() {
let mut service = CoordinatorService::new(7);
let tenant = TenantId::from("tenant");
let project = ProjectId::from("project");
let process = ProcessId::from("vp-current");
let main_task = TaskInstanceId::from("ti:vp-current:main");
let scope = MainScope {
tenant: tenant.clone(),
project: project.clone(),
process: process.clone(),
task_definition: TaskDefinitionId::from("build"),
task_instance: main_task.clone(),
epoch: 7,
launch_id: 1,
};
service
.coordinator
.start_process(tenant.clone(), project.clone(), process.clone());
let process_key = process_control_key(&tenant, &project, &process);
service.main_runtime.controls.insert(
process_key.clone(),
CoordinatorMainControl {
task_definition: scope.task_definition.clone(),
task_instance: main_task.clone(),
abort: Arc::new(AtomicBool::new(false)),
debug: Arc::new(WasmDebugControl::default()),
state: "running".to_owned(),
stopped_probe_symbol: None,
handles: Arc::new(Mutex::new(HashMap::new())),
launch_id: 1,
},
);
service.debug_epochs.insert(process_key.clone(), 2);
service.debug_epoch_runtime.insert(
process_key.clone(),
super::super::debug::DebugEpochRuntime {
epoch: 2,
command: "resume".to_owned(),
expected: BTreeSet::new(),
acknowledgements: BTreeMap::new(),
deadline: std::time::Instant::now(),
},
);
service.record_coordinator_main_completion(
scope,
Ok(WasmTaskResult::completed(
main_task,
TaskBoundaryValue::SmallJson(serde_json::Value::Null),
)),
);
assert!(service
.coordinator
.active_process(&tenant, &project, &process)
.is_none());
let CoordinatorResponse::ProcessStatuses { processes, .. } = service
.handle_list_processes(
tenant.as_str().to_owned(),
project.as_str().to_owned(),
"user".to_owned(),
)
.unwrap()
else {
panic!("expected process statuses");
};
assert!(processes.is_empty());
assert!(!service.main_runtime.controls.contains_key(&process_key));
assert!(!service.debug_epochs.contains_key(&process_key));
assert!(!service.debug_epoch_runtime.contains_key(&process_key));
assert!(service.process_aborts.contains(&process_key));
}
}

View file

@ -1,7 +1,7 @@
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use disasmer_core::{ use clusterflux_core::{
generate_opaque_token, verify_node_request_signature, Actor, ArtifactId, CredentialKind, generate_opaque_token, verify_node_request_signature, Actor, ArtifactId, CredentialKind,
Digest, NodeCapabilities, NodeDescriptor, NodeId, NodeSignedRequest, ProjectId, Digest, NodeCapabilities, NodeDescriptor, NodeId, NodeSignedRequest, ProjectId,
SourceProviderKind, TenantId, UserId, SourceProviderKind, TenantId, UserId,
@ -15,6 +15,38 @@ use super::{
}; };
impl CoordinatorService { impl CoordinatorService {
pub fn set_node_stale_after_seconds(&mut self, seconds: u64) {
self.node_stale_after_seconds = seconds.max(1);
}
pub(super) fn liveness_now_epoch_seconds(&self) -> u64 {
#[cfg(test)]
if let Some(now) = self.server_time_override {
return now;
}
unix_timestamp_seconds()
}
pub(super) fn node_is_live(&self, node: &NodeId) -> bool {
self.node_last_seen_epoch_seconds
.get(node)
.is_some_and(|last_seen| {
self.liveness_now_epoch_seconds().saturating_sub(*last_seen)
<= self.node_stale_after_seconds
})
}
pub(super) fn live_node_descriptors(&self) -> Vec<NodeDescriptor> {
self.node_descriptors
.values()
.cloned()
.map(|mut descriptor| {
descriptor.online = self.node_is_live(&descriptor.id);
descriptor
})
.collect()
}
pub(super) fn handle_attach_node( pub(super) fn handle_attach_node(
&mut self, &mut self,
tenant: String, tenant: String,
@ -138,7 +170,7 @@ impl CoordinatorService {
self.enrollment_grants self.enrollment_grants
.get_mut(&grant_key) .get_mut(&grant_key)
.ok_or(CoordinatorError::Enrollment( .ok_or(CoordinatorError::Enrollment(
disasmer_core::EnrollmentError::Expired, clusterflux_core::EnrollmentError::Expired,
))?; ))?;
let credential = self.coordinator.exchange_node_enrollment_grant( let credential = self.coordinator.exchange_node_enrollment_grant(
grant, grant,
@ -182,7 +214,7 @@ impl CoordinatorService {
source_snapshots: Vec<Digest>, source_snapshots: Vec<Digest>,
artifact_locations: Vec<String>, artifact_locations: Vec<String>,
direct_connectivity: bool, direct_connectivity: bool,
online: bool, _online: bool,
) -> Result<CoordinatorResponse, CoordinatorServiceError> { ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant); let tenant = TenantId::new(tenant);
let project = ProjectId::new(project); let project = ProjectId::new(project);
@ -239,6 +271,7 @@ impl CoordinatorService {
self.artifact_registry self.artifact_registry
.reconcile_node_retention(&node, &artifact_locations); .reconcile_node_retention(&node, &artifact_locations);
let online = self.node_is_live(&node);
self.node_descriptors.insert( self.node_descriptors.insert(
node.clone(), node.clone(),
NodeDescriptor { NodeDescriptor {
@ -270,10 +303,9 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let descriptors = self let descriptors = self
.node_descriptors .live_node_descriptors()
.values() .into_iter()
.filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project) .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
.cloned()
.collect(); .collect();
Ok(CoordinatorResponse::NodeDescriptors { descriptors, actor }) Ok(CoordinatorResponse::NodeDescriptors { descriptors, actor })
} }
@ -289,13 +321,14 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let node = NodeId::new(node); let node = NodeId::new(node);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
}; };
self.coordinator.revoke_node_credential(&context, &node)?; self.coordinator.revoke_node_credential(&context, &node)?;
let descriptor_removed = self.node_descriptors.remove(&node).is_some(); let descriptor_removed = self.node_descriptors.remove(&node).is_some();
self.node_last_seen_epoch_seconds.remove(&node);
self.artifact_registry.garbage_collect_node(&node); self.artifact_registry.garbage_collect_node(&node);
let queued_assignments_removed = self let queued_assignments_removed = self
.task_assignments .task_assignments
@ -395,6 +428,12 @@ impl CoordinatorService {
} }
self.node_replay_nonces self.node_replay_nonces
.insert(replay_key, now_epoch_seconds); .insert(replay_key, now_epoch_seconds);
let seen_at = self.liveness_now_epoch_seconds();
self.node_last_seen_epoch_seconds
.insert(node.clone(), seen_at);
if let Some(descriptor) = self.node_descriptors.get_mut(node) {
descriptor.online = true;
}
Ok(()) Ok(())
} }
} }

View file

@ -1,4 +1,4 @@
use disasmer_core::{ use clusterflux_core::{
Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind, Actor, DownloadPolicy, PanelEvent, PanelEventKind, PanelState, PanelWidget, PanelWidgetKind,
ProcessId, ProjectId, RateLimit, TenantId, UserId, ProcessId, ProjectId, RateLimit, TenantId, UserId,
}; };
@ -241,11 +241,11 @@ impl CoordinatorService {
} }
let mut actions = vec![ let mut actions = vec![
disasmer_core::ControlPlaneAction::DebugProcess, clusterflux_core::ControlPlaneAction::DebugProcess,
disasmer_core::ControlPlaneAction::CancelProcess, clusterflux_core::ControlPlaneAction::CancelProcess,
]; ];
if let Some(task) = last_task.clone() { if let Some(task) = last_task.clone() {
actions.push(disasmer_core::ControlPlaneAction::RestartTask(task)); actions.push(clusterflux_core::ControlPlaneAction::RestartTask(task));
} }
panel.set_control_plane_actions(actions); panel.set_control_plane_actions(actions);
@ -255,7 +255,7 @@ impl CoordinatorService {
.find_map(|event| event.artifact_path.as_ref()) .find_map(|event| event.artifact_path.as_ref())
{ {
let artifact = artifact_id_from_path(path); let artifact = artifact_id_from_path(path);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant, tenant,
project, project,
actor: Actor::User(actor_user), actor: Actor::User(actor_user),

View file

@ -1,10 +1,11 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{ use clusterflux_core::{
AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest, AgentSignedRequest, ArtifactId, CheckpointBoundary, CredentialKind, DefaultScheduler, Digest,
NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskCheckpoint, TaskDispatch, NodeId, PlacementRequest, ProcessId, ProjectId, Scheduler, TaskBoundaryValue, TaskCheckpoint,
TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath, WasmTaskInvocation, TaskDispatch, TaskInstanceId, TaskSpec, TenantId, VfsManifest, VfsObject, VfsPath,
WasmTaskInvocation,
}; };
use crate::CoordinatorError; use crate::CoordinatorError;
@ -15,6 +16,7 @@ use super::{
}; };
use super::processes::*; use super::processes::*;
use super::protocol::{TaskAttemptSnapshot, TaskAttemptState};
impl CoordinatorService { impl CoordinatorService {
pub(super) fn capture_task_restart_checkpoint( pub(super) fn capture_task_restart_checkpoint(
@ -24,7 +26,7 @@ impl CoordinatorService {
let task_spec = &assignment.task_spec; let task_spec = &assignment.task_spec;
let environment_digest = task_spec.environment_digest.clone().unwrap_or_else(|| { let environment_digest = task_spec.environment_digest.clone().unwrap_or_else(|| {
task_spec.environment.as_ref().map_or_else( task_spec.environment.as_ref().map_or_else(
|| Digest::sha256("disasmer.environment.unconstrained.v1"), || Digest::sha256("clusterflux.environment.unconstrained.v1"),
|environment| { |environment| {
Digest::sha256( Digest::sha256(
serde_json::to_vec(environment) serde_json::to_vec(environment)
@ -34,7 +36,7 @@ impl CoordinatorService {
) )
}); });
let task_entrypoint = match &task_spec.dispatch { let task_entrypoint = match &task_spec.dispatch {
disasmer_core::TaskDispatch::CoordinatorNodeWasm { export, .. } => export clusterflux_core::TaskDispatch::CoordinatorNodeWasm { export, .. } => export
.clone() .clone()
.or_else(|| assignment_task_descriptor(assignment)?.get("export")?.as_str().map(str::to_owned)) .or_else(|| assignment_task_descriptor(assignment)?.get("export")?.as_str().map(str::to_owned))
.ok_or_else(|| { .ok_or_else(|| {
@ -132,6 +134,11 @@ impl CoordinatorService {
self.task_restart_checkpoints.remove(&expired); self.task_restart_checkpoints.remove(&expired);
} }
} }
while self.task_restart_checkpoint_order.len() > super::MAX_RESTART_CHECKPOINTS_TOTAL {
if let Some(expired) = self.task_restart_checkpoint_order.pop_front() {
self.task_restart_checkpoints.remove(&expired);
}
}
Ok(()) Ok(())
} }
@ -139,9 +146,9 @@ impl CoordinatorService {
&mut self, &mut self,
tenant: String, tenant: String,
project: String, project: String,
environment: Option<disasmer_core::EnvironmentRequirements>, environment: Option<clusterflux_core::EnvironmentRequirements>,
environment_digest: Option<Digest>, environment_digest: Option<Digest>,
required_capabilities: Vec<disasmer_core::Capability>, required_capabilities: Vec<clusterflux_core::Capability>,
dependency_cache: Option<Digest>, dependency_cache: Option<Digest>,
source_snapshot: Option<Digest>, source_snapshot: Option<Digest>,
required_artifacts: Vec<String>, required_artifacts: Vec<String>,
@ -155,6 +162,7 @@ impl CoordinatorService {
project: project.clone(), project: project.clone(),
environment, environment,
environment_digest, environment_digest,
environment_cache_required: false,
required_capabilities: required_capabilities.into_iter().collect(), required_capabilities: required_capabilities.into_iter().collect(),
dependency_cache, dependency_cache,
source_snapshot, source_snapshot,
@ -169,7 +177,7 @@ impl CoordinatorService {
policy_allowed: self.admission.workflow_placement_allowed, policy_allowed: self.admission.workflow_placement_allowed,
prefer_node: prefer_node.map(NodeId::new), prefer_node: prefer_node.map(NodeId::new),
}; };
let nodes = self.node_descriptors.values().cloned().collect::<Vec<_>>(); let nodes = self.live_node_descriptors();
let placement = DefaultScheduler.place(&nodes, &request)?; let placement = DefaultScheduler.place(&nodes, &request)?;
Ok(CoordinatorResponse::TaskPlacement { placement }) Ok(CoordinatorResponse::TaskPlacement { placement })
} }
@ -191,7 +199,7 @@ impl CoordinatorService {
if matches!( if matches!(
&task_spec.dispatch, &task_spec.dispatch,
TaskDispatch::CoordinatorNodeWasm { TaskDispatch::CoordinatorNodeWasm {
abi: disasmer_core::WasmExportAbi::EntrypointV1, abi: clusterflux_core::WasmExportAbi::EntrypointV1,
.. ..
} }
) { ) {
@ -324,6 +332,7 @@ impl CoordinatorService {
let active = self let active = self
.coordinator .coordinator
.active_process(&tenant, &project, &process) .active_process(&tenant, &project, &process)
.cloned()
.ok_or_else(|| { .ok_or_else(|| {
CoordinatorError::Unauthorized( CoordinatorError::Unauthorized(
"task launch requires an active coordinator-side virtual process".to_owned(), "task launch requires an active coordinator-side virtual process".to_owned(),
@ -435,6 +444,8 @@ impl CoordinatorService {
project: project.clone(), project: project.clone(),
environment: task_spec.environment.clone(), environment: task_spec.environment.clone(),
environment_digest: task_spec.environment_digest.clone(), environment_digest: task_spec.environment_digest.clone(),
environment_cache_required: task_spec.environment_id.is_some()
&& task_spec.environment.is_none(),
required_capabilities: task_spec.required_capabilities.clone(), required_capabilities: task_spec.required_capabilities.clone(),
dependency_cache: task_spec.dependency_cache.clone(), dependency_cache: task_spec.dependency_cache.clone(),
source_snapshot: task_spec.source_snapshot.clone(), source_snapshot: task_spec.source_snapshot.clone(),
@ -446,7 +457,7 @@ impl CoordinatorService {
policy_allowed: self.admission.workflow_placement_allowed, policy_allowed: self.admission.workflow_placement_allowed,
prefer_node: None, prefer_node: None,
}; };
let nodes = self.node_descriptors.values().cloned().collect::<Vec<_>>(); let nodes = self.live_node_descriptors();
let placement = match DefaultScheduler.place(&nodes, &request) { let placement = match DefaultScheduler.place(&nodes, &request) {
Ok(placement) => placement, Ok(placement) => placement,
Err(err) if wait_for_node => { Err(err) if wait_for_node => {
@ -458,6 +469,7 @@ impl CoordinatorService {
let charged_spawns = let charged_spawns =
self.quota self.quota
.charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?; .charge_workflow_spawn(&tenant, &project, now_epoch_seconds)?;
self.begin_task_attempt(&task_spec, None, Some(&artifact_path), true)?;
self.pending_task_launches.push_back(PendingTaskLaunch { self.pending_task_launches.push_back(PendingTaskLaunch {
tenant: tenant.clone(), tenant: tenant.clone(),
project: project.clone(), project: project.clone(),
@ -494,6 +506,12 @@ impl CoordinatorService {
task_spec, task_spec,
wasm_module_base64, wasm_module_base64,
}; };
self.begin_task_attempt(
&assignment.task_spec,
Some(assignment.node.clone()),
Some(&assignment.artifact_path),
false,
)?;
self.capture_task_restart_checkpoint(&assignment)?; self.capture_task_restart_checkpoint(&assignment)?;
let task_key = task_control_key(&tenant, &project, &process, &placement.node, &task); let task_key = task_control_key(&tenant, &project, &process, &placement.node, &task);
self.task_placements self.task_placements
@ -518,7 +536,7 @@ impl CoordinatorService {
tenant: &TenantId, tenant: &TenantId,
project: &ProjectId, project: &ProjectId,
process: &ProcessId, process: &ProcessId,
task_instance: &disasmer_core::TaskInstanceId, task_instance: &clusterflux_core::TaskInstanceId,
) -> bool { ) -> bool {
self.active_tasks.iter().any( self.active_tasks.iter().any(
|(task_tenant, task_project, task_process, _, existing_instance)| { |(task_tenant, task_project, task_process, _, existing_instance)| {
@ -532,12 +550,137 @@ impl CoordinatorService {
&& &pending.project == project && &pending.project == project
&& &pending.process == process && &pending.process == process
&& &pending.task == task_instance && &pending.task == task_instance
}) || self.task_events.iter().any(|event| { }) || (!self.restart_launches.contains(&task_restart_key(
tenant,
project,
process,
task_instance,
)) && self.task_events.iter().any(|event| {
&event.tenant == tenant &event.tenant == tenant
&& &event.project == project && &event.project == project
&& &event.process == process && &event.process == process
&& &event.task == task_instance && &event.task == task_instance
}) }))
}
pub(super) fn begin_task_attempt(
&mut self,
task_spec: &TaskSpec,
node: Option<NodeId>,
artifact_path: Option<&str>,
queued: bool,
) -> Result<String, CoordinatorServiceError> {
let key = task_restart_key(
&task_spec.tenant,
&task_spec.project,
&task_spec.process,
&task_spec.task_instance,
);
while !self.task_attempts.contains_key(&key)
&& self.task_attempts.len() >= super::MAX_TASK_ATTEMPT_HISTORIES
{
let removable = self.task_attempts.iter().find_map(|(candidate, attempts)| {
attempts
.iter()
.all(|attempt| {
!matches!(
&attempt.state,
TaskAttemptState::Queued
| TaskAttemptState::Running
| TaskAttemptState::FailedAwaitingAction
)
})
.then(|| candidate.clone())
});
let Some(removable) = removable else {
return Err(CoordinatorServiceError::Protocol(
"task attempt history capacity is exhausted by active attempts".to_owned(),
));
};
self.task_attempts.remove(&removable);
}
let attempts = self.task_attempts.entry(key).or_default();
for attempt in attempts.iter_mut() {
attempt.current = false;
}
let attempt_id = clusterflux_core::generate_opaque_token("ta")
.map_err(CoordinatorServiceError::Protocol)?;
let attempt_number = u32::try_from(attempts.len() + 1).unwrap_or(u32::MAX);
let mut argument_summary = task_spec
.args
.iter()
.map(|argument| {
let mut value = serde_json::to_string(argument)
.unwrap_or_else(|_| "<invalid canonical argument>".to_owned());
value.truncate(value.len().min(1024));
value
})
.collect::<Vec<_>>();
argument_summary.truncate(64);
let mut handle_summary = task_spec
.required_artifacts
.iter()
.map(|artifact| format!("artifact:{artifact}"))
.collect::<Vec<_>>();
for argument in &task_spec.args {
if let TaskBoundaryValue::Structured(boundary) = argument {
handle_summary.extend(boundary.handles.iter().map(|handle| format!("{handle:?}")));
}
}
handle_summary.truncate(256);
attempts.push(TaskAttemptSnapshot {
process: task_spec.process.clone(),
task: task_spec.task_instance.clone(),
attempt_id: attempt_id.clone(),
attempt_number,
task_definition: task_spec.task_definition.clone(),
display_name: task_spec.task_definition.as_str().replace(['_', '-'], " "),
state: if queued {
TaskAttemptState::Queued
} else {
TaskAttemptState::Running
},
current: true,
node,
environment_id: task_spec.environment_id.clone(),
environment_digest: task_spec.environment_digest.clone(),
argument_summary,
handle_summary,
command_state: Some(if queued { "queued" } else { "running" }.to_owned()),
vfs_checkpoint: format!("vfs-epoch:{}", task_spec.vfs_epoch),
probe_symbol: None,
source_path: None,
source_line: None,
restart_compatible: true,
failure_policy: task_spec.failure_policy,
artifact_path: artifact_path.and_then(|path| VfsPath::new(path).ok()),
artifact_digest: None,
artifact_size_bytes: None,
status_code: None,
error: None,
});
if attempts.len() > 128 {
attempts.remove(0);
}
Ok(attempt_id)
}
pub(super) fn assign_task_attempt(&mut self, task_spec: &TaskSpec, node: NodeId) {
let key = task_restart_key(
&task_spec.tenant,
&task_spec.project,
&task_spec.process,
&task_spec.task_instance,
);
if let Some(attempt) = self
.task_attempts
.get_mut(&key)
.and_then(|attempts| attempts.iter_mut().rev().find(|attempt| attempt.current))
{
attempt.node = Some(node);
attempt.state = TaskAttemptState::Running;
attempt.command_state = Some("running".to_owned());
}
} }
} }

View file

@ -1,7 +1,7 @@
use std::collections::{BTreeSet, VecDeque}; use std::collections::{BTreeSet, VecDeque};
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use disasmer_core::{ use clusterflux_core::{
AgentId, AgentSignedRequest, CredentialKind, DefaultScheduler, Digest, NodeId, AgentId, AgentSignedRequest, CredentialKind, DefaultScheduler, Digest, NodeId,
PlacementRequest, ProcessId, ProjectId, RendezvousRequest, Scheduler, SourcePreparation, PlacementRequest, ProcessId, ProjectId, RendezvousRequest, Scheduler, SourcePreparation,
TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId, TaskCheckpoint, TaskInstanceId, TaskSpec, TenantId, UserId,
@ -129,6 +129,7 @@ impl CoordinatorService {
task_spec: pending.task_spec, task_spec: pending.task_spec,
wasm_module_base64: pending.wasm_module_base64, wasm_module_base64: pending.wasm_module_base64,
}; };
self.assign_task_attempt(&assignment.task_spec, assignment.node.clone());
self.capture_task_restart_checkpoint(&assignment)?; self.capture_task_restart_checkpoint(&assignment)?;
let task_key = task_control_key( let task_key = task_control_key(
&pending.tenant, &pending.tenant,
@ -148,9 +149,9 @@ impl CoordinatorService {
pub(super) fn handle_request_rendezvous( pub(super) fn handle_request_rendezvous(
&mut self, &mut self,
scope: disasmer_core::DataPlaneScope, scope: clusterflux_core::DataPlaneScope,
source: disasmer_core::NodeEndpoint, source: clusterflux_core::NodeEndpoint,
destination: disasmer_core::NodeEndpoint, destination: clusterflux_core::NodeEndpoint,
direct_connectivity: bool, direct_connectivity: bool,
failure_reason: String, failure_reason: String,
) -> Result<CoordinatorResponse, CoordinatorServiceError> { ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
@ -179,7 +180,7 @@ impl CoordinatorService {
&mut self, &mut self,
tenant: String, tenant: String,
project: String, project: String,
provider: disasmer_core::SourceProviderKind, provider: clusterflux_core::SourceProviderKind,
) -> Result<CoordinatorResponse, CoordinatorServiceError> { ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant); let tenant = TenantId::new(tenant);
let project = ProjectId::new(project); let project = ProjectId::new(project);
@ -189,6 +190,7 @@ impl CoordinatorService {
project, project,
environment: None, environment: None,
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: preparation.required_capabilities.clone(), required_capabilities: preparation.required_capabilities.clone(),
dependency_cache: None, dependency_cache: None,
source_snapshot: None, source_snapshot: None,
@ -197,7 +199,7 @@ impl CoordinatorService {
policy_allowed: true, policy_allowed: true,
prefer_node: None, prefer_node: None,
}; };
let nodes = self.node_descriptors.values().cloned().collect::<Vec<_>>(); let nodes = self.live_node_descriptors();
let disposition = match DefaultScheduler.place(&nodes, &request) { let disposition = match DefaultScheduler.place(&nodes, &request) {
Ok(placement) => SourcePreparationDisposition::Assigned { Ok(placement) => SourcePreparationDisposition::Assigned {
node: placement.node, node: placement.node,
@ -223,7 +225,7 @@ impl CoordinatorService {
tenant: String, tenant: String,
project: String, project: String,
node: String, node: String,
provider: disasmer_core::SourceProviderKind, provider: clusterflux_core::SourceProviderKind,
source_snapshot: Digest, source_snapshot: Digest,
) -> Result<CoordinatorResponse, CoordinatorServiceError> { ) -> Result<CoordinatorResponse, CoordinatorServiceError> {
let tenant = TenantId::new(tenant); let tenant = TenantId::new(tenant);
@ -369,6 +371,18 @@ impl CoordinatorService {
self.task_events.retain(|event| { self.task_events.retain(|event| {
event.tenant != tenant || event.project != project || event.process != process event.tenant != tenant || event.project != project || event.process != process
}); });
self.task_attempts
.retain(|(attempt_tenant, attempt_project, attempt_process, _), _| {
attempt_tenant != &tenant
|| attempt_project != &project
|| attempt_process != &process
});
self.restart_launches
.retain(|(attempt_tenant, attempt_project, attempt_process, _)| {
attempt_tenant != &tenant
|| attempt_project != &project
|| attempt_process != &process
});
self.debug_audit_events.retain(|event| { self.debug_audit_events.retain(|event| {
event.tenant != tenant || event.project != project || event.process != process event.tenant != tenant || event.project != project || event.process != process
}); });
@ -480,6 +494,12 @@ impl CoordinatorService {
}); });
} }
} }
let process_key = process_control_key(&tenant, &project, &process);
if cancelled_tasks.is_empty() && !self.main_runtime.controls.contains_key(&process_key) {
self.coordinator
.abort_process(&tenant, &project, &process)?;
self.clear_operator_panel_state(&tenant, &project, &process);
}
Ok(CoordinatorResponse::ProcessCancellationRequested { Ok(CoordinatorResponse::ProcessCancellationRequested {
process, process,
cancelled_tasks, cancelled_tasks,
@ -723,15 +743,16 @@ impl CoordinatorService {
) )
.into()); .into());
} }
let canonical_scope = clusterflux_core::AgentWorkflowRequestScope::new(
tenant.clone(),
project.clone(),
request_kind,
process.clone(),
task.cloned(),
)
.map_err(CoordinatorError::Unauthorized)?;
let record = self.coordinator.authorize_agent_project_run( let record = self.coordinator.authorize_agent_project_run(
disasmer_core::AgentWorkflowScope { canonical_scope.for_agent(&agent),
tenant,
project,
agent: &agent,
request_kind,
process,
task,
},
agent_public_key_fingerprint.as_ref(), agent_public_key_fingerprint.as_ref(),
request_payload_digest, request_payload_digest,
&signature, &signature,

View file

@ -1,6 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use disasmer_core::{ use clusterflux_core::{
AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind, AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind,
DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements, DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements,
LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest, LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest,
@ -22,6 +22,13 @@ pub struct TaskReplacementBundle {
pub wasm_module_base64: String, pub wasm_module_base64: String,
} }
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskFailureResolution {
AcceptFailure,
Cancel,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum CoordinatorRequest { pub enum CoordinatorRequest {
@ -306,6 +313,14 @@ pub enum CoordinatorRequest {
#[serde(default)] #[serde(default)]
replacement_bundle: Option<TaskReplacementBundle>, replacement_bundle: Option<TaskReplacementBundle>,
}, },
ResolveTaskFailure {
tenant: String,
project: String,
actor_user: String,
process: String,
task: String,
resolution: TaskFailureResolution,
},
DebugAttach { DebugAttach {
tenant: String, tenant: String,
project: String, project: String,
@ -444,6 +459,12 @@ pub enum CoordinatorRequest {
#[serde(default)] #[serde(default)]
process: Option<String>, process: Option<String>,
}, },
ListTaskSnapshots {
tenant: String,
project: String,
actor_user: String,
process: String,
},
JoinTask { JoinTask {
tenant: String, tenant: String,
project: String, project: String,
@ -507,7 +528,7 @@ impl CoordinatorRequest {
pub fn operation(&self) -> Result<String, String> { pub fn operation(&self) -> Result<String, String> {
serde_json::to_value(self) serde_json::to_value(self)
.map_err(|err| format!("failed to encode coordinator request operation: {err}")) .map_err(|err| format!("failed to encode coordinator request operation: {err}"))
.map(|value| disasmer_core::coordinator_payload_operation(&value)) .map(|value| clusterflux_core::coordinator_payload_operation(&value))
} }
} }
@ -580,6 +601,11 @@ pub enum AuthenticatedCoordinatorRequest {
#[serde(default)] #[serde(default)]
replacement_bundle: Option<TaskReplacementBundle>, replacement_bundle: Option<TaskReplacementBundle>,
}, },
ResolveTaskFailure {
process: String,
task: String,
resolution: TaskFailureResolution,
},
DebugAttach { DebugAttach {
process: String, process: String,
}, },
@ -607,6 +633,9 @@ pub enum AuthenticatedCoordinatorRequest {
#[serde(default)] #[serde(default)]
process: Option<String>, process: Option<String>,
}, },
ListTaskSnapshots {
process: String,
},
JoinTask { JoinTask {
process: String, process: String,
task: String, task: String,

View file

@ -31,9 +31,11 @@ pub struct TaskCompletionEvent {
pub process: ProcessId, pub process: ProcessId,
pub node: NodeId, pub node: NodeId,
pub executor: TaskExecutor, pub executor: TaskExecutor,
pub task_definition: disasmer_core::TaskDefinitionId, pub task_definition: clusterflux_core::TaskDefinitionId,
pub task: TaskInstanceId, pub task: TaskInstanceId,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub attempt_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placement: Option<Placement>, pub placement: Option<Placement>,
pub terminal_state: TaskTerminalState, pub terminal_state: TaskTerminalState,
pub status_code: Option<i32>, pub status_code: Option<i32>,
@ -49,6 +51,46 @@ pub struct TaskCompletionEvent {
pub result: Option<TaskBoundaryValue>, pub result: Option<TaskBoundaryValue>,
} }
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskAttemptState {
Queued,
Running,
FailedAwaitingAction,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskAttemptSnapshot {
pub process: ProcessId,
pub task: TaskInstanceId,
pub attempt_id: String,
pub attempt_number: u32,
pub task_definition: clusterflux_core::TaskDefinitionId,
pub display_name: String,
pub state: TaskAttemptState,
pub current: bool,
pub node: Option<NodeId>,
pub environment_id: Option<String>,
pub environment_digest: Option<Digest>,
pub argument_summary: Vec<String>,
pub handle_summary: Vec<String>,
pub command_state: Option<String>,
pub vfs_checkpoint: String,
pub probe_symbol: Option<String>,
pub source_path: Option<String>,
pub source_line: Option<u32>,
pub restart_compatible: bool,
pub failure_policy: clusterflux_core::TaskFailurePolicy,
pub artifact_path: Option<VfsPath>,
pub artifact_digest: Option<Digest>,
pub artifact_size_bytes: Option<u64>,
pub status_code: Option<i32>,
pub error: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugAuditEvent { pub struct DebugAuditEvent {
pub tenant: TenantId, pub tenant: TenantId,
@ -115,7 +157,7 @@ pub enum DebugAcknowledgementState {
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct DebugParticipantAcknowledgement { pub struct DebugParticipantAcknowledgement {
pub node: NodeId, pub node: NodeId,
pub task_definition: disasmer_core::TaskDefinitionId, pub task_definition: clusterflux_core::TaskDefinitionId,
pub task: TaskInstanceId, pub task: TaskInstanceId,
pub epoch: u64, pub epoch: u64,
pub state: DebugAcknowledgementState, pub state: DebugAcknowledgementState,
@ -132,7 +174,7 @@ pub struct DebugParticipantAcknowledgement {
pub struct VirtualProcessStatus { pub struct VirtualProcessStatus {
pub process: ProcessId, pub process: ProcessId,
pub state: String, pub state: String,
pub main_task_definition: Option<disasmer_core::TaskDefinitionId>, pub main_task_definition: Option<clusterflux_core::TaskDefinitionId>,
pub main_task_instance: Option<TaskInstanceId>, pub main_task_instance: Option<TaskInstanceId>,
pub main_state: Option<String>, pub main_state: Option<String>,
pub main_wait_state: Option<String>, pub main_wait_state: Option<String>,
@ -226,7 +268,7 @@ pub enum CoordinatorResponse {
node: NodeId, node: NodeId,
tenant: TenantId, tenant: TenantId,
project: ProjectId, project: ProjectId,
credential: disasmer_core::NodeCredential, credential: clusterflux_core::NodeCredential,
}, },
NodeHeartbeat { NodeHeartbeat {
node: NodeId, node: NodeId,
@ -261,7 +303,7 @@ pub enum CoordinatorResponse {
}, },
MainLaunched { MainLaunched {
process: ProcessId, process: ProcessId,
task_definition: disasmer_core::TaskDefinitionId, task_definition: clusterflux_core::TaskDefinitionId,
task_instance: TaskInstanceId, task_instance: TaskInstanceId,
actor: WorkflowActor, actor: WorkflowActor,
state: String, state: String,
@ -349,7 +391,8 @@ pub enum CoordinatorResponse {
TaskRestart { TaskRestart {
process: ProcessId, process: ProcessId,
task: TaskInstanceId, task: TaskInstanceId,
restarted_task_instance: Option<disasmer_core::TaskInstanceId>, restarted_task_instance: Option<clusterflux_core::TaskInstanceId>,
restarted_attempt_id: Option<String>,
actor: UserId, actor: UserId,
accepted: bool, accepted: bool,
clean_boundary_available: bool, clean_boundary_available: bool,
@ -420,6 +463,7 @@ pub enum CoordinatorResponse {
expected_tasks: Vec<TaskCancellationTarget>, expected_tasks: Vec<TaskCancellationTarget>,
acknowledgements: Vec<DebugParticipantAcknowledgement>, acknowledgements: Vec<DebugParticipantAcknowledgement>,
fully_frozen: bool, fully_frozen: bool,
partially_frozen: bool,
fully_resumed: bool, fully_resumed: bool,
failed: bool, failed: bool,
failure_messages: Vec<String>, failure_messages: Vec<String>,
@ -450,6 +494,15 @@ pub enum CoordinatorResponse {
TaskEvents { TaskEvents {
events: Vec<TaskCompletionEvent>, events: Vec<TaskCompletionEvent>,
}, },
TaskSnapshots {
snapshots: Vec<TaskAttemptSnapshot>,
},
TaskFailureResolved {
process: ProcessId,
task: TaskInstanceId,
attempt_id: String,
resolution: TaskFailureResolution,
},
TaskJoined { TaskJoined {
join: TaskJoinResult, join: TaskJoinResult,
}, },

View file

@ -1,6 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use disasmer_core::{LimitError, LimitKind, ProjectId, ResourceLimits, ResourceMeter, TenantId}; use clusterflux_core::{LimitError, LimitKind, ProjectId, ResourceLimits, ResourceMeter, TenantId};
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct CoordinatorQuotaConfiguration { pub struct CoordinatorQuotaConfiguration {

View file

@ -0,0 +1,486 @@
use std::collections::BTreeMap;
use clusterflux_core::{ProjectId, TenantId, UserId};
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactRelayConfiguration {
pub enabled: bool,
pub max_artifact_bytes: u64,
pub max_active_per_project: usize,
pub max_active_per_tenant: usize,
pub max_active_per_account: usize,
pub max_active_global: usize,
pub max_period_bytes_per_project: u64,
pub max_period_bytes_per_tenant: u64,
pub max_period_bytes_per_account: u64,
pub max_period_bytes_global: u64,
pub period_seconds: u64,
pub framing_overhead_bytes: u64,
pub max_tracked_scopes: usize,
}
impl ArtifactRelayConfiguration {
pub fn unlimited() -> Self {
Self {
enabled: true,
max_artifact_bytes: u64::MAX,
max_active_per_project: usize::MAX,
max_active_per_tenant: usize::MAX,
max_active_per_account: usize::MAX,
max_active_global: usize::MAX,
max_period_bytes_per_project: u64::MAX,
max_period_bytes_per_tenant: u64::MAX,
max_period_bytes_per_account: u64::MAX,
max_period_bytes_global: u64::MAX,
period_seconds: u64::MAX,
framing_overhead_bytes: 512,
max_tracked_scopes: usize::MAX,
}
}
pub fn validate(&self) -> Result<(), ArtifactRelayError> {
if self.period_seconds == 0
|| self.max_active_per_project == 0
|| self.max_active_per_tenant == 0
|| self.max_active_per_account == 0
|| self.max_active_global == 0
|| self.max_tracked_scopes == 0
{
return Err(ArtifactRelayError::InvalidConfiguration);
}
Ok(())
}
}
impl Default for ArtifactRelayConfiguration {
fn default() -> Self {
Self::unlimited()
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct ArtifactRelayUsage {
pub active_transfers: usize,
pub ingress_bytes: u64,
pub egress_bytes: u64,
pub abandoned_or_failed_bytes: u64,
pub reserved_bytes: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum RelayFinishReason {
Completed,
Failed,
Cancelled,
Expired,
}
#[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum ArtifactRelayError {
#[error("artifact relay is disabled by hosted policy")]
Disabled,
#[error("artifact exceeds the configured relay size limit")]
ArtifactTooLarge,
#[error("artifact relay concurrency limit reached for {0}")]
Concurrency(&'static str),
#[error("artifact relay period byte budget exhausted for {0}")]
PeriodBudget(&'static str),
#[error("artifact relay retained scope state limit reached")]
StateLimit,
#[error("artifact relay reservation is missing or expired")]
MissingReservation,
#[error("artifact relay configuration contains a zero bound")]
InvalidConfiguration,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct RelayScope {
tenant: TenantId,
project: ProjectId,
account: UserId,
}
#[derive(Clone, Debug)]
struct RelayReservation {
scope: RelayScope,
remaining_reserved_bytes: u64,
ingress_bytes: u64,
egress_bytes: u64,
expires_at_epoch_seconds: u64,
}
#[derive(Clone, Debug)]
pub(super) struct ArtifactRelayLedger {
configuration: ArtifactRelayConfiguration,
reservations: BTreeMap<String, RelayReservation>,
period: u64,
project_used: BTreeMap<(TenantId, ProjectId), u64>,
tenant_used: BTreeMap<TenantId, u64>,
account_used: BTreeMap<(TenantId, UserId), u64>,
global_used: u64,
ingress_used: u64,
egress_used: u64,
abandoned_or_failed_used: u64,
}
impl Default for ArtifactRelayLedger {
fn default() -> Self {
Self::new(ArtifactRelayConfiguration::default())
}
}
impl ArtifactRelayLedger {
pub(super) fn new(configuration: ArtifactRelayConfiguration) -> Self {
Self {
configuration,
reservations: BTreeMap::new(),
period: 0,
project_used: BTreeMap::new(),
tenant_used: BTreeMap::new(),
account_used: BTreeMap::new(),
global_used: 0,
ingress_used: 0,
egress_used: 0,
abandoned_or_failed_used: 0,
}
}
pub(super) fn configure(
&mut self,
configuration: ArtifactRelayConfiguration,
) -> Result<(), ArtifactRelayError> {
configuration.validate()?;
if !self.reservations.is_empty() {
return Err(ArtifactRelayError::Concurrency("configuration change"));
}
*self = Self::new(configuration);
Ok(())
}
fn prepare_period(&mut self, now_epoch_seconds: u64) {
let period = now_epoch_seconds / self.configuration.period_seconds.max(1);
if self.period != period {
self.period = period;
self.project_used.clear();
self.tenant_used.clear();
self.account_used.clear();
self.global_used = 0;
self.ingress_used = 0;
self.egress_used = 0;
self.abandoned_or_failed_used = 0;
}
}
pub(super) fn estimated_wire_bytes(&self, artifact_bytes: u64, chunk_bytes: u64) -> u64 {
let encoded = artifact_bytes
.saturating_add(2)
.saturating_div(3)
.saturating_mul(4);
let chunks = artifact_bytes
.saturating_add(chunk_bytes.saturating_sub(1))
.saturating_div(chunk_bytes.max(1))
.max(1);
encoded.saturating_mul(2).saturating_add(
chunks
.saturating_mul(2)
.saturating_mul(self.configuration.framing_overhead_bytes),
)
}
pub(super) fn framing_overhead_bytes(&self) -> u64 {
self.configuration.framing_overhead_bytes
}
pub(super) fn reserve(
&mut self,
key: String,
tenant: TenantId,
project: ProjectId,
account: UserId,
artifact_bytes: u64,
chunk_bytes: u64,
expires_at_epoch_seconds: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.expire(now_epoch_seconds);
self.prepare_period(now_epoch_seconds);
if !self.configuration.enabled {
return Err(ArtifactRelayError::Disabled);
}
if artifact_bytes > self.configuration.max_artifact_bytes {
return Err(ArtifactRelayError::ArtifactTooLarge);
}
let scope = RelayScope {
tenant,
project,
account,
};
self.check_concurrency(&scope)?;
self.check_tracked_scope_capacity(&scope)?;
let reserved = self.estimated_wire_bytes(artifact_bytes, chunk_bytes);
self.check_budget(&scope, reserved)?;
self.reservations.insert(
key,
RelayReservation {
scope,
remaining_reserved_bytes: reserved,
ingress_bytes: 0,
egress_bytes: 0,
expires_at_epoch_seconds,
},
);
Ok(())
}
pub(super) fn rekey(&mut self, from: &str, to: String) -> Result<(), ArtifactRelayError> {
let reservation = self
.reservations
.remove(from)
.ok_or(ArtifactRelayError::MissingReservation)?;
self.reservations.insert(to, reservation);
Ok(())
}
fn check_concurrency(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
if self.reservations.len() >= self.configuration.max_active_global {
return Err(ArtifactRelayError::Concurrency("global"));
}
let project = self
.reservations
.values()
.filter(|reservation| {
reservation.scope.tenant == scope.tenant
&& reservation.scope.project == scope.project
})
.count();
if project >= self.configuration.max_active_per_project {
return Err(ArtifactRelayError::Concurrency("project"));
}
let tenant = self
.reservations
.values()
.filter(|reservation| reservation.scope.tenant == scope.tenant)
.count();
if tenant >= self.configuration.max_active_per_tenant {
return Err(ArtifactRelayError::Concurrency("tenant"));
}
let account = self
.reservations
.values()
.filter(|reservation| {
reservation.scope.tenant == scope.tenant
&& reservation.scope.account == scope.account
})
.count();
if account >= self.configuration.max_active_per_account {
return Err(ArtifactRelayError::Concurrency("account"));
}
Ok(())
}
fn check_tracked_scope_capacity(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
let project_key = (scope.tenant.clone(), scope.project.clone());
let account_key = (scope.tenant.clone(), scope.account.clone());
let new_scopes = usize::from(!self.project_used.contains_key(&project_key))
+ usize::from(!self.tenant_used.contains_key(&scope.tenant))
+ usize::from(!self.account_used.contains_key(&account_key));
let tracked = self.project_used.len() + self.tenant_used.len() + self.account_used.len();
if tracked.saturating_add(new_scopes) > self.configuration.max_tracked_scopes {
return Err(ArtifactRelayError::StateLimit);
}
Ok(())
}
fn reserved_for(&self, scope: &RelayScope) -> (u64, u64, u64, u64) {
let mut project = 0_u64;
let mut tenant = 0_u64;
let mut account = 0_u64;
let mut global = 0_u64;
for reservation in self.reservations.values() {
let bytes = reservation.remaining_reserved_bytes;
global = global.saturating_add(bytes);
if reservation.scope.tenant == scope.tenant {
tenant = tenant.saturating_add(bytes);
if reservation.scope.project == scope.project {
project = project.saturating_add(bytes);
}
if reservation.scope.account == scope.account {
account = account.saturating_add(bytes);
}
}
}
(project, tenant, account, global)
}
fn check_budget(&self, scope: &RelayScope, additional: u64) -> Result<(), ArtifactRelayError> {
let (project_reserved, tenant_reserved, account_reserved, global_reserved) =
self.reserved_for(scope);
let project_used = self
.project_used
.get(&(scope.tenant.clone(), scope.project.clone()))
.copied()
.unwrap_or(0);
let tenant_used = self.tenant_used.get(&scope.tenant).copied().unwrap_or(0);
let account_used = self
.account_used
.get(&(scope.tenant.clone(), scope.account.clone()))
.copied()
.unwrap_or(0);
for (used, reserved, limit, label) in [
(
project_used,
project_reserved,
self.configuration.max_period_bytes_per_project,
"project",
),
(
tenant_used,
tenant_reserved,
self.configuration.max_period_bytes_per_tenant,
"tenant",
),
(
account_used,
account_reserved,
self.configuration.max_period_bytes_per_account,
"account",
),
(
self.global_used,
global_reserved,
self.configuration.max_period_bytes_global,
"global",
),
] {
if used.saturating_add(reserved).saturating_add(additional) > limit {
return Err(ArtifactRelayError::PeriodBudget(label));
}
}
Ok(())
}
fn charge(
&mut self,
key: &str,
bytes: u64,
ingress: bool,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.prepare_period(now_epoch_seconds);
let reservation = self
.reservations
.get(key)
.cloned()
.ok_or(ArtifactRelayError::MissingReservation)?;
let additional = bytes.saturating_sub(reservation.remaining_reserved_bytes);
if additional > 0 {
self.check_budget(&reservation.scope, additional)?;
}
let reservation = self
.reservations
.get_mut(key)
.ok_or(ArtifactRelayError::MissingReservation)?;
reservation.remaining_reserved_bytes =
reservation.remaining_reserved_bytes.saturating_sub(bytes);
if ingress {
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
self.ingress_used = self.ingress_used.saturating_add(bytes);
} else {
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
self.egress_used = self.egress_used.saturating_add(bytes);
}
let project_key = (
reservation.scope.tenant.clone(),
reservation.scope.project.clone(),
);
let account_key = (
reservation.scope.tenant.clone(),
reservation.scope.account.clone(),
);
*self.project_used.entry(project_key).or_default() = self
.project_used
.get(&(
reservation.scope.tenant.clone(),
reservation.scope.project.clone(),
))
.copied()
.unwrap_or(0)
.saturating_add(bytes);
*self
.tenant_used
.entry(reservation.scope.tenant.clone())
.or_default() = self
.tenant_used
.get(&reservation.scope.tenant)
.copied()
.unwrap_or(0)
.saturating_add(bytes);
*self.account_used.entry(account_key).or_default() = self
.account_used
.get(&(
reservation.scope.tenant.clone(),
reservation.scope.account.clone(),
))
.copied()
.unwrap_or(0)
.saturating_add(bytes);
self.global_used = self.global_used.saturating_add(bytes);
Ok(())
}
pub(super) fn charge_ingress(
&mut self,
key: &str,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.charge(key, bytes, true, now_epoch_seconds)
}
pub(super) fn charge_egress(
&mut self,
key: &str,
bytes: u64,
now_epoch_seconds: u64,
) -> Result<(), ArtifactRelayError> {
self.charge(key, bytes, false, now_epoch_seconds)
}
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
if let Some(reservation) = self.reservations.remove(key) {
if reason != RelayFinishReason::Completed {
self.abandoned_or_failed_used = self
.abandoned_or_failed_used
.saturating_add(reservation.ingress_bytes)
.saturating_add(reservation.egress_bytes);
}
}
}
pub(super) fn expire(&mut self, now_epoch_seconds: u64) {
let expired = self
.reservations
.iter()
.filter(|(_, reservation)| reservation.expires_at_epoch_seconds < now_epoch_seconds)
.map(|(key, _)| key.clone())
.collect::<Vec<_>>();
for key in expired {
self.finish(&key, RelayFinishReason::Expired);
}
}
pub(super) fn usage(&self) -> ArtifactRelayUsage {
ArtifactRelayUsage {
active_transfers: self.reservations.len(),
ingress_bytes: self.ingress_used,
egress_bytes: self.egress_used,
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
reserved_bytes: self
.reservations
.values()
.map(|reservation| reservation.remaining_reserved_bytes)
.fold(0_u64, u64::saturating_add),
}
}
}

View file

@ -11,7 +11,8 @@ impl CoordinatorService {
"failed to canonicalize coordinator request for authentication: {error}" "failed to canonicalize coordinator request for authentication: {error}"
)) ))
})?; })?;
let request_payload_digest = disasmer_core::signed_request_payload_digest(&request_payload); let request_payload_digest =
clusterflux_core::signed_request_payload_digest(&request_payload);
match request { match request {
CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong { CoordinatorRequest::Ping => Ok(CoordinatorResponse::Pong {
epoch: self.coordinator.coordinator_epoch(), epoch: self.coordinator.coordinator_epoch(),
@ -143,7 +144,7 @@ impl CoordinatorService {
CoordinatorRequest::ListProjects { tenant, actor_user } => { CoordinatorRequest::ListProjects { tenant, actor_user } => {
let tenant = TenantId::new(tenant); let tenant = TenantId::new(tenant);
let actor = UserId::new(actor_user); let actor = UserId::new(actor_user);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant: tenant.clone(), tenant: tenant.clone(),
project: ProjectId::from("__project_listing__"), project: ProjectId::from("__project_listing__"),
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -209,7 +210,7 @@ impl CoordinatorService {
let tenant = TenantId::new(tenant); let tenant = TenantId::new(tenant);
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(user); let actor = UserId::new(user);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant, tenant,
project, project,
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -229,7 +230,7 @@ impl CoordinatorService {
let project = ProjectId::new(project); let project = ProjectId::new(project);
let actor = UserId::new(user); let actor = UserId::new(user);
let agent = AgentId::new(agent); let agent = AgentId::new(agent);
let context = disasmer_core::AuthContext { let context = clusterflux_core::AuthContext {
tenant, tenant,
project, project,
actor: Actor::User(actor.clone()), actor: Actor::User(actor.clone()),
@ -426,6 +427,16 @@ impl CoordinatorService {
task, task,
replacement_bundle, replacement_bundle,
), ),
CoordinatorRequest::ResolveTaskFailure {
tenant,
project,
actor_user,
process,
task,
resolution,
} => self.handle_resolve_task_failure(
tenant, project, actor_user, process, task, resolution,
),
request @ (CoordinatorRequest::DebugAttach { .. } request @ (CoordinatorRequest::DebugAttach { .. }
| CoordinatorRequest::SetDebugBreakpoints { .. } | CoordinatorRequest::SetDebugBreakpoints { .. }
| CoordinatorRequest::InspectDebugBreakpoints { .. } | CoordinatorRequest::InspectDebugBreakpoints { .. }
@ -444,6 +455,12 @@ impl CoordinatorService {
actor_user, actor_user,
process, process,
} => self.handle_list_task_events(tenant, project, actor_user, process), } => self.handle_list_task_events(tenant, project, actor_user, process),
CoordinatorRequest::ListTaskSnapshots {
tenant,
project,
actor_user,
process,
} => self.handle_list_task_snapshots(tenant, project, actor_user, process),
CoordinatorRequest::JoinTask { CoordinatorRequest::JoinTask {
tenant, tenant,
project, project,
@ -708,6 +725,18 @@ impl CoordinatorService {
task, task,
replacement_bundle, replacement_bundle,
), ),
AuthenticatedCoordinatorRequest::ResolveTaskFailure {
process,
task,
resolution,
} => self.handle_resolve_task_failure(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
task,
resolution,
),
request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. } request @ (AuthenticatedCoordinatorRequest::DebugAttach { .. }
| AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. } | AuthenticatedCoordinatorRequest::SetDebugBreakpoints { .. }
| AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. } | AuthenticatedCoordinatorRequest::InspectDebugBreakpoints { .. }
@ -727,6 +756,13 @@ impl CoordinatorService {
actor.as_str().to_owned(), actor.as_str().to_owned(),
process, process,
), ),
AuthenticatedCoordinatorRequest::ListTaskSnapshots { process } => self
.handle_list_task_snapshots(
context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(),
actor.as_str().to_owned(),
process,
),
AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task( AuthenticatedCoordinatorRequest::JoinTask { process, task } => self.handle_join_task(
context.tenant.as_str().to_owned(), context.tenant.as_str().to_owned(),
context.project.as_str().to_owned(), context.project.as_str().to_owned(),

View file

@ -1,4 +1,4 @@
use disasmer_core::{NodeId, NodeSignedRequest}; use clusterflux_core::{NodeId, NodeSignedRequest};
use crate::CoordinatorError; use crate::CoordinatorError;
@ -18,7 +18,7 @@ impl CoordinatorService {
"failed to canonicalize signed node request: {error}" "failed to canonicalize signed node request: {error}"
)) ))
})?; })?;
let payload_digest = disasmer_core::signed_request_payload_digest(&request_payload); let payload_digest = clusterflux_core::signed_request_payload_digest(&request_payload);
let signed_node = NodeId::new(signed_node); let signed_node = NodeId::new(signed_node);
if request_node != signed_node { if request_node != signed_node {
return Err(CoordinatorError::Unauthorized( return Err(CoordinatorError::Unauthorized(

View file

@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::{ use clusterflux_core::{
admin_request_proof, agent_ed25519_public_key_from_private_key, coordinator_wire_request, admin_request_proof, agent_ed25519_public_key_from_private_key, coordinator_wire_request,
derive_ed25519_private_key_from_seed, node_ed25519_public_key_from_private_key, derive_ed25519_private_key_from_seed, node_ed25519_public_key_from_private_key,
sign_agent_workflow_request, sign_node_request, signed_request_payload_digest, sign_agent_workflow_request, sign_node_request, signed_request_payload_digest,
@ -64,7 +64,7 @@ fn runtime_service_uses_memory_only_when_database_url_is_absent() {
#[test] #[test]
fn postgres_backed_runtime_service_survives_restart_without_live_process_state() { fn postgres_backed_runtime_service_survives_restart_without_live_process_state() {
let Ok(database_url) = std::env::var("DISASMER_TEST_POSTGRES_SERVICE") else { let Ok(database_url) = std::env::var("CLUSTERFLUX_TEST_POSTGRES_SERVICE") else {
return; return;
}; };
let mut clean_store = crate::PostgresDurableStore::connect(&database_url).unwrap(); let mut clean_store = crate::PostgresDurableStore::connect(&database_url).unwrap();
@ -202,7 +202,7 @@ fn assert_agent_workflow_actor(actor: &WorkflowActor, fingerprint: &Digest) {
const TEST_AGENT_PRIVATE_KEY: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc="; const TEST_AGENT_PRIVATE_KEY: &str = "ed25519:BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc=";
static TEST_NODE_REQUEST_NONCE: AtomicU64 = AtomicU64::new(1); static TEST_NODE_REQUEST_NONCE: AtomicU64 = AtomicU64::new(1);
const TEST_WASM_MODULE: &[u8] = b"disasmer-test-wasm-module"; const TEST_WASM_MODULE: &[u8] = b"clusterflux-test-wasm-module";
fn test_wasm_module_base64() -> String { fn test_wasm_module_base64() -> String {
BASE64_STANDARD.encode(TEST_WASM_MODULE) BASE64_STANDARD.encode(TEST_WASM_MODULE)
@ -243,13 +243,13 @@ fn test_edited_task_bundle(compatibility: &Digest, edit_marker: &str) -> (String
"result_schema": "u32", "result_schema": "u32",
"required_capabilities": [], "required_capabilities": [],
"restart_compatibility_hash": compatibility, "restart_compatibility_hash": compatibility,
"abi_version": disasmer_core::WASM_TASK_ABI_VERSION, "abi_version": clusterflux_core::WASM_TASK_ABI_VERSION,
"probe_symbol": "disasmer.probe.compile", "probe_symbol": "clusterflux.probe.compile",
})) }))
.unwrap(); .unwrap();
let mut module = b"\0asm\x01\0\0\0".to_vec(); let mut module = b"\0asm\x01\0\0\0".to_vec();
append_test_custom_section(&mut module, "disasmer.tasks", &descriptor); append_test_custom_section(&mut module, "clusterflux.tasks", &descriptor);
append_test_custom_section(&mut module, "disasmer.edit", edit_marker.as_bytes()); append_test_custom_section(&mut module, "clusterflux.edit", edit_marker.as_bytes());
let digest = Digest::sha256(&module); let digest = Digest::sha256(&module);
(BASE64_STANDARD.encode(module), digest) (BASE64_STANDARD.encode(module), digest)
} }
@ -286,8 +286,8 @@ fn test_task_spec_instance(
tenant: TenantId::from(tenant), tenant: TenantId::from(tenant),
project: ProjectId::from(project), project: ProjectId::from(project),
process: ProcessId::from(process), process: ProcessId::from(process),
task_definition: disasmer_core::TaskDefinitionId::from(task_definition), task_definition: clusterflux_core::TaskDefinitionId::from(task_definition),
task_instance: disasmer_core::TaskInstanceId::from(task_instance), task_instance: clusterflux_core::TaskInstanceId::from(task_instance),
dispatch: TaskDispatch::CoordinatorNodeWasm { dispatch: TaskDispatch::CoordinatorNodeWasm {
export: Some(task_definition.to_owned()), export: Some(task_definition.to_owned()),
abi: WasmExportAbi::TaskV1, abi: WasmExportAbi::TaskV1,
@ -301,6 +301,7 @@ fn test_task_spec_instance(
required_artifacts: Vec::new(), required_artifacts: Vec::new(),
args: Vec::new(), args: Vec::new(),
vfs_epoch: epoch, vfs_epoch: epoch,
failure_policy: Default::default(),
bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)), bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
} }
} }
@ -2640,10 +2641,10 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
&ProcessId::from("process"), &ProcessId::from("process"),
), ),
super::main_runtime::CoordinatorMainControl { super::main_runtime::CoordinatorMainControl {
task_definition: disasmer_core::TaskDefinitionId::from("completed-main"), task_definition: clusterflux_core::TaskDefinitionId::from("completed-main"),
task_instance: TaskInstanceId::from("ti:process:main"), task_instance: TaskInstanceId::from("ti:process:main"),
abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), abort: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)),
debug: std::sync::Arc::new(disasmer_wasm_runtime::WasmDebugControl::default()), debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
state: "completed".to_owned(), state: "completed".to_owned(),
stopped_probe_symbol: None, stopped_probe_symbol: None,
handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
@ -2660,13 +2661,13 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
project: "project".to_owned(), project: "project".to_owned(),
actor_user: "user".to_owned(), actor_user: "user".to_owned(),
process: "process".to_owned(), process: "process".to_owned(),
probe_symbols: vec!["disasmer.probe.compile_linux".to_owned()], probe_symbols: vec!["clusterflux.probe.compile_linux".to_owned()],
}) })
.unwrap() .unwrap()
else { else {
panic!("expected debug breakpoints response"); panic!("expected debug breakpoints response");
}; };
assert_eq!(probe_symbols, ["disasmer.probe.compile_linux"]); assert_eq!(probe_symbols, ["clusterflux.probe.compile_linux"]);
assert_eq!(hit_epoch, None); assert_eq!(hit_epoch, None);
let CoordinatorResponse::DebugProbeHit { let CoordinatorResponse::DebugProbeHit {
@ -2681,7 +2682,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
process: "process".to_owned(), process: "process".to_owned(),
node: "worker".to_owned(), node: "worker".to_owned(),
task: "compile-linux".to_owned(), task: "compile-linux".to_owned(),
probe_symbol: "disasmer.probe.compile_linux".to_owned(), probe_symbol: "clusterflux.probe.compile_linux".to_owned(),
}) })
.unwrap() .unwrap()
else { else {
@ -2689,7 +2690,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
}; };
assert!(breakpoint_matched); assert!(breakpoint_matched);
assert_eq!(debug_epoch, Some(1)); assert_eq!(debug_epoch, Some(1));
assert_eq!(probe_symbol, "disasmer.probe.compile_linux"); assert_eq!(probe_symbol, "clusterflux.probe.compile_linux");
let CoordinatorResponse::DebugBreakpoints { let CoordinatorResponse::DebugBreakpoints {
hit_epoch, hit_epoch,
@ -2711,7 +2712,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
assert_eq!(hit_task, Some(TaskInstanceId::from("compile-linux"))); assert_eq!(hit_task, Some(TaskInstanceId::from("compile-linux")));
assert_eq!( assert_eq!(
hit_probe_symbol.as_deref(), hit_probe_symbol.as_deref(),
Some("disasmer.probe.compile_linux") Some("clusterflux.probe.compile_linux")
); );
let CoordinatorResponse::DebugCommand { let CoordinatorResponse::DebugCommand {
@ -2780,7 +2781,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
.unwrap_err(); .unwrap_err();
assert!(early_resume assert!(early_resume
.to_string() .to_string()
.contains("have not acknowledged frozen state")); .contains("no settled frozen participant set"));
service service
.handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState { .handle_signed_node_request_auto(CoordinatorRequest::ReportDebugState {
@ -3111,6 +3112,7 @@ fn service_reports_task_restart_boundary_through_public_api() {
completed_event_observed, completed_event_observed,
requires_whole_process_restart, requires_whole_process_restart,
restarted_task_instance, restarted_task_instance,
restarted_attempt_id,
message, message,
audit_event, audit_event,
used_debug_read_bytes, used_debug_read_bytes,
@ -3133,7 +3135,9 @@ fn service_reports_task_restart_boundary_through_public_api() {
assert!(!active_task); assert!(!active_task);
assert!(completed_event_observed); assert!(completed_event_observed);
assert!(!requires_whole_process_restart); assert!(!requires_whole_process_restart);
let restarted_task_instance = restarted_task_instance.expect("restart must return new id"); let restarted_task_instance = restarted_task_instance.expect("restart returns logical id");
let restarted_attempt_id = restarted_attempt_id.expect("restart returns a new attempt id");
assert!(restarted_attempt_id.starts_with("ta_"));
assert!(message.contains("from clean VFS entry boundary epoch 7")); assert!(message.contains("from clean VFS entry boundary epoch 7"));
let CoordinatorResponse::TaskAssignment { let CoordinatorResponse::TaskAssignment {
assignment: Some(restarted_assignment), assignment: Some(restarted_assignment),
@ -3148,7 +3152,7 @@ fn service_reports_task_restart_boundary_through_public_api() {
panic!("expected restarted task assignment"); panic!("expected restarted task assignment");
}; };
assert_eq!(restarted_assignment.task, restarted_task_instance); assert_eq!(restarted_assignment.task, restarted_task_instance);
assert_ne!(restarted_assignment.task, TaskInstanceId::from("task")); assert_eq!(restarted_assignment.task, TaskInstanceId::from("task"));
let mut expected_task_spec = initial_assignment.task_spec.clone(); let mut expected_task_spec = initial_assignment.task_spec.clone();
expected_task_spec.task_instance = restarted_task_instance; expected_task_spec.task_instance = restarted_task_instance;
assert_eq!(restarted_assignment.task_spec, expected_task_spec); assert_eq!(restarted_assignment.task_spec, expected_task_spec);
@ -3404,10 +3408,10 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
service.main_runtime.controls.insert( service.main_runtime.controls.insert(
process_key.clone(), process_key.clone(),
super::main_runtime::CoordinatorMainControl { super::main_runtime::CoordinatorMainControl {
task_definition: disasmer_core::TaskDefinitionId::from("build"), task_definition: clusterflux_core::TaskDefinitionId::from("build"),
task_instance: TaskInstanceId::from("ti:process-a:main"), task_instance: TaskInstanceId::from("ti:process-a:main"),
abort: std::sync::Arc::clone(&retired_main_abort), abort: std::sync::Arc::clone(&retired_main_abort),
debug: std::sync::Arc::new(disasmer_wasm_runtime::WasmDebugControl::default()), debug: std::sync::Arc::new(clusterflux_wasm_runtime::WasmDebugControl::default()),
state: "running".to_owned(), state: "running".to_owned(),
stopped_probe_symbol: None, stopped_probe_symbol: None,
handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), handles: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())),
@ -3448,7 +3452,7 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
} }
#[test] #[test]
fn cooperative_cancel_keeps_process_identity_until_explicit_abort_releases_slot() { fn quiescent_cooperative_cancel_releases_slot_immediately() {
let mut service = CoordinatorService::new(17); let mut service = CoordinatorService::new(17);
service service
.handle_request(CoordinatorRequest::StartProcess { .handle_request(CoordinatorRequest::StartProcess {
@ -3468,8 +3472,9 @@ fn cooperative_cancel_keeps_process_identity_until_explicit_abort_releases_slot(
process: ProcessId::from("process-a"), process: ProcessId::from("process-a"),
node: NodeId::from("node"), node: NodeId::from("node"),
executor: TaskExecutor::Node, executor: TaskExecutor::Node,
task_definition: disasmer_core::TaskDefinitionId::from("old-task"), task_definition: clusterflux_core::TaskDefinitionId::from("old-task"),
task: TaskInstanceId::from("ti:process-a:old"), task: TaskInstanceId::from("ti:process-a:old"),
attempt_id: None,
placement: None, placement: None,
terminal_state: TaskTerminalState::Completed, terminal_state: TaskTerminalState::Completed,
status_code: Some(0), status_code: Some(0),
@ -3506,49 +3511,6 @@ fn cooperative_cancel_keeps_process_identity_until_explicit_abort_releases_slot(
}) })
.unwrap(); .unwrap();
let CoordinatorResponse::ProcessStatuses { processes, .. } = service
.handle_request(CoordinatorRequest::ListProcesses {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
})
.unwrap()
else {
panic!("expected live process statuses");
};
assert_eq!(processes.len(), 1);
assert_eq!(processes[0].process, ProcessId::from("process-a"));
assert_eq!(processes[0].state, "cancelling");
let blocked = service
.handle_request(CoordinatorRequest::StartProcess {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: None,
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
process: "process-b".to_owned(),
restart: false,
})
.unwrap_err();
assert!(blocked
.to_string()
.contains("already has active virtual process process-a"));
let CoordinatorResponse::ProcessAborted { process, .. } = service
.handle_request(CoordinatorRequest::AbortProcess {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
process: "process-a".to_owned(),
})
.unwrap()
else {
panic!("expected forced process abort");
};
assert_eq!(process, ProcessId::from("process-a"));
let CoordinatorResponse::ProcessStatuses { processes, .. } = service let CoordinatorResponse::ProcessStatuses { processes, .. } = service
.handle_request(CoordinatorRequest::ListProcesses { .handle_request(CoordinatorRequest::ListProcesses {
tenant: "tenant".to_owned(), tenant: "tenant".to_owned(),
@ -3674,7 +3636,7 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() {
); );
assert_eq!( assert_eq!(
service.task_events[0].task_definition, service.task_events[0].task_definition,
disasmer_core::TaskDefinitionId::from("compile") clusterflux_core::TaskDefinitionId::from("compile")
); );
assert!(service.task_restart_checkpoints.is_empty()); assert!(service.task_restart_checkpoints.is_empty());
} }
@ -3987,20 +3949,6 @@ fn service_download_links_are_scoped_and_streaming_is_metered() {
.unwrap_err(); .unwrap_err();
assert!(cross_actor_open.to_string().contains("token is invalid")); assert!(cross_actor_open.to_string().contains("token is invalid"));
service.set_server_time(71);
let expired = service
.handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
artifact: "app.txt".to_owned(),
max_bytes: 64,
token_digest: link.scoped_token_digest.clone(),
chunk_bytes: 1,
})
.unwrap_err();
assert!(expired.to_string().contains("expired"));
service.set_server_time(11); service.set_server_time(11);
upload_pending_artifact_transfer(&mut service, "node", artifact_bytes); upload_pending_artifact_transfer(&mut service, "node", artifact_bytes);
let CoordinatorResponse::ArtifactDownloadStream { let CoordinatorResponse::ArtifactDownloadStream {
@ -4032,6 +3980,36 @@ fn service_download_links_are_scoped_and_streaming_is_metered() {
); );
assert_eq!(content_source, "retaining_node_reverse_stream"); assert_eq!(content_source, "retaining_node_reverse_stream");
let CoordinatorResponse::ArtifactDownloadLink {
link: expiring_link,
} = service
.handle_request(CoordinatorRequest::CreateArtifactDownloadLink {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
artifact: "app.txt".to_owned(),
max_bytes: 64,
ttl_seconds: 1,
})
.unwrap()
else {
panic!("expected expiring download link");
};
service.set_server_time(13);
let expired = service
.handle_request(CoordinatorRequest::OpenArtifactDownloadStream {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: "user".to_owned(),
artifact: "app.txt".to_owned(),
max_bytes: 64,
token_digest: expiring_link.scoped_token_digest,
chunk_bytes: 1,
})
.unwrap_err();
assert!(expired.to_string().contains("expired"));
service.set_server_time(11);
let cross_actor_revoke = service let cross_actor_revoke = service
.handle_request(CoordinatorRequest::RevokeArtifactDownloadLink { .handle_request(CoordinatorRequest::RevokeArtifactDownloadLink {
tenant: "tenant".to_owned(), tenant: "tenant".to_owned(),
@ -4732,8 +4710,8 @@ fn coordinator_side_task_launch_queues_worker_assignment() {
tenant: TenantId::from("tenant"), tenant: TenantId::from("tenant"),
project: ProjectId::from("project"), project: ProjectId::from("project"),
process: ProcessId::from("vp-control"), process: ProcessId::from("vp-control"),
task_definition: disasmer_core::TaskDefinitionId::from("compile-linux"), task_definition: clusterflux_core::TaskDefinitionId::from("compile-linux"),
task_instance: disasmer_core::TaskInstanceId::from("compile-linux-1"), task_instance: clusterflux_core::TaskInstanceId::from("compile-linux-1"),
dispatch: TaskDispatch::CoordinatorNodeWasm { dispatch: TaskDispatch::CoordinatorNodeWasm {
export: Some("compile-linux".to_owned()), export: Some("compile-linux".to_owned()),
abi: WasmExportAbi::TaskV1, abi: WasmExportAbi::TaskV1,
@ -4747,6 +4725,7 @@ fn coordinator_side_task_launch_queues_worker_assignment() {
required_artifacts: vec![ArtifactId::from("bootstrap-artifact")], required_artifacts: vec![ArtifactId::from("bootstrap-artifact")],
args: vec![TaskBoundaryValue::SmallJson(serde_json::json!("test"))], args: vec![TaskBoundaryValue::SmallJson(serde_json::json!("test"))],
vfs_epoch: epoch, vfs_epoch: epoch,
failure_policy: Default::default(),
bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)), bundle_digest: Some(Digest::sha256(TEST_WASM_MODULE)),
}; };
@ -4843,7 +4822,6 @@ fn coordinator_side_task_launch_queues_worker_assignment() {
}; };
assert_eq!(join.state, TaskJoinState::Pending); assert_eq!(join.state, TaskJoinState::Pending);
assert!(!join.remote_completion_observed); assert!(!join.remote_completion_observed);
assert!(join.message.contains("waiting for signed node"));
let CoordinatorResponse::TaskAssignment { assignment } = service let CoordinatorResponse::TaskAssignment { assignment } = service
.handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment { .handle_signed_node_request_auto(CoordinatorRequest::PollTaskAssignment {
@ -5067,11 +5045,11 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
assert_eq!(task, TaskInstanceId::from(instance)); assert_eq!(task, TaskInstanceId::from(instance));
assert_eq!( assert_eq!(
assignment.task_spec.task_definition, assignment.task_spec.task_definition,
disasmer_core::TaskDefinitionId::from("compile") clusterflux_core::TaskDefinitionId::from("compile")
); );
assert_eq!( assert_eq!(
assignment.task_spec.task_instance, assignment.task_spec.task_instance,
disasmer_core::TaskInstanceId::from(instance) clusterflux_core::TaskInstanceId::from(instance)
); );
} }
for instance in ["compile-1", "compile-2"] { for instance in ["compile-1", "compile-2"] {
@ -5159,7 +5137,7 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
}; };
assert_eq!( assert_eq!(
join.task_instance, join.task_instance,
disasmer_core::TaskInstanceId::from(instance) clusterflux_core::TaskInstanceId::from(instance)
); );
assert_eq!( assert_eq!(
join.result, join.result,
@ -5197,6 +5175,7 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
let CoordinatorResponse::TaskRestart { let CoordinatorResponse::TaskRestart {
accepted, accepted,
restarted_task_instance, restarted_task_instance,
restarted_attempt_id,
.. ..
} = service } = service
.handle_request(CoordinatorRequest::RestartTask { .handle_request(CoordinatorRequest::RestartTask {
@ -5215,8 +5194,10 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
panic!("expected task restart response"); panic!("expected task restart response");
}; };
assert!(accepted); assert!(accepted);
let restarted = restarted_task_instance.expect("restart creates a new instance"); let restarted = restarted_task_instance.expect("restart returns the logical instance");
assert_ne!(restarted, TaskInstanceId::from("compile-1")); let restarted_attempt_id = restarted_attempt_id.expect("restart creates a new attempt");
assert!(restarted_attempt_id.starts_with("ta_"));
assert_eq!(restarted, TaskInstanceId::from("compile-1"));
assert_ne!(restarted, TaskInstanceId::from("compile-2")); assert_ne!(restarted, TaskInstanceId::from("compile-2"));
let CoordinatorResponse::TaskAssignment { let CoordinatorResponse::TaskAssignment {
assignment: Some(restarted_assignment), assignment: Some(restarted_assignment),
@ -5233,7 +5214,7 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
assert_eq!(restarted_assignment.task, restarted); assert_eq!(restarted_assignment.task, restarted);
assert_eq!( assert_eq!(
restarted_assignment.task_spec.task_definition, restarted_assignment.task_spec.task_definition,
disasmer_core::TaskDefinitionId::from("compile") clusterflux_core::TaskDefinitionId::from("compile")
); );
assert_eq!( assert_eq!(
restarted_assignment.task_spec.args, restarted_assignment.task_spec.args,
@ -5432,8 +5413,9 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() {
process: ProcessId::from("vp-other"), process: ProcessId::from("vp-other"),
node: NodeId::from("worker-other"), node: NodeId::from("worker-other"),
executor: TaskExecutor::Node, executor: TaskExecutor::Node,
task_definition: disasmer_core::TaskDefinitionId::from("other"), task_definition: clusterflux_core::TaskDefinitionId::from("other"),
task: TaskInstanceId::new(format!("other-{index}")), task: TaskInstanceId::new(format!("other-{index}")),
attempt_id: None,
placement: None, placement: None,
terminal_state: TaskTerminalState::Completed, terminal_state: TaskTerminalState::Completed,
status_code: Some(0), status_code: Some(0),
@ -5490,7 +5472,7 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() {
assert!( assert!(
super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY super::MAX_NODE_REPLAY_NONCES_PER_AUTHORITY
>= super::NODE_SIGNATURE_WINDOW_SECONDS as usize * 2 * 1_000 >= super::NODE_SIGNATURE_WINDOW_SECONDS as usize * 2 * 1_000
/ disasmer_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS as usize / clusterflux_core::MIN_SIGNED_NODE_POLL_INTERVAL_MS as usize
+ 64, + 64,
"the bounded node replay window must sustain artifact and assignment polls at the protocol minimum with control-message headroom" "the bounded node replay window must sustain artifact and assignment polls at the protocol minimum with control-message headroom"
); );
@ -5616,6 +5598,82 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() {
assert_eq!(join.state, TaskJoinState::Pending); assert_eq!(join.state, TaskJoinState::Pending);
} }
#[test]
fn coordinator_rejects_named_environment_without_requirements() {
let mut service = CoordinatorService::new(10);
service
.handle_request(CoordinatorRequest::AttachNode {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "uncached-worker".to_owned(),
public_key: test_node_public_key("uncached-worker"),
})
.unwrap();
service
.handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: "uncached-worker".to_owned(),
capabilities: linux_capabilities(),
cached_environment_digests: Vec::new(),
dependency_cache_digests: Vec::new(),
source_snapshots: Vec::new(),
artifact_locations: Vec::new(),
direct_connectivity: true,
online: true,
})
.unwrap();
let CoordinatorResponse::ProcessStarted { epoch, .. } = service
.handle_request(CoordinatorRequest::StartProcess {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: None,
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
process: "vp-environment".to_owned(),
restart: false,
})
.unwrap()
else {
panic!("expected process start");
};
service
.handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
node: "uncached-worker".to_owned(),
process: "vp-environment".to_owned(),
epoch,
})
.unwrap();
let mut task_spec = test_task_spec(
"tenant",
"project",
"vp-environment",
"compile-linux",
epoch,
[],
);
task_spec.environment_id = Some("missing-environment".to_owned());
task_spec.environment_digest = Some(Digest::sha256("missing-environment"));
let error = service
.handle_request(CoordinatorRequest::LaunchTask {
task_spec,
tenant: "tenant".to_owned(),
project: "project".to_owned(),
actor_user: Some("user".to_owned()),
actor_agent: None,
agent_public_key_fingerprint: None,
agent_signature: None,
wait_for_node: false,
artifact_path: "/vfs/artifacts/environment-output.txt".to_owned(),
wasm_module_base64: test_wasm_module_base64(),
})
.unwrap_err();
assert!(error.to_string().contains("named environment cache"));
}
#[test] #[test]
fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() { fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
let mut service = CoordinatorService::new(10); let mut service = CoordinatorService::new(10);
@ -5918,6 +5976,29 @@ fn service_rejects_task_completion_outside_node_scope() {
assert!(events.is_empty()); assert!(events.is_empty());
} }
#[test]
fn service_rejects_task_event_access_using_retained_process_scope() {
let mut service = CoordinatorService::new(1);
service.process_scope_history.push_back((
TenantId::from("tenant-a"),
ProjectId::from("project-a"),
ProcessId::from("process"),
));
let error = service
.handle_request(CoordinatorRequest::ListTaskEvents {
tenant: "tenant-b".to_owned(),
project: "project-b".to_owned(),
actor_user: "user".to_owned(),
process: Some("process".to_owned()),
})
.unwrap_err();
assert!(error
.to_string()
.contains("outside the virtual process tenant/project scope"));
}
#[test] #[test]
fn service_rejects_node_capability_report_outside_enrollment_scope() { fn service_rejects_node_capability_report_outside_enrollment_scope() {
let mut service = CoordinatorService::new(1); let mut service = CoordinatorService::new(1);

View file

@ -1,4 +1,4 @@
use disasmer_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE}; use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;

View file

@ -1,6 +1,6 @@
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
use disasmer_core::{Actor, AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId}; use clusterflux_core::{Actor, AuthContext, CredentialKind, Digest, ProjectId, TenantId, UserId};
use crate::{CliSessionRecord, Coordinator, CoordinatorError, CredentialRecord}; use crate::{CliSessionRecord, Coordinator, CoordinatorError, CredentialRecord};
@ -83,7 +83,8 @@ impl Coordinator {
.is_some_and(|expires_at| expires_at <= now_epoch_seconds) .is_some_and(|expires_at| expires_at <= now_epoch_seconds)
{ {
return Err(CoordinatorError::Unauthorized( return Err(CoordinatorError::Unauthorized(
"CLI session credential has expired; run disasmer login --browser again".to_owned(), "CLI session credential has expired; run clusterflux login --browser again"
.to_owned(),
)); ));
} }
if record.credential_kind != CredentialKind::CliDeviceSession { if record.credential_kind != CredentialKind::CliDeviceSession {

View file

@ -1,5 +1,5 @@
[package] [package]
name = "disasmer-core" name = "clusterflux-core"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
license.workspace = true license.workspace = true

View file

@ -40,7 +40,7 @@ pub fn admin_request_proof_from_token_digest(
) -> Digest { ) -> Digest {
let issued_at_epoch_seconds = issued_at_epoch_seconds.to_string(); let issued_at_epoch_seconds = issued_at_epoch_seconds.to_string();
Digest::from_parts([ Digest::from_parts([
b"disasmer-admin-request-proof:v1".as_slice(), b"clusterflux-admin-request-proof:v1".as_slice(),
admin_token_digest.as_str().as_bytes(), admin_token_digest.as_str().as_bytes(),
operation.as_bytes(), operation.as_bytes(),
tenant.as_bytes(), tenant.as_bytes(),
@ -272,6 +272,115 @@ pub struct AgentWorkflowScope<'a> {
pub task: Option<&'a TaskInstanceId>, pub task: Option<&'a TaskInstanceId>,
} }
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AgentWorkflowRequestScope {
pub tenant: TenantId,
pub project: ProjectId,
pub request_kind: String,
pub process: ProcessId,
pub task: Option<TaskInstanceId>,
}
impl AgentWorkflowRequestScope {
pub fn new(
tenant: TenantId,
project: ProjectId,
request_kind: impl Into<String>,
process: ProcessId,
task: Option<TaskInstanceId>,
) -> Result<Self, String> {
let request_kind = request_kind.into();
match request_kind.as_str() {
"start_process" if task.is_some() => {
return Err("start_process agent scope must not contain a task instance".to_owned())
}
"launch_task" if task.is_none() => {
return Err("launch_task agent scope requires a task instance".to_owned())
}
"start_process" | "launch_task" => {}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
}
Ok(Self {
tenant,
project,
request_kind,
process,
task,
})
}
pub fn for_agent<'a>(&'a self, agent: &'a AgentId) -> AgentWorkflowScope<'a> {
AgentWorkflowScope {
tenant: &self.tenant,
project: &self.project,
agent,
request_kind: &self.request_kind,
process: &self.process,
task: self.task.as_ref(),
}
}
}
pub fn agent_workflow_request_scope_from_payload(
payload: &Value,
) -> Result<AgentWorkflowRequestScope, String> {
let object = payload
.as_object()
.ok_or_else(|| "agent workflow request payload must be an object".to_owned())?;
let request_kind = object
.get("type")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow request type is missing".to_owned())?;
let tenant = object
.get("tenant")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow tenant is missing".to_owned())?;
let project = object
.get("project")
.and_then(Value::as_str)
.ok_or_else(|| "agent workflow project is missing".to_owned())?;
let (process, task) = match request_kind {
"start_process" => (
object
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "start_process agent scope is missing process".to_owned())?,
None,
),
"launch_task" => {
let task_spec = object
.get("task_spec")
.and_then(Value::as_object)
.ok_or_else(|| "launch_task agent scope is missing task_spec".to_owned())?;
let process = task_spec
.get("process")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing process".to_owned())?;
let task = task_spec
.get("task_instance")
.and_then(Value::as_str)
.ok_or_else(|| "launch_task agent scope is missing task_instance".to_owned())?;
(process, Some(TaskInstanceId::from(task)))
}
_ => {
return Err(format!(
"request kind `{request_kind}` is not an agent workflow operation"
))
}
};
AgentWorkflowRequestScope::new(
TenantId::from(tenant),
ProjectId::from(project),
request_kind,
ProcessId::from(process),
task,
)
}
pub fn sign_agent_workflow_request( pub fn sign_agent_workflow_request(
private_key: &str, private_key: &str,
scope: AgentWorkflowScope<'_>, scope: AgentWorkflowScope<'_>,
@ -402,7 +511,7 @@ fn agent_workflow_signature_message(
let issued_at = issued_at_epoch_seconds.to_string(); let issued_at = issued_at_epoch_seconds.to_string();
let task = scope.task.map(TaskInstanceId::as_str).unwrap_or(""); let task = scope.task.map(TaskInstanceId::as_str).unwrap_or("");
let parts = [ let parts = [
"disasmer-agent-workflow-signature:v2", "clusterflux-agent-workflow-signature:v2",
scope.tenant.as_str(), scope.tenant.as_str(),
scope.project.as_str(), scope.project.as_str(),
scope.agent.as_str(), scope.agent.as_str(),
@ -432,7 +541,7 @@ fn node_request_signature_message(
) -> Vec<u8> { ) -> Vec<u8> {
let issued_at = issued_at_epoch_seconds.to_string(); let issued_at = issued_at_epoch_seconds.to_string();
let parts = [ let parts = [
"disasmer-node-request-signature:v2", "clusterflux-node-request-signature:v2",
node.as_str(), node.as_str(),
request_kind, request_kind,
payload_digest.as_str(), payload_digest.as_str(),

View file

@ -136,7 +136,7 @@ impl BundleIdentityInputs {
task_abi: self.task_abi.clone(), task_abi: self.task_abi.clone(),
entrypoints, entrypoints,
default_entrypoint: self.default_entrypoint.clone(), default_entrypoint: self.default_entrypoint.clone(),
boundary: "disasmer_task_exports".to_owned(), boundary: "clusterflux_task_exports".to_owned(),
}, },
source_metadata: BundleSourceMetadata { source_metadata: BundleSourceMetadata {
source_provider_manifest: self.source_provider_manifest.clone(), source_provider_manifest: self.source_provider_manifest.clone(),
@ -202,7 +202,7 @@ pub fn discover_source_debug_probes(
return None; return None;
}; };
let function_name = function.sig.ident.to_string(); let function_name = function.sig.ident.to_string();
let task = disasmer_probe_task(function)?; let task = clusterflux_probe_task(function)?;
let (function_index, (line_index, _)) = function_starts let (function_index, (line_index, _)) = function_starts
.iter() .iter()
.enumerate() .enumerate()
@ -261,7 +261,7 @@ fn parse_rust_function_name(line: &str) -> Option<String> {
(!name.is_empty()).then_some(name) (!name.is_empty()).then_some(name)
} }
fn disasmer_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> { fn clusterflux_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> {
for attribute in &function.attrs { for attribute in &function.attrs {
let mut segments = attribute.path().segments.iter(); let mut segments = attribute.path().segments.iter();
let Some(namespace) = segments.next() else { let Some(namespace) = segments.next() else {
@ -270,7 +270,7 @@ fn disasmer_probe_task(function: &syn::ItemFn) -> Option<TaskDefinitionId> {
let Some(kind) = segments.next() else { let Some(kind) = segments.next() else {
continue; continue;
}; };
if namespace.ident != "disasmer" || segments.next().is_some() { if namespace.ident != "clusterflux" || segments.next().is_some() {
continue; continue;
} }
let function_name = function.sig.ident.to_string(); let function_name = function.sig.ident.to_string();
@ -425,19 +425,19 @@ mod tests {
fn source_debug_probe_metadata_maps_function_ranges_to_tasks() { fn source_debug_probe_metadata_maps_function_ranges_to_tasks() {
let probes = discover_source_debug_probes( let probes = discover_source_debug_probes(
"src/build.rs", "src/build.rs",
r#"#[disasmer::main] r#"#[clusterflux::main]
fn build_main() { fn build_main() {
let linux = compile_linux(); let linux = compile_linux();
} }
#[disasmer::task] #[clusterflux::task]
fn compile_linux() { fn compile_linux() {
println!("linux"); println!("linux");
} }
fn helper_without_runtime_probe() {} fn helper_without_runtime_probe() {}
#[disasmer::task(name = "release")] #[clusterflux::task(name = "release")]
fn package_release() { fn package_release() {
println!("package"); println!("package");
} }

View file

@ -62,10 +62,12 @@ pub struct DebugEpoch {
#[derive(Clone, Debug, Error, PartialEq, Eq)] #[derive(Clone, Debug, Error, PartialEq, Eq)]
pub enum DebugEpochError { pub enum DebugEpochError {
#[error("participant `{task}` cannot freeze, so all-stop failed")] #[error("no participant could freeze; `{task}` rejected the freeze request")]
CannotFreeze { task: TaskInstanceId }, CannotFreeze { task: TaskInstanceId },
#[error("participant `{0}` is not part of this debug epoch")] #[error("participant `{0}` is not part of this debug epoch")]
UnknownParticipant(TaskInstanceId), UnknownParticipant(TaskInstanceId),
#[error("participant `{0}` did not acknowledge frozen state in this debug epoch")]
ParticipantNotFrozen(TaskInstanceId),
} }
impl DebugEpoch { impl DebugEpoch {
@ -75,28 +77,38 @@ impl DebugEpoch {
reason: DebugStopReason, reason: DebugStopReason,
participants: Vec<DebugParticipant>, participants: Vec<DebugParticipant>,
) -> Result<Self, DebugEpochError> { ) -> Result<Self, DebugEpochError> {
for participant in &participants { let first_rejected = participants
if matches!( .iter()
participant.kind, .find(|participant| {
DebugParticipantKind::WasmTask | DebugParticipantKind::ControlledNativeCommand matches!(participant.state, DebugRuntimeState::Running) && !participant.can_freeze
) && !participant.can_freeze })
{ .map(|participant| participant.task.clone());
return Err(DebugEpochError::CannotFreeze { let participants: BTreeMap<_, _> = participants
task: participant.task.clone(),
});
}
}
let participants = participants
.into_iter() .into_iter()
.map(|mut participant| { .map(|mut participant| {
if matches!(participant.state, DebugRuntimeState::Running) { if matches!(participant.state, DebugRuntimeState::Running) {
participant.state = DebugRuntimeState::Frozen; participant.state = if participant.can_freeze {
DebugRuntimeState::Frozen
} else {
DebugRuntimeState::Failed(
"participant did not acknowledge frozen state before the debug deadline"
.to_owned(),
)
};
} }
(participant.task.clone(), participant) (participant.task.clone(), participant)
}) })
.collect(); .collect();
if !participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
{
return Err(DebugEpochError::CannotFreeze {
task: first_rejected.unwrap_or_else(|| TaskInstanceId::from("debug-epoch")),
});
}
Ok(Self { Ok(Self {
process, process,
epoch, epoch,
@ -126,6 +138,9 @@ impl DebugEpoch {
.participants .participants
.get(task) .get(task)
.ok_or_else(|| DebugEpochError::UnknownParticipant(task.clone()))?; .ok_or_else(|| DebugEpochError::UnknownParticipant(task.clone()))?;
if participant.state != DebugRuntimeState::Frozen {
return Err(DebugEpochError::ParticipantNotFrozen(task.clone()));
}
Ok(ThreadInspection { Ok(ThreadInspection {
task: participant.task.clone(), task: participant.task.clone(),
name: participant.name.clone(), name: participant.name.clone(),
@ -150,6 +165,22 @@ impl DebugEpoch {
.map(|participant| participant.name.clone()) .map(|participant| participant.name.clone())
.collect() .collect()
} }
pub fn all_threads_stopped(&self) -> bool {
!self.participants.is_empty()
&& self
.participants
.values()
.all(|participant| participant.state == DebugRuntimeState::Frozen)
}
pub fn partially_frozen(&self) -> bool {
!self.all_threads_stopped()
&& self
.participants
.values()
.any(|participant| participant.state == DebugRuntimeState::Frozen)
}
} }
#[cfg(test)] #[cfg(test)]
@ -203,7 +234,7 @@ mod tests {
} }
#[test] #[test]
fn debug_epoch_reports_freeze_failure_instead_of_claiming_all_stop() { fn debug_epoch_reports_failure_when_no_participant_can_freeze() {
let error = DebugEpoch::pause( let error = DebugEpoch::pause(
ProcessId::from("process"), ProcessId::from("process"),
1, 1,
@ -218,6 +249,34 @@ mod tests {
assert!(matches!(error, DebugEpochError::CannotFreeze { .. })); assert!(matches!(error, DebugEpochError::CannotFreeze { .. }));
} }
#[test]
fn debug_epoch_keeps_frozen_participants_when_another_participant_fails() {
let epoch = DebugEpoch::pause(
ProcessId::from("process"),
1,
vec![
participant("main", DebugParticipantKind::WasmTask, true),
participant(
"native",
DebugParticipantKind::ControlledNativeCommand,
false,
),
],
)
.unwrap();
assert!(epoch.partially_frozen());
assert!(!epoch.all_threads_stopped());
assert!(matches!(
epoch.participant_state(&TaskInstanceId::from("native")),
Some(DebugRuntimeState::Failed(_))
));
assert!(matches!(
epoch.inspection(&TaskInstanceId::from("native")),
Err(DebugEpochError::ParticipantNotFrozen(_))
));
}
#[test] #[test]
fn continue_resumes_every_frozen_participant() { fn continue_resumes_every_frozen_participant() {
let mut epoch = DebugEpoch::pause( let mut epoch = DebugEpoch::pause(

View file

@ -129,7 +129,7 @@ pub fn diagnose_environment_references(
.filter(|reference| !known.contains(reference.name.as_str())) .filter(|reference| !known.contains(reference.name.as_str()))
.map(|reference| EnvironmentDiagnostic { .map(|reference| EnvironmentDiagnostic {
message: format!( message: format!(
"missing Disasmer environment `{}`; expected envs/{}/Containerfile or envs/{}/Dockerfile", "missing Clusterflux environment `{}`; expected envs/{}/Containerfile or envs/{}/Dockerfile",
reference.name, reference.name, reference.name reference.name, reference.name, reference.name
), ),
reference, reference,

View file

@ -78,7 +78,7 @@ pub enum TaskBoundaryValue {
pub enum TaskBoundaryHandle { pub enum TaskBoundaryHandle {
SourceSnapshot(crate::Digest), SourceSnapshot(crate::Digest),
Blob(crate::Digest), Blob(crate::Digest),
Artifact(crate::ArtifactId), Artifact(crate::ArtifactHandle),
VfsManifest(crate::Digest), VfsManifest(crate::Digest),
} }
@ -89,6 +89,168 @@ pub struct StructuredTaskBoundary {
pub handles: Vec<TaskBoundaryHandle>, pub handles: Vec<TaskBoundaryHandle>,
} }
const TASK_HANDLE_PLACEHOLDER_KEY: &str = "$task_handle";
impl TaskBoundaryHandle {
fn kind(&self) -> &'static str {
match self {
Self::SourceSnapshot(_) => "source_snapshot",
Self::Blob(_) => "blob",
Self::Artifact(_) => "artifact",
Self::VfsManifest(_) => "vfs_manifest",
}
}
fn validate(&self) -> Result<(), String> {
match self {
Self::SourceSnapshot(digest) | Self::Blob(digest) | Self::VfsManifest(digest) => {
if !digest.is_valid_sha256() {
return Err(format!("{} handle has an invalid digest", self.kind()));
}
}
Self::Artifact(artifact) => {
if artifact.id.as_str().trim().is_empty() || artifact.id.as_str().len() > 256 {
return Err("artifact handle has an invalid ID".to_owned());
}
if !artifact.digest.is_valid_sha256() {
return Err("artifact handle has an invalid digest".to_owned());
}
}
}
Ok(())
}
fn materialized_value(&self) -> serde_json::Value {
match self {
Self::SourceSnapshot(digest) | Self::Blob(digest) | Self::VfsManifest(digest) => {
serde_json::json!({ "digest": digest })
}
Self::Artifact(artifact) => serde_json::json!({
"id": artifact.id,
"digest": artifact.digest,
"size_bytes": artifact.size_bytes,
}),
}
}
}
impl StructuredTaskBoundary {
pub fn validate(&self) -> Result<(), String> {
if self.handles.len() > 256 {
return Err("structured task argument exceeds 256 handles".to_owned());
}
for handle in &self.handles {
handle.validate()?;
}
let mut used = vec![false; self.handles.len()];
validate_task_handle_placeholders(&self.value, &self.handles, &mut used)?;
if let Some(index) = used.iter().position(|used| !used) {
return Err(format!(
"structured task argument contains unused handle-table entry {index}"
));
}
Ok(())
}
pub fn materialize(&self) -> Result<serde_json::Value, String> {
self.validate()?;
let mut value = self.value.clone();
materialize_task_handle_placeholders(&mut value, &self.handles)?;
Ok(value)
}
}
fn validate_task_handle_placeholders(
value: &serde_json::Value,
handles: &[TaskBoundaryHandle],
used: &mut [bool],
) -> Result<(), String> {
match value {
serde_json::Value::Array(values) => {
for value in values {
validate_task_handle_placeholders(value, handles, used)?;
}
}
serde_json::Value::Object(object) => {
if let Some(placeholder) = object.get(TASK_HANDLE_PLACEHOLDER_KEY) {
if object.len() != 1 {
return Err("task handle placeholder must be the only object field".to_owned());
}
let placeholder = placeholder.as_object().ok_or_else(|| {
"task handle placeholder payload must be an object".to_owned()
})?;
if placeholder.len() != 2
|| !placeholder.contains_key("index")
|| !placeholder.contains_key("kind")
{
return Err(
"task handle placeholder must contain exactly index and kind".to_owned(),
);
}
let index = placeholder["index"]
.as_u64()
.and_then(|index| usize::try_from(index).ok())
.ok_or_else(|| "task handle placeholder index is invalid".to_owned())?;
let kind = placeholder["kind"]
.as_str()
.ok_or_else(|| "task handle placeholder kind is invalid".to_owned())?;
let handle = handles.get(index).ok_or_else(|| {
format!("task handle placeholder index {index} is out of range")
})?;
if handle.kind() != kind {
return Err(format!(
"task handle placeholder {index} expects kind {kind}, but the table contains {}",
handle.kind()
));
}
if std::mem::replace(&mut used[index], true) {
return Err(format!(
"task handle placeholder index {index} is used more than once"
));
}
} else {
for value in object.values() {
validate_task_handle_placeholders(value, handles, used)?;
}
}
}
_ => {}
}
Ok(())
}
fn materialize_task_handle_placeholders(
value: &mut serde_json::Value,
handles: &[TaskBoundaryHandle],
) -> Result<(), String> {
match value {
serde_json::Value::Array(values) => {
for value in values {
materialize_task_handle_placeholders(value, handles)?;
}
}
serde_json::Value::Object(object) => {
if let Some(placeholder) = object.get(TASK_HANDLE_PLACEHOLDER_KEY) {
let index = placeholder
.get("index")
.and_then(serde_json::Value::as_u64)
.and_then(|index| usize::try_from(index).ok())
.ok_or_else(|| "validated task handle placeholder lost its index".to_owned())?;
*value = handles
.get(index)
.ok_or_else(|| "validated task handle placeholder became invalid".to_owned())?
.materialized_value();
} else {
for value in object.values_mut() {
materialize_task_handle_placeholders(value, handles)?;
}
}
}
_ => {}
}
Ok(())
}
impl TaskBoundaryValue { impl TaskBoundaryValue {
pub fn required_artifacts(&self) -> Vec<ArtifactId> { pub fn required_artifacts(&self) -> Vec<ArtifactId> {
match self { match self {
@ -97,7 +259,7 @@ impl TaskBoundaryValue {
.handles .handles
.iter() .iter()
.filter_map(|handle| match handle { .filter_map(|handle| match handle {
TaskBoundaryHandle::Artifact(artifact) => Some(artifact.clone()), TaskBoundaryHandle::Artifact(artifact) => Some(artifact.id.clone()),
_ => None, _ => None,
}) })
.collect(), .collect(),
@ -131,6 +293,8 @@ pub struct WasmHostTaskStartRequest {
pub task_definition: TaskDefinitionId, pub task_definition: TaskDefinitionId,
pub environment_id: Option<String>, pub environment_id: Option<String>,
pub args: Vec<TaskBoundaryValue>, pub args: Vec<TaskBoundaryValue>,
#[serde(default)]
pub failure_policy: TaskFailurePolicy,
} }
impl WasmHostTaskStartRequest { impl WasmHostTaskStartRequest {
@ -241,8 +405,8 @@ impl WasmHostCommandRequest {
fn validate_command_working_directory(path: &str) -> Result<(), String> { fn validate_command_working_directory(path: &str) -> Result<(), String> {
let permitted_root = path == "/workspace" let permitted_root = path == "/workspace"
|| path.starts_with("/workspace/") || path.starts_with("/workspace/")
|| path == "/disasmer/output" || path == "/clusterflux/output"
|| path.starts_with("/disasmer/output/"); || path.starts_with("/clusterflux/output/");
if !permitted_root if !permitted_root
|| path.len() > 4096 || path.len() > 4096
|| path.contains('\0') || path.contains('\0')
@ -250,7 +414,7 @@ fn validate_command_working_directory(path: &str) -> Result<(), String> {
|| path.split('/').any(|component| component == "..") || path.split('/').any(|component| component == "..")
{ {
return Err( return Err(
"Wasm command working directory must stay under /workspace or /disasmer/output" "Wasm command working directory must stay under /workspace or /clusterflux/output"
.to_owned(), .to_owned(),
); );
} }
@ -470,6 +634,9 @@ impl WasmTaskInvocation {
if self.task_instance.as_str().len() > 192 { if self.task_instance.as_str().len() > 192 {
return Err("Wasm task invocation has an invalid task-instance id".to_owned()); return Err("Wasm task invocation has an invalid task-instance id".to_owned());
} }
for argument in &self.args {
argument.validate()?;
}
let encoded = serde_json::to_vec(self).map_err(|error| error.to_string())?; let encoded = serde_json::to_vec(self).map_err(|error| error.to_string())?;
if encoded.len() > MAX_WASM_TASK_ENVELOPE_BYTES { if encoded.len() > MAX_WASM_TASK_ENVELOPE_BYTES {
return Err(format!( return Err(format!(
@ -542,6 +709,17 @@ impl WasmTaskResult {
} }
impl TaskBoundaryValue { impl TaskBoundaryValue {
pub fn validate(&self) -> Result<(), String> {
match self {
Self::SmallJson(_) => Ok(()),
Self::Structured(structured) => structured.validate(),
Self::SourceSnapshot(digest) | Self::Blob(digest) | Self::VfsManifest(digest) => {
TaskBoundaryHandle::SourceSnapshot(digest.clone()).validate()
}
Self::Artifact(artifact) => TaskBoundaryHandle::Artifact(artifact.clone()).validate(),
}
}
pub fn reject_host_only(type_name: &str) -> Result<Self, String> { pub fn reject_host_only(type_name: &str) -> Result<Self, String> {
Err(format!( Err(format!(
"task boundary value `{type_name}` is host-only; use small serialized data or handles" "task boundary value `{type_name}` is host-only; use small serialized data or handles"
@ -589,9 +767,19 @@ pub struct TaskSpec {
pub required_artifacts: Vec<ArtifactId>, pub required_artifacts: Vec<ArtifactId>,
pub args: Vec<TaskBoundaryValue>, pub args: Vec<TaskBoundaryValue>,
pub vfs_epoch: u64, pub vfs_epoch: u64,
#[serde(default)]
pub failure_policy: TaskFailurePolicy,
pub bundle_digest: Option<Digest>, pub bundle_digest: Option<Digest>,
} }
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TaskFailurePolicy {
#[default]
FailFast,
AwaitOperator,
}
impl TaskSpec { impl TaskSpec {
pub fn product_mode_uses_remote_dispatch(&self) -> bool { pub fn product_mode_uses_remote_dispatch(&self) -> bool {
self.dispatch.is_product_remote_dispatch() self.dispatch.is_product_remote_dispatch()
@ -683,6 +871,55 @@ mod tests {
assert!(error.contains("host-only")); assert!(error.contains("host-only"));
} }
#[test]
fn structured_handle_placeholders_are_strict_and_table_authoritative() {
let artifact = ArtifactHandle {
id: ArtifactId::from("real.bin"),
digest: Digest::sha256("real bytes"),
size_bytes: 10,
};
let boundary = StructuredTaskBoundary {
value: serde_json::json!({
"label": "release",
"artifact": { "$task_handle": { "index": 0, "kind": "artifact" } }
}),
handles: vec![TaskBoundaryHandle::Artifact(artifact.clone())],
};
boundary.validate().unwrap();
assert_eq!(
boundary.materialize().unwrap()["artifact"]["id"],
"real.bin"
);
assert!(!boundary.value.to_string().contains("real.bin"));
let mut wrong_kind = boundary.clone();
wrong_kind.value["artifact"]["$task_handle"]["kind"] =
serde_json::Value::String("blob".to_owned());
assert!(wrong_kind.validate().unwrap_err().contains("expects kind"));
let mut out_of_range = boundary.clone();
out_of_range.value["artifact"]["$task_handle"]["index"] = serde_json::json!(1);
assert!(out_of_range
.validate()
.unwrap_err()
.contains("out of range"));
let unused = StructuredTaskBoundary {
value: serde_json::json!({ "label": "release" }),
handles: boundary.handles.clone(),
};
assert!(unused.validate().unwrap_err().contains("unused"));
let malformed = StructuredTaskBoundary {
value: serde_json::json!({
"$task_handle": { "index": 0, "kind": "artifact", "id": "forged.bin" }
}),
handles: boundary.handles,
};
assert!(malformed.validate().unwrap_err().contains("exactly"));
}
#[test] #[test]
fn product_task_spec_has_no_local_function_dispatch_variant() { fn product_task_spec_has_no_local_function_dispatch_variant() {
let spec = TaskSpec { let spec = TaskSpec {
@ -704,6 +941,7 @@ mod tests {
required_artifacts: Vec::new(), required_artifacts: Vec::new(),
args: vec![TaskBoundaryValue::SmallJson(serde_json::json!("src"))], args: vec![TaskBoundaryValue::SmallJson(serde_json::json!("src"))],
vfs_epoch: 7, vfs_epoch: 7,
failure_policy: Default::default(),
bundle_digest: Some(Digest::sha256("bundle")), bundle_digest: Some(Digest::sha256("bundle")),
}; };

View file

@ -25,13 +25,13 @@ pub use artifact::{
}; };
pub use auth::{ pub use auth::{
admin_request_proof, admin_request_proof_from_token_digest, admin_request_proof, admin_request_proof_from_token_digest,
agent_ed25519_public_key_from_private_key, derive_ed25519_private_key_from_seed, agent_ed25519_public_key_from_private_key, agent_workflow_request_scope_from_payload,
node_capability_policy_digest, node_ed25519_public_key_from_private_key, derive_ed25519_private_key_from_seed, node_capability_policy_digest,
sign_agent_workflow_request, sign_node_request, signed_request_payload_digest, node_ed25519_public_key_from_private_key, sign_agent_workflow_request, sign_node_request,
verify_agent_workflow_signature, verify_node_request_signature, Action, Actor, signed_request_payload_digest, verify_agent_workflow_signature, verify_node_request_signature,
AgentSignedRequest, AgentWorkflowScope, AuthContext, Authorization, BrowserLoginFlow, Action, Actor, AgentSignedRequest, AgentWorkflowRequestScope, AgentWorkflowScope, AuthContext,
CredentialKind, EnrollmentError, EnrollmentGrant, IdentityKind, NodeCredential, Authorization, BrowserLoginFlow, CredentialKind, EnrollmentError, EnrollmentGrant,
NodeSignedRequest, PublicKeyIdentity, Scope, IdentityKind, NodeCredential, NodeSignedRequest, PublicKeyIdentity, Scope,
}; };
#[cfg(not(target_arch = "wasm32"))] #[cfg(not(target_arch = "wasm32"))]
pub use auth::{generate_ed25519_private_key, generate_opaque_token}; pub use auth::{generate_ed25519_private_key, generate_opaque_token};
@ -57,13 +57,13 @@ pub use environment::{
pub use execution::{ pub use execution::{
CommandBackendKind, CommandInvocation, CommandNetworkPolicy, CommandPlan, GuestRuntimeKind, CommandBackendKind, CommandInvocation, CommandNetworkPolicy, CommandPlan, GuestRuntimeKind,
NativeCommandPolicy, StructuredTaskBoundary, TaskBoundaryHandle, TaskBoundaryValue, NativeCommandPolicy, StructuredTaskBoundary, TaskBoundaryHandle, TaskBoundaryValue,
TaskDispatch, TaskJoinResult, TaskJoinState, TaskSpec, WasmExportAbi, WasmHostCommandRequest, TaskDispatch, TaskFailurePolicy, TaskJoinResult, TaskJoinState, TaskSpec, WasmExportAbi,
WasmHostCommandResult, WasmHostDebugProbeRequest, WasmHostDebugProbeResult, WasmHostCommandRequest, WasmHostCommandResult, WasmHostDebugProbeRequest,
WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult, WasmHostTaskControlRequest, WasmHostDebugProbeResult, WasmHostSourceSnapshotRequest, WasmHostSourceSnapshotResult,
WasmHostTaskControlResult, WasmHostTaskHandle, WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskControlRequest, WasmHostTaskControlResult, WasmHostTaskHandle,
WasmHostTaskStartRequest, WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmHostTaskJoinRequest, WasmHostTaskJoinResult, WasmHostTaskStartRequest,
WasmTaskInvocation, WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WasmHostVfsOperation, WasmHostVfsRequest, WasmHostVfsResult, WasmTaskInvocation,
WASM_TASK_ABI_VERSION, WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
}; };
pub use ids::{ pub use ids::{
AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId, AgentId, ArtifactId, NodeId, ProcessId, ProjectId, TaskDefinitionId, TaskInstanceId, TenantId,

View file

@ -27,13 +27,13 @@ pub struct ProjectModel {
pub enum ProjectModelError { pub enum ProjectModelError {
#[error("environment discovery failed: {0}")] #[error("environment discovery failed: {0}")]
Environment(String), Environment(String),
#[error("Disasmer entrypoint discovery failed: {0}")] #[error("Clusterflux entrypoint discovery failed: {0}")]
EntrypointDiscovery(String), EntrypointDiscovery(String),
#[error( #[error(
"no Disasmer entrypoint is declared; add `#[disasmer::main]` to a function under src/" "no Clusterflux entrypoint is declared; add `#[clusterflux::main]` to a function under src/"
)] )]
NoEntrypoints, NoEntrypoints,
#[error("unknown Disasmer entrypoint `{name}`; available entrypoints: {available:?}")] #[error("unknown Clusterflux entrypoint `{name}`; available entrypoints: {available:?}")]
UnknownEntrypoint { UnknownEntrypoint {
name: String, name: String,
available: Vec<String>, available: Vec<String>,
@ -155,7 +155,7 @@ fn collect_entrypoint_items(
.iter() .iter()
.map(|segment| segment.ident.to_string()) .map(|segment| segment.ident.to_string())
.collect::<Vec<_>>(); .collect::<Vec<_>>();
segments.as_slice() == ["disasmer", "main"] segments.as_slice() == ["clusterflux", "main"]
}) else { }) else {
continue; continue;
}; };
@ -233,7 +233,7 @@ mod tests {
.unwrap(); .unwrap();
fs::write( fs::write(
temp.path().join("src/main.rs"), temp.path().join("src/main.rs"),
"#[disasmer::main]\npub fn build_main() {}\n", "#[clusterflux::main]\npub fn build_main() {}\n",
) )
.unwrap(); .unwrap();
@ -250,12 +250,12 @@ mod tests {
fs::create_dir_all(temp.path().join("src/nested")).unwrap(); fs::create_dir_all(temp.path().join("src/nested")).unwrap();
fs::write( fs::write(
temp.path().join("src/lib.rs"), temp.path().join("src/lib.rs"),
"#[disasmer::main(name = \"check\")]\npub fn test_main() {}\n", "#[clusterflux::main(name = \"check\")]\npub fn test_main() {}\n",
) )
.unwrap(); .unwrap();
fs::write( fs::write(
temp.path().join("src/nested/release.rs"), temp.path().join("src/nested/release.rs"),
"#[disasmer::main]\npub fn release_main() {}\n", "#[clusterflux::main]\npub fn release_main() {}\n",
) )
.unwrap(); .unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap(); let model = ProjectModel::discover_without_config(temp.path()).unwrap();
@ -276,7 +276,7 @@ mod tests {
fs::create_dir_all(temp.path().join("src")).unwrap(); fs::create_dir_all(temp.path().join("src")).unwrap();
fs::write( fs::write(
temp.path().join("src/main.rs"), temp.path().join("src/main.rs"),
"#[disasmer::main]\npub fn build_main() {}\n", "#[clusterflux::main]\npub fn build_main() {}\n",
) )
.unwrap(); .unwrap();
let model = ProjectModel::discover_without_config(temp.path()).unwrap(); let model = ProjectModel::discover_without_config(temp.path()).unwrap();

View file

@ -28,6 +28,8 @@ pub struct PlacementRequest {
pub project: ProjectId, pub project: ProjectId,
pub environment: Option<EnvironmentRequirements>, pub environment: Option<EnvironmentRequirements>,
pub environment_digest: Option<Digest>, pub environment_digest: Option<Digest>,
#[serde(default)]
pub environment_cache_required: bool,
pub required_capabilities: BTreeSet<Capability>, pub required_capabilities: BTreeSet<Capability>,
pub dependency_cache: Option<Digest>, pub dependency_cache: Option<Digest>,
pub source_snapshot: Option<Digest>, pub source_snapshot: Option<Digest>,
@ -139,6 +141,17 @@ fn compatibility(
} }
} }
} }
if request.environment_cache_required {
match request.environment_digest.as_ref() {
Some(digest) if !node.cached_environments.contains(digest) => {
reasons.push(format!(
"required named environment cache {digest} is unavailable"
));
}
None => reasons.push("required named environment cache digest is missing".to_owned()),
Some(_) => {}
}
}
let source_transfer_required = request let source_transfer_required = request
.source_snapshot .source_snapshot
.as_ref() .as_ref()
@ -259,6 +272,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::linux_container()), environment: Some(EnvironmentRequirements::linux_container()),
environment_digest: Some(Digest::sha256("env")), environment_digest: Some(Digest::sha256("env")),
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]), required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: Some(Digest::sha256("deps")), dependency_cache: Some(Digest::sha256("deps")),
source_snapshot: Some(Digest::sha256("source")), source_snapshot: Some(Digest::sha256("source")),
@ -283,6 +297,30 @@ mod tests {
.any(|reason| reason.contains("dependency"))); .any(|reason| reason.contains("dependency")));
} }
#[test]
fn scheduler_requires_requested_named_environment_cache() {
let request = PlacementRequest {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
environment: None,
environment_digest: Some(Digest::sha256("missing-environment")),
environment_cache_required: true,
required_capabilities: BTreeSet::new(),
dependency_cache: None,
source_snapshot: None,
required_artifacts: BTreeSet::new(),
quota_available: true,
policy_allowed: true,
prefer_node: None,
};
let error = DefaultScheduler
.place(&[node("uncached", false)], &request)
.unwrap_err();
assert!(error.message.contains("named environment cache"));
}
#[test] #[test]
fn scheduler_failure_names_missing_constraint() { fn scheduler_failure_names_missing_constraint() {
let mut request = PlacementRequest { let mut request = PlacementRequest {
@ -290,6 +328,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: None, environment: None,
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::WindowsCommandDev]), required_capabilities: BTreeSet::from([Capability::WindowsCommandDev]),
dependency_cache: None, dependency_cache: None,
source_snapshot: None, source_snapshot: None,
@ -314,6 +353,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: Some(EnvironmentRequirements::windows_command_dev()), environment: Some(EnvironmentRequirements::windows_command_dev()),
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::new(), required_capabilities: BTreeSet::new(),
dependency_cache: None, dependency_cache: None,
source_snapshot: None, source_snapshot: None,
@ -344,6 +384,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: None, environment: None,
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]), required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None, dependency_cache: None,
source_snapshot: Some(Digest::sha256("source")), source_snapshot: Some(Digest::sha256("source")),
@ -378,6 +419,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: None, environment: None,
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]), required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None, dependency_cache: None,
source_snapshot: None, source_snapshot: None,
@ -403,6 +445,7 @@ mod tests {
project: ProjectId::from("project"), project: ProjectId::from("project"),
environment: None, environment: None,
environment_digest: None, environment_digest: None,
environment_cache_required: false,
required_capabilities: BTreeSet::from([Capability::Command]), required_capabilities: BTreeSet::from([Capability::Command]),
dependency_cache: None, dependency_cache: None,
source_snapshot: None, source_snapshot: None,

View file

@ -1,20 +1,20 @@
[package] [package]
name = "disasmer-dap" name = "clusterflux-dap"
version = "0.1.0" version = "0.1.0"
edition.workspace = true edition.workspace = true
license.workspace = true license.workspace = true
repository.workspace = true repository.workspace = true
[[bin]] [[bin]]
name = "disasmer-debug-dap" name = "clusterflux-debug-dap"
path = "src/main.rs" path = "src/main.rs"
[dependencies] [dependencies]
anyhow.workspace = true anyhow.workspace = true
base64.workspace = true base64.workspace = true
disasmer-core = { path = "../disasmer-core" } clusterflux-core = { path = "../clusterflux-core" }
disasmer-control = { path = "../disasmer-control" } clusterflux-control = { path = "../clusterflux-control" }
disasmer-node = { path = "../disasmer-node" } clusterflux-node = { path = "../clusterflux-node" }
serde_json.workspace = true serde_json.workspace = true
[dev-dependencies] [dev-dependencies]

View file

@ -1,7 +1,7 @@
use std::io::{self, BufReader}; use std::io::{self, BufReader};
use anyhow::Result; use anyhow::Result;
use disasmer_core::DebugRuntimeState; use clusterflux_core::DebugRuntimeState;
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::breakpoints::{ use crate::breakpoints::{
@ -83,7 +83,7 @@ pub(crate) fn run_adapter() -> Result<()> {
.and_then(Value::as_bool) .and_then(Value::as_bool)
.unwrap_or(false); .unwrap_or(false);
if let Some(process_id) = args.get("processId").and_then(Value::as_str) { if let Some(process_id) = args.get("processId").and_then(Value::as_str) {
state.process = disasmer_core::ProcessId::new(process_id); state.process = clusterflux_core::ProcessId::new(process_id);
} }
if command == "attach" { if command == "attach" {
state.session_mode = DapSessionMode::Attach; state.session_mode = DapSessionMode::Attach;
@ -178,12 +178,12 @@ pub(crate) fn run_adapter() -> Result<()> {
writer.response(&request, true, json!({}))?; writer.response(&request, true, json!({}))?;
let session_message = if state.session_mode == DapSessionMode::Attach { let session_message = if state.session_mode == DapSessionMode::Attach {
format!( format!(
"Attached to Disasmer virtual process {} with {:?} runtime\n", "Attached to Clusterflux virtual process {} with {:?} runtime\n",
state.process, state.runtime_backend state.process, state.runtime_backend
) )
} else { } else {
format!( format!(
"Disasmer bundle refreshed for entry `{}`; starting virtual process {} with {:?} runtime\n", "Clusterflux bundle refreshed for entry `{}`; starting virtual process {} with {:?} runtime\n",
state.entry, state.process, state.runtime_backend state.entry, state.process, state.runtime_backend
) )
}; };
@ -200,9 +200,9 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "breakpoint", "reason": "breakpoint",
"description": "Disasmer Debug Epoch all-stop confirmed by every active participant", "description": "Clusterflux Debug Epoch all-stop confirmed by every active participant",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
continue; continue;
@ -223,9 +223,9 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "breakpoint", "reason": "breakpoint",
"description": "Disasmer Debug Epoch all-stop", "description": "Clusterflux Debug Epoch all-stop",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
} }
@ -255,27 +255,35 @@ pub(crate) fn run_adapter() -> Result<()> {
"stackTrace" => { "stackTrace" => {
let thread = request_thread(&request, &state); let thread = request_thread(&request, &state);
let source_path = crate::source::stack_source_path(&state); let source_path = crate::source::stack_source_path(&state);
let frame_name = thread let stack_frames = if state.runtime_backend != RuntimeBackend::Simulated
.runtime_stack_frames && thread.runtime_stack_frames.is_empty()
.first() {
.cloned() Vec::new()
.unwrap_or_else(|| format!("{}::run", thread.name)); } else {
let frame_name = thread
.runtime_stack_frames
.first()
.cloned()
.unwrap_or_else(|| format!("{}::run", thread.name));
vec![json!({
"id": thread.frame_id,
"name": frame_name,
"line": thread.line,
"column": 1,
"source": {
"name": crate::source::source_name(&source_path),
"path": source_path,
"presentationHint": "normal"
}
})]
};
let total_frames = stack_frames.len();
writer.response( writer.response(
&request, &request,
true, true,
json!({ json!({
"stackFrames": [{ "stackFrames": stack_frames,
"id": thread.frame_id, "totalFrames": total_frames
"name": frame_name,
"line": thread.line,
"column": 1,
"source": {
"name": crate::source::source_name(&source_path),
"path": source_path,
"presentationHint": "normal"
}
}],
"totalFrames": 1
}), }),
)?; )?;
} }
@ -319,7 +327,7 @@ pub(crate) fn run_adapter() -> Result<()> {
"presentationHint": "arguments" "presentationHint": "arguments"
}, },
{ {
"name": "Disasmer Runtime", "name": "Clusterflux Runtime",
"variablesReference": thread.runtime_ref, "variablesReference": thread.runtime_ref,
"expensive": false "expensive": false
}, },
@ -379,9 +387,9 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "pause", "reason": "pause",
"description": "Disasmer Debug Epoch pause", "description": "Clusterflux Debug Epoch pause",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
} }
@ -429,9 +437,23 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "breakpoint", "reason": "breakpoint",
"description": "Disasmer Debug Epoch all-stop confirmed by every active participant", "description": "Clusterflux Debug Epoch all-stop confirmed by every active participant",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Exception(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
writer.output("stderr", format!("{}\n", state.command_status))?;
writer.event(
"stopped",
json!({
"reason": "exception",
"description": "Task failed and is awaiting operator action",
"threadId": stopped_thread,
"allThreadsStopped": false,
}), }),
)?; )?;
} }
@ -475,9 +497,9 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "breakpoint", "reason": "breakpoint",
"description": "Disasmer Debug Epoch all-stop", "description": "Clusterflux Debug Epoch all-stop",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
} }
@ -512,9 +534,9 @@ pub(crate) fn run_adapter() -> Result<()> {
"stopped", "stopped",
json!({ json!({
"reason": "step", "reason": "step",
"description": format!("Disasmer Debug Epoch {description}"), "description": format!("Clusterflux Debug Epoch {description}"),
"threadId": thread_id, "threadId": thread_id,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
} }
@ -565,7 +587,7 @@ pub(crate) fn run_adapter() -> Result<()> {
"reason": "breakpoint", "reason": "breakpoint",
"description": "Restarted main from the rebuilt bundle; every active participant confirmed all-stop", "description": "Restarted main from the rebuilt bundle; every active participant confirmed all-stop",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}), }),
)?; )?;
} }
@ -610,7 +632,20 @@ pub(crate) fn run_adapter() -> Result<()> {
"reason": "breakpoint", "reason": "breakpoint",
"description": "Restarted task reached a Wasm probe and every active participant confirmed all-stop", "description": "Restarted task reached a Wasm probe and every active participant confirmed all-stop",
"threadId": stopped_thread, "threadId": stopped_thread,
"allThreadsStopped": true, "allThreadsStopped": state.debug_all_threads_stopped,
}),
)?;
}
Ok(RuntimeContinuationOutcome::Exception(record)) => {
state.apply_runtime_record(record);
let stopped_thread = stopped_thread_for_breakpoint(&state);
writer.event(
"stopped",
json!({
"reason": "exception",
"description": "Replacement attempt failed and is awaiting operator action",
"threadId": stopped_thread,
"allThreadsStopped": false,
}), }),
)?; )?;
} }
@ -712,15 +747,19 @@ fn restart_task_through_coordinator(state: &mut AdapterState, thread_id: i64) ->
)); ));
} }
let restarted_task = record.restarted_task_instance.ok_or_else(|| { let restarted_task = record.restarted_task_instance.ok_or_else(|| {
anyhow::anyhow!("coordinator accepted task restart without a new task-instance ID") anyhow::anyhow!("coordinator accepted task restart without its stable task-instance ID")
})?; })?;
if let Some(thread) = state.threads.get_mut(&thread_id) { if let Some(thread) = state.threads.get_mut(&thread_id) {
thread.task = restarted_task.clone(); thread.task = restarted_task.clone();
thread.attempt_id = record.restarted_attempt_id.clone().ok_or_else(|| {
anyhow::anyhow!("coordinator accepted task restart without a new attempt ID")
})?;
} }
state.debug_probes = state.debug_probes =
crate::breakpoints::load_bundle_debug_probes(&state.project, &state.source_path); crate::breakpoints::load_bundle_debug_probes(&state.project, &state.source_path);
Ok(format!( Ok(format!(
"Coordinator restarted task `{task}` as `{restarted_task}` from the rebuilt bundle and a clean runtime checkpoint boundary" "Coordinator restarted logical task `{restarted_task}` as attempt `{}` from the rebuilt bundle and a clean runtime checkpoint boundary",
record.restarted_attempt_id.as_deref().unwrap_or("unknown")
)) ))
} }
@ -739,6 +778,7 @@ fn record_coordinator_debug_epoch(
.unwrap_or_else(|| state.threads[&default_thread_id(state)].task.clone()); .unwrap_or_else(|| state.threads[&default_thread_id(state)].task.clone());
let record = create_debug_epoch(state, &stopped_task, reason)?; let record = create_debug_epoch(state, &stopped_task, reason)?;
let status = wait_for_debug_epoch_frozen(state, record.epoch)?; let status = wait_for_debug_epoch_frozen(state, record.epoch)?;
state.debug_all_threads_stopped = status.fully_frozen;
state.epoch = record.epoch; state.epoch = record.epoch;
state.coordinator_debug_epoch = Some(record.epoch); state.coordinator_debug_epoch = Some(record.epoch);
let previous_status = state.command_status.clone(); let previous_status = state.command_status.clone();
@ -808,7 +848,7 @@ pub(crate) fn runtime_backend_from_launch_arg(
Some("live-services") => Ok(RuntimeBackend::LiveServices), Some("live-services") => Ok(RuntimeBackend::LiveServices),
Some(value) if is_explicit_demo_backend(value) => Ok(RuntimeBackend::Simulated), Some(value) if is_explicit_demo_backend(value) => Ok(RuntimeBackend::Simulated),
Some(value) => Err(format!( Some(value) => Err(format!(
"unsupported Disasmer runtimeBackend `{value}`; use local-services, live-services, or explicit demo" "unsupported Clusterflux runtimeBackend `{value}`; use local-services, live-services, or explicit demo"
)), )),
} }
} }

View file

@ -1,6 +1,6 @@
use std::fs; use std::fs;
use disasmer_core::{ use clusterflux_core::{
discover_source_debug_probes, BundleDebugProbe, DebugRuntimeState, TaskInstanceId, discover_source_debug_probes, BundleDebugProbe, DebugRuntimeState, TaskInstanceId,
}; };
use serde_json::{json, Value}; use serde_json::{json, Value};
@ -63,11 +63,11 @@ pub(crate) fn resolve_breakpoints(
&& source_function_name_at_line(state, line).is_some()); && source_function_name_at_line(state, line).is_some());
let message = match probe { let message = match probe {
Some(probe) => format!( Some(probe) => format!(
"Mapped to Disasmer debug probe {} for task {}", "Mapped to Clusterflux debug probe {} for task {}",
probe.id, probe.task probe.id, probe.task
), ),
None if verified => "Mapped to Disasmer virtual source location".to_owned(), None if verified => "Mapped to Clusterflux virtual source location".to_owned(),
None => "No Disasmer debug probe metadata covers this source line".to_owned(), None => "No Clusterflux debug probe metadata covers this source line".to_owned(),
}; };
ResolvedBreakpoint { ResolvedBreakpoint {
id: index + 1, id: index + 1,

View file

@ -1,6 +1,6 @@
use std::collections::BTreeMap; use std::collections::BTreeMap;
use disasmer_core::{DebugRuntimeState, TaskInstanceId}; use clusterflux_core::{DebugRuntimeState, TaskInstanceId};
use crate::virtual_model::{AdapterState, VirtualThread}; use crate::virtual_model::{AdapterState, VirtualThread};
@ -42,7 +42,8 @@ fn thread(id: i64, task: &str, name: &str, line: i64) -> VirtualThread {
vfs_ref: 7000 + id, vfs_ref: 7000 + id,
command_ref: 8000 + id, command_ref: 8000 + id,
task: TaskInstanceId::from(task), task: TaskInstanceId::from(task),
task_definition: disasmer_core::TaskDefinitionId::from(task), attempt_id: format!("demo-attempt-{id}"),
task_definition: clusterflux_core::TaskDefinitionId::from(task),
name: name.to_owned(), name: name.to_owned(),
line, line,
state: DebugRuntimeState::Running, state: DebugRuntimeState::Running,
@ -59,5 +60,9 @@ fn thread(id: i64, task: &str, name: &str, line: i64) -> VirtualThread {
runtime_task_args: Vec::new(), runtime_task_args: Vec::new(),
runtime_handles: Vec::new(), runtime_handles: Vec::new(),
runtime_command_status: None, runtime_command_status: None,
runtime_node: Some("demo-node".to_owned()),
runtime_environment: Some("demo".to_owned()),
runtime_vfs_checkpoint: Some("demo-vfs".to_owned()),
restart_compatible: Some(true),
} }
} }

View file

@ -29,9 +29,9 @@ use variables::variables_response;
use virtual_model::{AdapterState, RuntimeLaunchRecord}; use virtual_model::{AdapterState, RuntimeLaunchRecord};
#[cfg(test)] #[cfg(test)]
use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD}; use clusterflux_core::{BundleDebugProbe, DebugRuntimeState, TaskInstanceId};
#[cfg(test)] #[cfg(test)]
use disasmer_core::{BundleDebugProbe, DebugRuntimeState, TaskInstanceId}; use demo_backend::{LINUX_THREAD, MAIN_THREAD, PACKAGE_THREAD, WINDOWS_THREAD};
#[cfg(test)] #[cfg(test)]
use virtual_model::{process_id, RuntimeBackend}; use virtual_model::{process_id, RuntimeBackend};

View file

@ -5,7 +5,7 @@ use std::time::{Duration, Instant};
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _}; use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
use disasmer_core::TaskInstanceId; use clusterflux_core::TaskInstanceId;
use serde_json::{json, Value}; use serde_json::{json, Value};
use crate::virtual_model::{AdapterState, RuntimeLaunchRecord}; use crate::virtual_model::{AdapterState, RuntimeLaunchRecord};
@ -83,6 +83,7 @@ pub(crate) struct DebugEpochStatusRecord {
pub(crate) expected_tasks: usize, pub(crate) expected_tasks: usize,
pub(crate) acknowledgements: Vec<Value>, pub(crate) acknowledgements: Vec<Value>,
pub(crate) fully_frozen: bool, pub(crate) fully_frozen: bool,
pub(crate) partially_frozen: bool,
pub(crate) fully_resumed: bool, pub(crate) fully_resumed: bool,
pub(crate) failed: bool, pub(crate) failed: bool,
pub(crate) failure_messages: Vec<String>, pub(crate) failure_messages: Vec<String>,
@ -92,6 +93,7 @@ pub(crate) struct DebugEpochStatusRecord {
pub(crate) struct TaskRestartRecord { pub(crate) struct TaskRestartRecord {
pub(crate) accepted: bool, pub(crate) accepted: bool,
pub(crate) restarted_task_instance: Option<TaskInstanceId>, pub(crate) restarted_task_instance: Option<TaskInstanceId>,
pub(crate) restarted_attempt_id: Option<String>,
pub(crate) clean_boundary_available: bool, pub(crate) clean_boundary_available: bool,
pub(crate) requires_whole_process_restart: bool, pub(crate) requires_whole_process_restart: bool,
pub(crate) active_task: bool, pub(crate) active_task: bool,
@ -101,6 +103,7 @@ pub(crate) struct TaskRestartRecord {
pub(crate) enum RuntimeContinuationOutcome { pub(crate) enum RuntimeContinuationOutcome {
Breakpoint(RuntimeLaunchRecord), Breakpoint(RuntimeLaunchRecord),
Exception(RuntimeLaunchRecord),
Terminal(RuntimeLaunchRecord), Terminal(RuntimeLaunchRecord),
} }
@ -109,9 +112,9 @@ pub(crate) fn run_local_services_runtime(
) -> Result<(RuntimeLaunchRecord, LocalRuntimeSession)> { ) -> Result<(RuntimeLaunchRecord, LocalRuntimeSession)> {
let repo = std::env::current_dir()?; let repo = std::env::current_dir()?;
let mut coordinator_command = local_tool_command( let mut coordinator_command = local_tool_command(
"DISASMER_COORDINATOR_BIN", "CLUSTERFLUX_COORDINATOR_BIN",
"disasmer-coordinator", "clusterflux-coordinator",
"disasmer-coordinator", "clusterflux-coordinator",
&repo, &repo,
); );
let mut coordinator = coordinator_command let mut coordinator = coordinator_command
@ -159,8 +162,12 @@ pub(crate) fn run_local_services_runtime(
.ok_or_else(|| anyhow!("local coordinator omitted the node enrollment grant"))? .ok_or_else(|| anyhow!("local coordinator omitted the node enrollment grant"))?
.to_owned(); .to_owned();
let mut worker_command = let mut worker_command = local_tool_command(
local_tool_command("DISASMER_NODE_BIN", "disasmer-node", "disasmer-node", &repo); "CLUSTERFLUX_NODE_BIN",
"clusterflux-node",
"clusterflux-node",
&repo,
);
let mut worker = ChildGuard::new( let mut worker = ChildGuard::new(
worker_command worker_command
.args([ .args([
@ -207,14 +214,19 @@ pub(crate) fn run_local_services_runtime(
} }
fn build_debug_bundle(state: &AdapterState, repo: &Path) -> Result<DebugBundle> { fn build_debug_bundle(state: &AdapterState, repo: &Path) -> Result<DebugBundle> {
let mut command = local_tool_command("DISASMER_CLI_BIN", "disasmer", "disasmer-cli", repo); let mut command = local_tool_command(
"CLUSTERFLUX_CLI_BIN",
"clusterflux",
"clusterflux-cli",
repo,
);
let output = command let output = command
.args(["build", "--project", &state.project, "--json"]) .args(["build", "--project", &state.project, "--json"])
.current_dir(repo) .current_dir(repo)
.output()?; .output()?;
if !output.status.success() { if !output.status.success() {
return Err(anyhow!( return Err(anyhow!(
"Disasmer bundle build failed before debug launch: {}", "Clusterflux bundle build failed before debug launch: {}",
String::from_utf8_lossy(&output.stderr).trim() String::from_utf8_lossy(&output.stderr).trim()
)); ));
} }
@ -298,7 +310,7 @@ fn launch_services_debug_entrypoint(
let probe_symbols = state.requested_probe_symbols(); let probe_symbols = state.requested_probe_symbols();
if probe_symbols.is_empty() { if probe_symbols.is_empty() {
return Err(anyhow!( return Err(anyhow!(
"no executable Disasmer probe corresponds to the configured source breakpoints" "no executable Clusterflux probe corresponds to the configured source breakpoints"
)); ));
} }
coordinator_request( coordinator_request(
@ -364,6 +376,14 @@ fn launch_services_debug_entrypoint(
.and_then(Value::as_u64) .and_then(Value::as_u64)
.ok_or_else(|| anyhow!("breakpoint status omitted the hit debug epoch"))?; .ok_or_else(|| anyhow!("breakpoint status omitted the hit debug epoch"))?;
let frozen = wait_for_debug_epoch_state_at(coordinator, state, debug_epoch, true)?; let frozen = wait_for_debug_epoch_state_at(coordinator, state, debug_epoch, true)?;
let fully_frozen = frozen
.get("fully_frozen")
.and_then(Value::as_bool)
.unwrap_or(false);
let partially_frozen = frozen
.get("partially_frozen")
.and_then(Value::as_bool)
.unwrap_or(false);
let expected = frozen let expected = frozen
.get("expected_tasks") .get("expected_tasks")
.and_then(Value::as_array) .and_then(Value::as_array)
@ -374,11 +394,12 @@ fn launch_services_debug_entrypoint(
.and_then(Value::as_array) .and_then(Value::as_array)
.map(Vec::len) .map(Vec::len)
.unwrap_or(0); .unwrap_or(0);
if expected == 0 || acknowledged != expected { if expected == 0 || (!fully_frozen && !partially_frozen) {
return Err(anyhow!( return Err(anyhow!(
"debug epoch {debug_epoch} claimed frozen without every active participant" "debug epoch {debug_epoch} settled without a frozen participant"
)); ));
} }
let task_snapshots = fetch_task_snapshots(coordinator, state)?;
let process_statuses = coordinator_request( let process_statuses = coordinator_request(
coordinator, coordinator,
client_user_request( client_user_request(
@ -418,6 +439,7 @@ fn launch_services_debug_entrypoint(
"debug_epoch": frozen, "debug_epoch": frozen,
"process_status": process_status, "process_status": process_status,
"process_statuses": process_statuses, "process_statuses": process_statuses,
"task_snapshots": task_snapshots,
}), }),
task_events: json!({ "events": [] }), task_events: json!({ "events": [] }),
placed_task_launched: true, placed_task_launched: true,
@ -439,7 +461,7 @@ fn launch_services_debug_entrypoint(
.get("hit_probe_symbol") .get("hit_probe_symbol")
.and_then(Value::as_str) .and_then(Value::as_str)
.map(str::to_owned), .map(str::to_owned),
all_participants_frozen: true, all_participants_frozen: fully_frozen && acknowledged == expected,
}) })
} }
@ -453,7 +475,7 @@ fn wait_for_local_node(
loop { loop {
if let Some(status) = worker.try_wait()? { if let Some(status) = worker.try_wait()? {
return Err(anyhow!( return Err(anyhow!(
"local Disasmer worker `{node}` exited before attaching ({status}){}", "local Clusterflux worker `{node}` exited before attaching ({status}){}",
child_stderr_suffix(worker) child_stderr_suffix(worker)
)); ));
} }
@ -481,7 +503,7 @@ fn wait_for_local_node(
let _ = worker.kill(); let _ = worker.kill();
let _ = worker.wait(); let _ = worker.wait();
return Err(anyhow!( return Err(anyhow!(
"local Disasmer worker `{node}` did not attach within 30 seconds{}", "local Clusterflux worker `{node}` did not attach within 30 seconds{}",
child_stderr_suffix(worker) child_stderr_suffix(worker)
)); ));
} }
@ -539,6 +561,17 @@ fn wait_for_debug_epoch_state_at(
}), }),
), ),
)?; )?;
let partially_frozen = frozen
&& response
.get("partially_frozen")
.and_then(Value::as_bool)
.unwrap_or(false);
if response.get("fully_frozen").and_then(Value::as_bool) == Some(true)
|| partially_frozen
|| (!frozen && response.get("fully_resumed").and_then(Value::as_bool) == Some(true))
{
return Ok(response);
}
if response.get("failed").and_then(Value::as_bool) == Some(true) { if response.get("failed").and_then(Value::as_bool) == Some(true) {
return Err(anyhow!( return Err(anyhow!(
"debug epoch {epoch} participant failed: {}", "debug epoch {epoch} participant failed: {}",
@ -557,9 +590,6 @@ fn wait_for_debug_epoch_state_at(
} else { } else {
"fully_resumed" "fully_resumed"
}; };
if response.get(ready_field).and_then(Value::as_bool) == Some(true) {
return Ok(response);
}
if Instant::now() >= deadline { if Instant::now() >= deadline {
return Err(anyhow!( return Err(anyhow!(
"debug epoch {epoch} did not reach {ready_field} within 60 seconds" "debug epoch {epoch} did not reach {ready_field} within 60 seconds"
@ -576,6 +606,50 @@ pub(crate) fn run_live_services_runtime(state: &AdapterState) -> Result<RuntimeL
launch_services_debug_entrypoint(&coordinator, state, &repo) launch_services_debug_entrypoint(&coordinator, state, &repo)
} }
fn fetch_task_snapshots(coordinator: &str, state: &AdapterState) -> Result<Value> {
coordinator_request(
coordinator,
client_user_request(
state,
json!({
"type": "list_task_snapshots",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
"process": state.process.as_str(),
}),
),
)
}
fn fetch_current_process_status(
coordinator: &str,
state: &AdapterState,
) -> Result<(Value, Option<Value>)> {
let statuses = coordinator_request(
coordinator,
client_user_request(
state,
json!({
"type": "list_processes",
"tenant": state.tenant,
"project": state.project_id,
"actor_user": state.actor_user,
}),
),
)?;
let current = statuses
.get("processes")
.and_then(Value::as_array)
.and_then(|processes| {
processes.iter().find(|process| {
process.get("process").and_then(Value::as_str) == Some(state.process.as_str())
})
})
.cloned();
Ok((statuses, current))
}
pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> { pub(crate) fn relaunch_services_main_runtime(state: &AdapterState) -> Result<RuntimeLaunchRecord> {
let coordinator = let coordinator =
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint); crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
@ -632,6 +706,23 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result<RuntimeLau
.and_then(Value::as_array) .and_then(Value::as_array)
.map(Vec::len) .map(Vec::len)
.unwrap_or(0); .unwrap_or(0);
let task_snapshots = fetch_task_snapshots(&coordinator, state)?;
let active_snapshot_count = task_snapshots
.get("snapshots")
.and_then(Value::as_array)
.map(|snapshots| {
snapshots
.iter()
.filter(|snapshot| {
snapshot.get("current").and_then(Value::as_bool) == Some(true)
&& matches!(
snapshot.get("state").and_then(Value::as_str),
Some("queued" | "running" | "failed_awaiting_action")
)
})
.count()
})
.unwrap_or(0);
let process_statuses = coordinator_request( let process_statuses = coordinator_request(
&coordinator, &coordinator,
client_user_request( client_user_request(
@ -674,9 +765,10 @@ pub(crate) fn attach_services_runtime(state: &AdapterState) -> Result<RuntimeLau
"debug_attach": debug_attach, "debug_attach": debug_attach,
"process_status": process_status, "process_status": process_status,
"process_statuses": process_statuses, "process_statuses": process_statuses,
"task_snapshots": task_snapshots,
}), }),
task_events: events, task_events: events,
placed_task_launched: active_main || event_count > 0, placed_task_launched: active_main || active_snapshot_count > 0,
status_code: None, status_code: None,
stdout_bytes: 0, stdout_bytes: 0,
stderr_bytes: 0, stderr_bytes: 0,
@ -775,6 +867,85 @@ pub(crate) fn wait_for_services_runtime_outcome(
if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) { if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
return Ok(outcome); return Ok(outcome);
} }
let task_snapshots = fetch_task_snapshots(&coordinator, state)?;
if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
let failed_task = failed_task.to_owned();
let failed_event = events
.get("events")
.and_then(Value::as_array)
.into_iter()
.flatten()
.rev()
.find(|event| {
event.get("task").and_then(Value::as_str) == Some(failed_task.as_str())
&& event.get("terminal_state").and_then(Value::as_str) == Some("failed")
})
.cloned()
.unwrap_or_else(|| json!({}));
let event_count = events
.get("events")
.and_then(Value::as_array)
.map(Vec::len)
.unwrap_or(state.runtime_event_count);
let (process_statuses, process_status) =
fetch_current_process_status(&coordinator, state)?;
return Ok(RuntimeContinuationOutcome::Exception(RuntimeLaunchRecord {
coordinator,
node: failed_event
.get("node")
.and_then(Value::as_str)
.unwrap_or("unknown")
.to_owned(),
node_report: json!({
"task_snapshots": task_snapshots,
"process_status": process_status,
"process_statuses": process_statuses,
"failed_awaiting_action": failed_task,
}),
task_events: events,
placed_task_launched: true,
status_code: failed_event
.get("status_code")
.and_then(Value::as_i64)
.map(|status| status as i32)
.or(Some(1)),
stdout_bytes: failed_event
.get("stdout_bytes")
.and_then(Value::as_u64)
.unwrap_or(0),
stderr_bytes: failed_event
.get("stderr_bytes")
.and_then(Value::as_u64)
.unwrap_or(0),
stdout_tail: failed_event
.get("stdout_tail")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned(),
stderr_tail: failed_event
.get("stderr_tail")
.and_then(Value::as_str)
.unwrap_or("task failed awaiting operator action")
.to_owned(),
stdout_truncated: failed_event
.get("stdout_truncated")
.and_then(Value::as_bool)
.unwrap_or(false),
stderr_truncated: failed_event
.get("stderr_truncated")
.and_then(Value::as_bool)
.unwrap_or(false),
artifact_path: failed_event
.get("artifact_path")
.and_then(Value::as_str)
.map(str::to_owned),
event_count,
debug_epoch: None,
stopped_task: Some(failed_task),
stopped_probe_symbol: None,
all_participants_frozen: false,
}));
}
let breakpoint = match coordinator_request( let breakpoint = match coordinator_request(
&coordinator, &coordinator,
@ -826,6 +997,13 @@ pub(crate) fn wait_for_services_runtime_outcome(
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or("unknown") .unwrap_or("unknown")
.to_owned(); .to_owned();
let task_snapshots = fetch_task_snapshots(&coordinator, state)?;
let (process_statuses, process_status) =
fetch_current_process_status(&coordinator, state)?;
let fully_frozen = frozen
.get("fully_frozen")
.and_then(Value::as_bool)
.unwrap_or(false);
return Ok(RuntimeContinuationOutcome::Breakpoint( return Ok(RuntimeContinuationOutcome::Breakpoint(
RuntimeLaunchRecord { RuntimeLaunchRecord {
coordinator, coordinator,
@ -833,6 +1011,9 @@ pub(crate) fn wait_for_services_runtime_outcome(
node_report: json!({ node_report: json!({
"breakpoint": breakpoint, "breakpoint": breakpoint,
"debug_epoch": frozen, "debug_epoch": frozen,
"task_snapshots": task_snapshots,
"process_status": process_status,
"process_statuses": process_statuses,
}), }),
task_events: json!({ "events": [] }), task_events: json!({ "events": [] }),
placed_task_launched: true, placed_task_launched: true,
@ -854,7 +1035,7 @@ pub(crate) fn wait_for_services_runtime_outcome(
.get("hit_probe_symbol") .get("hit_probe_symbol")
.and_then(Value::as_str) .and_then(Value::as_str)
.map(str::to_owned), .map(str::to_owned),
all_participants_frozen: true, all_participants_frozen: fully_frozen,
}, },
)); ));
} }
@ -870,6 +1051,23 @@ pub(crate) fn wait_for_services_runtime_outcome(
} }
} }
pub(crate) fn failed_awaiting_action_snapshot(task_snapshots: &Value) -> Option<(&str, &str)> {
task_snapshots
.get("snapshots")?
.as_array()?
.iter()
.find(|snapshot| {
snapshot.get("current").and_then(Value::as_bool) == Some(true)
&& snapshot.get("state").and_then(Value::as_str) == Some("failed_awaiting_action")
})
.and_then(|snapshot| {
Some((
snapshot.get("task")?.as_str()?,
snapshot.get("attempt_id")?.as_str()?,
))
})
}
fn terminal_runtime_outcome( fn terminal_runtime_outcome(
coordinator: &str, coordinator: &str,
state: &AdapterState, state: &AdapterState,

Some files were not shown because too many files have changed in this diff Show more