michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-e47f9c27bbeb

Browse source 
Source commit: e47f9c27bbebe6759f6b6fe7b12fd00bb20d116d

Public tree identity: sha256:4c1af1dfd67d3f2b5088531ea8727b11124b013d2251a4d5088f51a6424c0196
This commit is contained in:
Clusterflux release 2026-07-24 15:52:30 +02:00
parent 3a4d4fa7ae
commit 9223c54939
30 changed files with 4195 additions and 434 deletions
Download patch file Download diff file Expand all files Collapse all files

41
crates/clusterflux-dap/src/adapter.rs
View file

@ -64,6 +64,7 @@ struct LaunchCompletion {
state: AdapterState,
local_runtime_session: Option<LocalRuntimeSession>,
breakpoint_revision: u64,
observation_diagnostics: Vec<String>,
}
pub(crate) fn run_adapter() -> Result<()> {
@ -108,6 +109,9 @@ pub(crate) fn run_adapter() -> Result<()> {
if let Some(session) = completion.local_runtime_session {
_local_runtime_session = Some(session);
}
for diagnostic in completion.observation_diagnostics {
writer.output("stderr", format!("{diagnostic}\n"))?;
}
runtime_started = true;
if state.breakpoints_installed {
emit_verified_breakpoints(&mut writer, &state)?;
@ -252,7 +256,12 @@ pub(crate) fn run_adapter() -> Result<()> {
revision,
result,
} => {
if generation == runtime_generation && revision == state.breakpoint_revision {
if breakpoint_update_is_current(
generation,
runtime_generation,
revision,
state.breakpoint_revision,
) {
match result {
Ok(()) => {
state.breakpoints_installed = true;
@ -478,6 +487,7 @@ pub(crate) fn run_adapter() -> Result<()> {
state: completed_state,
local_runtime_session: None,
breakpoint_revision,
observation_diagnostics: Vec::new(),
}
})
.map_err(|error| format!("services runtime attach failed: {error:#}"))
@ -486,6 +496,8 @@ pub(crate) fn run_adapter() -> Result<()> {
RuntimeBackend::LocalServices => {
run_local_services_runtime(&completed_state)
.map(|(record, session)| {
let observation_diagnostics =
runtime_observation_diagnostics(&record);
completed_state.apply_runtime_record(record);
let breakpoint_revision =
completed_state.breakpoint_revision;
@ -493,6 +505,7 @@ pub(crate) fn run_adapter() -> Result<()> {
state: completed_state,
local_runtime_session: Some(session),
breakpoint_revision,
observation_diagnostics,
}
})
.map_err(|error| {
@ -502,6 +515,8 @@ pub(crate) fn run_adapter() -> Result<()> {
RuntimeBackend::LiveServices => {
run_live_services_runtime(&completed_state)
.map(|record| {
let observation_diagnostics =
runtime_observation_diagnostics(&record);
completed_state.apply_runtime_record(record);
let breakpoint_revision =
completed_state.breakpoint_revision;
@ -509,6 +524,7 @@ pub(crate) fn run_adapter() -> Result<()> {
state: completed_state,
local_runtime_session: None,
breakpoint_revision,
observation_diagnostics,
}
})
.map_err(|error| {
@ -1051,6 +1067,29 @@ fn spawn_runtime_observer(
});
}
pub(crate) fn breakpoint_update_is_current(
update_generation: u64,
runtime_generation: u64,
update_revision: u64,
current_revision: u64,
) -> bool {
update_generation == runtime_generation && update_revision == current_revision
}
fn runtime_observation_diagnostics(
record: &crate::virtual_model::RuntimeLaunchRecord,
) -> Vec<String> {
record
.node_report
.get("observation_diagnostics")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::to_owned)
.collect()
}
fn emit_verified_breakpoints(writer: &mut DapWriter, state: &AdapterState) -> Result<()> {
if !state.breakpoints_installed {
return Ok(());

135
crates/clusterflux-dap/src/runtime_client.rs
View file

@ -539,7 +539,24 @@ fn new_launch_attempt_id() -> String {
fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) -> Result<()> {
static INJECTED_INSTALL_FAILURE: AtomicBool = AtomicBool::new(false);
if std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some()
if std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_REVISION")
.ok()
.and_then(|value| value.parse::<u64>().ok())
== Some(state.breakpoint_revision)
{
let delay_ms = std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_DELAY_MS")
.ok()
.and_then(|value| value.parse::<u64>().ok())
.unwrap_or(750);
std::thread::sleep(Duration::from_millis(delay_ms));
}
let revision_failure =
std::env::var("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE_REVISION")
.ok()
.and_then(|value| value.parse::<u64>().ok())
== Some(state.breakpoint_revision);
if (revision_failure
|| std::env::var_os("CLUSTERFLUX_TEST_DAP_BREAKPOINT_INSTALLATION_FAILURE").is_some())
&& !state.breakpoints.is_empty()
&& !INJECTED_INSTALL_FAILURE.swap(true, Ordering::SeqCst)
{
@ -557,6 +574,7 @@ fn set_services_debug_breakpoints_at(coordinator: &str, state: &AdapterState) ->
"project": state.project_id,
"actor_user": state.actor_user,
"process": state.process.as_str(),
"revision": state.breakpoint_revision,
"probe_symbols": state.requested_probe_symbols(),
}),
),
@ -1026,6 +1044,12 @@ pub(crate) fn observe_services_runtime(
let inject_connection_loss =
std::env::var("CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS").as_deref() == Ok("1");
let mut connection_loss_injected = false;
let inject_snapshot_failure =
std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_SNAPSHOT_FAILURE").is_some();
let mut snapshot_failure_injected = false;
let inject_process_status_failure =
std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_PROCESS_STATUS_FAILURE").is_some();
let mut process_status_failure_injected = false;
let inject_fallback_failure =
std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE").is_some();
let mut fallback_failure_stage = 0_u8;
@ -1095,11 +1119,49 @@ pub(crate) fn observe_services_runtime(
continue;
}
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
let task_snapshots = fetch_task_snapshots_in(current_session, state)
.unwrap_or_else(|_| json!({ "snapshots": [] }));
let (process_statuses, process_status) =
fetch_current_process_status_in(current_session, state)
.unwrap_or_else(|_| (json!({ "processes": [] }), None));
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
snapshot_failure_injected = true;
Err(anyhow!("injected task snapshot transport failure"))
} else {
fetch_task_snapshots_in(current_session, state)
};
let task_snapshots = match snapshot_request {
Ok(snapshots) => snapshots,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime terminal snapshot observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
let process_status_request =
if inject_process_status_failure && !process_status_failure_injected {
process_status_failure_injected = true;
Err(anyhow!("injected process-status transport failure"))
} else {
fetch_current_process_status_in(current_session, state)
};
let (process_statuses, process_status) = match process_status_request {
Ok(status) => status,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime terminal process observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
if !has_current_runtime_task(&task_snapshots) {
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
record.node_report = json!({
@ -1113,7 +1175,13 @@ pub(crate) fn observe_services_runtime(
return Ok(());
}
}
let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
let snapshot_request = if inject_snapshot_failure && !snapshot_failure_injected {
snapshot_failure_injected = true;
Err(anyhow!("injected task snapshot transport failure"))
} else {
fetch_task_snapshots_in(current_session, state)
};
let task_snapshots = match snapshot_request {
Ok(snapshots) => snapshots,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
@ -1128,22 +1196,28 @@ pub(crate) fn observe_services_runtime(
continue;
}
};
let (process_statuses, process_status) =
match fetch_current_process_status_in(current_session, state) {
Ok(status) => status,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime process observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
let process_status_request =
if inject_process_status_failure && !process_status_failure_injected {
process_status_failure_injected = true;
Err(anyhow!("injected process-status transport failure"))
} else {
fetch_current_process_status_in(current_session, state)
};
let (process_statuses, process_status) = match process_status_request {
Ok(status) => status,
Err(error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime process observation failed: {error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
reconnect_delay = Duration::from_millis(100);
if let Some((failed_task, _attempt_id)) = failed_awaiting_action_snapshot(&task_snapshots) {
let failed_task = failed_task.to_owned();
@ -1277,8 +1351,21 @@ pub(crate) fn observe_services_runtime(
}
};
if let Some(mut outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
let task_snapshots = fetch_task_snapshots_in(current_session, state)
.unwrap_or_else(|_| json!({ "snapshots": [] }));
let task_snapshots = match fetch_task_snapshots_in(current_session, state) {
Ok(snapshots) => snapshots,
Err(snapshot_error) => {
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
"runtime fallback terminal snapshot observation failed: {snapshot_error:#}; reconnecting in {} ms",
reconnect_delay.as_millis()
))) {
return Ok(());
}
session = None;
std::thread::sleep(reconnect_delay);
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
continue;
}
};
if !has_current_runtime_task(&task_snapshots) {
if let RuntimeContinuationOutcome::Terminal(record) = &mut outcome {
record.node_report = json!({

4
crates/clusterflux-dap/src/runtime_client/debug_protocol.rs
View file

@ -173,7 +173,9 @@ pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestart
restarted_task_instance: response
.get("restarted_task_instance")
.and_then(Value::as_str)
.map(TaskInstanceId::new),
.map(TaskInstanceId::try_new)
.transpose()
.map_err(|error| anyhow!("malformed restarted task instance: {error}"))?,
restarted_attempt_id: response
.get("restarted_attempt_id")
.and_then(Value::as_str)

7
crates/clusterflux-dap/src/tests.rs
View file

@ -45,6 +45,13 @@ fn initialize_capabilities_use_standard_dap_flags() {
.all(|key| key.starts_with("supports") && !key.contains("clusterflux")));
}
#[test]
fn stale_breakpoint_success_and_failure_completions_are_not_current() {
assert!(crate::adapter::breakpoint_update_is_current(4, 4, 9, 9));
assert!(!crate::adapter::breakpoint_update_is_current(4, 4, 8, 9));
assert!(!crate::adapter::breakpoint_update_is_current(3, 4, 9, 9));
}
#[test]
fn runtime_backend_defaults_to_local_services_and_requires_explicit_demo() {
assert_eq!(

81
crates/clusterflux-dap/src/virtual_model.rs
View file

@ -313,7 +313,10 @@ impl AdapterState {
self.debug_all_threads_stopped = record.all_participants_frozen;
self.epoch = record.debug_epoch.unwrap_or(self.epoch);
self.coordinator_debug_epoch = record.debug_epoch;
self.stopped_task = record.stopped_task.as_deref().map(TaskInstanceId::new);
self.stopped_task = record
.stopped_task
.as_deref()
.and_then(|task| TaskInstanceId::try_new(task).ok());
self.stopped_probe_symbol = record.stopped_probe_symbol.clone();
if let Some(acknowledgements) = record
.node_report
@ -330,6 +333,12 @@ impl AdapterState {
else {
continue;
};
let Ok(task_id) = TaskInstanceId::try_new(task) else {
continue;
};
let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
continue;
};
let coordinator_main = acknowledgement.get("node").and_then(Value::as_str)
== Some("coordinator-main");
let thread_id = if coordinator_main {
@ -339,14 +348,19 @@ impl AdapterState {
.values()
.find(|thread| thread.task.as_str() == task)
.map(|thread| thread.id)
.unwrap_or_else(|| self.insert_runtime_thread(task, task_definition))
.unwrap_or_else(|| {
self.insert_runtime_thread(
task_id.clone(),
task_definition_id.clone(),
)
})
};
let thread = self
.threads
.get_mut(&thread_id)
.expect("runtime thread was found or inserted");
thread.task = TaskInstanceId::new(task);
thread.task_definition = TaskDefinitionId::new(task_definition);
thread.task = task_id;
thread.task_definition = task_definition_id;
thread.runtime_stack_frames = acknowledgement
.get("stack_frames")
.and_then(Value::as_array)
@ -428,18 +442,22 @@ impl AdapterState {
let _ = crate::view_state::write_view_state(self, &record);
}
fn insert_runtime_thread(&mut self, task: &str, task_definition: &str) -> i64 {
fn insert_runtime_thread(
&mut self,
task: TaskInstanceId,
task_definition: TaskDefinitionId,
) -> i64 {
let id = self.allocate_runtime_thread_id();
let line = self
.debug_probes
.iter()
.find(|probe| {
probe.task.as_str() == task_definition || probe.function == task_definition
probe.task == task_definition || probe.function == task_definition.as_str()
})
.map(|probe| i64::from(probe.line_start))
.unwrap_or(1);
let definition_name = task_definition.replace(['_', '-'], " ");
let name = if task == task_definition {
let definition_name = task_definition.as_str().replace(['_', '-'], " ");
let name = if task.as_str() == task_definition.as_str() {
definition_name
} else {
format!("{definition_name} ({task})")
@ -457,9 +475,9 @@ impl AdapterState {
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::new(task),
task,
attempt_id: "unknown-attempt".to_owned(),
task_definition: TaskDefinitionId::new(task_definition),
task_definition,
name,
line,
state: DebugRuntimeState::Running,
@ -725,7 +743,12 @@ pub(crate) fn process_id(project: &str, entry: &str) -> ProcessId {
ProcessId::new(format!("vp-{suffix}"))
}
fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> VirtualThread {
fn coordinator_main_thread(
entry: &str,
task: TaskInstanceId,
task_definition: TaskDefinitionId,
) -> VirtualThread {
let name = format!("{entry} coordinator main ({task})");
VirtualThread {
id: MAIN_THREAD,
frame_id: 1_001,
@ -737,10 +760,10 @@ fn coordinator_main_thread(entry: &str, task: &str, task_definition: &str) -> Vi
target_ref: 4_501,
vfs_ref: 5_001,
command_ref: 5_501,
task: TaskInstanceId::from(task),
task,
attempt_id: "main:unknown".to_owned(),
task_definition: TaskDefinitionId::from(task_definition),
name: format!("{entry} coordinator main ({task})"),
task_definition,
name,
line: 0,
state: DebugRuntimeState::Running,
freeze_supported: true,
@ -776,10 +799,14 @@ fn coordinator_threads_from_events(
.expect("launch thread model should include main");
if let Some(status) = process_status {
if let Some(task) = status.get("main_task_instance").and_then(Value::as_str) {
main.task = TaskInstanceId::from(task);
if let Ok(task) = TaskInstanceId::try_new(task) {
main.task = task;
}
}
if let Some(definition) = status.get("main_task_definition").and_then(Value::as_str) {
main.task_definition = TaskDefinitionId::from(definition);
if let Ok(definition) = TaskDefinitionId::try_new(definition) {
main.task_definition = definition;
}
}
main.name = format!("{entry} coordinator main ({})", main.task);
main.state = if status
@ -841,6 +868,12 @@ fn coordinator_threads_from_snapshots(
.get("main_task_definition")
.and_then(Value::as_str)
.unwrap_or(entry);
let (Ok(task), Ok(task_definition)) = (
TaskInstanceId::try_new(task),
TaskDefinitionId::try_new(task_definition),
) else {
return threads;
};
let mut main = coordinator_main_thread(entry, task, task_definition);
main.attempt_id = format!(
"main:{}",
@ -895,6 +928,12 @@ fn coordinator_threads_from_snapshots(
let Some(task_definition) = snapshot.get("task_definition").and_then(Value::as_str) else {
continue;
};
let Ok(task_id) = TaskInstanceId::try_new(task) else {
continue;
};
let Ok(task_definition_id) = TaskDefinitionId::try_new(task_definition) else {
continue;
};
let attempt_id = snapshot
.get("attempt_id")
.and_then(Value::as_str)
@ -945,9 +984,9 @@ fn coordinator_threads_from_snapshots(
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::from(task),
task: task_id,
attempt_id,
task_definition: TaskDefinitionId::from(task_definition),
task_definition: task_definition_id,
name: format!("{display_name} ({task}, attempt {short_attempt})"),
line: snapshot
.get("source_line")
@ -994,6 +1033,8 @@ fn coordinator_threads_from_snapshots(
fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Option<VirtualThread> {
let task = event.get("task").and_then(Value::as_str)?;
let task_definition = event.get("task_definition").and_then(Value::as_str)?;
let task_id = TaskInstanceId::try_new(task).ok()?;
let task_definition_id = TaskDefinitionId::try_new(task_definition).ok()?;
let definition_name = task_definition.replace(['_', '-'], " ");
let name = if task == task_definition {
definition_name
@ -1039,13 +1080,13 @@ fn coordinator_event_thread(id: i64, _event_index: usize, event: &Value) -> Opti
target_ref: 6000 + id,
vfs_ref: 7000 + id,
command_ref: 8000 + id,
task: TaskInstanceId::from(task),
task: task_id,
attempt_id: event
.get("attempt_id")
.and_then(Value::as_str)
.unwrap_or("unknown-attempt")
.to_owned(),
task_definition: TaskDefinitionId::from(task_definition),
task_definition: task_definition_id,
name,
line: 0,
state,