243 lines
8 KiB
Rust
243 lines
8 KiB
Rust
use std::time::{Duration, Instant};
|
|
|
|
use anyhow::{anyhow, Result};
|
|
use clusterflux_core::TaskInstanceId;
|
|
use serde_json::{json, Value};
|
|
|
|
use crate::virtual_model::AdapterState;
|
|
|
|
use super::{
|
|
client_user_request, coordinator_request, DebugEpochRecord, DebugEpochStatusRecord,
|
|
TaskRestartRecord,
|
|
};
|
|
|
|
pub(super) fn coordinator_debug_epoch_request(
|
|
state: &AdapterState,
|
|
payload: Value,
|
|
) -> Result<Value> {
|
|
let coordinator =
|
|
crate::view_state::normalize_coordinator_endpoint(&state.coordinator_endpoint);
|
|
coordinator_request(&coordinator, payload)
|
|
}
|
|
|
|
pub(super) fn parse_debug_epoch_response(response: Value) -> Result<DebugEpochRecord> {
|
|
let epoch = response
|
|
.get("epoch")
|
|
.and_then(Value::as_u64)
|
|
.ok_or_else(|| anyhow!("coordinator debug epoch response did not include an epoch"))?;
|
|
let command = response
|
|
.get("command")
|
|
.and_then(Value::as_str)
|
|
.ok_or_else(|| anyhow!("coordinator debug epoch response did not include a command"))?
|
|
.to_owned();
|
|
let affected_tasks = response
|
|
.get("affected_tasks")
|
|
.and_then(Value::as_array)
|
|
.map(Vec::len)
|
|
.unwrap_or(0);
|
|
Ok(DebugEpochRecord {
|
|
epoch,
|
|
command,
|
|
affected_tasks,
|
|
})
|
|
}
|
|
|
|
pub(super) fn wait_for_debug_epoch_state(
|
|
state: &AdapterState,
|
|
epoch: u64,
|
|
frozen: bool,
|
|
) -> Result<DebugEpochStatusRecord> {
|
|
let deadline = Instant::now() + Duration::from_secs(60);
|
|
loop {
|
|
let response = match coordinator_debug_epoch_request(
|
|
state,
|
|
client_user_request(
|
|
state,
|
|
json!({
|
|
"type": "inspect_debug_epoch",
|
|
"tenant": state.tenant,
|
|
"project": state.project_id,
|
|
"actor_user": state.actor_user,
|
|
"process": state.process.as_str(),
|
|
"epoch": epoch,
|
|
}),
|
|
),
|
|
) {
|
|
Ok(response) => response,
|
|
Err(error) if !frozen && debug_epoch_was_released(&error, state, epoch) => {
|
|
return Ok(DebugEpochStatusRecord {
|
|
epoch,
|
|
command: "resume".to_owned(),
|
|
expected_tasks: 0,
|
|
acknowledgements: Vec::new(),
|
|
fully_frozen: false,
|
|
partially_frozen: false,
|
|
fully_resumed: true,
|
|
failed: false,
|
|
failure_messages: Vec::new(),
|
|
});
|
|
}
|
|
Err(error) => return Err(error),
|
|
};
|
|
let status = parse_debug_epoch_status(response)?;
|
|
if frozen && (status.fully_frozen || status.partially_frozen) {
|
|
return Ok(status);
|
|
}
|
|
if status.failed {
|
|
return Err(anyhow!(
|
|
"debug epoch {epoch} participant failed: {}",
|
|
status.failure_messages.join("; ")
|
|
));
|
|
}
|
|
if !frozen && status.fully_resumed {
|
|
return Ok(status);
|
|
}
|
|
if Instant::now() >= deadline {
|
|
return Err(anyhow!(
|
|
"debug epoch {epoch} did not receive {}/{} signed participant acknowledgements for {} within 60 seconds",
|
|
status.acknowledgements.len(),
|
|
status.expected_tasks,
|
|
if frozen { "frozen state" } else { "resumed state" }
|
|
));
|
|
}
|
|
std::thread::sleep(Duration::from_millis(100));
|
|
}
|
|
}
|
|
|
|
fn debug_epoch_was_released(error: &anyhow::Error, state: &AdapterState, epoch: u64) -> bool {
|
|
format!("{error:#}").contains(&format!(
|
|
"debug epoch {epoch} is not active for {}",
|
|
state.process
|
|
))
|
|
}
|
|
|
|
fn parse_debug_epoch_status(response: Value) -> Result<DebugEpochStatusRecord> {
|
|
let epoch = response
|
|
.get("epoch")
|
|
.and_then(Value::as_u64)
|
|
.ok_or_else(|| anyhow!("coordinator debug epoch status omitted epoch"))?;
|
|
let command = response
|
|
.get("command")
|
|
.and_then(Value::as_str)
|
|
.ok_or_else(|| anyhow!("coordinator debug epoch status omitted command"))?
|
|
.to_owned();
|
|
let expected_tasks = response
|
|
.get("expected_tasks")
|
|
.and_then(Value::as_array)
|
|
.map(Vec::len)
|
|
.unwrap_or(0);
|
|
let acknowledgements = response
|
|
.get("acknowledgements")
|
|
.and_then(Value::as_array)
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
let failure_messages = response
|
|
.get("failure_messages")
|
|
.and_then(Value::as_array)
|
|
.into_iter()
|
|
.flatten()
|
|
.filter_map(Value::as_str)
|
|
.map(str::to_owned)
|
|
.collect();
|
|
Ok(DebugEpochStatusRecord {
|
|
epoch,
|
|
command,
|
|
expected_tasks,
|
|
acknowledgements,
|
|
fully_frozen: response
|
|
.get("fully_frozen")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(false),
|
|
partially_frozen: response
|
|
.get("partially_frozen")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(false),
|
|
fully_resumed: response
|
|
.get("fully_resumed")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(false),
|
|
failed: response
|
|
.get("failed")
|
|
.and_then(Value::as_bool)
|
|
.unwrap_or(false),
|
|
failure_messages,
|
|
})
|
|
}
|
|
|
|
pub(crate) fn parse_task_restart_response(response: Value) -> Result<TaskRestartRecord> {
|
|
Ok(TaskRestartRecord {
|
|
accepted: response
|
|
.get("accepted")
|
|
.and_then(Value::as_bool)
|
|
.ok_or_else(|| anyhow!("coordinator task restart response did not include accepted"))?,
|
|
restarted_task_instance: response
|
|
.get("restarted_task_instance")
|
|
.and_then(Value::as_str)
|
|
.map(TaskInstanceId::new),
|
|
restarted_attempt_id: response
|
|
.get("restarted_attempt_id")
|
|
.and_then(Value::as_str)
|
|
.map(str::to_owned),
|
|
clean_boundary_available: response
|
|
.get("clean_boundary_available")
|
|
.and_then(Value::as_bool)
|
|
.ok_or_else(|| {
|
|
anyhow!("coordinator task restart response did not include clean_boundary_available")
|
|
})?,
|
|
requires_whole_process_restart: response
|
|
.get("requires_whole_process_restart")
|
|
.and_then(Value::as_bool)
|
|
.ok_or_else(|| {
|
|
anyhow!(
|
|
"coordinator task restart response did not include requires_whole_process_restart"
|
|
)
|
|
})?,
|
|
active_task: response
|
|
.get("active_task")
|
|
.and_then(Value::as_bool)
|
|
.ok_or_else(|| anyhow!("coordinator task restart response did not include active_task"))?,
|
|
completed_event_observed: response
|
|
.get("completed_event_observed")
|
|
.and_then(Value::as_bool)
|
|
.ok_or_else(|| {
|
|
anyhow!("coordinator task restart response did not include completed_event_observed")
|
|
})?,
|
|
message: response
|
|
.get("message")
|
|
.and_then(Value::as_str)
|
|
.ok_or_else(|| anyhow!("coordinator task restart response did not include message"))?
|
|
.to_owned(),
|
|
})
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use anyhow::anyhow;
|
|
|
|
use super::debug_epoch_was_released;
|
|
use crate::virtual_model::AdapterState;
|
|
|
|
#[test]
|
|
fn accepts_epoch_release_after_an_accepted_resume() {
|
|
let state = AdapterState {
|
|
process: "vp-completed".into(),
|
|
..AdapterState::default()
|
|
};
|
|
|
|
assert!(debug_epoch_was_released(
|
|
&anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-completed"),
|
|
&state,
|
|
3,
|
|
));
|
|
assert!(!debug_epoch_was_released(
|
|
&anyhow!("coordinator protocol error: debug epoch 4 is not active for vp-completed"),
|
|
&state,
|
|
3,
|
|
));
|
|
assert!(!debug_epoch_was_released(
|
|
&anyhow!("coordinator protocol error: debug epoch 3 is not active for vp-other"),
|
|
&state,
|
|
3,
|
|
));
|
|
}
|
|
}
|