Publish Clusterflux release candidate 6e83783
This commit is contained in:
parent
4d75257c6e
commit
f3640643bd
16 changed files with 4667 additions and 106 deletions
|
|
@ -3,10 +3,10 @@ use std::collections::BTreeMap;
|
|||
use clusterflux_core::{
|
||||
AgentId, AgentSignedRequest, ArtifactId, Authorization, Capability, CredentialKind,
|
||||
DataPlaneScope, Digest, DirectBulkTransferPlan, DownloadLink, EnvironmentRequirements,
|
||||
LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId, NodeSignedRequest,
|
||||
PanelEventKind, PanelState, Placement, ProcessId, ProjectId, ResourceLimits, SourcePreparation,
|
||||
SourceProviderKind, TaskBoundaryValue, TaskInstanceId, TaskJoinResult, TaskSpec, TenantId,
|
||||
UserId, VfsPath,
|
||||
LaunchAttemptId, LimitKind, NodeCapabilities, NodeDescriptor, NodeEndpoint, NodeId,
|
||||
NodeSignedRequest, PanelEventKind, PanelState, Placement, ProcessId, ProjectId, RequestId,
|
||||
ResourceLimits, SourcePreparation, SourceProviderKind, TaskBoundaryValue, TaskDefinitionId,
|
||||
TaskInstanceId, TaskJoinResult, TaskSpec, TenantId, UserId, VfsPath,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
|
|
@ -531,6 +531,13 @@ pub enum CoordinatorRequest {
|
|||
}
|
||||
|
||||
impl CoordinatorRequest {
|
||||
pub fn validate_external_identifiers(&self) -> Result<(), String> {
|
||||
let value = serde_json::to_value(self).map_err(|error| {
|
||||
format!("failed to validate coordinator request identifiers: {error}")
|
||||
})?;
|
||||
validate_identifier_tree(&value, "request")
|
||||
}
|
||||
|
||||
pub fn operation(&self) -> Result<String, String> {
|
||||
serde_json::to_value(self)
|
||||
.map_err(|err| format!("failed to encode coordinator request operation: {err}"))
|
||||
|
|
@ -538,6 +545,118 @@ impl CoordinatorRequest {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum ExternalIdentifierKind {
|
||||
Agent,
|
||||
Artifact,
|
||||
LaunchAttempt,
|
||||
Node,
|
||||
Process,
|
||||
Project,
|
||||
Request,
|
||||
TaskDefinition,
|
||||
TaskInstance,
|
||||
Tenant,
|
||||
User,
|
||||
}
|
||||
|
||||
fn identifier_kind(field: &str) -> Option<ExternalIdentifierKind> {
|
||||
match field {
|
||||
"tenant" | "target_tenant" => Some(ExternalIdentifierKind::Tenant),
|
||||
"project" => Some(ExternalIdentifierKind::Project),
|
||||
"actor_user" | "user" => Some(ExternalIdentifierKind::User),
|
||||
"agent" | "actor_agent" => Some(ExternalIdentifierKind::Agent),
|
||||
"node" | "prefer_node" | "receiver_node" => Some(ExternalIdentifierKind::Node),
|
||||
"process" => Some(ExternalIdentifierKind::Process),
|
||||
"task" | "parent_task" | "stopped_task" | "task_instance" => {
|
||||
Some(ExternalIdentifierKind::TaskInstance)
|
||||
}
|
||||
"task_definition" => Some(ExternalIdentifierKind::TaskDefinition),
|
||||
"artifact" | "required_artifacts" | "artifact_locations" => {
|
||||
Some(ExternalIdentifierKind::Artifact)
|
||||
}
|
||||
"launch_attempt" => Some(ExternalIdentifierKind::LaunchAttempt),
|
||||
"request_id" | "session_secret" | "transaction_id" | "polling_secret" | "transfer_id"
|
||||
| "enrollment_grant" | "widget_id" | "environment_id" | "admin_nonce" => {
|
||||
Some(ExternalIdentifierKind::Request)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_identifier_tree(value: &serde_json::Value, path: &str) -> Result<(), String> {
|
||||
match value {
|
||||
serde_json::Value::Object(fields) => {
|
||||
for (field, value) in fields {
|
||||
let field_path = format!("{path}.{field}");
|
||||
if let Some(kind) = identifier_kind(field) {
|
||||
validate_identifier_value(kind, value, &field_path)?;
|
||||
}
|
||||
validate_identifier_tree(value, &field_path)?;
|
||||
}
|
||||
}
|
||||
serde_json::Value::Array(values) => {
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
validate_identifier_tree(value, &format!("{path}[{index}]"))?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_identifier_value(
|
||||
kind: ExternalIdentifierKind,
|
||||
value: &serde_json::Value,
|
||||
path: &str,
|
||||
) -> Result<(), String> {
|
||||
if value.is_null() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(values) = value.as_array() {
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
validate_identifier_value(kind, value, &format!("{path}[{index}]"))?;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let value = value
|
||||
.as_str()
|
||||
.ok_or_else(|| format!("external identifier {path} must be a string"))?
|
||||
.to_owned();
|
||||
let result = match kind {
|
||||
ExternalIdentifierKind::Agent => AgentId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Artifact => ArtifactId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::LaunchAttempt => LaunchAttemptId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Node => NodeId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Process => ProcessId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Project => ProjectId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Request => RequestId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::TaskDefinition => TaskDefinitionId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::TaskInstance => TaskInstanceId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::Tenant => TenantId::try_new(value).map(|_| ()),
|
||||
ExternalIdentifierKind::User => UserId::try_new(value).map(|_| ()),
|
||||
};
|
||||
result.map_err(|error| format!("malformed external identifier {path}: {error}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod external_identifier_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn nested_authenticated_identifiers_are_validated() {
|
||||
let request = CoordinatorRequest::Authenticated {
|
||||
session_secret: "cli_session_valid".to_owned(),
|
||||
request: AuthenticatedCoordinatorRequest::AbortProcess {
|
||||
process: "bad process".to_owned(),
|
||||
launch_attempt: Some("attempt".to_owned()),
|
||||
},
|
||||
};
|
||||
let error = request.validate_external_identifiers().unwrap_err();
|
||||
assert!(error.contains("request.request.process"));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum AuthenticatedCoordinatorRequest {
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ impl CoordinatorService {
|
|||
&mut self,
|
||||
request: CoordinatorRequest,
|
||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
||||
request
|
||||
.validate_external_identifiers()
|
||||
.map_err(CoordinatorServiceError::Protocol)?;
|
||||
self.pump_main_runtime_commands();
|
||||
let request_payload = serde_json::to_value(&request).map_err(|error| {
|
||||
CoordinatorServiceError::Protocol(format!(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use clusterflux_core::{COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
|
||||
use clusterflux_core::{RequestId, COORDINATOR_PROTOCOL_VERSION, COORDINATOR_WIRE_REQUEST_TYPE};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -44,9 +44,9 @@ impl CoordinatorRequestEnvelope {
|
|||
self.protocol_version, COORDINATOR_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
if self.request_id.trim().is_empty() {
|
||||
return Err("coordinator wire request_id must be non-empty".to_owned());
|
||||
}
|
||||
RequestId::try_new(self.request_id.clone())
|
||||
.map_err(|error| format!("malformed coordinator wire request_id: {error}"))?;
|
||||
self.payload.validate_external_identifiers()?;
|
||||
let payload_operation = self.payload.operation()?;
|
||||
if self.operation != payload_operation {
|
||||
return Err(format!(
|
||||
|
|
|
|||
|
|
@ -1,18 +1,73 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const MAX_EXTERNAL_ID_BYTES: usize = 255;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct IdParseError {
|
||||
id_type: &'static str,
|
||||
reason: &'static str,
|
||||
}
|
||||
|
||||
impl IdParseError {
|
||||
fn new(id_type: &'static str, reason: &'static str) -> Self {
|
||||
Self { id_type, reason }
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for IdParseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{} is invalid: {}", self.id_type, self.reason)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for IdParseError {}
|
||||
|
||||
fn validate_id(value: &str, id_type: &'static str) -> Result<(), IdParseError> {
|
||||
if value.is_empty() || value.trim().is_empty() {
|
||||
return Err(IdParseError::new(
|
||||
id_type,
|
||||
"value must not be empty or whitespace-only",
|
||||
));
|
||||
}
|
||||
if value.len() > MAX_EXTERNAL_ID_BYTES {
|
||||
return Err(IdParseError::new(
|
||||
id_type,
|
||||
"value exceeds the 255-byte limit",
|
||||
));
|
||||
}
|
||||
if value.chars().any(char::is_control) {
|
||||
return Err(IdParseError::new(
|
||||
id_type,
|
||||
"control characters are forbidden",
|
||||
));
|
||||
}
|
||||
if !value.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric()
|
||||
|| matches!(byte, b'-' | b'_' | b'.' | b':' | b'/' | b'@' | b'+')
|
||||
}) {
|
||||
return Err(IdParseError::new(
|
||||
id_type,
|
||||
"only ASCII letters, digits, '-', '_', '.', ':', '/', '@', and '+' are allowed",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
macro_rules! id_type {
|
||||
($name:ident) => {
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub struct $name(String);
|
||||
|
||||
impl $name {
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
pub fn try_new(value: impl Into<String>) -> Result<Self, IdParseError> {
|
||||
let value = value.into();
|
||||
assert!(
|
||||
!value.trim().is_empty(),
|
||||
concat!(stringify!($name), " cannot be empty")
|
||||
);
|
||||
Self(value)
|
||||
validate_id(&value, stringify!($name))?;
|
||||
Ok(Self(value))
|
||||
}
|
||||
|
||||
/// Constructs an identifier from a trusted, internally generated value.
|
||||
pub fn new(value: impl Into<String>) -> Self {
|
||||
Self::try_new(value).unwrap_or_else(|error| panic!("{error}"))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
|
|
@ -44,3 +99,39 @@ id_type!(TaskDefinitionId);
|
|||
id_type!(TaskInstanceId);
|
||||
id_type!(TenantId);
|
||||
id_type!(UserId);
|
||||
|
||||
pub type DebugSessionId = ProcessId;
|
||||
pub type RequestId = LaunchAttemptId;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn assert_hostile_values<T>(parse: impl Fn(String) -> Result<T, IdParseError>) {
|
||||
for value in [
|
||||
String::new(),
|
||||
" ".to_owned(),
|
||||
"bad\0id".to_owned(),
|
||||
"bad id".to_owned(),
|
||||
"x".repeat(MAX_EXTERNAL_ID_BYTES + 1),
|
||||
] {
|
||||
assert!(parse(value).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_external_identifier_type_rejects_hostile_values() {
|
||||
assert_hostile_values(AgentId::try_new);
|
||||
assert_hostile_values(ArtifactId::try_new);
|
||||
assert_hostile_values(DebugSessionId::try_new);
|
||||
assert_hostile_values(NodeId::try_new);
|
||||
assert_hostile_values(ProcessId::try_new);
|
||||
assert_hostile_values(LaunchAttemptId::try_new);
|
||||
assert_hostile_values(ProjectId::try_new);
|
||||
assert_hostile_values(RequestId::try_new);
|
||||
assert_hostile_values(TaskDefinitionId::try_new);
|
||||
assert_hostile_values(TaskInstanceId::try_new);
|
||||
assert_hostile_values(TenantId::try_new);
|
||||
assert_hostile_values(UserId::try_new);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -66,8 +66,9 @@ pub use execution::{
|
|||
WasmTaskOutcome, WasmTaskResult, MAX_WASM_TASK_ENVELOPE_BYTES, WASM_TASK_ABI_VERSION,
|
||||
};
|
||||
pub use ids::{
|
||||
AgentId, ArtifactId, LaunchAttemptId, NodeId, ProcessId, ProjectId, TaskDefinitionId,
|
||||
TaskInstanceId, TenantId, UserId,
|
||||
AgentId, ArtifactId, DebugSessionId, IdParseError, LaunchAttemptId, NodeId, ProcessId,
|
||||
ProjectId, RequestId, TaskDefinitionId, TaskInstanceId, TenantId, UserId,
|
||||
MAX_EXTERNAL_ID_BYTES,
|
||||
};
|
||||
pub use limits::{
|
||||
LargeArgumentPolicy, LimitError, LimitKind, LogBuffer, LogRecord, ResourceLimits,
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ enum AdapterEvent {
|
|||
},
|
||||
BreakpointsUpdated {
|
||||
generation: u64,
|
||||
revision: u64,
|
||||
result: std::result::Result<(), String>,
|
||||
},
|
||||
}
|
||||
|
|
@ -62,6 +63,7 @@ enum AdapterEvent {
|
|||
struct LaunchCompletion {
|
||||
state: AdapterState,
|
||||
local_runtime_session: Option<LocalRuntimeSession>,
|
||||
breakpoint_revision: u64,
|
||||
}
|
||||
|
||||
pub(crate) fn run_adapter() -> Result<()> {
|
||||
|
|
@ -93,8 +95,13 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
runtime_starting = false;
|
||||
match result {
|
||||
Ok(mut completion) => {
|
||||
completion.state.breakpoints = state.breakpoints.clone();
|
||||
completion.state.breakpoints_installed = true;
|
||||
let requested_breakpoints = state.breakpoints.clone();
|
||||
let requested_revision = state.breakpoint_revision;
|
||||
let installed_revision = completion.breakpoint_revision;
|
||||
completion.state.breakpoints = requested_breakpoints;
|
||||
completion.state.breakpoint_revision = requested_revision;
|
||||
completion.state.breakpoints_installed =
|
||||
installed_revision == requested_revision;
|
||||
let previous_threads = state.threads.clone();
|
||||
state = completion.state;
|
||||
emit_thread_lifecycle(&mut writer, &previous_threads, &state.threads)?;
|
||||
|
|
@ -102,7 +109,24 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
_local_runtime_session = Some(session);
|
||||
}
|
||||
runtime_started = true;
|
||||
emit_verified_breakpoints(&mut writer, &state)?;
|
||||
if state.breakpoints_installed {
|
||||
emit_verified_breakpoints(&mut writer, &state)?;
|
||||
} else {
|
||||
let breakpoint_state = state.clone();
|
||||
let breakpoint_tx = event_tx.clone();
|
||||
let revision = state.breakpoint_revision;
|
||||
thread::spawn(move || {
|
||||
let result = set_services_debug_breakpoints(&breakpoint_state)
|
||||
.map_err(|error| {
|
||||
format!("coordinator breakpoint update failed: {error:#}")
|
||||
});
|
||||
let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
|
||||
generation,
|
||||
revision,
|
||||
result,
|
||||
});
|
||||
});
|
||||
}
|
||||
writer.output(
|
||||
"console",
|
||||
if state.session_mode == DapSessionMode::Attach {
|
||||
|
|
@ -127,15 +151,7 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
Err(message) => {
|
||||
runtime_started = false;
|
||||
writer.output("stderr", format!("{message}\n"))?;
|
||||
writer.event(
|
||||
"stopped",
|
||||
json!({
|
||||
"reason": "exception",
|
||||
"description": message,
|
||||
"threadId": MAIN_THREAD,
|
||||
"allThreadsStopped": false,
|
||||
}),
|
||||
)?;
|
||||
writer.event("terminated", json!({ "restart": false }))?;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
|
|
@ -231,8 +247,12 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
}
|
||||
continue;
|
||||
}
|
||||
AdapterEvent::BreakpointsUpdated { generation, result } => {
|
||||
if generation == runtime_generation {
|
||||
AdapterEvent::BreakpointsUpdated {
|
||||
generation,
|
||||
revision,
|
||||
result,
|
||||
} => {
|
||||
if generation == runtime_generation && revision == state.breakpoint_revision {
|
||||
match result {
|
||||
Ok(()) => {
|
||||
state.breakpoints_installed = true;
|
||||
|
|
@ -306,7 +326,16 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
if let Some(process_id) = args.get("processId").and_then(Value::as_str) {
|
||||
state.process = clusterflux_core::ProcessId::new(process_id);
|
||||
match clusterflux_core::ProcessId::try_new(process_id.to_owned()) {
|
||||
Ok(process) => state.process = process,
|
||||
Err(error) => {
|
||||
writer.error_response(
|
||||
&request,
|
||||
format!("invalid DAP processId: {error}"),
|
||||
)?;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if command == "attach" {
|
||||
state.session_mode = DapSessionMode::Attach;
|
||||
|
|
@ -338,6 +367,8 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
requested_source_path,
|
||||
requested_lines,
|
||||
);
|
||||
state.breakpoint_revision = state.breakpoint_revision.saturating_add(1);
|
||||
let breakpoint_revision = state.breakpoint_revision;
|
||||
let coordinator_backed = matches!(
|
||||
state.runtime_backend,
|
||||
RuntimeBackend::LocalServices | RuntimeBackend::LiveServices
|
||||
|
|
@ -371,8 +402,11 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
set_services_debug_breakpoints(&breakpoint_state).map_err(|error| {
|
||||
format!("coordinator breakpoint update failed: {error:#}")
|
||||
});
|
||||
let _ = breakpoint_tx
|
||||
.send(AdapterEvent::BreakpointsUpdated { generation, result });
|
||||
let _ = breakpoint_tx.send(AdapterEvent::BreakpointsUpdated {
|
||||
generation,
|
||||
revision: breakpoint_revision,
|
||||
result,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -433,11 +467,17 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
let mut completed_state = launch_state;
|
||||
let result = if completed_state.session_mode == DapSessionMode::Attach {
|
||||
attach_services_runtime(&completed_state)
|
||||
.map(|record| {
|
||||
.and_then(|record| {
|
||||
completed_state.apply_attach_record(record);
|
||||
set_services_debug_breakpoints(&completed_state)?;
|
||||
Ok(())
|
||||
})
|
||||
.map(|()| {
|
||||
let breakpoint_revision = completed_state.breakpoint_revision;
|
||||
LaunchCompletion {
|
||||
state: completed_state,
|
||||
local_runtime_session: None,
|
||||
breakpoint_revision,
|
||||
}
|
||||
})
|
||||
.map_err(|error| format!("services runtime attach failed: {error:#}"))
|
||||
|
|
@ -447,9 +487,12 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
run_local_services_runtime(&completed_state)
|
||||
.map(|(record, session)| {
|
||||
completed_state.apply_runtime_record(record);
|
||||
let breakpoint_revision =
|
||||
completed_state.breakpoint_revision;
|
||||
LaunchCompletion {
|
||||
state: completed_state,
|
||||
local_runtime_session: Some(session),
|
||||
breakpoint_revision,
|
||||
}
|
||||
})
|
||||
.map_err(|error| {
|
||||
|
|
@ -460,9 +503,12 @@ pub(crate) fn run_adapter() -> Result<()> {
|
|||
run_live_services_runtime(&completed_state)
|
||||
.map(|record| {
|
||||
completed_state.apply_runtime_record(record);
|
||||
let breakpoint_revision =
|
||||
completed_state.breakpoint_revision;
|
||||
LaunchCompletion {
|
||||
state: completed_state,
|
||||
local_runtime_session: None,
|
||||
breakpoint_revision,
|
||||
}
|
||||
})
|
||||
.map_err(|error| {
|
||||
|
|
|
|||
|
|
@ -422,8 +422,31 @@ fn launch_services_debug_entrypoint(
|
|||
}
|
||||
// The launch acknowledgement is the commit point. Failures while fetching
|
||||
// observation state after this line must not abort a process that is running.
|
||||
let task_snapshots = fetch_task_snapshots(coordinator, state)?;
|
||||
let (process_statuses, process_status) = fetch_current_process_status(coordinator, state)?;
|
||||
let mut observation_diagnostics = Vec::new();
|
||||
let inject_post_commit_observation_failure =
|
||||
std::env::var_os("CLUSTERFLUX_TEST_DAP_POST_COMMIT_OBSERVATION_FAILURE").is_some();
|
||||
let task_snapshots = (if inject_post_commit_observation_failure {
|
||||
Err(anyhow!("injected post-commit task observation failure"))
|
||||
} else {
|
||||
fetch_task_snapshots(coordinator, state)
|
||||
})
|
||||
.unwrap_or_else(|error| {
|
||||
observation_diagnostics.push(format!(
|
||||
"initial task observation failed after main_launched: {error:#}"
|
||||
));
|
||||
json!({ "snapshots": [] })
|
||||
});
|
||||
let (process_statuses, process_status) = (if inject_post_commit_observation_failure {
|
||||
Err(anyhow!("injected post-commit process observation failure"))
|
||||
} else {
|
||||
fetch_current_process_status(coordinator, state)
|
||||
})
|
||||
.unwrap_or_else(|error| {
|
||||
observation_diagnostics.push(format!(
|
||||
"initial process observation failed after main_launched: {error:#}"
|
||||
));
|
||||
(json!({ "processes": [] }), None)
|
||||
});
|
||||
let node = process_status
|
||||
.as_ref()
|
||||
.and_then(|status| status.get("connected_nodes"))
|
||||
|
|
@ -441,6 +464,7 @@ fn launch_services_debug_entrypoint(
|
|||
"process_status": process_status,
|
||||
"process_statuses": process_statuses,
|
||||
"task_snapshots": task_snapshots,
|
||||
"observation_diagnostics": observation_diagnostics,
|
||||
}),
|
||||
task_events: json!({ "events": [] }),
|
||||
placed_task_launched: true,
|
||||
|
|
@ -1002,6 +1026,12 @@ pub(crate) fn observe_services_runtime(
|
|||
let inject_connection_loss =
|
||||
std::env::var("CLUSTERFLUX_TEST_DAP_OBSERVER_CONNECTION_LOSS").as_deref() == Ok("1");
|
||||
let mut connection_loss_injected = false;
|
||||
let inject_fallback_failure =
|
||||
std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_FALLBACK_FAILURE").is_some();
|
||||
let mut fallback_failure_stage = 0_u8;
|
||||
let inject_debug_epoch_wait_failure =
|
||||
std::env::var_os("CLUSTERFLUX_TEST_DAP_OBSERVER_DEBUG_EPOCH_WAIT_FAILURE").is_some();
|
||||
let mut debug_epoch_wait_failure_injected = false;
|
||||
loop {
|
||||
if cancelled.load(Ordering::Acquire) {
|
||||
return Ok(());
|
||||
|
|
@ -1191,16 +1221,22 @@ pub(crate) fn observe_services_runtime(
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
let breakpoint = match current_session.request(client_user_request(
|
||||
state,
|
||||
json!({
|
||||
"type": "inspect_debug_breakpoints",
|
||||
"tenant": state.tenant,
|
||||
"project": state.project_id,
|
||||
"actor_user": state.actor_user,
|
||||
"process": state.process.as_str(),
|
||||
}),
|
||||
)) {
|
||||
let breakpoint_request = if inject_fallback_failure && fallback_failure_stage == 0 {
|
||||
fallback_failure_stage = 1;
|
||||
Err(anyhow!("injected breakpoint inspection transport failure"))
|
||||
} else {
|
||||
current_session.request(client_user_request(
|
||||
state,
|
||||
json!({
|
||||
"type": "inspect_debug_breakpoints",
|
||||
"tenant": state.tenant,
|
||||
"project": state.project_id,
|
||||
"actor_user": state.actor_user,
|
||||
"process": state.process.as_str(),
|
||||
}),
|
||||
))
|
||||
};
|
||||
let breakpoint = match breakpoint_request {
|
||||
Ok(breakpoint) => breakpoint,
|
||||
Err(inspect_error) => {
|
||||
// Main completion records its terminal event and clears ephemeral
|
||||
|
|
@ -1208,16 +1244,36 @@ pub(crate) fn observe_services_runtime(
|
|||
// the event read above and this inspection request. Re-read the
|
||||
// durable event stream before treating the missing debug state as
|
||||
// an adapter failure.
|
||||
let events = current_session.request(client_user_request(
|
||||
state,
|
||||
json!({
|
||||
"type": "list_task_events",
|
||||
"tenant": state.tenant,
|
||||
"project": state.project_id,
|
||||
"actor_user": state.actor_user,
|
||||
"process": state.process.as_str(),
|
||||
}),
|
||||
))?;
|
||||
let fallback_request = if inject_fallback_failure && fallback_failure_stage == 1 {
|
||||
fallback_failure_stage = 2;
|
||||
Err(anyhow!("injected fallback event transport failure"))
|
||||
} else {
|
||||
current_session.request(client_user_request(
|
||||
state,
|
||||
json!({
|
||||
"type": "list_task_events",
|
||||
"tenant": state.tenant,
|
||||
"project": state.project_id,
|
||||
"actor_user": state.actor_user,
|
||||
"process": state.process.as_str(),
|
||||
}),
|
||||
))
|
||||
};
|
||||
let events = match fallback_request {
|
||||
Ok(events) => events,
|
||||
Err(fallback_error) => {
|
||||
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
|
||||
"runtime breakpoint and fallback event observation failed: {inspect_error:#}; {fallback_error:#}; reconnecting in {} ms",
|
||||
reconnect_delay.as_millis()
|
||||
))) {
|
||||
return Ok(());
|
||||
}
|
||||
session = None;
|
||||
std::thread::sleep(reconnect_delay);
|
||||
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(outcome) = terminal_runtime_outcome(&coordinator, state, &events) {
|
||||
emit(outcome);
|
||||
return Ok(());
|
||||
|
|
@ -1236,7 +1292,28 @@ pub(crate) fn observe_services_runtime(
|
|||
};
|
||||
if let Some(epoch) = breakpoint.get("hit_epoch").and_then(Value::as_u64) {
|
||||
if epoch > previous_debug_epoch {
|
||||
let frozen = wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)?;
|
||||
let frozen_request =
|
||||
if inject_debug_epoch_wait_failure && !debug_epoch_wait_failure_injected {
|
||||
debug_epoch_wait_failure_injected = true;
|
||||
Err(anyhow!("injected Debug Epoch wait transport failure"))
|
||||
} else {
|
||||
wait_for_debug_epoch_state_at(&coordinator, state, epoch, true)
|
||||
};
|
||||
let frozen = match frozen_request {
|
||||
Ok(frozen) => frozen,
|
||||
Err(error) => {
|
||||
if !emit(RuntimeContinuationOutcome::Diagnostic(format!(
|
||||
"runtime Debug Epoch observation failed: {error:#}; reconnecting in {} ms",
|
||||
reconnect_delay.as_millis()
|
||||
))) {
|
||||
return Ok(());
|
||||
}
|
||||
session = None;
|
||||
std::thread::sleep(reconnect_delay);
|
||||
reconnect_delay = (reconnect_delay * 2).min(Duration::from_secs(5));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let node = frozen
|
||||
.get("acknowledgements")
|
||||
.and_then(Value::as_array)
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ pub(crate) struct AdapterState {
|
|||
pub(crate) debug_probes: Vec<BundleDebugProbe>,
|
||||
pub(crate) breakpoints: Vec<i64>,
|
||||
pub(crate) breakpoints_installed: bool,
|
||||
pub(crate) breakpoint_revision: u64,
|
||||
pub(crate) threads: BTreeMap<i64, VirtualThread>,
|
||||
}
|
||||
|
||||
|
|
@ -133,6 +134,7 @@ impl Default for AdapterState {
|
|||
debug_probes: Vec::new(),
|
||||
breakpoints: Vec::new(),
|
||||
breakpoints_installed: false,
|
||||
breakpoint_revision: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -198,18 +200,24 @@ impl AdapterState {
|
|||
}
|
||||
self.coordinator_endpoint =
|
||||
crate::view_state::normalize_coordinator_endpoint(&session.coordinator);
|
||||
self.tenant = TenantId::new(session.tenant);
|
||||
self.project_id = ProjectId::new(session.project);
|
||||
self.actor_user = UserId::new(session.user);
|
||||
self.tenant = TenantId::try_new(session.tenant)
|
||||
.map_err(|error| anyhow!("invalid tenant in CLI session: {error}"))?;
|
||||
self.project_id = ProjectId::try_new(session.project)
|
||||
.map_err(|error| anyhow!("invalid project in CLI session: {error}"))?;
|
||||
self.actor_user = UserId::try_new(session.user)
|
||||
.map_err(|error| anyhow!("invalid user in CLI session: {error}"))?;
|
||||
self.client_session_secret = Some(session.session_secret);
|
||||
} else {
|
||||
self.coordinator_endpoint = crate::view_state::normalize_coordinator_endpoint(
|
||||
coordinator_endpoint.as_deref().unwrap_or("127.0.0.1:0"),
|
||||
);
|
||||
if let Some(scope) = project_scope {
|
||||
self.tenant = TenantId::new(scope.tenant);
|
||||
self.project_id = ProjectId::new(scope.project);
|
||||
self.actor_user = UserId::new(scope.user);
|
||||
self.tenant = TenantId::try_new(scope.tenant)
|
||||
.map_err(|error| anyhow!("invalid tenant in project scope: {error}"))?;
|
||||
self.project_id = ProjectId::try_new(scope.project)
|
||||
.map_err(|error| anyhow!("invalid project in project scope: {error}"))?;
|
||||
self.actor_user = UserId::try_new(scope.user)
|
||||
.map_err(|error| anyhow!("invalid user in project scope: {error}"))?;
|
||||
}
|
||||
self.client_session_secret = None;
|
||||
}
|
||||
|
|
@ -230,6 +238,7 @@ impl AdapterState {
|
|||
self.epoch = 0;
|
||||
self.breakpoints.clear();
|
||||
self.breakpoints_installed = self.runtime_backend == RuntimeBackend::Simulated;
|
||||
self.breakpoint_revision = 0;
|
||||
self.threads = if self.runtime_backend == RuntimeBackend::Simulated {
|
||||
launch_threads(&self.entry)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ use std::time::{Duration, Instant};
|
|||
use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine as _};
|
||||
use clusterflux_core::{
|
||||
sign_node_request, signed_request_payload_digest, ArtifactId, Digest, NodeCapabilities, NodeId,
|
||||
TaskSpec, MIN_SIGNED_NODE_POLL_INTERVAL_MS,
|
||||
ProcessId, ProjectId, RequestId, TaskInstanceId, TaskSpec, TenantId,
|
||||
MIN_SIGNED_NODE_POLL_INTERVAL_MS,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
|
|
@ -222,7 +223,7 @@ fn worker_loop(
|
|||
let expiry = Instant::now()
|
||||
.checked_add(Duration::from_secs(retention_limits.restart_pin_seconds))
|
||||
.unwrap_or_else(Instant::now);
|
||||
restart_pins.insert(ArtifactId::new(artifact), expiry);
|
||||
restart_pins.insert(ArtifactId::try_new(artifact.to_owned())?, expiry);
|
||||
}
|
||||
println!("{}", serde_json::to_string(&report)?);
|
||||
std::io::stdout().flush()?;
|
||||
|
|
@ -289,7 +290,7 @@ fn service_pending_artifact_transfer(
|
|||
return Ok(false);
|
||||
};
|
||||
let transfer_id = required_string(transfer, "transfer_id")?;
|
||||
let artifact = ArtifactId::new(required_string(transfer, "artifact")?);
|
||||
let artifact = ArtifactId::try_new(required_string(transfer, "artifact")?)?;
|
||||
let expected_digest: Digest = serde_json::from_value(
|
||||
transfer
|
||||
.get("expected_digest")
|
||||
|
|
@ -369,9 +370,11 @@ pub(crate) fn runtime_task_from_assignment(
|
|||
.cloned()
|
||||
.ok_or("task assignment missing task_spec")?,
|
||||
)?;
|
||||
let process = ProcessId::try_new(required_string(value, "process")?)?;
|
||||
let task = TaskInstanceId::try_new(required_string(value, "task")?)?;
|
||||
Ok(RuntimeTask {
|
||||
process: required_string(value, "process")?,
|
||||
task: required_string(value, "task")?,
|
||||
process: process.to_string(),
|
||||
task: task.to_string(),
|
||||
epoch: value.get("epoch").and_then(Value::as_u64),
|
||||
bundle_digest: task_spec.bundle_digest.clone(),
|
||||
task_spec: Some(task_spec),
|
||||
|
|
@ -570,6 +573,12 @@ fn parse_args() -> Result<Args, Box<dyn std::error::Error>> {
|
|||
}
|
||||
}
|
||||
|
||||
TenantId::try_new(tenant.clone())?;
|
||||
ProjectId::try_new(project.clone())?;
|
||||
NodeId::try_new(node.clone())?;
|
||||
if let Some(grant) = enrollment_grant.as_ref() {
|
||||
RequestId::try_new(grant.clone())?;
|
||||
}
|
||||
Ok(Args {
|
||||
coordinator: coordinator.ok_or("--coordinator is required")?,
|
||||
tenant,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue