Publish Clusterflux 45bbc21
This commit is contained in:
parent
eb1077f380
commit
3f88254c70
28 changed files with 896 additions and 167 deletions
|
|
@ -33,6 +33,7 @@ pub use service::{
|
|||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ActiveProcess {
|
||||
pub id: ProcessId,
|
||||
pub launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
|
||||
pub tenant: TenantId,
|
||||
pub project: ProjectId,
|
||||
pub connected_nodes: BTreeSet<NodeId>,
|
||||
|
|
@ -322,9 +323,20 @@ impl Coordinator {
|
|||
tenant: TenantId,
|
||||
project: ProjectId,
|
||||
id: ProcessId,
|
||||
) -> ActiveProcess {
|
||||
self.start_process_for_launch_attempt(tenant, project, id, None)
|
||||
}
|
||||
|
||||
pub fn start_process_for_launch_attempt(
|
||||
&mut self,
|
||||
tenant: TenantId,
|
||||
project: ProjectId,
|
||||
id: ProcessId,
|
||||
launch_attempt: Option<clusterflux_core::LaunchAttemptId>,
|
||||
) -> ActiveProcess {
|
||||
let process = ActiveProcess {
|
||||
id: id.clone(),
|
||||
launch_attempt,
|
||||
tenant,
|
||||
project,
|
||||
connected_nodes: BTreeSet::new(),
|
||||
|
|
@ -531,6 +543,32 @@ impl Coordinator {
|
|||
.expect("active process was checked immediately before removal"))
|
||||
}
|
||||
|
||||
pub fn abort_process_for_launch_attempt(
|
||||
&mut self,
|
||||
tenant: &TenantId,
|
||||
project: &ProjectId,
|
||||
process: &ProcessId,
|
||||
launch_attempt: &clusterflux_core::LaunchAttemptId,
|
||||
) -> Result<ActiveProcess, CoordinatorError> {
|
||||
let key = (tenant.clone(), project.clone(), process.clone());
|
||||
let active = self.active_processes.get(&key).ok_or_else(|| {
|
||||
CoordinatorError::Unauthorized(
|
||||
"launch rollback requires an active virtual process".to_owned(),
|
||||
)
|
||||
})?;
|
||||
if active.launch_attempt.as_ref() != Some(launch_attempt) {
|
||||
return Err(CoordinatorError::Unauthorized(format!(
|
||||
"launch rollback denied: attempt {} does not own process {}",
|
||||
launch_attempt.as_str(),
|
||||
process.as_str()
|
||||
)));
|
||||
}
|
||||
Ok(self
|
||||
.active_processes
|
||||
.remove(&key)
|
||||
.expect("active process was checked immediately before removal"))
|
||||
}
|
||||
|
||||
pub fn active_process_count(&self) -> usize {
|
||||
self.active_processes.len()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -273,6 +273,7 @@ impl CoordinatorService {
|
|||
agent_signature: Option<AgentSignedRequest>,
|
||||
request_payload_digest: Option<&Digest>,
|
||||
process: String,
|
||||
launch_attempt: Option<String>,
|
||||
restart: bool,
|
||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
||||
let tenant = TenantId::new(tenant);
|
||||
|
|
@ -392,10 +393,18 @@ impl CoordinatorService {
|
|||
self.debug_audit_events.retain(|event| {
|
||||
event.tenant != tenant || event.project != project || event.process != process
|
||||
});
|
||||
self.coordinator
|
||||
.start_process(tenant, project, process.clone());
|
||||
let active = self.coordinator.start_process_for_launch_attempt(
|
||||
tenant,
|
||||
project,
|
||||
process.clone(),
|
||||
launch_attempt.map(clusterflux_core::LaunchAttemptId::new),
|
||||
);
|
||||
Ok(CoordinatorResponse::ProcessStarted {
|
||||
process,
|
||||
launch_attempt: active
|
||||
.launch_attempt
|
||||
.as_ref()
|
||||
.map(|attempt| attempt.as_str().to_owned()),
|
||||
epoch: self.coordinator.coordinator_epoch(),
|
||||
actor,
|
||||
charged_spawns,
|
||||
|
|
@ -519,6 +528,7 @@ impl CoordinatorService {
|
|||
project: String,
|
||||
actor_user: String,
|
||||
process: String,
|
||||
launch_attempt: Option<String>,
|
||||
) -> Result<CoordinatorResponse, CoordinatorServiceError> {
|
||||
let tenant = TenantId::new(tenant);
|
||||
let project = ProjectId::new(project);
|
||||
|
|
@ -534,6 +544,17 @@ impl CoordinatorService {
|
|||
})?;
|
||||
debug_assert_eq!(active.tenant, tenant);
|
||||
debug_assert_eq!(active.project, project);
|
||||
let launch_attempt = launch_attempt.map(clusterflux_core::LaunchAttemptId::new);
|
||||
if let Some(expected) = launch_attempt.as_ref() {
|
||||
if active.launch_attempt.as_ref() != Some(expected) {
|
||||
return Err(CoordinatorError::Unauthorized(format!(
|
||||
"launch rollback denied: attempt {} does not own process {}",
|
||||
expected.as_str(),
|
||||
process.as_str()
|
||||
))
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
let process_key = process_control_key(&tenant, &project, &process);
|
||||
self.process_cancellations.remove(&process_key);
|
||||
|
|
@ -575,8 +596,17 @@ impl CoordinatorService {
|
|||
}
|
||||
}
|
||||
|
||||
self.coordinator
|
||||
.abort_process(&tenant, &project, &process)?;
|
||||
if let Some(launch_attempt) = launch_attempt.as_ref() {
|
||||
self.coordinator.abort_process_for_launch_attempt(
|
||||
&tenant,
|
||||
&project,
|
||||
&process,
|
||||
launch_attempt,
|
||||
)?;
|
||||
} else {
|
||||
self.coordinator
|
||||
.abort_process(&tenant, &project, &process)?;
|
||||
}
|
||||
let active_restart_tasks = aborted_tasks
|
||||
.iter()
|
||||
.map(|target| target.task.clone())
|
||||
|
|
|
|||
|
|
@ -262,6 +262,8 @@ pub enum CoordinatorRequest {
|
|||
#[serde(default)]
|
||||
agent_signature: Option<AgentSignedRequest>,
|
||||
process: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
launch_attempt: Option<String>,
|
||||
#[serde(default)]
|
||||
restart: bool,
|
||||
},
|
||||
|
|
@ -288,6 +290,8 @@ pub enum CoordinatorRequest {
|
|||
project: String,
|
||||
actor_user: String,
|
||||
process: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
launch_attempt: Option<String>,
|
||||
},
|
||||
ListProcesses {
|
||||
tenant: String,
|
||||
|
|
@ -569,6 +573,8 @@ pub enum AuthenticatedCoordinatorRequest {
|
|||
},
|
||||
StartProcess {
|
||||
process: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
launch_attempt: Option<String>,
|
||||
#[serde(default)]
|
||||
restart: bool,
|
||||
},
|
||||
|
|
@ -594,6 +600,8 @@ pub enum AuthenticatedCoordinatorRequest {
|
|||
},
|
||||
AbortProcess {
|
||||
process: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
launch_attempt: Option<String>,
|
||||
},
|
||||
ListProcesses,
|
||||
QuotaStatus,
|
||||
|
|
|
|||
|
|
@ -344,6 +344,8 @@ pub enum CoordinatorResponse {
|
|||
},
|
||||
ProcessStarted {
|
||||
process: ProcessId,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
launch_attempt: Option<String>,
|
||||
epoch: u64,
|
||||
actor: WorkflowActor,
|
||||
charged_spawns: u64,
|
||||
|
|
|
|||
|
|
@ -370,6 +370,7 @@ impl CoordinatorService {
|
|||
agent_public_key_fingerprint,
|
||||
agent_signature,
|
||||
process,
|
||||
launch_attempt,
|
||||
restart,
|
||||
} => self.handle_start_process(
|
||||
tenant,
|
||||
|
|
@ -380,6 +381,7 @@ impl CoordinatorService {
|
|||
agent_signature,
|
||||
Some(&request_payload_digest),
|
||||
process,
|
||||
launch_attempt,
|
||||
restart,
|
||||
),
|
||||
CoordinatorRequest::ReconnectNode { .. } => self.reject_unsigned_node_request(),
|
||||
|
|
@ -401,7 +403,8 @@ impl CoordinatorService {
|
|||
project,
|
||||
actor_user,
|
||||
process,
|
||||
} => self.handle_abort_process(tenant, project, actor_user, process),
|
||||
launch_attempt,
|
||||
} => self.handle_abort_process(tenant, project, actor_user, process, launch_attempt),
|
||||
CoordinatorRequest::ListProcesses {
|
||||
tenant,
|
||||
project,
|
||||
|
|
@ -689,18 +692,22 @@ impl CoordinatorService {
|
|||
actor.as_str().to_owned(),
|
||||
node,
|
||||
),
|
||||
AuthenticatedCoordinatorRequest::StartProcess { process, restart } => self
|
||||
.handle_start_process(
|
||||
context.tenant.as_str().to_owned(),
|
||||
context.project.as_str().to_owned(),
|
||||
Some(actor.as_str().to_owned()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
process,
|
||||
restart,
|
||||
),
|
||||
AuthenticatedCoordinatorRequest::StartProcess {
|
||||
process,
|
||||
launch_attempt,
|
||||
restart,
|
||||
} => self.handle_start_process(
|
||||
context.tenant.as_str().to_owned(),
|
||||
context.project.as_str().to_owned(),
|
||||
Some(actor.as_str().to_owned()),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
process,
|
||||
launch_attempt,
|
||||
restart,
|
||||
),
|
||||
request @ AuthenticatedCoordinatorRequest::ScheduleTask { .. } => {
|
||||
self.handle_authenticated_schedule_task(&context, request)
|
||||
}
|
||||
|
|
@ -714,11 +721,15 @@ impl CoordinatorService {
|
|||
actor.as_str().to_owned(),
|
||||
process,
|
||||
),
|
||||
AuthenticatedCoordinatorRequest::AbortProcess { process } => self.handle_abort_process(
|
||||
AuthenticatedCoordinatorRequest::AbortProcess {
|
||||
process,
|
||||
launch_attempt,
|
||||
} => self.handle_abort_process(
|
||||
context.tenant.as_str().to_owned(),
|
||||
context.project.as_str().to_owned(),
|
||||
actor.as_str().to_owned(),
|
||||
process,
|
||||
launch_attempt,
|
||||
),
|
||||
AuthenticatedCoordinatorRequest::ListProcesses => self.handle_list_processes(
|
||||
context.tenant.as_str().to_owned(),
|
||||
|
|
|
|||
|
|
@ -96,6 +96,7 @@ fn postgres_backed_runtime_service_survives_restart_without_live_process_state()
|
|||
.handle_request(CoordinatorRequest::Authenticated {
|
||||
session_secret: session_secret.to_owned(),
|
||||
request: AuthenticatedCoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
process: "process-ephemeral".to_owned(),
|
||||
restart: false,
|
||||
},
|
||||
|
|
@ -433,7 +434,9 @@ fn with_signed_agent_workflow(
|
|||
let signature = signed_agent_workflow_request(&request, request_kind, process, task, nonce);
|
||||
match &mut request {
|
||||
CoordinatorRequest::StartProcess {
|
||||
agent_signature, ..
|
||||
launch_attempt: None,
|
||||
agent_signature,
|
||||
..
|
||||
}
|
||||
| CoordinatorRequest::LaunchTask {
|
||||
agent_signature, ..
|
||||
|
|
@ -862,10 +865,16 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() {
|
|||
};
|
||||
assert_eq!(placement.node, NodeId::from("session-node"));
|
||||
|
||||
let CoordinatorResponse::ProcessStarted { process, actor, .. } = service
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
process,
|
||||
actor,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::Authenticated {
|
||||
session_secret: "cli-session-secret".to_owned(),
|
||||
request: AuthenticatedCoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
process: "vp-session".to_owned(),
|
||||
restart: false,
|
||||
},
|
||||
|
|
@ -993,6 +1002,7 @@ fn authenticated_envelope_derives_user_scope_from_cli_session() {
|
|||
.handle_request(CoordinatorRequest::Authenticated {
|
||||
session_secret: "victim-cli-session-secret".to_owned(),
|
||||
request: AuthenticatedCoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
process: "vp-victim".to_owned(),
|
||||
restart: false,
|
||||
},
|
||||
|
|
@ -1398,6 +1408,7 @@ fn service_reports_and_enforces_public_admin_tenant_suspension() {
|
|||
|
||||
let start = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: Some("attempt-a".to_owned()),
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1536,6 +1547,7 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() {
|
|||
|
||||
let fingerprint_only = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1551,6 +1563,7 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() {
|
|||
let wrong_fingerprint = service
|
||||
.handle_request(with_signed_agent_workflow(
|
||||
CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1592,10 +1605,13 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() {
|
|||
.unwrap();
|
||||
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
actor: start_actor, ..
|
||||
launch_attempt: None,
|
||||
actor: start_actor,
|
||||
..
|
||||
} = service
|
||||
.handle_request(with_signed_agent_workflow(
|
||||
CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1619,6 +1635,7 @@ fn service_runs_agent_workflows_with_scoped_key_attribution() {
|
|||
let replay = service
|
||||
.handle_request(with_signed_agent_workflow(
|
||||
CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1728,6 +1745,7 @@ fn signed_node_and_agent_requests_reject_body_modification() {
|
|||
})
|
||||
.unwrap();
|
||||
let original_agent_request = CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1746,6 +1764,7 @@ fn signed_node_and_agent_requests_reject_body_modification() {
|
|||
);
|
||||
let mut modified_agent_request = original_agent_request;
|
||||
let CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
restart,
|
||||
agent_signature: request_signature,
|
||||
..
|
||||
|
|
@ -1789,8 +1808,13 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() {
|
|||
})
|
||||
.unwrap();
|
||||
|
||||
let CoordinatorResponse::ProcessStarted { charged_spawns, .. } = service
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
charged_spawns,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1887,6 +1911,7 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() {
|
|||
|
||||
let other_project_process = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "other-project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1899,7 +1924,10 @@ fn service_checks_spawn_quota_before_process_or_task_work_starts() {
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
other_project_process,
|
||||
CoordinatorResponse::ProcessStarted { .. }
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert_eq!(
|
||||
service.quota.used_workflow_spawns(
|
||||
|
|
@ -1935,6 +1963,7 @@ fn project_quota_resets_at_the_configured_window_boundary() {
|
|||
service.set_server_time(59);
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: Some("attempt-b".to_owned()),
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1947,6 +1976,7 @@ fn project_quota_resets_at_the_configured_window_boundary() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::AbortProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
|
|
@ -1956,6 +1986,7 @@ fn project_quota_resets_at_the_configured_window_boundary() {
|
|||
|
||||
let exhausted = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: Some("attempt-b".to_owned()),
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1971,6 +2002,7 @@ fn project_quota_resets_at_the_configured_window_boundary() {
|
|||
service.set_server_time(60);
|
||||
let started = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -1984,6 +2016,7 @@ fn project_quota_resets_at_the_configured_window_boundary() {
|
|||
assert!(matches!(
|
||||
started,
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
charged_spawns: 1,
|
||||
..
|
||||
}
|
||||
|
|
@ -2073,6 +2106,7 @@ fn signed_node_log_ingestion_checks_scoped_quota_before_accepting_bytes() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -2148,6 +2182,7 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() {
|
|||
|
||||
let started = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -2161,6 +2196,7 @@ fn service_attaches_node_starts_process_and_records_scoped_task_event() {
|
|||
assert_eq!(
|
||||
started,
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
process: ProcessId::from("process"),
|
||||
epoch: 7,
|
||||
actor: WorkflowActor {
|
||||
|
|
@ -2409,6 +2445,7 @@ fn service_delivers_cancellation_to_connected_node_and_records_terminal_state()
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -2542,6 +2579,7 @@ fn service_authorizes_debug_attach_through_public_api() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -2664,6 +2702,7 @@ fn debug_epoch_commands_are_polled_by_signed_active_task_nodes() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: Some("user".to_owned()),
|
||||
|
|
@ -2991,6 +3030,7 @@ fn service_reports_task_restart_boundary_through_public_api() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3263,6 +3303,7 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() {
|
|||
}
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3382,6 +3423,7 @@ fn service_cancels_whole_process_and_blocks_new_task_launches() {
|
|||
|
||||
service
|
||||
.handle_request(CoordinatorRequest::AbortProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
|
|
@ -3413,6 +3455,7 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
let mut service = CoordinatorService::new(7);
|
||||
let started = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3425,11 +3468,15 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
started,
|
||||
CoordinatorResponse::ProcessStarted { .. }
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
..
|
||||
}
|
||||
));
|
||||
|
||||
let same_without_restart = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3446,6 +3493,7 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
|
||||
let other_process = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3460,6 +3508,27 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
.to_string()
|
||||
.contains("already has active virtual process"));
|
||||
|
||||
let wrong_attempt_abort = service
|
||||
.handle_request(CoordinatorRequest::AbortProcess {
|
||||
launch_attempt: Some("attempt-b".to_owned()),
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process-a".to_owned(),
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(wrong_attempt_abort
|
||||
.to_string()
|
||||
.contains("does not own process process-a"));
|
||||
assert!(service
|
||||
.coordinator
|
||||
.active_process(
|
||||
&TenantId::from("tenant"),
|
||||
&ProjectId::from("project"),
|
||||
&ProcessId::from("process-a")
|
||||
)
|
||||
.is_some());
|
||||
|
||||
let retired_main_abort = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let process_key = process_control_key(
|
||||
&TenantId::from("tenant"),
|
||||
|
|
@ -3481,6 +3550,7 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
);
|
||||
let restarted = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: Some("attempt-a-restart".to_owned()),
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3494,6 +3564,7 @@ fn service_rejects_second_active_process_unless_restarting_same_process() {
|
|||
assert_eq!(
|
||||
restarted,
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: Some("attempt-a-restart".to_owned()),
|
||||
process: ProcessId::from("process-a"),
|
||||
epoch: 7,
|
||||
actor: WorkflowActor {
|
||||
|
|
@ -3517,6 +3588,7 @@ fn quiescent_cooperative_cancel_releases_slot_immediately() {
|
|||
let mut service = CoordinatorService::new(17);
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3586,6 +3658,7 @@ fn quiescent_cooperative_cancel_releases_slot_immediately() {
|
|||
|
||||
let started = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3598,7 +3671,10 @@ fn quiescent_cooperative_cancel_releases_slot_immediately() {
|
|||
.unwrap();
|
||||
assert!(matches!(
|
||||
started,
|
||||
CoordinatorResponse::ProcessStarted { .. }
|
||||
CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
..
|
||||
}
|
||||
));
|
||||
assert!(service.task_events.iter().all(|event| {
|
||||
event.tenant != TenantId::from("tenant")
|
||||
|
|
@ -3625,6 +3701,7 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -3655,6 +3732,7 @@ fn aborted_process_accepts_signed_terminal_event_for_issued_task() {
|
|||
|
||||
let CoordinatorResponse::ProcessAborted { aborted_tasks, .. } = service
|
||||
.handle_request(CoordinatorRequest::AbortProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
|
|
@ -3731,6 +3809,7 @@ fn service_download_links_are_scoped_and_streaming_is_metered() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -4127,6 +4206,7 @@ fn download_quota_cannot_be_reset_by_reconnect_retry_or_scope_switch() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -4325,6 +4405,7 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -4470,6 +4551,7 @@ fn windows_task_events_share_the_virtual_process_scope() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -4734,8 +4816,13 @@ fn coordinator_side_task_launch_queues_worker_assignment() {
|
|||
online: true,
|
||||
})
|
||||
.unwrap();
|
||||
let CoordinatorResponse::ProcessStarted { epoch, .. } = service
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
epoch,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -5059,6 +5146,7 @@ fn same_definition_instances_join_correctly_when_they_complete_in_reverse_order(
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: Some("user".to_owned()),
|
||||
|
|
@ -5385,6 +5473,7 @@ fn long_lived_process_state_reaches_a_scoped_bounded_steady_state() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant-soak".to_owned(),
|
||||
project: "project-soak".to_owned(),
|
||||
actor_user: Some("user-soak".to_owned()),
|
||||
|
|
@ -5581,6 +5670,7 @@ fn signed_active_wasm_task_can_spawn_and_join_child_in_its_process_only() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: Some("user".to_owned()),
|
||||
|
|
@ -5691,8 +5781,13 @@ fn coordinator_rejects_named_environment_without_requirements() {
|
|||
online: true,
|
||||
})
|
||||
.unwrap();
|
||||
let CoordinatorResponse::ProcessStarted { epoch, .. } = service
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
epoch,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -5747,6 +5842,7 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
|
|||
let mut service = CoordinatorService::new(10);
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -5786,8 +5882,13 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
|
|||
#[test]
|
||||
fn coordinator_side_task_launch_can_wait_for_capable_worker() {
|
||||
let mut service = CoordinatorService::new(11);
|
||||
let CoordinatorResponse::ProcessStarted { epoch, .. } = service
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
epoch,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -5996,6 +6097,7 @@ fn service_rejects_task_completion_outside_node_scope() {
|
|||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "tenant-b".to_owned(),
|
||||
project: "project-b".to_owned(),
|
||||
actor_user: None,
|
||||
|
|
@ -6300,6 +6402,7 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
|
|||
write_coordinator_wire_request(
|
||||
&mut stream,
|
||||
&CoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
tenant: "victim-tenant".to_owned(),
|
||||
project: "victim-project".to_owned(),
|
||||
actor_user: Some("forged-user".to_owned()),
|
||||
|
|
@ -6326,6 +6429,7 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
|
|||
&CoordinatorRequest::Authenticated {
|
||||
session_secret: "strict-stream-session".to_owned(),
|
||||
request: AuthenticatedCoordinatorRequest::StartProcess {
|
||||
launch_attempt: None,
|
||||
process: "vp-authenticated".to_owned(),
|
||||
restart: false,
|
||||
},
|
||||
|
|
@ -6335,8 +6439,12 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
|
|||
|
||||
line.clear();
|
||||
reader.read_line(&mut line).unwrap();
|
||||
let CoordinatorResponse::ProcessStarted { process, actor, .. } =
|
||||
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
|
||||
let CoordinatorResponse::ProcessStarted {
|
||||
launch_attempt: None,
|
||||
process,
|
||||
actor,
|
||||
..
|
||||
} = serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
|
||||
else {
|
||||
panic!("expected authenticated strict process start");
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue