Expose public task restart boundary
This commit is contained in:
parent
f586b48893
commit
0b14889c79
5 changed files with 419 additions and 8 deletions
|
|
@ -221,6 +221,13 @@ pub enum CoordinatorRequest {
|
|||
node: String,
|
||||
task: String,
|
||||
},
|
||||
RestartTask {
|
||||
tenant: String,
|
||||
project: String,
|
||||
actor_user: String,
|
||||
process: String,
|
||||
task: String,
|
||||
},
|
||||
DebugAttach {
|
||||
tenant: String,
|
||||
project: String,
|
||||
|
|
@ -558,6 +565,17 @@ pub enum CoordinatorResponse {
|
|||
task: TaskId,
|
||||
cancel_requested: bool,
|
||||
},
|
||||
TaskRestart {
|
||||
process: ProcessId,
|
||||
task: TaskId,
|
||||
actor: UserId,
|
||||
accepted: bool,
|
||||
clean_boundary_available: bool,
|
||||
active_task: bool,
|
||||
completed_event_observed: bool,
|
||||
requires_whole_process_restart: bool,
|
||||
message: String,
|
||||
},
|
||||
DebugCommand {
|
||||
process: ProcessId,
|
||||
task: TaskId,
|
||||
|
|
@ -1571,6 +1589,72 @@ impl CoordinatorService {
|
|||
cancel_requested,
|
||||
})
|
||||
}
|
||||
CoordinatorRequest::RestartTask {
|
||||
tenant,
|
||||
project,
|
||||
actor_user,
|
||||
process,
|
||||
task,
|
||||
} => {
|
||||
let tenant = TenantId::new(tenant);
|
||||
let project = ProjectId::new(project);
|
||||
let actor = UserId::new(actor_user);
|
||||
let process = ProcessId::new(process);
|
||||
let task = TaskId::new(task);
|
||||
let context = disasmer_core::AuthContext {
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
actor: Actor::User(actor.clone()),
|
||||
};
|
||||
self.coordinator.upsert_user(
|
||||
tenant.clone(),
|
||||
actor.clone(),
|
||||
CredentialKind::BrowserSession,
|
||||
);
|
||||
let authorization = self.coordinator.authorize_debug_attach(&context, &process);
|
||||
if !authorization.allowed {
|
||||
return Err(CoordinatorError::Unauthorized(format!(
|
||||
"task restart denied: {}",
|
||||
authorization.reason
|
||||
))
|
||||
.into());
|
||||
}
|
||||
let active_key = self
|
||||
.active_tasks
|
||||
.iter()
|
||||
.find(|(task_tenant, task_project, task_process, _, task_id)| {
|
||||
task_tenant == &tenant
|
||||
&& task_project == &project
|
||||
&& task_process == &process
|
||||
&& task_id == &task
|
||||
})
|
||||
.cloned();
|
||||
let active_task = active_key.is_some();
|
||||
let completed_event_observed = self.task_events.iter().any(|event| {
|
||||
event.tenant == tenant
|
||||
&& event.project == project
|
||||
&& event.process == process
|
||||
&& event.task == task
|
||||
});
|
||||
let message = if active_task {
|
||||
"selected task is still active; clean task restart requires a captured checkpoint boundary"
|
||||
} else if completed_event_observed {
|
||||
"selected task has only terminal event metadata; restart requires a captured checkpoint boundary"
|
||||
} else {
|
||||
"selected task is not known in the active process; restart the whole virtual process or inspect task list"
|
||||
};
|
||||
Ok(CoordinatorResponse::TaskRestart {
|
||||
process,
|
||||
task,
|
||||
actor,
|
||||
accepted: false,
|
||||
clean_boundary_available: false,
|
||||
active_task,
|
||||
completed_event_observed,
|
||||
requires_whole_process_restart: true,
|
||||
message: message.to_owned(),
|
||||
})
|
||||
}
|
||||
CoordinatorRequest::DebugAttach {
|
||||
tenant,
|
||||
project,
|
||||
|
|
@ -3160,6 +3244,171 @@ mod tests {
|
|||
assert!(authorization.reason.contains("tenant or project"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_reports_task_restart_boundary_through_public_api() {
|
||||
let mut service = CoordinatorService::new(7);
|
||||
service
|
||||
.handle_request(CoordinatorRequest::CreateProject {
|
||||
tenant: "tenant".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
name: "Demo".to_owned(),
|
||||
})
|
||||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::StartProcess {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
restart: false,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let denied = service
|
||||
.handle_request(CoordinatorRequest::RestartTask {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "other-user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
})
|
||||
.unwrap_err();
|
||||
assert!(denied.to_string().contains("task restart denied"));
|
||||
|
||||
let CoordinatorResponse::TaskRestart {
|
||||
accepted,
|
||||
clean_boundary_available,
|
||||
active_task,
|
||||
completed_event_observed,
|
||||
requires_whole_process_restart,
|
||||
message,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::RestartTask {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected task restart response");
|
||||
};
|
||||
assert!(!accepted);
|
||||
assert!(!clean_boundary_available);
|
||||
assert!(!active_task);
|
||||
assert!(!completed_event_observed);
|
||||
assert!(requires_whole_process_restart);
|
||||
assert!(message.contains("not known"));
|
||||
|
||||
service
|
||||
.handle_request(CoordinatorRequest::AttachNode {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
node: "worker-linux".to_owned(),
|
||||
public_key: "node-public-key".to_owned(),
|
||||
})
|
||||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::ReportNodeCapabilities {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
node: "worker-linux".to_owned(),
|
||||
capabilities: linux_capabilities(),
|
||||
cached_environment_digests: Vec::new(),
|
||||
dependency_cache_digests: Vec::new(),
|
||||
source_snapshots: Vec::new(),
|
||||
artifact_locations: Vec::new(),
|
||||
direct_connectivity: true,
|
||||
online: true,
|
||||
})
|
||||
.unwrap();
|
||||
service
|
||||
.handle_request(CoordinatorRequest::LaunchTask {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
environment: None,
|
||||
environment_digest: None,
|
||||
required_capabilities: vec![Capability::Command],
|
||||
dependency_cache: None,
|
||||
source_snapshot: None,
|
||||
required_artifacts: Vec::new(),
|
||||
quota_available: true,
|
||||
policy_allowed: true,
|
||||
command: "cargo".to_owned(),
|
||||
command_args: vec!["test".to_owned()],
|
||||
artifact_path: "/vfs/artifacts/output.txt".to_owned(),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let CoordinatorResponse::TaskRestart {
|
||||
active_task,
|
||||
completed_event_observed,
|
||||
message,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::RestartTask {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected active task restart response");
|
||||
};
|
||||
assert!(active_task);
|
||||
assert!(!completed_event_observed);
|
||||
assert!(message.contains("still active"));
|
||||
|
||||
service
|
||||
.handle_request(CoordinatorRequest::TaskCompleted {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
node: "worker-linux".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
terminal_state: Some(TaskTerminalState::Completed),
|
||||
status_code: Some(0),
|
||||
stdout_bytes: 0,
|
||||
stderr_bytes: 0,
|
||||
stdout_tail: String::new(),
|
||||
stderr_tail: String::new(),
|
||||
stdout_truncated: false,
|
||||
stderr_truncated: false,
|
||||
artifact_path: None,
|
||||
artifact_digest: None,
|
||||
artifact_size_bytes: None,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let CoordinatorResponse::TaskRestart {
|
||||
active_task,
|
||||
completed_event_observed,
|
||||
message,
|
||||
..
|
||||
} = service
|
||||
.handle_request(CoordinatorRequest::RestartTask {
|
||||
tenant: "tenant".to_owned(),
|
||||
project: "project".to_owned(),
|
||||
actor_user: "user".to_owned(),
|
||||
process: "process".to_owned(),
|
||||
task: "task".to_owned(),
|
||||
})
|
||||
.unwrap()
|
||||
else {
|
||||
panic!("expected completed task restart response");
|
||||
};
|
||||
assert!(!active_task);
|
||||
assert!(completed_event_observed);
|
||||
assert!(message.contains("terminal event metadata"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_cancels_whole_process_and_blocks_new_task_launches() {
|
||||
let mut service = CoordinatorService::new(7);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue