Public release release-7fcdc75d8eaf

Source commit: 7fcdc75d8eaf4dc03b09086568dbb184f903f6a4

Public tree identity: sha256:de9eec9b4e2f4cba2fa32b6ecb4b3424cce793a0f8a969707cc9c4565acdbb4e

crates/clusterflux-coordinator/src/service/logs.rs

@@ -248,10 +248,19 @@ impl CoordinatorService {
             self.notify_coordinator_main_waiters(&event);
         }
         self.maybe_retire_terminal_process(&event.tenant, &event.project, &event.process)?;
+        let events_recorded = self
+            .task_events
+            .iter()
+            .filter(|recorded| {
+                recorded.tenant == event.tenant
+                    && recorded.project == event.project
+                    && recorded.process == event.process
+            })
+            .count();
         Ok(CoordinatorResponse::TaskRecorded {
             process: event.process,
             task: event.task,
-            events_recorded: self.task_events.len(),
+            events_recorded,
         })
     }
 
@@ -613,6 +622,14 @@ impl CoordinatorService {
         self.coordinator.abort_process(tenant, project, process)?;
         self.clear_debug_state_for_process(tenant, project, process);
         self.clear_operator_panel_state(tenant, project, process);
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(tenant, project);
+        self.artifact_registry.enforce_project_metadata_limit(
+            tenant,
+            project,
+            &pinned,
+            &protected_processes,
+        );
         Ok(true)
     }
 
@@ -623,8 +640,30 @@ impl CoordinatorService {
         let now_epoch_seconds = self.current_epoch_seconds()?;
         self.artifact_registry
             .expire_download_links(now_epoch_seconds);
+        let tenant = flush.tenant.clone();
+        let project = flush.project.clone();
+        let (pinned, protected_processes) =
+            self.artifact_retention_guards_for_project(&tenant, &project);
+        self.artifact_registry
+            .flush_metadata_with_protected_processes(flush, &pinned, &protected_processes)
+            .map(|_| ())
+            .map_err(CoordinatorServiceError::Protocol)
+    }
+
+    fn artifact_retention_guards_for_project(
+        &self,
+        tenant: &TenantId,
+        project: &ProjectId,
+    ) -> (
+        std::collections::BTreeSet<ArtifactScopeKey>,
+        std::collections::BTreeSet<ProcessId>,
+    ) {
         let mut pinned = std::collections::BTreeSet::new();
         for checkpoint in self.task_restart_checkpoints.values() {
+            if &checkpoint.assignment.tenant != tenant || &checkpoint.assignment.project != project
+            {
+                continue;
+            }
             for artifact in &checkpoint.assignment.task_spec.required_artifacts {
                 pinned.insert(ArtifactScopeKey::from_refs(
                     &checkpoint.assignment.tenant,
@@ -634,6 +673,9 @@ impl CoordinatorService {
             }
         }
         for pending in &self.pending_task_launches {
+            if &pending.tenant != tenant || &pending.project != project {
+                continue;
+            }
             for artifact in &pending.task_spec.required_artifacts {
                 pinned.insert(ArtifactScopeKey::from_refs(
                     &pending.tenant,
@@ -642,10 +684,13 @@ impl CoordinatorService {
                 ));
             }
         }
-        self.artifact_registry
-            .flush_metadata_bounded(flush, &pinned)
-            .map(|_| ())
-            .map_err(CoordinatorServiceError::Protocol)
+        let protected_processes = self
+            .coordinator
+            .active_processes_for_project(tenant, project)
+            .into_iter()
+            .map(|process| process.id)
+            .collect();
+        (pinned, protected_processes)
     }
 
     fn task_is_known_or_active(

crates/clusterflux-coordinator/src/service/nodes.rs

@@ -300,8 +300,8 @@ impl CoordinatorService {
             node_scope,
             NodeDescriptor {
                 id: node.clone(),
-                tenant,
-                project,
+                tenant: tenant.clone(),
+                project: project.clone(),
                 capabilities,
                 cached_environments: cached_environment_digests.into_iter().collect(),
                 dependency_caches: dependency_cache_digests.into_iter().collect(),
@@ -311,9 +311,14 @@ impl CoordinatorService {
                 online,
             },
         );
+        let node_descriptors = self
+            .node_descriptors
+            .values()
+            .filter(|descriptor| descriptor.tenant == tenant && descriptor.project == project)
+            .count();
         Ok(CoordinatorResponse::NodeCapabilitiesRecorded {
             node,
-            node_descriptors: self.node_descriptors.len(),
+            node_descriptors,
         })
     }
 

crates/clusterflux-coordinator/src/service/process_launch.rs

@@ -545,13 +545,22 @@ impl CoordinatorService {
                     task_spec,
                     wasm_module_base64,
                 });
+                let queued_tasks = self
+                    .pending_task_launches
+                    .iter()
+                    .filter(|pending| {
+                        pending.tenant == tenant
+                            && pending.project == project
+                            && pending.process == process
+                    })
+                    .count();
                 return Ok(CoordinatorResponse::TaskQueued {
                     process,
                     task,
                     actor,
                     reason,
                     charged_spawns,
-                    queued_tasks: self.pending_task_launches.len(),
+                    queued_tasks,
                 });
             }
             Err(err) => return Err(err.into()),

crates/clusterflux-coordinator/src/service/tests.rs

@@ -512,6 +512,20 @@ fn service_with_completed_main_and_final_child(
             public_key: test_node_public_key(node.as_str()),
         })
         .unwrap();
+    service
+        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
+            tenant: tenant.to_string(),
+            project: project.to_string(),
+            node: node.to_string(),
+            capabilities: linux_capabilities(),
+            cached_environment_digests: Vec::new(),
+            dependency_cache_digests: Vec::new(),
+            source_snapshots: Vec::new(),
+            artifact_locations: Vec::new(),
+            direct_connectivity: true,
+            online: true,
+        })
+        .unwrap();
     service
         .handle_request(CoordinatorRequest::StartProcess {
             launch_attempt: None,
@@ -4549,6 +4563,81 @@ fn completed_main_await_operator_blocks_retirement_until_each_resolution() {
     }
 }
 
+#[test]
+fn completed_main_failed_child_restarted_successfully_retires_with_successful_current_attempt() {
+    let mut service = service_with_completed_main_and_final_child(
+        clusterflux_core::TaskFailurePolicy::AwaitOperator,
+    );
+    complete_terminal_matrix_child(&mut service, TaskTerminalState::Failed);
+    let tenant = TenantId::from("tenant");
+    let project = ProjectId::from("project");
+    let process = ProcessId::from("terminal-matrix");
+    let task = TaskInstanceId::from("final-child");
+
+    let CoordinatorResponse::TaskRestart { accepted, .. } = service
+        .handle_request(CoordinatorRequest::RestartTask {
+            tenant: tenant.to_string(),
+            project: project.to_string(),
+            actor_user: "user".to_owned(),
+            process: process.to_string(),
+            task: task.to_string(),
+            replacement_bundle: None,
+        })
+        .unwrap()
+    else {
+        panic!("expected task restart");
+    };
+    assert!(accepted);
+
+    service
+        .handle_signed_node_request_auto(CoordinatorRequest::TaskCompleted {
+            tenant: tenant.to_string(),
+            project: project.to_string(),
+            process: process.to_string(),
+            node: "worker".to_owned(),
+            task: task.to_string(),
+            terminal_state: Some(TaskTerminalState::Completed),
+            status_code: Some(0),
+            stdout_bytes: 2,
+            stderr_bytes: 0,
+            stdout_tail: "ok".to_owned(),
+            stderr_tail: String::new(),
+            stdout_truncated: false,
+            stderr_truncated: false,
+            artifact_path: None,
+            artifact_digest: None,
+            artifact_size_bytes: None,
+            result: Some(TaskBoundaryValue::SmallJson(json!("ok"))),
+        })
+        .unwrap();
+
+    assert!(service
+        .coordinator
+        .active_process(&tenant, &project, &process)
+        .is_none());
+    let attempts = service
+        .task_attempts
+        .get(&super::keys::task_restart_key(
+            &tenant, &project, &process, &task,
+        ))
+        .unwrap();
+    assert!(
+        attempts
+            .iter()
+            .any(|attempt| !attempt.current
+                && attempt.state == TaskAttemptState::FailedAwaitingAction)
+    );
+    assert!(attempts
+        .iter()
+        .any(|attempt| attempt.current && attempt.state == TaskAttemptState::Completed));
+    assert_eq!(
+        service
+            .task_join_result(tenant, project, process, task)
+            .state,
+        TaskJoinState::Completed
+    );
+}
+
 #[test]
 fn completed_main_failed_child_does_not_abort_another_active_child() {
     let mut service =
@@ -5570,6 +5659,29 @@ fn retained_artifact_reverse_stream_is_chunked_below_control_frame_limit() {
 #[test]
 fn windows_task_events_share_the_virtual_process_scope() {
     let mut service = CoordinatorService::new(7);
+    service.record_task_completion_event(TaskCompletionEvent {
+        tenant: TenantId::from("other-tenant"),
+        project: ProjectId::from("other-project"),
+        process: ProcessId::from("other-process"),
+        node: NodeId::from("other-node"),
+        executor: super::TaskExecutor::Node,
+        task_definition: TaskDefinitionId::from("other-task"),
+        task: TaskInstanceId::from("other-task"),
+        attempt_id: None,
+        placement: None,
+        terminal_state: TaskTerminalState::Completed,
+        status_code: Some(0),
+        stdout_bytes: 0,
+        stderr_bytes: 0,
+        stdout_tail: String::new(),
+        stderr_tail: String::new(),
+        stdout_truncated: false,
+        stderr_truncated: false,
+        artifact_path: None,
+        artifact_digest: None,
+        artifact_size_bytes: None,
+        result: None,
+    });
     service
         .handle_request(CoordinatorRequest::AttachNode {
             tenant: "tenant".to_owned(),
@@ -5670,6 +5782,28 @@ fn windows_task_events_share_the_virtual_process_scope() {
 #[test]
 fn service_schedules_task_across_reported_node_descriptors() {
     let mut service = CoordinatorService::new(7);
+    service
+        .handle_request(CoordinatorRequest::AttachNode {
+            tenant: "other-tenant".to_owned(),
+            project: "other-project".to_owned(),
+            node: "other-node".to_owned(),
+            public_key: test_node_public_key("other-node"),
+        })
+        .unwrap();
+    service
+        .handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
+            tenant: "other-tenant".to_owned(),
+            project: "other-project".to_owned(),
+            node: "other-node".to_owned(),
+            capabilities: linux_capabilities(),
+            cached_environment_digests: Vec::new(),
+            dependency_cache_digests: Vec::new(),
+            source_snapshots: Vec::new(),
+            artifact_locations: Vec::new(),
+            direct_connectivity: false,
+            online: true,
+        })
+        .unwrap();
     for node in ["cold-node", "warm-node"] {
         service
             .handle_request(CoordinatorRequest::AttachNode {
@@ -6957,6 +7091,52 @@ fn coordinator_side_task_launch_fails_cleanly_without_capable_worker() {
 #[test]
 fn coordinator_side_task_launch_can_wait_for_capable_worker() {
     let mut service = CoordinatorService::new(11);
+    let CoordinatorResponse::ProcessStarted {
+        epoch: other_epoch, ..
+    } = service
+        .handle_request(CoordinatorRequest::StartProcess {
+            launch_attempt: None,
+            tenant: "other-tenant".to_owned(),
+            project: "other-project".to_owned(),
+            actor_user: None,
+            actor_agent: None,
+            agent_public_key_fingerprint: None,
+            agent_signature: None,
+            process: "other-vp-wait".to_owned(),
+            restart: false,
+        })
+        .unwrap()
+    else {
+        panic!("expected unrelated process start");
+    };
+    let CoordinatorResponse::TaskQueued {
+        queued_tasks: other_queued_tasks,
+        ..
+    } = service
+        .handle_authorized_test_task_launch(CoordinatorRequest::LaunchTask {
+            task_spec: test_task_spec(
+                "other-tenant",
+                "other-project",
+                "other-vp-wait",
+                "other-compile",
+                other_epoch,
+                [Capability::Command],
+            ),
+            tenant: "other-tenant".to_owned(),
+            project: "other-project".to_owned(),
+            actor_user: Some("other-user".to_owned()),
+            actor_agent: None,
+            agent_public_key_fingerprint: None,
+            agent_signature: None,
+            wait_for_node: true,
+            artifact_path: "/vfs/artifacts/other-wait-output.txt".to_owned(),
+            wasm_module_base64: test_wasm_module_base64(),
+        })
+        .unwrap()
+    else {
+        panic!("expected unrelated queued task launch");
+    };
+    assert_eq!(other_queued_tasks, 1);
     let CoordinatorResponse::ProcessStarted {
         launch_attempt: None,
         epoch,