Update public backend API surface

Private source commit: ba3f7ce2b6d9
This commit is contained in:
Clusterflux Release 2026-07-26 17:22:33 +02:00
parent 784463622c
commit 26fdcb9d84
34 changed files with 5473 additions and 72 deletions

View file

@ -829,6 +829,7 @@ fn signed_node_request_auto(request: CoordinatorRequest) -> CoordinatorRequest {
| CoordinatorRequest::ReportDebugState { node, .. }
| CoordinatorRequest::ReportDebugProbeHit { node, .. }
| CoordinatorRequest::ReportTaskLog { node, .. }
| CoordinatorRequest::ReportTaskLogChunk { node, .. }
| CoordinatorRequest::ReportVfsMetadata { node, .. }
| CoordinatorRequest::TaskCompleted { node, .. } => node.clone(),
CoordinatorRequest::RequestRendezvous { source, .. } => source.node.to_string(),
@ -874,6 +875,9 @@ fn signed_node_request_auto_with_private_key(
(node.clone(), "report_debug_probe_hit")
}
CoordinatorRequest::ReportTaskLog { node, .. } => (node.clone(), "report_task_log"),
CoordinatorRequest::ReportTaskLogChunk { node, .. } => {
(node.clone(), "report_task_log_chunk")
}
CoordinatorRequest::ReportVfsMetadata { node, .. } => (node.clone(), "report_vfs_metadata"),
CoordinatorRequest::TaskCompleted { node, .. } => (node.clone(), "task_completed"),
_ => panic!("test helper only signs node-originated requests"),
@ -7850,12 +7854,16 @@ fn strict_service_stream_rejects_body_authority_and_accepts_cli_session() {
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let CoordinatorResponse::Error { message } =
let CoordinatorResponse::Error { error } =
serde_json::from_str::<CoordinatorResponse>(&line).unwrap()
else {
panic!("expected strict body-authority denial");
};
assert!(message.contains("request-body identity fields are not authority"));
assert!(error
.message
.contains("request-body identity fields are not authority"));
assert_eq!(error.request_id, "strict-stream-forged");
assert_eq!(error.code, clusterflux_core::ApiErrorCode::Forbidden);
write_coordinator_wire_request(
&mut stream,
@ -7911,10 +7919,14 @@ fn service_stream_rejects_invalid_versioned_envelope_metadata() {
let mut line = String::new();
reader.read_line(&mut line).unwrap();
let response = serde_json::from_str::<CoordinatorResponse>(&line).unwrap();
let CoordinatorResponse::Error { message } = response else {
let CoordinatorResponse::Error { error } = response else {
panic!("expected invalid wire envelope response");
};
assert!(message.contains("operation attach_node does not match payload operation ping"));
assert!(error
.message
.contains("operation attach_node does not match payload operation ping"));
assert_eq!(error.request_id, "bad-operation");
assert_eq!(error.code, clusterflux_core::ApiErrorCode::ValidationError);
stream.shutdown(std::net::Shutdown::Both).unwrap();
server.join().unwrap();
@ -8011,3 +8023,697 @@ fn coordinator_generates_and_bounds_node_enrollment_grants() {
assert_ne!(first_grant, second_grant);
assert_eq!(expires_at_epoch_seconds, 100 + 15 * 60);
}
#[test]
fn web_process_summaries_are_scoped_paginated_and_retain_authoritative_terminal_state() {
let mut service = CoordinatorService::new(7);
for (tenant, project, user, secret) in [
("tenant-a", "project-a", "user-a", "session-a"),
("tenant-b", "project-b", "user-b", "session-b"),
] {
service
.issue_cli_session(
TenantId::from(tenant),
ProjectId::from(project),
UserId::from(user),
secret,
None,
)
.unwrap();
}
service.set_server_time(100);
service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::StartProcess {
launch_attempt: None,
process: "process-one".to_owned(),
restart: false,
},
})
.unwrap();
let CoordinatorResponse::ProcessSummaries {
processes,
next_cursor,
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: None,
limit: 1,
},
})
.unwrap()
else {
panic!("expected process summaries");
};
assert_eq!(processes.len(), 1);
assert_eq!(processes[0].process, ProcessId::from("process-one"));
assert_eq!(processes[0].lifecycle, ProcessLifecycleState::Active);
assert_eq!(processes[0].activity, ProcessActivityState::Running);
assert_eq!(processes[0].started_at_epoch_seconds, 100);
assert!(next_cursor.is_none());
let CoordinatorResponse::ProcessSummaries { processes, .. } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-b".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: None,
limit: 10,
},
})
.unwrap()
else {
panic!("expected scoped process summaries");
};
assert!(processes.is_empty());
service.set_server_time(120);
service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::AbortProcess {
process: "process-one".to_owned(),
launch_attempt: None,
},
})
.unwrap();
let CoordinatorResponse::ProcessSummaries { processes, .. } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: None,
limit: 10,
},
})
.unwrap()
else {
panic!("expected terminal process summary");
};
assert_eq!(
processes[0].lifecycle,
ProcessLifecycleState::RecentTerminal
);
assert_eq!(processes[0].activity, ProcessActivityState::Cancelled);
assert_eq!(
processes[0].final_result,
Some(ProcessFinalResult::Cancelled)
);
assert_eq!(processes[0].ended_at_epoch_seconds, Some(120));
service.set_server_time(130);
service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::StartProcess {
launch_attempt: None,
process: "process-two".to_owned(),
restart: false,
},
})
.unwrap();
let CoordinatorResponse::ProcessSummaries {
processes,
next_cursor: Some(cursor),
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: None,
limit: 1,
},
})
.unwrap()
else {
panic!("expected first process summary page");
};
assert_eq!(processes[0].process, ProcessId::from("process-two"));
let CoordinatorResponse::ProcessSummaries {
processes,
next_cursor: None,
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: Some(cursor),
limit: 1,
},
})
.unwrap()
else {
panic!("expected final process summary page");
};
assert_eq!(processes[0].process, ProcessId::from("process-one"));
let oversized = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListProcessSummaries {
cursor: None,
limit: 101,
},
})
.unwrap_err();
assert!(oversized.to_string().contains("limit"));
assert!(oversized.to_string().contains("100"));
}
#[test]
fn web_node_summaries_are_scoped_paginated_and_hard_bounded() {
let mut service = CoordinatorService::new(7);
service
.issue_cli_session(
TenantId::from("tenant"),
ProjectId::from("project"),
UserId::from("user"),
"session",
None,
)
.unwrap();
for node in ["node-a", "node-b", "node-c"] {
enroll_test_node(
&mut service,
"tenant",
"project",
node,
&test_node_public_key(node),
);
service
.handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
tenant: "tenant".to_owned(),
project: "project".to_owned(),
node: node.to_owned(),
capabilities: linux_capabilities(),
cached_environment_digests: Vec::new(),
dependency_cache_digests: Vec::new(),
source_snapshots: Vec::new(),
artifact_locations: Vec::new(),
direct_connectivity: true,
online: true,
})
.unwrap();
}
let CoordinatorResponse::NodeSummaries {
nodes,
next_cursor: Some(cursor),
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session".to_owned(),
request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
cursor: None,
limit: 2,
},
})
.unwrap()
else {
panic!("expected first node-summary page");
};
assert_eq!(
nodes
.iter()
.map(|node| node.id.as_str())
.collect::<Vec<_>>(),
["node-a", "node-b"]
);
let CoordinatorResponse::NodeSummaries {
nodes,
next_cursor: None,
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session".to_owned(),
request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
cursor: Some(cursor),
limit: 2,
},
})
.unwrap()
else {
panic!("expected final node-summary page");
};
assert_eq!(nodes.len(), 1);
assert_eq!(nodes[0].id, NodeId::from("node-c"));
let oversized = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session".to_owned(),
request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
cursor: None,
limit: 201,
},
})
.unwrap_err();
assert!(oversized.to_string().contains("limit"));
assert!(oversized.to_string().contains("200"));
}
#[test]
fn web_artifact_queries_are_scoped_paginated_and_track_retention_availability() {
let mut service = CoordinatorService::new(7);
for (tenant, project, user, secret, node) in [
("tenant-a", "project-a", "user-a", "session-a", "node-a"),
("tenant-b", "project-b", "user-b", "session-b", "node-b"),
] {
service
.issue_cli_session(
TenantId::from(tenant),
ProjectId::from(project),
UserId::from(user),
secret,
None,
)
.unwrap();
enroll_test_node(
&mut service,
tenant,
project,
node,
&test_node_public_key(node),
);
}
service.set_server_time(100);
for (tenant, project, node) in [
("tenant-a", "project-a", "node-a"),
("tenant-b", "project-b", "node-b"),
] {
service
.handle_signed_node_request_auto(CoordinatorRequest::ReportNodeCapabilities {
tenant: tenant.to_owned(),
project: project.to_owned(),
node: node.to_owned(),
capabilities: linux_capabilities(),
cached_environment_digests: Vec::new(),
dependency_cache_digests: Vec::new(),
source_snapshots: Vec::new(),
artifact_locations: vec!["shared-artifact".to_owned()],
direct_connectivity: true,
online: false,
})
.unwrap();
}
let CoordinatorResponse::NodeSummaries { nodes, .. } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
cursor: None,
limit: 200,
},
})
.unwrap()
else {
panic!("expected node summaries");
};
assert_eq!(nodes.len(), 1);
assert_eq!(nodes[0].id, NodeId::from("node-a"));
assert!(nodes[0].online);
assert!(!nodes[0].stale);
assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100));
assert_eq!(nodes[0].capabilities.os, Os::Linux);
for (tenant, project, node, digest) in [
("tenant-a", "project-a", "node-a", "tenant-a-bytes"),
("tenant-b", "project-b", "node-b", "tenant-b-bytes"),
] {
service.artifact_registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("shared-artifact"),
tenant: TenantId::from(tenant),
project: ProjectId::from(project),
process: ProcessId::from("process-one"),
producer_task: TaskInstanceId::from("task-one"),
retaining_node: NodeId::from(node),
digest: Digest::sha256(digest),
size: digest.len() as u64,
});
}
service.artifact_registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("second-artifact"),
tenant: TenantId::from("tenant-a"),
project: ProjectId::from("project-a"),
process: ProcessId::from("process-two"),
producer_task: TaskInstanceId::from("task-two"),
retaining_node: NodeId::from("node-a"),
digest: Digest::sha256("second"),
size: 6,
});
let CoordinatorResponse::Artifacts {
artifacts,
next_cursor: Some(cursor),
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListArtifacts {
process: None,
cursor: None,
limit: 1,
},
})
.unwrap()
else {
panic!("expected first artifact page");
};
assert_eq!(artifacts.len(), 1);
assert_eq!(artifacts[0].id, ArtifactId::from("second-artifact"));
assert_eq!(artifacts[0].availability, ArtifactAvailability::Available);
let CoordinatorResponse::Artifacts {
artifacts,
next_cursor: None,
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListArtifacts {
process: None,
cursor: Some(cursor),
limit: 1,
},
})
.unwrap()
else {
panic!("expected final artifact page");
};
assert_eq!(artifacts[0].id, ArtifactId::from("shared-artifact"));
assert_eq!(artifacts[0].digest, Digest::sha256("tenant-a-bytes"));
let cross_tenant = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::GetArtifact {
artifact: "tenant-b-only".to_owned(),
},
})
.unwrap_err();
assert!(cross_tenant.to_string().contains("does not exist"));
let CoordinatorResponse::Artifact { artifact } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-b".to_owned(),
request: AuthenticatedCoordinatorRequest::GetArtifact {
artifact: "shared-artifact".to_owned(),
},
})
.unwrap()
else {
panic!("expected tenant-b artifact");
};
assert_eq!(artifact.digest, Digest::sha256("tenant-b-bytes"));
service.set_server_time(131);
let CoordinatorResponse::NodeSummaries { nodes, .. } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListNodeSummaries {
cursor: None,
limit: 200,
},
})
.unwrap()
else {
panic!("expected stale node summary");
};
assert!(!nodes[0].online);
assert!(nodes[0].stale);
assert_eq!(nodes[0].last_seen_epoch_seconds, Some(100));
let CoordinatorResponse::Artifact { artifact } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::GetArtifact {
artifact: "shared-artifact".to_owned(),
},
})
.unwrap()
else {
panic!("expected offline artifact metadata");
};
assert_eq!(artifact.availability, ArtifactAvailability::NodeOffline);
assert!(!artifact.downloadable_now);
service
.artifact_registry
.sync_to_explicit_store(
&TenantId::from("tenant-a"),
&ProjectId::from("project-a"),
&ArtifactId::from("shared-artifact"),
"store://tenant-a/shared-artifact",
)
.unwrap();
let CoordinatorResponse::Artifact { artifact } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::GetArtifact {
artifact: "shared-artifact".to_owned(),
},
})
.unwrap()
else {
panic!("expected explicitly retained artifact metadata");
};
assert_eq!(artifact.availability, ArtifactAvailability::Available);
assert_eq!(
artifact.retention_state,
ArtifactRetentionState::ExplicitStorage
);
assert!(artifact.downloadable_now);
}
#[test]
fn web_recent_logs_are_signed_scoped_cursor_safe_and_memory_bounded() {
let mut service = CoordinatorService::new(7);
for (tenant, project, user, secret, node, process) in [
(
"tenant-a",
"project-a",
"user-a",
"session-a",
"node-a",
"process-shared",
),
(
"tenant-b",
"project-b",
"user-b",
"session-b",
"node-b",
"process-b",
),
] {
service
.issue_cli_session(
TenantId::from(tenant),
ProjectId::from(project),
UserId::from(user),
secret,
None,
)
.unwrap();
enroll_test_node(
&mut service,
tenant,
project,
node,
&test_node_public_key(node),
);
service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: secret.to_owned(),
request: AuthenticatedCoordinatorRequest::StartProcess {
launch_attempt: None,
process: process.to_owned(),
restart: false,
},
})
.unwrap();
service
.handle_signed_node_request_auto(CoordinatorRequest::ReconnectNode {
tenant: tenant.to_owned(),
project: project.to_owned(),
node: node.to_owned(),
process: process.to_owned(),
epoch: 7,
})
.unwrap();
}
service.set_server_time(100);
let first = service
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
tenant: "tenant-a".to_owned(),
project: "project-a".to_owned(),
process: "process-shared".to_owned(),
node: "node-a".to_owned(),
task: "task-one".to_owned(),
stream: TaskLogStream::Stdout,
offset: 0,
source_bytes: 5,
text: "hello".to_owned(),
truncated: false,
})
.unwrap();
let CoordinatorResponse::TaskLogChunkRecorded {
sequence: Some(first_sequence),
next_offset: 5,
..
} = first
else {
panic!("expected first live log sequence");
};
let retry = service
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
tenant: "tenant-a".to_owned(),
project: "project-a".to_owned(),
process: "process-shared".to_owned(),
node: "node-a".to_owned(),
task: "task-one".to_owned(),
stream: TaskLogStream::Stdout,
offset: 0,
source_bytes: 5,
text: "hello".to_owned(),
truncated: false,
})
.unwrap();
assert!(matches!(
retry,
CoordinatorResponse::TaskLogChunkRecorded { sequence: None, .. }
));
service
.handle_signed_node_request_auto(CoordinatorRequest::ReportTaskLogChunk {
tenant: "tenant-a".to_owned(),
project: "project-a".to_owned(),
process: "process-shared".to_owned(),
node: "node-a".to_owned(),
task: "task-one".to_owned(),
stream: TaskLogStream::Stdout,
offset: 8,
source_bytes: 2,
text: "ok".to_owned(),
truncated: false,
})
.unwrap();
let CoordinatorResponse::RecentLogs {
entries,
next_sequence: Some(cursor),
history_truncated: false,
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListRecentLogs {
process: "process-shared".to_owned(),
task: None,
after_sequence: None,
limit: 2,
},
})
.unwrap()
else {
panic!("expected first recent-log page");
};
assert_eq!(entries.len(), 2);
assert_eq!(entries[0].sequence, first_sequence);
assert_eq!(entries[0].text, "hello");
assert!(entries[1].text.contains("3 bytes"));
assert!(entries[1].truncated);
let CoordinatorResponse::RecentLogs { entries, .. } = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListRecentLogs {
process: "process-shared".to_owned(),
task: None,
after_sequence: Some(cursor),
limit: 2,
},
})
.unwrap()
else {
panic!("expected second recent-log page");
};
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].text, "ok");
let cross_tenant = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-b".to_owned(),
request: AuthenticatedCoordinatorRequest::ListRecentLogs {
process: "process-shared".to_owned(),
task: None,
after_sequence: None,
limit: 10,
},
})
.unwrap_err();
assert!(cross_tenant.to_string().contains("outside"));
let oversized = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListRecentLogs {
process: "process-shared".to_owned(),
task: None,
after_sequence: None,
limit: 201,
},
})
.unwrap_err();
assert!(oversized.to_string().contains("limit"));
assert!(oversized.to_string().contains("200"));
service
.handle_report_task_log_chunk(
"tenant-b".to_owned(),
"project-b".to_owned(),
"process-b".to_owned(),
"node-b".to_owned(),
"task-b".to_owned(),
TaskLogStream::Stderr,
0,
1,
"b".to_owned(),
false,
)
.unwrap();
for offset in 10..310 {
service
.handle_report_task_log_chunk(
"tenant-a".to_owned(),
"project-a".to_owned(),
"process-shared".to_owned(),
"node-a".to_owned(),
"task-one".to_owned(),
TaskLogStream::Stdout,
offset,
1,
"x".to_owned(),
false,
)
.unwrap();
}
let tenant_a_logs =
&service.recent_logs[&(TenantId::from("tenant-a"), ProjectId::from("project-a"))];
assert!(tenant_a_logs.len() <= MAX_RECENT_LOG_ENTRIES_PER_PROCESS);
let tenant_b_logs =
&service.recent_logs[&(TenantId::from("tenant-b"), ProjectId::from("project-b"))];
assert_eq!(tenant_b_logs.len(), 1);
assert_eq!(tenant_b_logs[0].text, "b");
let CoordinatorResponse::RecentLogs {
entries,
history_truncated,
..
} = service
.handle_request(CoordinatorRequest::Authenticated {
session_secret: "session-a".to_owned(),
request: AuthenticatedCoordinatorRequest::ListRecentLogs {
process: "process-shared".to_owned(),
task: None,
after_sequence: None,
limit: 200,
},
})
.unwrap()
else {
panic!("expected bounded recent-log response");
};
assert_eq!(entries.len(), 200);
assert!(history_truncated);
}