michel/clusterflux-public
1
0
Fork 0
Code Issues Pull requests Projects Releases 21 Packages Wiki Activity Actions

Public release release-ea887c8f56cd

Browse source 
Source commit: ea887c8f56cd53985a1179b13e5f1b85c485f584

Public tree identity: sha256:b96a97aacdbc8fd4fc2ea20d0e1450280d46db3f24aabe1475e5fbe20e05f255
This commit is contained in:
Clusterflux release 2026-07-25 02:55:35 +02:00
parent 9223c54939
commit 2a0f7ded04
37 changed files with 3145 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

461
crates/clusterflux-core/src/artifact.rs
View file

@ -134,6 +134,27 @@ pub struct ArtifactMetadata {
pub coordinator_has_large_bytes: bool,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct ArtifactScopeKey {
pub tenant: TenantId,
pub project: ProjectId,
pub artifact: ArtifactId,
}
impl ArtifactScopeKey {
pub fn new(tenant: TenantId, project: ProjectId, artifact: ArtifactId) -> Self {
Self {
tenant,
project,
artifact,
}
}
pub fn from_refs(tenant: &TenantId, project: &ProjectId, artifact: &ArtifactId) -> Self {
Self::new(tenant.clone(), project.clone(), artifact.clone())
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ArtifactFlush {
pub id: ArtifactId,
@ -148,7 +169,7 @@ pub struct ArtifactFlush {
#[derive(Clone, Debug, Default)]
pub struct ArtifactRegistry {
artifacts: BTreeMap<ArtifactId, ArtifactMetadata>,
artifacts: BTreeMap<ArtifactScopeKey, ArtifactMetadata>,
issued_download_links: BTreeMap<Digest, IssuedDownloadLink>,
next_epoch: u64,
}
@ -162,9 +183,10 @@ impl ArtifactRegistry {
pub fn flush_metadata_bounded(
&mut self,
flush: ArtifactFlush,
pinned: &BTreeSet<ArtifactId>,
pinned: &BTreeSet<ArtifactScopeKey>,
) -> Result<ArtifactMetadata, String> {
let replacing_existing = self.artifacts.contains_key(&flush.id);
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
let replacing_existing = self.artifacts.contains_key(&key);
while !replacing_existing
&& self
.artifacts
@ -184,13 +206,25 @@ impl ArtifactRegistry {
metadata.tenant == flush.tenant
&& metadata.project == flush.project
&& metadata.process == flush.process
&& !pinned.contains(&metadata.id)
&& !self.issued_download_links.values().any(|issued| {
issued.link.artifact == metadata.id
})
&& !pinned.contains(&ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
))
&& !self.issued_download_links.values().any(|issued| {
issued.link.tenant == metadata.tenant
&& issued.link.project == metadata.project
&& issued.link.artifact == metadata.id
})
})
.min_by_key(|metadata| metadata.flushed_epoch)
.map(|metadata| metadata.id.clone())
.map(|metadata| {
ArtifactScopeKey::from_refs(
&metadata.tenant,
&metadata.project,
&metadata.id,
)
})
.ok_or_else(|| {
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
.to_owned()
@ -212,36 +246,47 @@ impl ArtifactRegistry {
explicit_locations: Vec::new(),
coordinator_has_large_bytes: false,
};
self.artifacts.insert(flush.id, metadata.clone());
self.artifacts.insert(key, metadata.clone());
Ok(metadata)
}
pub fn sync_to_explicit_store(
&mut self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
location: impl Into<String>,
) -> Result<(), ArtifactUnavailable> {
let metadata = self
.artifacts
.get_mut(artifact)
.get_mut(&ArtifactScopeKey::from_refs(tenant, project, artifact))
.ok_or(ArtifactUnavailable)?;
metadata.explicit_locations.push(location.into());
Ok(())
}
pub fn garbage_collect_node(&mut self, node: &NodeId) {
for metadata in self.artifacts.values_mut() {
pub fn garbage_collect_node(&mut self, tenant: &TenantId, project: &ProjectId, node: &NodeId) {
for metadata in self
.artifacts
.values_mut()
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
{
metadata.retaining_nodes.remove(node);
}
}
pub fn reconcile_node_retention(
&mut self,
tenant: &TenantId,
project: &ProjectId,
node: &NodeId,
retained_artifacts: &BTreeSet<ArtifactId>,
) {
for (artifact, metadata) in &mut self.artifacts {
if retained_artifacts.contains(artifact) {
for (key, metadata) in &mut self.artifacts {
if &key.tenant != tenant || &key.project != project {
continue;
}
if retained_artifacts.contains(&key.artifact) {
metadata.retaining_nodes.insert(node.clone());
} else {
metadata.retaining_nodes.remove(node);
@ -249,8 +294,14 @@ impl ArtifactRegistry {
}
}
pub fn metadata(&self, artifact: &ArtifactId) -> Option<&ArtifactMetadata> {
self.artifacts.get(artifact)
pub fn metadata(
&self,
tenant: &TenantId,
project: &ProjectId,
artifact: &ArtifactId,
) -> Option<&ArtifactMetadata> {
self.artifacts
.get(&ArtifactScopeKey::from_refs(tenant, project, artifact))
}
pub fn download_action(
@ -261,7 +312,11 @@ impl ArtifactRegistry {
) -> Result<DownloadAction, DownloadError> {
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let scope = Scope {
tenant: metadata.tenant.clone(),
@ -315,7 +370,11 @@ impl ArtifactRegistry {
self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
Ok(metadata.size)
}
@ -333,7 +392,11 @@ impl ArtifactRegistry {
if self
.issued_download_links
.values()
.filter(|issued| issued.link.artifact == *artifact)
.filter(|issued| {
issued.link.tenant == context.tenant
&& issued.link.project == context.project
&& issued.link.artifact == *artifact
})
.count()
>= MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT
{
@ -345,7 +408,11 @@ impl ArtifactRegistry {
let action = self.download_action(context, artifact, policy)?;
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
let expires_at_epoch_seconds = now_epoch_seconds.saturating_add(ttl_seconds);
let policy_context_digest =
@ -398,7 +465,11 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact || issued.link.actor != context.actor {
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.actor != context.actor
{
return Err(DownloadError::InvalidToken);
}
self.download_action(
@ -434,7 +505,9 @@ impl ArtifactRegistry {
.issued_download_links
.get(presented_token_digest)
.ok_or(DownloadError::InvalidToken)?;
if issued.link.artifact != *artifact
if issued.link.tenant != context.tenant
|| issued.link.project != context.project
|| issued.link.artifact != *artifact
|| issued.link.max_bytes != policy.max_bytes
|| issued.link.actor != context.actor
{
@ -452,7 +525,11 @@ impl ArtifactRegistry {
}
let metadata = self
.artifacts
.get(artifact)
.get(&ArtifactScopeKey::from_refs(
&context.tenant,
&context.project,
artifact,
))
.ok_or(DownloadError::NotFound)?;
if download_policy_context_digest(metadata, &action.source, policy)
!= issued.link.policy_context_digest
@ -477,7 +554,11 @@ impl ArtifactRegistry {
) -> Result<(), DownloadError> {
let metadata = self
.artifacts
.get(&stream.link.artifact)
.get(&ArtifactScopeKey::from_refs(
&stream.link.tenant,
&stream.link.project,
&stream.link.artifact,
))
.ok_or(DownloadError::NotFound)?;
if !source_is_available(metadata, &stream.link.source) {
return Err(DownloadError::Unavailable);
@ -596,7 +677,13 @@ mod tests {
#[test]
fn flush_publishes_metadata_without_coordinator_bytes() {
let registry = registry_with_artifact();
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(!metadata.coordinator_has_large_bytes);
assert_eq!(metadata.id, ArtifactId::from("artifact"));
@ -615,7 +702,11 @@ mod tests {
#[test]
fn unsynced_node_loss_surfaces_as_unavailable() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -636,9 +727,20 @@ mod tests {
#[test]
fn signed_node_retention_inventory_removes_garbage_collected_location() {
let mut registry = registry_with_artifact();
registry.reconcile_node_retention(&NodeId::from("node"), &BTreeSet::new());
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
&BTreeSet::new(),
);
let metadata = registry.metadata(&ArtifactId::from("artifact")).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap();
assert!(metadata.retaining_nodes.is_empty());
}
@ -648,11 +750,26 @@ mod tests {
let mut registry = registry_with_artifact();
let node = NodeId::from("node");
let artifact = ArtifactId::from("artifact");
registry.garbage_collect_node(&node);
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
);
registry.reconcile_node_retention(&node, &BTreeSet::from([artifact.clone()]));
registry.reconcile_node_retention(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&node,
&BTreeSet::from([artifact.clone()]),
);
let metadata = registry.metadata(&artifact).unwrap();
let metadata = registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&artifact,
)
.unwrap();
assert!(metadata.retaining_nodes.contains(&node));
}
@ -660,9 +777,18 @@ mod tests {
fn explicit_user_storage_location_survives_node_retention_loss() {
let mut registry = registry_with_artifact();
registry
.sync_to_explicit_store(&ArtifactId::from("artifact"), "s3://bucket/app")
.sync_to_explicit_store(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
"s3://bucket/app",
)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -683,7 +809,11 @@ mod tests {
);
assert!(
!registry
.metadata(&ArtifactId::from("artifact"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact"),
)
.unwrap()
.coordinator_has_large_bytes
);
@ -714,7 +844,7 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
@ -734,13 +864,206 @@ mod tests {
)
.unwrap_err();
assert!(matches!(error, DownloadError::Unauthorized(_)));
assert_eq!(error, DownloadError::NotFound);
}
#[test]
fn duplicate_artifact_ids_are_isolated_across_metadata_links_retention_and_limits() {
let artifact = ArtifactId::from("shared-artifact");
let tenant_a = TenantId::from("tenant-a");
let project_a = ProjectId::from("project-a");
let tenant_b = TenantId::from("tenant-b");
let project_b = ProjectId::from("project-b");
let project_c = ProjectId::from("project-c");
let node_a = NodeId::from("node-a");
let node_b = NodeId::from("node-b");
let node_c = NodeId::from("node-c");
let mut registry = ArtifactRegistry::default();
for (tenant, project, process, node, content, size) in [
(&tenant_a, &project_a, "process-a", &node_a, "bytes-a", 17),
(&tenant_b, &project_b, "process-b", &node_b, "bytes-b", 29),
(&tenant_a, &project_c, "process-c", &node_c, "bytes-c", 31),
] {
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant.clone(),
project: project.clone(),
process: ProcessId::from(process),
producer_task: TaskInstanceId::from("producer"),
retaining_node: node.clone(),
digest: Digest::sha256(content),
size,
});
}
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
assert_eq!(
registry
.metadata(&tenant_a, &project_c, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-c")
);
registry.flush_metadata(ArtifactFlush {
id: artifact.clone(),
tenant: tenant_a.clone(),
project: project_a.clone(),
process: ProcessId::from("process-a"),
producer_task: TaskInstanceId::from("producer-repeat"),
retaining_node: node_a.clone(),
digest: Digest::sha256("bytes-a-repeat"),
size: 18,
});
assert_eq!(registry.artifact_count(), 3);
assert_eq!(
registry
.metadata(&tenant_a, &project_a, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-a-repeat")
);
assert_eq!(
registry
.metadata(&tenant_b, &project_b, &artifact)
.unwrap()
.digest,
Digest::sha256("bytes-b")
);
registry
.sync_to_explicit_store(&tenant_a, &project_a, &artifact, "store://tenant-a")
.unwrap();
registry.garbage_collect_node(&tenant_a, &project_a, &node_a);
let context_a = AuthContext {
tenant: tenant_a.clone(),
project: project_a.clone(),
actor: Actor::User(UserId::from("user")),
};
let context_b = AuthContext {
tenant: tenant_b.clone(),
project: project_b.clone(),
actor: Actor::User(UserId::from("user")),
};
let policy = DownloadPolicy { max_bytes: 100 };
assert_eq!(
registry
.download_action(&context_a, &artifact, &policy)
.unwrap()
.source,
StorageLocation::ExplicitStore("store://tenant-a".to_owned())
);
assert_eq!(
registry
.download_action(&context_b, &artifact, &policy)
.unwrap()
.source,
StorageLocation::RetainedNode(node_b)
);
let first_a = registry
.create_download_link(&context_a, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
let first_b = registry
.create_download_link(&context_b, &artifact, &policy, "same-nonce", 10, 60)
.unwrap();
assert_ne!(
first_a.scoped_token_digest, first_b.scoped_token_digest,
"the full artifact scope must contribute to download tokens"
);
for index in 1..MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT {
registry
.create_download_link(
&context_a,
&artifact,
&policy,
&format!("tenant-a-{index}"),
10,
60,
)
.unwrap();
}
assert!(matches!(
registry.create_download_link(
&context_a,
&artifact,
&policy,
"tenant-a-over-limit",
10,
60,
),
Err(DownloadError::Usage(_))
));
registry
.create_download_link(
&context_b,
&artifact,
&policy,
"tenant-b-still-independent",
10,
60,
)
.unwrap();
registry
.revoke_download_link(&context_a, &artifact, &first_a.scoped_token_digest)
.unwrap();
let limits = ResourceLimits {
limits: BTreeMap::from([(LimitKind::ArtifactDownloadBytes, 100)]),
};
let mut meter = ResourceMeter::default();
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_b.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.expect("revoking tenant A's link must not revoke tenant B's link");
assert_eq!(
registry
.open_download_stream(
DownloadStreamRequest {
context: &context_b,
artifact: &artifact,
policy: &policy,
presented_token_digest: &first_a.scoped_token_digest,
now_epoch_seconds: 11,
limits: &limits,
},
&mut meter,
)
.unwrap_err(),
DownloadError::InvalidToken
);
}
#[test]
fn download_link_is_not_created_when_artifact_is_unavailable_or_too_large() {
let mut registry = registry_with_artifact();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
let context = AuthContext {
tenant: TenantId::from("tenant"),
project: ProjectId::from("project"),
@ -1019,10 +1342,24 @@ mod tests {
#[test]
fn artifact_metadata_is_bounded_without_evicting_pins() {
let mut registry = ArtifactRegistry::default();
registry.flush_metadata(ArtifactFlush {
id: ArtifactId::from("artifact-0"),
tenant: TenantId::from("other-tenant"),
project: ProjectId::from("other-project"),
process: ProcessId::from("process"),
producer_task: TaskInstanceId::from("other-task"),
retaining_node: NodeId::from("other-node"),
digest: Digest::sha256("other-content"),
size: 1,
});
let mut pinned = BTreeSet::new();
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
let id = ArtifactId::new(format!("artifact-{index}"));
pinned.insert(id.clone());
pinned.insert(ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
id.clone(),
));
registry
.flush_metadata_bounded(
ArtifactFlush {
@ -1039,7 +1376,10 @@ mod tests {
)
.unwrap();
}
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
let next = ArtifactFlush {
id: ArtifactId::from("artifact-next"),
tenant: TenantId::from("tenant"),
@ -1055,13 +1395,42 @@ mod tests {
.unwrap_err()
.contains("pinned"));
pinned.remove(&ArtifactId::from("artifact-0"));
pinned.remove(&ArtifactScopeKey::new(
TenantId::from("tenant"),
ProjectId::from("project"),
ArtifactId::from("artifact-0"),
));
registry.flush_metadata_bounded(next, &pinned).unwrap();
assert_eq!(registry.artifact_count(), MAX_ARTIFACT_METADATA_PER_PROCESS);
assert!(registry.metadata(&ArtifactId::from("artifact-0")).is_none());
assert_eq!(
registry.artifact_count(),
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
);
assert!(registry
.metadata(&ArtifactId::from("artifact-next"))
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-0"),
)
.is_none());
assert!(registry
.metadata(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&ArtifactId::from("artifact-next"),
)
.is_some());
assert_eq!(
registry
.metadata(
&TenantId::from("other-tenant"),
&ProjectId::from("other-project"),
&ArtifactId::from("artifact-0"),
)
.unwrap()
.digest,
Digest::sha256("other-content"),
"eviction and pins in one scope must not affect an equal ID in another scope"
);
}
#[test]
@ -1150,7 +1519,11 @@ mod tests {
registry
.stream_download_chunk(&mut stream, &limits, &mut meter, 16)
.unwrap();
registry.garbage_collect_node(&NodeId::from("node"));
registry.garbage_collect_node(
&TenantId::from("tenant"),
&ProjectId::from("project"),
&NodeId::from("node"),
);
assert_eq!(
registry