Public release release-e7c2b3ac175d
Source commit: e7c2b3ac175db426791c02779841c5df1d0ffdff Public tree identity: sha256:4f0b7cd828932c06d56f2406788f89b2050ef56c1e1a4f76f55341d387d78e46
This commit is contained in:
Clusterflux release
parent
9f43c6276a
commit
cfd4f19da2
16 changed files with 1386 additions and 313 deletions
|
|
@ -195,7 +195,34 @@ impl ArtifactRegistry {
|
|||
protected_processes: &BTreeSet<ProcessId>,
|
||||
) -> Result<ArtifactMetadata, String> {
|
||||
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
|
||||
self.next_epoch += 1;
|
||||
let replacing = self.artifacts.contains_key(&key);
|
||||
let retained_for_project = self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| metadata.tenant == flush.tenant && metadata.project == flush.project)
|
||||
.count();
|
||||
let eviction = if replacing || retained_for_project < MAX_ARTIFACT_METADATA_PER_PROJECT {
|
||||
None
|
||||
} else {
|
||||
let mut protected_processes = protected_processes.clone();
|
||||
protected_processes.insert(flush.process.clone());
|
||||
Some(
|
||||
self.project_metadata_eviction_candidate(
|
||||
&flush.tenant,
|
||||
&flush.project,
|
||||
pinned,
|
||||
&protected_processes,
|
||||
)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"artifact metadata capacity of {MAX_ARTIFACT_METADATA_PER_PROJECT} is \
|
||||
exhausted by active or retained artifacts"
|
||||
)
|
||||
})?,
|
||||
)
|
||||
};
|
||||
|
||||
self.next_epoch = self.next_epoch.saturating_add(1);
|
||||
let metadata = ArtifactMetadata {
|
||||
id: flush.id.clone(),
|
||||
tenant: flush.tenant,
|
||||
|
|
@ -210,15 +237,10 @@ impl ArtifactRegistry {
|
|||
explicit_locations: Vec::new(),
|
||||
coordinator_has_large_bytes: false,
|
||||
};
|
||||
if let Some(eviction) = eviction {
|
||||
self.artifacts.remove(&eviction);
|
||||
}
|
||||
self.artifacts.insert(key, metadata.clone());
|
||||
let mut protected_processes = protected_processes.clone();
|
||||
protected_processes.insert(metadata.process.clone());
|
||||
self.enforce_project_metadata_limit(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
pinned,
|
||||
&protected_processes,
|
||||
);
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
|
|
@ -237,30 +259,12 @@ impl ArtifactRegistry {
|
|||
.count()
|
||||
> MAX_ARTIFACT_METADATA_PER_PROJECT
|
||||
{
|
||||
let candidate = self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| {
|
||||
&metadata.tenant == tenant
|
||||
&& &metadata.project == project
|
||||
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
&metadata.id,
|
||||
))
|
||||
&& !protected_processes.contains(&metadata.process)
|
||||
&& metadata.explicit_locations.is_empty()
|
||||
&& !self.issued_download_links.values().any(|issued| {
|
||||
!issued.revoked
|
||||
&& issued.link.tenant == metadata.tenant
|
||||
&& issued.link.project == metadata.project
|
||||
&& issued.link.artifact == metadata.id
|
||||
})
|
||||
})
|
||||
.min_by_key(|metadata| metadata.flushed_epoch)
|
||||
.map(|metadata| {
|
||||
ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
|
||||
});
|
||||
let candidate = self.project_metadata_eviction_candidate(
|
||||
tenant,
|
||||
project,
|
||||
pinned,
|
||||
protected_processes,
|
||||
);
|
||||
let Some(candidate) = candidate else {
|
||||
break;
|
||||
};
|
||||
|
|
@ -270,6 +274,38 @@ impl ArtifactRegistry {
|
|||
evicted
|
||||
}
|
||||
|
||||
fn project_metadata_eviction_candidate(
|
||||
&self,
|
||||
tenant: &TenantId,
|
||||
project: &ProjectId,
|
||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
||||
protected_processes: &BTreeSet<ProcessId>,
|
||||
) -> Option<ArtifactScopeKey> {
|
||||
self.artifacts
|
||||
.values()
|
||||
.filter(|metadata| {
|
||||
&metadata.tenant == tenant
|
||||
&& &metadata.project == project
|
||||
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
&metadata.id,
|
||||
))
|
||||
&& !protected_processes.contains(&metadata.process)
|
||||
&& metadata.explicit_locations.is_empty()
|
||||
&& !self.issued_download_links.values().any(|issued| {
|
||||
!issued.revoked
|
||||
&& issued.link.tenant == metadata.tenant
|
||||
&& issued.link.project == metadata.project
|
||||
&& issued.link.artifact == metadata.id
|
||||
})
|
||||
})
|
||||
.min_by_key(|metadata| metadata.flushed_epoch)
|
||||
.map(|metadata| {
|
||||
ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn sync_to_explicit_store(
|
||||
&mut self,
|
||||
tenant: &TenantId,
|
||||
|
|
@ -1485,6 +1521,87 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_capacity_rejects_atomically_when_every_entry_is_protected() {
|
||||
let mut registry = ArtifactRegistry::default();
|
||||
let tenant = TenantId::from("tenant");
|
||||
let project = ProjectId::from("project");
|
||||
let active_process = ProcessId::from("active-process");
|
||||
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
|
||||
registry.flush_metadata(ArtifactFlush {
|
||||
id: ArtifactId::new(format!("artifact-{index}")),
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: active_process.clone(),
|
||||
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256(format!("content-{index}")),
|
||||
size: 1,
|
||||
});
|
||||
}
|
||||
|
||||
let replacement = ArtifactFlush {
|
||||
id: ArtifactId::from("artifact-0"),
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: active_process.clone(),
|
||||
producer_task: TaskInstanceId::from("replacement-task"),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256("replacement"),
|
||||
size: 2,
|
||||
};
|
||||
registry
|
||||
.flush_metadata_with_protected_processes(
|
||||
replacement,
|
||||
&BTreeSet::new(),
|
||||
&BTreeSet::from([active_process.clone()]),
|
||||
)
|
||||
.expect("replacement must remain possible at the metadata bound");
|
||||
assert_eq!(
|
||||
registry
|
||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
|
||||
.unwrap()
|
||||
.digest,
|
||||
Digest::sha256("replacement")
|
||||
);
|
||||
|
||||
let epoch_before_rejection = registry.next_epoch;
|
||||
let result = registry.flush_metadata_with_protected_processes(
|
||||
ArtifactFlush {
|
||||
id: ArtifactId::from("artifact-rejected"),
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: active_process.clone(),
|
||||
producer_task: TaskInstanceId::from("rejected-task"),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256("rejected"),
|
||||
size: 3,
|
||||
},
|
||||
&BTreeSet::new(),
|
||||
&BTreeSet::from([active_process]),
|
||||
);
|
||||
|
||||
assert!(result
|
||||
.unwrap_err()
|
||||
.contains("exhausted by active or retained artifacts"));
|
||||
assert_eq!(registry.next_epoch, epoch_before_rejection);
|
||||
assert_eq!(
|
||||
registry.metadata_for_project(&tenant, &project).count(),
|
||||
MAX_ARTIFACT_METADATA_PER_PROJECT
|
||||
);
|
||||
assert!(registry
|
||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-rejected"))
|
||||
.is_none());
|
||||
assert_eq!(
|
||||
registry
|
||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-0"))
|
||||
.unwrap()
|
||||
.digest,
|
||||
Digest::sha256("replacement"),
|
||||
"rejection must not modify existing metadata"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn download_stream_accounts_usage_before_and_during_streaming() {
|
||||
let mut registry = registry_with_artifact();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue