Public release release-7fcdc75d8eaf
Source commit: 7fcdc75d8eaf4dc03b09086568dbb184f903f6a4 Public tree identity: sha256:de9eec9b4e2f4cba2fa32b6ecb4b3424cce793a0f8a969707cc9c4565acdbb4e
This commit is contained in:
Clusterflux release
parent
2a0f7ded04
commit
f4590ca576
10 changed files with 521 additions and 109 deletions
|
|
@ -10,7 +10,7 @@ use crate::{
|
|||
|
||||
const MAX_ISSUED_DOWNLOAD_LINKS_PER_ARTIFACT: usize = 32;
|
||||
const DOWNLOAD_LINK_TOMBSTONE_SECONDS: u64 = 15 * 60;
|
||||
const MAX_ARTIFACT_METADATA_PER_PROCESS: usize = 256;
|
||||
const MAX_ARTIFACT_METADATA_PER_PROJECT: usize = 1_024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
|
|
@ -184,53 +184,17 @@ impl ArtifactRegistry {
|
|||
&mut self,
|
||||
flush: ArtifactFlush,
|
||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
||||
) -> Result<ArtifactMetadata, String> {
|
||||
self.flush_metadata_with_protected_processes(flush, pinned, &BTreeSet::new())
|
||||
}
|
||||
|
||||
pub fn flush_metadata_with_protected_processes(
|
||||
&mut self,
|
||||
flush: ArtifactFlush,
|
||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
||||
protected_processes: &BTreeSet<ProcessId>,
|
||||
) -> Result<ArtifactMetadata, String> {
|
||||
let key = ArtifactScopeKey::from_refs(&flush.tenant, &flush.project, &flush.id);
|
||||
let replacing_existing = self.artifacts.contains_key(&key);
|
||||
while !replacing_existing
|
||||
&& self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| {
|
||||
metadata.tenant == flush.tenant
|
||||
&& metadata.project == flush.project
|
||||
&& metadata.process == flush.process
|
||||
})
|
||||
.count()
|
||||
>= MAX_ARTIFACT_METADATA_PER_PROCESS
|
||||
{
|
||||
let candidate = self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| {
|
||||
metadata.tenant == flush.tenant
|
||||
&& metadata.project == flush.project
|
||||
&& metadata.process == flush.process
|
||||
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
&metadata.id,
|
||||
))
|
||||
&& !self.issued_download_links.values().any(|issued| {
|
||||
issued.link.tenant == metadata.tenant
|
||||
&& issued.link.project == metadata.project
|
||||
&& issued.link.artifact == metadata.id
|
||||
})
|
||||
})
|
||||
.min_by_key(|metadata| metadata.flushed_epoch)
|
||||
.map(|metadata| {
|
||||
ArtifactScopeKey::from_refs(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
&metadata.id,
|
||||
)
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
"artifact metadata retention limit reached and every retained object is pinned by active work, restart state, or a download"
|
||||
.to_owned()
|
||||
})?;
|
||||
self.artifacts.remove(&candidate);
|
||||
}
|
||||
self.next_epoch += 1;
|
||||
let metadata = ArtifactMetadata {
|
||||
id: flush.id.clone(),
|
||||
|
|
@ -247,9 +211,65 @@ impl ArtifactRegistry {
|
|||
coordinator_has_large_bytes: false,
|
||||
};
|
||||
self.artifacts.insert(key, metadata.clone());
|
||||
let mut protected_processes = protected_processes.clone();
|
||||
protected_processes.insert(metadata.process.clone());
|
||||
self.enforce_project_metadata_limit(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
pinned,
|
||||
&protected_processes,
|
||||
);
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub fn enforce_project_metadata_limit(
|
||||
&mut self,
|
||||
tenant: &TenantId,
|
||||
project: &ProjectId,
|
||||
pinned: &BTreeSet<ArtifactScopeKey>,
|
||||
protected_processes: &BTreeSet<ProcessId>,
|
||||
) -> usize {
|
||||
let mut evicted = 0;
|
||||
while self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| &metadata.tenant == tenant && &metadata.project == project)
|
||||
.count()
|
||||
> MAX_ARTIFACT_METADATA_PER_PROJECT
|
||||
{
|
||||
let candidate = self
|
||||
.artifacts
|
||||
.values()
|
||||
.filter(|metadata| {
|
||||
&metadata.tenant == tenant
|
||||
&& &metadata.project == project
|
||||
&& !pinned.contains(&ArtifactScopeKey::from_refs(
|
||||
&metadata.tenant,
|
||||
&metadata.project,
|
||||
&metadata.id,
|
||||
))
|
||||
&& !protected_processes.contains(&metadata.process)
|
||||
&& metadata.explicit_locations.is_empty()
|
||||
&& !self.issued_download_links.values().any(|issued| {
|
||||
!issued.revoked
|
||||
&& issued.link.tenant == metadata.tenant
|
||||
&& issued.link.project == metadata.project
|
||||
&& issued.link.artifact == metadata.id
|
||||
})
|
||||
})
|
||||
.min_by_key(|metadata| metadata.flushed_epoch)
|
||||
.map(|metadata| {
|
||||
ArtifactScopeKey::from_refs(&metadata.tenant, &metadata.project, &metadata.id)
|
||||
});
|
||||
let Some(candidate) = candidate else {
|
||||
break;
|
||||
};
|
||||
self.artifacts.remove(&candidate);
|
||||
evicted += 1;
|
||||
}
|
||||
evicted
|
||||
}
|
||||
|
||||
pub fn sync_to_explicit_store(
|
||||
&mut self,
|
||||
tenant: &TenantId,
|
||||
|
|
@ -1340,8 +1360,10 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn artifact_metadata_is_bounded_without_evicting_pins() {
|
||||
fn artifact_metadata_is_bounded_per_project_without_evicting_live_or_retained_state() {
|
||||
let mut registry = ArtifactRegistry::default();
|
||||
let tenant = TenantId::from("tenant");
|
||||
let project = ProjectId::from("project");
|
||||
registry.flush_metadata(ArtifactFlush {
|
||||
id: ArtifactId::from("artifact-0"),
|
||||
tenant: TenantId::from("other-tenant"),
|
||||
|
|
@ -1352,72 +1374,92 @@ mod tests {
|
|||
digest: Digest::sha256("other-content"),
|
||||
size: 1,
|
||||
});
|
||||
let mut pinned = BTreeSet::new();
|
||||
for index in 0..MAX_ARTIFACT_METADATA_PER_PROCESS {
|
||||
for index in 0..MAX_ARTIFACT_METADATA_PER_PROJECT {
|
||||
let id = ArtifactId::new(format!("artifact-{index}"));
|
||||
pinned.insert(ArtifactScopeKey::new(
|
||||
TenantId::from("tenant"),
|
||||
ProjectId::from("project"),
|
||||
id.clone(),
|
||||
));
|
||||
registry
|
||||
.flush_metadata_bounded(
|
||||
ArtifactFlush {
|
||||
id,
|
||||
tenant: TenantId::from("tenant"),
|
||||
project: ProjectId::from("project"),
|
||||
process: ProcessId::from("process"),
|
||||
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256(format!("content-{index}")),
|
||||
size: 1,
|
||||
},
|
||||
&pinned,
|
||||
)
|
||||
.unwrap();
|
||||
registry.flush_metadata(ArtifactFlush {
|
||||
id,
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: ProcessId::new(if index == 3 {
|
||||
"active-process-3".to_owned()
|
||||
} else {
|
||||
format!("completed-process-{index}")
|
||||
}),
|
||||
producer_task: TaskInstanceId::new(format!("task-{index}")),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256(format!("content-{index}")),
|
||||
size: 1,
|
||||
});
|
||||
}
|
||||
assert_eq!(
|
||||
registry.artifact_count(),
|
||||
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
|
||||
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
|
||||
);
|
||||
let pinned = BTreeSet::from([ArtifactScopeKey::new(
|
||||
tenant.clone(),
|
||||
project.clone(),
|
||||
ArtifactId::from("artifact-0"),
|
||||
)]);
|
||||
registry
|
||||
.sync_to_explicit_store(
|
||||
&tenant,
|
||||
&project,
|
||||
&ArtifactId::from("artifact-1"),
|
||||
"store://retained-export",
|
||||
)
|
||||
.unwrap();
|
||||
let context = AuthContext {
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
actor: Actor::User(UserId::from("user")),
|
||||
};
|
||||
registry
|
||||
.create_download_link(
|
||||
&context,
|
||||
&ArtifactId::from("artifact-2"),
|
||||
&DownloadPolicy { max_bytes: 1 },
|
||||
"active-download",
|
||||
10,
|
||||
60,
|
||||
)
|
||||
.unwrap();
|
||||
let protected_processes = BTreeSet::from([
|
||||
ProcessId::from("active-process-3"),
|
||||
ProcessId::from("active-process"),
|
||||
]);
|
||||
let next = ArtifactFlush {
|
||||
id: ArtifactId::from("artifact-next"),
|
||||
tenant: TenantId::from("tenant"),
|
||||
project: ProjectId::from("project"),
|
||||
process: ProcessId::from("process"),
|
||||
tenant: tenant.clone(),
|
||||
project: project.clone(),
|
||||
process: ProcessId::from("active-process"),
|
||||
producer_task: TaskInstanceId::from("task-next"),
|
||||
retaining_node: NodeId::from("node"),
|
||||
digest: Digest::sha256("next"),
|
||||
size: 1,
|
||||
};
|
||||
assert!(registry
|
||||
.flush_metadata_bounded(next.clone(), &pinned)
|
||||
.unwrap_err()
|
||||
.contains("pinned"));
|
||||
|
||||
pinned.remove(&ArtifactScopeKey::new(
|
||||
TenantId::from("tenant"),
|
||||
ProjectId::from("project"),
|
||||
ArtifactId::from("artifact-0"),
|
||||
));
|
||||
registry.flush_metadata_bounded(next, &pinned).unwrap();
|
||||
registry
|
||||
.flush_metadata_with_protected_processes(next, &pinned, &protected_processes)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
registry.artifact_count(),
|
||||
MAX_ARTIFACT_METADATA_PER_PROCESS + 1
|
||||
MAX_ARTIFACT_METADATA_PER_PROJECT + 1
|
||||
);
|
||||
for id in ["artifact-0", "artifact-1", "artifact-2", "artifact-3"] {
|
||||
assert!(
|
||||
registry
|
||||
.metadata(&tenant, &project, &ArtifactId::from(id))
|
||||
.is_some(),
|
||||
"{id} was evicted despite being pinned, exported, downloaded, or live"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
registry
|
||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-4"))
|
||||
.is_none(),
|
||||
"the oldest completed unprotected metadata should be evicted"
|
||||
);
|
||||
assert!(registry
|
||||
.metadata(
|
||||
&TenantId::from("tenant"),
|
||||
&ProjectId::from("project"),
|
||||
&ArtifactId::from("artifact-0"),
|
||||
)
|
||||
.is_none());
|
||||
assert!(registry
|
||||
.metadata(
|
||||
&TenantId::from("tenant"),
|
||||
&ProjectId::from("project"),
|
||||
&ArtifactId::from("artifact-next"),
|
||||
)
|
||||
.metadata(&tenant, &project, &ArtifactId::from("artifact-next"))
|
||||
.is_some());
|
||||
assert_eq!(
|
||||
registry
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue