740 lines
26 KiB
Rust
740 lines
26 KiB
Rust
use std::collections::BTreeMap;
|
|
|
|
use clusterflux_core::{ProjectId, TenantId, UserId};
|
|
use serde::{Deserialize, Serialize};
|
|
use thiserror::Error;
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct ArtifactRelayConfiguration {
|
|
pub enabled: bool,
|
|
pub max_artifact_bytes: u64,
|
|
pub max_active_per_project: usize,
|
|
pub max_active_per_tenant: usize,
|
|
pub max_active_per_account: usize,
|
|
pub max_active_global: usize,
|
|
pub max_period_bytes_per_project: u64,
|
|
pub max_period_bytes_per_tenant: u64,
|
|
pub max_period_bytes_per_account: u64,
|
|
pub period_seconds: u64,
|
|
pub framing_overhead_bytes: u64,
|
|
pub max_tracked_scopes: usize,
|
|
}
|
|
|
|
impl ArtifactRelayConfiguration {
|
|
pub fn unlimited() -> Self {
|
|
Self {
|
|
enabled: true,
|
|
max_artifact_bytes: u64::MAX,
|
|
max_active_per_project: usize::MAX,
|
|
max_active_per_tenant: usize::MAX,
|
|
max_active_per_account: usize::MAX,
|
|
max_active_global: usize::MAX,
|
|
max_period_bytes_per_project: u64::MAX,
|
|
max_period_bytes_per_tenant: u64::MAX,
|
|
max_period_bytes_per_account: u64::MAX,
|
|
period_seconds: u64::MAX,
|
|
framing_overhead_bytes: 512,
|
|
max_tracked_scopes: usize::MAX,
|
|
}
|
|
}
|
|
|
|
pub fn validate(&self) -> Result<(), ArtifactRelayError> {
|
|
if self.period_seconds == 0
|
|
|| self.max_active_per_project == 0
|
|
|| self.max_active_per_tenant == 0
|
|
|| self.max_active_per_account == 0
|
|
|| self.max_active_global == 0
|
|
|| self.max_tracked_scopes == 0
|
|
{
|
|
return Err(ArtifactRelayError::InvalidConfiguration);
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
impl Default for ArtifactRelayConfiguration {
|
|
fn default() -> Self {
|
|
Self::unlimited()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
|
pub struct ArtifactRelayUsage {
|
|
pub active_transfers: usize,
|
|
pub ingress_bytes: u64,
|
|
pub egress_bytes: u64,
|
|
pub abandoned_or_failed_bytes: u64,
|
|
pub reserved_bytes: u64,
|
|
pub lifetime_ingress_bytes: u64,
|
|
pub lifetime_egress_bytes: u64,
|
|
pub lifetime_abandoned_or_failed_bytes: u64,
|
|
pub completed_transfers: u64,
|
|
pub failed_transfers: u64,
|
|
pub cancelled_transfers: u64,
|
|
pub expired_transfers: u64,
|
|
pub tracked_scopes: usize,
|
|
pub max_active_global: usize,
|
|
pub max_tracked_scopes: usize,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct ArtifactRelayDurableState {
|
|
pub reservations: BTreeMap<String, ArtifactRelayReservationState>,
|
|
pub period: u64,
|
|
pub project_used: Vec<(TenantId, ProjectId, u64)>,
|
|
pub tenant_used: Vec<(TenantId, u64)>,
|
|
pub account_used: Vec<(TenantId, UserId, u64)>,
|
|
pub ingress_used: u64,
|
|
pub egress_used: u64,
|
|
pub abandoned_or_failed_used: u64,
|
|
#[serde(default)]
|
|
pub lifetime_ingress_bytes: u64,
|
|
#[serde(default)]
|
|
pub lifetime_egress_bytes: u64,
|
|
#[serde(default)]
|
|
pub lifetime_abandoned_or_failed_bytes: u64,
|
|
#[serde(default)]
|
|
pub completed_transfers: u64,
|
|
#[serde(default)]
|
|
pub failed_transfers: u64,
|
|
#[serde(default)]
|
|
pub cancelled_transfers: u64,
|
|
#[serde(default)]
|
|
pub expired_transfers: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct ArtifactRelayReservationState {
|
|
pub tenant: TenantId,
|
|
pub project: ProjectId,
|
|
pub account: UserId,
|
|
pub remaining_reserved_bytes: u64,
|
|
pub ingress_bytes: u64,
|
|
pub egress_bytes: u64,
|
|
pub expires_at_epoch_seconds: u64,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
|
pub(super) enum RelayFinishReason {
|
|
Completed,
|
|
Failed,
|
|
Cancelled,
|
|
Expired,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Error, PartialEq, Eq)]
|
|
pub enum ArtifactRelayError {
|
|
#[error("artifact relay is disabled by hosted policy")]
|
|
Disabled,
|
|
#[error("artifact exceeds the configured relay size limit")]
|
|
ArtifactTooLarge,
|
|
#[error("artifact relay concurrency limit reached for {0}")]
|
|
Concurrency(&'static str),
|
|
#[error("artifact relay is temporarily at process capacity")]
|
|
Capacity,
|
|
#[error("artifact relay period byte budget exhausted for {0}")]
|
|
PeriodBudget(&'static str),
|
|
#[error("artifact relay retained scope state limit reached")]
|
|
StateLimit,
|
|
#[error("artifact relay reservation is missing or expired")]
|
|
MissingReservation,
|
|
#[error("artifact relay configuration contains a zero bound")]
|
|
InvalidConfiguration,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
|
struct RelayScope {
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
account: UserId,
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
struct RelayReservation {
|
|
scope: RelayScope,
|
|
remaining_reserved_bytes: u64,
|
|
ingress_bytes: u64,
|
|
egress_bytes: u64,
|
|
expires_at_epoch_seconds: u64,
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub(super) struct ArtifactRelayLedger {
|
|
configuration: ArtifactRelayConfiguration,
|
|
reservations: BTreeMap<String, RelayReservation>,
|
|
period: u64,
|
|
project_used: BTreeMap<(TenantId, ProjectId), u64>,
|
|
tenant_used: BTreeMap<TenantId, u64>,
|
|
account_used: BTreeMap<(TenantId, UserId), u64>,
|
|
ingress_used: u64,
|
|
egress_used: u64,
|
|
abandoned_or_failed_used: u64,
|
|
lifetime_ingress_bytes: u64,
|
|
lifetime_egress_bytes: u64,
|
|
lifetime_abandoned_or_failed_bytes: u64,
|
|
completed_transfers: u64,
|
|
failed_transfers: u64,
|
|
cancelled_transfers: u64,
|
|
expired_transfers: u64,
|
|
}
|
|
|
|
impl Default for ArtifactRelayLedger {
|
|
fn default() -> Self {
|
|
Self::new(ArtifactRelayConfiguration::default())
|
|
}
|
|
}
|
|
|
|
impl ArtifactRelayLedger {
|
|
pub(super) fn new(configuration: ArtifactRelayConfiguration) -> Self {
|
|
Self {
|
|
configuration,
|
|
reservations: BTreeMap::new(),
|
|
period: 0,
|
|
project_used: BTreeMap::new(),
|
|
tenant_used: BTreeMap::new(),
|
|
account_used: BTreeMap::new(),
|
|
ingress_used: 0,
|
|
egress_used: 0,
|
|
abandoned_or_failed_used: 0,
|
|
lifetime_ingress_bytes: 0,
|
|
lifetime_egress_bytes: 0,
|
|
lifetime_abandoned_or_failed_bytes: 0,
|
|
completed_transfers: 0,
|
|
failed_transfers: 0,
|
|
cancelled_transfers: 0,
|
|
expired_transfers: 0,
|
|
}
|
|
}
|
|
|
|
pub(super) fn configure(
|
|
&mut self,
|
|
configuration: ArtifactRelayConfiguration,
|
|
) -> Result<(), ArtifactRelayError> {
|
|
configuration.validate()?;
|
|
self.configuration = configuration;
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) fn from_durable(
|
|
configuration: ArtifactRelayConfiguration,
|
|
state: ArtifactRelayDurableState,
|
|
) -> Self {
|
|
Self {
|
|
configuration,
|
|
reservations: state
|
|
.reservations
|
|
.into_iter()
|
|
.map(|(key, reservation)| {
|
|
(
|
|
key,
|
|
RelayReservation {
|
|
scope: RelayScope {
|
|
tenant: reservation.tenant,
|
|
project: reservation.project,
|
|
account: reservation.account,
|
|
},
|
|
remaining_reserved_bytes: reservation.remaining_reserved_bytes,
|
|
ingress_bytes: reservation.ingress_bytes,
|
|
egress_bytes: reservation.egress_bytes,
|
|
expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
|
|
},
|
|
)
|
|
})
|
|
.collect(),
|
|
period: state.period,
|
|
project_used: state
|
|
.project_used
|
|
.into_iter()
|
|
.map(|(tenant, project, bytes)| ((tenant, project), bytes))
|
|
.collect(),
|
|
tenant_used: state.tenant_used.into_iter().collect(),
|
|
account_used: state
|
|
.account_used
|
|
.into_iter()
|
|
.map(|(tenant, account, bytes)| ((tenant, account), bytes))
|
|
.collect(),
|
|
ingress_used: state.ingress_used,
|
|
egress_used: state.egress_used,
|
|
abandoned_or_failed_used: state.abandoned_or_failed_used,
|
|
lifetime_ingress_bytes: state.lifetime_ingress_bytes,
|
|
lifetime_egress_bytes: state.lifetime_egress_bytes,
|
|
lifetime_abandoned_or_failed_bytes: state.lifetime_abandoned_or_failed_bytes,
|
|
completed_transfers: state.completed_transfers,
|
|
failed_transfers: state.failed_transfers,
|
|
cancelled_transfers: state.cancelled_transfers,
|
|
expired_transfers: state.expired_transfers,
|
|
}
|
|
}
|
|
|
|
pub(super) fn durable_state(&self) -> ArtifactRelayDurableState {
|
|
ArtifactRelayDurableState {
|
|
reservations: self
|
|
.reservations
|
|
.iter()
|
|
.map(|(key, reservation)| {
|
|
(
|
|
key.clone(),
|
|
ArtifactRelayReservationState {
|
|
tenant: reservation.scope.tenant.clone(),
|
|
project: reservation.scope.project.clone(),
|
|
account: reservation.scope.account.clone(),
|
|
remaining_reserved_bytes: reservation.remaining_reserved_bytes,
|
|
ingress_bytes: reservation.ingress_bytes,
|
|
egress_bytes: reservation.egress_bytes,
|
|
expires_at_epoch_seconds: reservation.expires_at_epoch_seconds,
|
|
},
|
|
)
|
|
})
|
|
.collect(),
|
|
period: self.period,
|
|
project_used: self
|
|
.project_used
|
|
.iter()
|
|
.map(|((tenant, project), bytes)| (tenant.clone(), project.clone(), *bytes))
|
|
.collect(),
|
|
tenant_used: self
|
|
.tenant_used
|
|
.iter()
|
|
.map(|(tenant, bytes)| (tenant.clone(), *bytes))
|
|
.collect(),
|
|
account_used: self
|
|
.account_used
|
|
.iter()
|
|
.map(|((tenant, account), bytes)| (tenant.clone(), account.clone(), *bytes))
|
|
.collect(),
|
|
ingress_used: self.ingress_used,
|
|
egress_used: self.egress_used,
|
|
abandoned_or_failed_used: self.abandoned_or_failed_used,
|
|
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
|
|
lifetime_egress_bytes: self.lifetime_egress_bytes,
|
|
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
|
|
completed_transfers: self.completed_transfers,
|
|
failed_transfers: self.failed_transfers,
|
|
cancelled_transfers: self.cancelled_transfers,
|
|
expired_transfers: self.expired_transfers,
|
|
}
|
|
}
|
|
|
|
pub(super) fn reconcile_after_restart(&mut self, now_epoch_seconds: u64) {
|
|
let reservations = self.reservations.keys().cloned().collect::<Vec<_>>();
|
|
for key in reservations {
|
|
self.finish(&key, RelayFinishReason::Failed);
|
|
}
|
|
self.prepare_period(now_epoch_seconds);
|
|
}
|
|
|
|
fn prepare_period(&mut self, now_epoch_seconds: u64) {
|
|
let period = now_epoch_seconds / self.configuration.period_seconds.max(1);
|
|
if self.period != period {
|
|
self.period = period;
|
|
self.project_used.clear();
|
|
self.tenant_used.clear();
|
|
self.account_used.clear();
|
|
self.ingress_used = 0;
|
|
self.egress_used = 0;
|
|
self.abandoned_or_failed_used = 0;
|
|
}
|
|
}
|
|
|
|
pub(super) fn estimated_wire_bytes(&self, artifact_bytes: u64, chunk_bytes: u64) -> u64 {
|
|
let encoded = artifact_bytes
|
|
.saturating_add(2)
|
|
.saturating_div(3)
|
|
.saturating_mul(4);
|
|
let chunks = artifact_bytes
|
|
.saturating_add(chunk_bytes.saturating_sub(1))
|
|
.saturating_div(chunk_bytes.max(1))
|
|
.max(1);
|
|
encoded.saturating_mul(2).saturating_add(
|
|
chunks
|
|
.saturating_mul(2)
|
|
.saturating_mul(self.configuration.framing_overhead_bytes),
|
|
)
|
|
}
|
|
|
|
pub(super) fn framing_overhead_bytes(&self) -> u64 {
|
|
self.configuration.framing_overhead_bytes
|
|
}
|
|
|
|
pub(super) fn reserve(
|
|
&mut self,
|
|
key: String,
|
|
tenant: TenantId,
|
|
project: ProjectId,
|
|
account: UserId,
|
|
artifact_bytes: u64,
|
|
chunk_bytes: u64,
|
|
expires_at_epoch_seconds: u64,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<(), ArtifactRelayError> {
|
|
self.expire(now_epoch_seconds);
|
|
self.prepare_period(now_epoch_seconds);
|
|
if !self.configuration.enabled {
|
|
return Err(ArtifactRelayError::Disabled);
|
|
}
|
|
if artifact_bytes > self.configuration.max_artifact_bytes {
|
|
return Err(ArtifactRelayError::ArtifactTooLarge);
|
|
}
|
|
let scope = RelayScope {
|
|
tenant,
|
|
project,
|
|
account,
|
|
};
|
|
self.check_concurrency(&scope)?;
|
|
self.check_tracked_scope_capacity(&scope)?;
|
|
let reserved = self.estimated_wire_bytes(artifact_bytes, chunk_bytes);
|
|
self.check_budget(&scope, reserved)?;
|
|
self.reservations.insert(
|
|
key,
|
|
RelayReservation {
|
|
scope,
|
|
remaining_reserved_bytes: reserved,
|
|
ingress_bytes: 0,
|
|
egress_bytes: 0,
|
|
expires_at_epoch_seconds,
|
|
},
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) fn rekey(&mut self, from: &str, to: String) -> Result<(), ArtifactRelayError> {
|
|
let reservation = self
|
|
.reservations
|
|
.remove(from)
|
|
.ok_or(ArtifactRelayError::MissingReservation)?;
|
|
self.reservations.insert(to, reservation);
|
|
Ok(())
|
|
}
|
|
|
|
fn check_concurrency(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
|
|
if self.reservations.len() >= self.configuration.max_active_global {
|
|
return Err(ArtifactRelayError::Capacity);
|
|
}
|
|
let project = self
|
|
.reservations
|
|
.values()
|
|
.filter(|reservation| {
|
|
reservation.scope.tenant == scope.tenant
|
|
&& reservation.scope.project == scope.project
|
|
})
|
|
.count();
|
|
if project >= self.configuration.max_active_per_project {
|
|
return Err(ArtifactRelayError::Concurrency("project"));
|
|
}
|
|
let tenant = self
|
|
.reservations
|
|
.values()
|
|
.filter(|reservation| reservation.scope.tenant == scope.tenant)
|
|
.count();
|
|
if tenant >= self.configuration.max_active_per_tenant {
|
|
return Err(ArtifactRelayError::Concurrency("tenant"));
|
|
}
|
|
let account = self
|
|
.reservations
|
|
.values()
|
|
.filter(|reservation| {
|
|
reservation.scope.tenant == scope.tenant
|
|
&& reservation.scope.account == scope.account
|
|
})
|
|
.count();
|
|
if account >= self.configuration.max_active_per_account {
|
|
return Err(ArtifactRelayError::Concurrency("account"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn check_tracked_scope_capacity(&self, scope: &RelayScope) -> Result<(), ArtifactRelayError> {
|
|
let project_key = (scope.tenant.clone(), scope.project.clone());
|
|
let account_key = (scope.tenant.clone(), scope.account.clone());
|
|
let new_scopes = usize::from(!self.project_used.contains_key(&project_key))
|
|
+ usize::from(!self.tenant_used.contains_key(&scope.tenant))
|
|
+ usize::from(!self.account_used.contains_key(&account_key));
|
|
let tracked = self.project_used.len() + self.tenant_used.len() + self.account_used.len();
|
|
if tracked.saturating_add(new_scopes) > self.configuration.max_tracked_scopes {
|
|
return Err(ArtifactRelayError::StateLimit);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn reserved_for(&self, scope: &RelayScope) -> (u64, u64, u64, u64) {
|
|
let mut project = 0_u64;
|
|
let mut tenant = 0_u64;
|
|
let mut account = 0_u64;
|
|
let mut global = 0_u64;
|
|
for reservation in self.reservations.values() {
|
|
let bytes = reservation.remaining_reserved_bytes;
|
|
global = global.saturating_add(bytes);
|
|
if reservation.scope.tenant == scope.tenant {
|
|
tenant = tenant.saturating_add(bytes);
|
|
if reservation.scope.project == scope.project {
|
|
project = project.saturating_add(bytes);
|
|
}
|
|
if reservation.scope.account == scope.account {
|
|
account = account.saturating_add(bytes);
|
|
}
|
|
}
|
|
}
|
|
(project, tenant, account, global)
|
|
}
|
|
|
|
fn check_budget(&self, scope: &RelayScope, additional: u64) -> Result<(), ArtifactRelayError> {
|
|
let (project_reserved, tenant_reserved, account_reserved, _) = self.reserved_for(scope);
|
|
let project_used = self
|
|
.project_used
|
|
.get(&(scope.tenant.clone(), scope.project.clone()))
|
|
.copied()
|
|
.unwrap_or(0);
|
|
let tenant_used = self.tenant_used.get(&scope.tenant).copied().unwrap_or(0);
|
|
let account_used = self
|
|
.account_used
|
|
.get(&(scope.tenant.clone(), scope.account.clone()))
|
|
.copied()
|
|
.unwrap_or(0);
|
|
for (used, reserved, limit, label) in [
|
|
(
|
|
project_used,
|
|
project_reserved,
|
|
self.configuration.max_period_bytes_per_project,
|
|
"project",
|
|
),
|
|
(
|
|
tenant_used,
|
|
tenant_reserved,
|
|
self.configuration.max_period_bytes_per_tenant,
|
|
"tenant",
|
|
),
|
|
(
|
|
account_used,
|
|
account_reserved,
|
|
self.configuration.max_period_bytes_per_account,
|
|
"account",
|
|
),
|
|
] {
|
|
if used.saturating_add(reserved).saturating_add(additional) > limit {
|
|
return Err(ArtifactRelayError::PeriodBudget(label));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn charge(
|
|
&mut self,
|
|
key: &str,
|
|
bytes: u64,
|
|
ingress: bool,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<(), ArtifactRelayError> {
|
|
self.prepare_period(now_epoch_seconds);
|
|
let reservation = self
|
|
.reservations
|
|
.get(key)
|
|
.cloned()
|
|
.ok_or(ArtifactRelayError::MissingReservation)?;
|
|
let additional = bytes.saturating_sub(reservation.remaining_reserved_bytes);
|
|
if additional > 0 {
|
|
self.check_budget(&reservation.scope, additional)?;
|
|
}
|
|
let reservation = self
|
|
.reservations
|
|
.get_mut(key)
|
|
.ok_or(ArtifactRelayError::MissingReservation)?;
|
|
reservation.remaining_reserved_bytes =
|
|
reservation.remaining_reserved_bytes.saturating_sub(bytes);
|
|
if ingress {
|
|
reservation.ingress_bytes = reservation.ingress_bytes.saturating_add(bytes);
|
|
self.ingress_used = self.ingress_used.saturating_add(bytes);
|
|
self.lifetime_ingress_bytes = self.lifetime_ingress_bytes.saturating_add(bytes);
|
|
} else {
|
|
reservation.egress_bytes = reservation.egress_bytes.saturating_add(bytes);
|
|
self.egress_used = self.egress_used.saturating_add(bytes);
|
|
self.lifetime_egress_bytes = self.lifetime_egress_bytes.saturating_add(bytes);
|
|
}
|
|
let project_key = (
|
|
reservation.scope.tenant.clone(),
|
|
reservation.scope.project.clone(),
|
|
);
|
|
let account_key = (
|
|
reservation.scope.tenant.clone(),
|
|
reservation.scope.account.clone(),
|
|
);
|
|
*self.project_used.entry(project_key).or_default() = self
|
|
.project_used
|
|
.get(&(
|
|
reservation.scope.tenant.clone(),
|
|
reservation.scope.project.clone(),
|
|
))
|
|
.copied()
|
|
.unwrap_or(0)
|
|
.saturating_add(bytes);
|
|
*self
|
|
.tenant_used
|
|
.entry(reservation.scope.tenant.clone())
|
|
.or_default() = self
|
|
.tenant_used
|
|
.get(&reservation.scope.tenant)
|
|
.copied()
|
|
.unwrap_or(0)
|
|
.saturating_add(bytes);
|
|
*self.account_used.entry(account_key).or_default() = self
|
|
.account_used
|
|
.get(&(
|
|
reservation.scope.tenant.clone(),
|
|
reservation.scope.account.clone(),
|
|
))
|
|
.copied()
|
|
.unwrap_or(0)
|
|
.saturating_add(bytes);
|
|
Ok(())
|
|
}
|
|
|
|
pub(super) fn charge_ingress(
|
|
&mut self,
|
|
key: &str,
|
|
bytes: u64,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<(), ArtifactRelayError> {
|
|
self.charge(key, bytes, true, now_epoch_seconds)
|
|
}
|
|
|
|
pub(super) fn charge_egress(
|
|
&mut self,
|
|
key: &str,
|
|
bytes: u64,
|
|
now_epoch_seconds: u64,
|
|
) -> Result<(), ArtifactRelayError> {
|
|
self.charge(key, bytes, false, now_epoch_seconds)
|
|
}
|
|
|
|
pub(super) fn finish(&mut self, key: &str, reason: RelayFinishReason) {
|
|
if let Some(reservation) = self.reservations.remove(key) {
|
|
if reason != RelayFinishReason::Completed {
|
|
let abandoned_or_failed_bytes = reservation
|
|
.ingress_bytes
|
|
.saturating_add(reservation.egress_bytes);
|
|
self.abandoned_or_failed_used = self
|
|
.abandoned_or_failed_used
|
|
.saturating_add(abandoned_or_failed_bytes);
|
|
self.lifetime_abandoned_or_failed_bytes = self
|
|
.lifetime_abandoned_or_failed_bytes
|
|
.saturating_add(abandoned_or_failed_bytes);
|
|
}
|
|
match reason {
|
|
RelayFinishReason::Completed => {
|
|
self.completed_transfers = self.completed_transfers.saturating_add(1);
|
|
}
|
|
RelayFinishReason::Failed => {
|
|
self.failed_transfers = self.failed_transfers.saturating_add(1);
|
|
}
|
|
RelayFinishReason::Cancelled => {
|
|
self.cancelled_transfers = self.cancelled_transfers.saturating_add(1);
|
|
}
|
|
RelayFinishReason::Expired => {
|
|
self.expired_transfers = self.expired_transfers.saturating_add(1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
pub(super) fn expire(&mut self, now_epoch_seconds: u64) {
|
|
let expired = self
|
|
.reservations
|
|
.iter()
|
|
.filter(|(_, reservation)| reservation.expires_at_epoch_seconds < now_epoch_seconds)
|
|
.map(|(key, _)| key.clone())
|
|
.collect::<Vec<_>>();
|
|
for key in expired {
|
|
self.finish(&key, RelayFinishReason::Expired);
|
|
}
|
|
}
|
|
|
|
pub(super) fn usage(&self) -> ArtifactRelayUsage {
|
|
ArtifactRelayUsage {
|
|
active_transfers: self.reservations.len(),
|
|
ingress_bytes: self.ingress_used,
|
|
egress_bytes: self.egress_used,
|
|
abandoned_or_failed_bytes: self.abandoned_or_failed_used,
|
|
lifetime_ingress_bytes: self.lifetime_ingress_bytes,
|
|
lifetime_egress_bytes: self.lifetime_egress_bytes,
|
|
lifetime_abandoned_or_failed_bytes: self.lifetime_abandoned_or_failed_bytes,
|
|
completed_transfers: self.completed_transfers,
|
|
failed_transfers: self.failed_transfers,
|
|
cancelled_transfers: self.cancelled_transfers,
|
|
expired_transfers: self.expired_transfers,
|
|
tracked_scopes: self.project_used.len()
|
|
+ self.tenant_used.len()
|
|
+ self.account_used.len(),
|
|
max_active_global: self.configuration.max_active_global,
|
|
max_tracked_scopes: self.configuration.max_tracked_scopes,
|
|
reserved_bytes: self
|
|
.reservations
|
|
.values()
|
|
.map(|reservation| reservation.remaining_reserved_bytes)
|
|
.fold(0_u64, u64::saturating_add),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn lifetime_metrics_survive_period_rollover_and_durable_round_trip() {
|
|
let mut ledger = ArtifactRelayLedger::new(ArtifactRelayConfiguration::unlimited());
|
|
ledger
|
|
.reserve(
|
|
"transfer".to_owned(),
|
|
TenantId::from("tenant"),
|
|
ProjectId::from("project"),
|
|
UserId::from("user"),
|
|
16,
|
|
16,
|
|
100,
|
|
1,
|
|
)
|
|
.unwrap();
|
|
ledger.charge_ingress("transfer", 24, 1).unwrap();
|
|
ledger.charge_egress("transfer", 24, 1).unwrap();
|
|
ledger.finish("transfer", RelayFinishReason::Completed);
|
|
|
|
let usage = ledger.usage();
|
|
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
assert_eq!(usage.completed_transfers, 1);
|
|
|
|
ledger.prepare_period(u64::MAX);
|
|
let usage = ledger.usage();
|
|
assert_eq!(usage.ingress_bytes, 0);
|
|
assert_eq!(usage.egress_bytes, 0);
|
|
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
|
|
let restored = ArtifactRelayLedger::from_durable(
|
|
ArtifactRelayConfiguration::unlimited(),
|
|
ledger.durable_state(),
|
|
);
|
|
let usage = restored.usage();
|
|
assert_eq!(usage.lifetime_ingress_bytes, 24);
|
|
assert_eq!(usage.lifetime_egress_bytes, 24);
|
|
assert_eq!(usage.completed_transfers, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn older_durable_relay_state_defaults_new_metrics() {
|
|
let state: ArtifactRelayDurableState = serde_json::from_value(serde_json::json!({
|
|
"reservations": {},
|
|
"period": 1,
|
|
"project_used": [],
|
|
"tenant_used": [],
|
|
"account_used": [],
|
|
"ingress_used": 3,
|
|
"egress_used": 4,
|
|
"abandoned_or_failed_used": 2
|
|
}))
|
|
.unwrap();
|
|
|
|
assert_eq!(state.lifetime_ingress_bytes, 0);
|
|
assert_eq!(state.lifetime_egress_bytes, 0);
|
|
assert_eq!(state.completed_transfers, 0);
|
|
}
|
|
}
|