v0.24.5
This commit is contained in:
gru committed
commit
5e23e9af02
240 files changed
+99957
No files matched your search
Vendored
+281
@@ -0,0 +1,281 @@
|
||||
impl Cache {
|
||||
pub fn new(revision: u64) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
store: Arc::new(Mutex::new(Store {
|
||||
revision,
|
||||
..Default::default()
|
||||
})),
|
||||
revision: AtomicU64::new(revision),
|
||||
cold: Arc::new(Mutex::new(ColdStore::default())),
|
||||
tier_settings: Mutex::new(Settings::default()),
|
||||
data_root: None,
|
||||
cold_tx: None,
|
||||
fills: Arc::new(Semaphore::new(FILL_SLOTS)),
|
||||
fill_bytes_active: AtomicU64::new(0),
|
||||
transient_bytes_active: Arc::new(AtomicU64::new(0)),
|
||||
cold_queue_items: AtomicU64::new(0),
|
||||
cold_queue_bytes: AtomicU64::new(0),
|
||||
cold_write_bytes_active: AtomicU64::new(0),
|
||||
cold_read_bytes_active: AtomicU64::new(0),
|
||||
revalidating: Mutex::new(HashSet::new()),
|
||||
fill_flights: Mutex::new(HashMap::new()),
|
||||
fill_flight_sequence: AtomicU64::new(0),
|
||||
collapse_waiters: AtomicU64::new(0),
|
||||
bans: Mutex::new(Vec::new()),
|
||||
ban_sequence: AtomicU64::new(0),
|
||||
proactive_scheduled_keys: Mutex::new(HashSet::new()),
|
||||
proactive_inflight: AtomicU64::new(0),
|
||||
cleanup_jobs: Mutex::new(HashMap::new()),
|
||||
cleanup_sequence: AtomicU64::new(0),
|
||||
counters: Counters::default(),
|
||||
})
|
||||
}
|
||||
pub fn start(
|
||||
revision: u64,
|
||||
data_root: Option<PathBuf>,
|
||||
settings: Settings,
|
||||
shutdown: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
let (tx, rx) = mpsc::channel(COLD_QUEUE);
|
||||
let cache = Arc::new(Self {
|
||||
store: Arc::new(Mutex::new(Store {
|
||||
revision,
|
||||
..Default::default()
|
||||
})),
|
||||
revision: AtomicU64::new(revision),
|
||||
cold: Arc::new(Mutex::new(ColdStore::default())),
|
||||
tier_settings: Mutex::new(settings),
|
||||
data_root,
|
||||
cold_tx: Some(tx),
|
||||
fills: Arc::new(Semaphore::new(FILL_SLOTS)),
|
||||
fill_bytes_active: AtomicU64::new(0),
|
||||
transient_bytes_active: Arc::new(AtomicU64::new(0)),
|
||||
cold_queue_items: AtomicU64::new(0),
|
||||
cold_queue_bytes: AtomicU64::new(0),
|
||||
cold_write_bytes_active: AtomicU64::new(0),
|
||||
cold_read_bytes_active: AtomicU64::new(0),
|
||||
revalidating: Mutex::new(HashSet::new()),
|
||||
fill_flights: Mutex::new(HashMap::new()),
|
||||
fill_flight_sequence: AtomicU64::new(0),
|
||||
collapse_waiters: AtomicU64::new(0),
|
||||
bans: Mutex::new(Vec::new()),
|
||||
ban_sequence: AtomicU64::new(0),
|
||||
proactive_scheduled_keys: Mutex::new(HashSet::new()),
|
||||
proactive_inflight: AtomicU64::new(0),
|
||||
cleanup_jobs: Mutex::new(HashMap::new()),
|
||||
cleanup_sequence: AtomicU64::new(0),
|
||||
counters: Counters::default(),
|
||||
});
|
||||
cache.load_cold_startup();
|
||||
let weak = Arc::downgrade(&cache);
|
||||
let worker_shutdown = shutdown.clone();
|
||||
tokio::spawn(async move { Self::cold_worker(weak, rx, worker_shutdown).await });
|
||||
let weak = Arc::downgrade(&cache);
|
||||
let maintenance_shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = maintenance_shutdown.cancelled() => break,
|
||||
_ = tick.tick() => {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
cache.demote_idle();
|
||||
cache.sweep_cold().await;
|
||||
cache.gc_banned(256);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let weak = Arc::downgrade(&cache);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
let interval = cache
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.cold_gc_interval_seconds;
|
||||
drop(cache);
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
_ = tokio::time::sleep(Duration::from_secs(interval)) => {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
cache.cold_gc_pass().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
cache
|
||||
}
|
||||
fn fill_headroom_bytes(max_bytes: usize) -> usize {
|
||||
// Allocation rounding is allowed a small global cushion, never a whole
|
||||
// max_object allocation. Use roughly 1/64 of HOT, cap it at 1 MiB and
|
||||
// never let the cushion itself exceed 1/8 of a very small HOT budget.
|
||||
let proportional = (max_bytes / 64).clamp(4 * 1024, 1024 * 1024);
|
||||
proportional.min((max_bytes / 8).max(1))
|
||||
}
|
||||
|
||||
fn try_reserve_transient(&self, bytes: usize, settings: &Settings) -> Option<TransientLease> {
|
||||
let bytes = u64::try_from(bytes).ok()?;
|
||||
let limit = u64::try_from(settings.transient_max_bytes).unwrap_or(u64::MAX);
|
||||
loop {
|
||||
let current = self.transient_bytes_active.load(Ordering::Relaxed);
|
||||
let next = current.checked_add(bytes)?;
|
||||
if next > limit {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.transient_bytes_active
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return Some(TransientLease(Arc::new(TransientLeaseInner {
|
||||
counter: self.transient_bytes_active.clone(),
|
||||
bytes,
|
||||
})));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve real allocation for an in-progress fill before growing its buffer.
|
||||
/// Existing HOT objects are LRU-evicted/demoted first so HOT + active fill
|
||||
/// allocation cannot race arbitrarily above the configured HOT budget.
|
||||
fn reserve_fill_allocation(&self, bytes: usize, settings: &Settings) -> Option<TransientLease> {
|
||||
if bytes == 0 {
|
||||
return self.try_reserve_transient(0, settings);
|
||||
}
|
||||
let lease = self.try_reserve_transient(bytes, settings)?;
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let fill_now = usize::try_from(self.fill_bytes_active.load(Ordering::Relaxed))
|
||||
.unwrap_or(usize::MAX);
|
||||
let allowed = settings
|
||||
.max_bytes
|
||||
.saturating_add(Self::fill_headroom_bytes(settings.max_bytes));
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let mut removed = 0usize;
|
||||
let mut released_hot = false;
|
||||
while store
|
||||
.bytes
|
||||
.saturating_add(fill_now)
|
||||
.saturating_add(bytes)
|
||||
> allowed
|
||||
{
|
||||
if removed >= settings.hot_eviction_max_objects {
|
||||
self.counters
|
||||
.hot_eviction_limited
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let Some(old) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
removed = removed.saturating_add(1);
|
||||
released_hot = true;
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&old.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store
|
||||
.key_generations
|
||||
.get(&old.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if !(settings.cold_enabled
|
||||
&& settings.cold_on_lru
|
||||
&& self.enqueue_demotion(
|
||||
key,
|
||||
old,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
))
|
||||
{
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let fits = store
|
||||
.bytes
|
||||
.saturating_add(fill_now)
|
||||
.saturating_add(bytes)
|
||||
<= allowed;
|
||||
if fits {
|
||||
self.fill_bytes_active
|
||||
.fetch_add(bytes as u64, Ordering::Relaxed);
|
||||
}
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
if fits {
|
||||
Some(lease)
|
||||
} else {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
drop(lease);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn configure(&self, settings: Settings) {
|
||||
*self.tier_settings.lock().unwrap_or_else(|e| e.into_inner()) = settings.clone();
|
||||
// Hot-applied limit reductions take effect immediately for published HOT
|
||||
// objects. Demotion is best effort and remains bounded by transient/COLD
|
||||
// queue budgets; otherwise the LRU object is dropped.
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let mut released_hot = false;
|
||||
while store.entries.len() > settings.max_entries || store.bytes > settings.max_bytes {
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let Some(old) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
released_hot = true;
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&old.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store
|
||||
.key_generations
|
||||
.get(&old.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if !(settings.cold_enabled
|
||||
&& settings.cold_on_lru
|
||||
&& self.enqueue_demotion(
|
||||
key,
|
||||
old,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
))
|
||||
{
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
}
|
||||
pub fn revision(&self) -> u64 {
|
||||
self.revision.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user