v0.24.7

src/cache/cold.rs

@@ -720,22 +720,37 @@ impl Cache {
         }
     }
 
-    fn enqueue_cold_delete(&self, paths: Vec<PathBuf>) {
+    fn enqueue_cold_delete(&self, mut paths: Vec<PathBuf>) {
         if paths.is_empty() {
             return;
         }
         if let Some(tx) = &self.cold_tx {
-            if tx.try_send(ColdCommand::Delete(paths.clone())).is_ok() {
-                return;
+            match tx.try_send(ColdCommand::Delete(paths)) {
+                Ok(()) => return,
+                Err(error) => {
+                    paths = match error.into_inner() {
+                        ColdCommand::Delete(paths) => paths,
+                        _ => unreachable!("enqueue_cold_delete only sends delete commands"),
+                    };
+                }
             }
         }
-        // Queue saturation must not block request processing. The index is already
-        // updated; a best-effort detached delete keeps physical cleanup asynchronous.
-        tokio::spawn(async move {
+        // Do not create an unbounded number of detached delete tasks when the
+        // writer queue is saturated. A tiny fallback pool absorbs short bursts;
+        // sustained saturation falls back to synchronous removal so stale files
+        // cannot be resurrected on restart at the cost of bounded backpressure.
+        if let Ok(permit) = self.cold_delete_fallbacks.clone().try_acquire_owned() {
+            tokio::spawn(async move {
+                let _permit = permit;
+                for path in paths {
+                    let _ = tokio::fs::remove_file(path).await;
+                }
+            });
+        } else {
             for path in paths {
-                let _ = tokio::fs::remove_file(path).await;
+                let _ = std::fs::remove_file(path);
             }
-        });
+        }
     }
 
     async fn remove_cold_candidates(
@@ -748,8 +763,13 @@ impl Cache {
         const SNAPSHOT_BATCH: usize = 256;
         const DELETE_BATCH: usize = 32;
 
+        let requested_limit = request
+            .and_then(|value| value.oldest)
+            .unwrap_or(max_delete)
+            .min(max_delete);
+        let manual = request.is_some();
         let now = SystemTime::now();
-        let mut candidates = Vec::new();
+        let mut candidates = BinaryHeap::with_capacity(requested_limit.saturating_add(1));
         let mut after_sequence = 0u64;
 
         // Copy candidate metadata in bounded chunks. Request-path lookups only
@@ -788,27 +808,32 @@ impl Cache {
                         })
                 });
                 if matches {
-                    candidates.push((sequence, key, hits, last_access, stored_at));
+                    let priority = if manual {
+                        CleanupPriority::Manual(stored_at, last_access, hits)
+                    } else {
+                        CleanupPriority::Automatic(hits, last_access, stored_at)
+                    };
+                    let candidate = CleanupCandidate {
+                        priority,
+                        sequence,
+                        key,
+                    };
+                    if candidates.len() < requested_limit {
+                        candidates.push(candidate);
+                    } else if candidates
+                        .peek()
+                        .is_some_and(|worst| candidate.cmp(worst).is_lt())
+                    {
+                        candidates.pop();
+                        candidates.push(candidate);
+                    }
                 }
             }
             tokio::task::yield_now().await;
         }
 
-        if request.is_some() {
-            candidates.sort_by_key(|(_, _, hits, last_access, stored_at)| {
-                (*stored_at, *last_access, *hits)
-            });
-        } else {
-            // Automatic GC prefers rarely used, least recently used, oldest data.
-            candidates.sort_by_key(|(_, _, hits, last_access, stored_at)| {
-                (*hits, *last_access, *stored_at)
-            });
-        }
-
-        let requested_limit = request
-            .and_then(|value| value.oldest)
-            .unwrap_or(max_delete)
-            .min(max_delete);
+        let mut candidates = candidates.into_vec();
+        candidates.sort_unstable();
         let mut result = ColdCleanupResult::default();
         let mut delete = Vec::new();
 
@@ -818,7 +843,7 @@ impl Cache {
             }
             {
                 let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
-                for (sequence, key, _, _, _) in chunk {
+                for candidate in chunk {
                     if result.removed_entries >= requested_limit {
                         break;
                     }
@@ -835,18 +860,21 @@ impl Cache {
                     }
                     if !cold
                         .entries
-                        .get(key)
-                        .is_some_and(|entry| entry.sequence == *sequence && entry.pinned_reads == 0)
+                        .get(&candidate.key)
+                        .is_some_and(|entry| entry.sequence == candidate.sequence && entry.pinned_reads == 0)
                     {
                         continue;
                     }
-                    if let Some(entry) = cold.remove(key) {
+                    if let Some(entry) = cold.remove(&candidate.key) {
                         result.removed_entries += 1;
                         result.freed_bytes = result.freed_bytes.saturating_add(entry.disk_bytes);
                         delete.extend([entry.meta_path, entry.body_path]);
                     }
                 }
             }
+            // Keep pending deletion metadata bounded to one selection batch; a
+            // 10k-entry manual cleanup must not build one large Vec<PathBuf>.
+            self.enqueue_cold_delete(std::mem::take(&mut delete));
             tokio::task::yield_now().await;
         }
 
@@ -855,11 +883,13 @@ impl Cache {
                 .cold_evictions
                 .fetch_add(result.removed_entries as u64, Ordering::Relaxed);
         }
-        self.enqueue_cold_delete(delete);
         result
     }
 
     async fn cold_gc_pass(&self) -> ColdCleanupResult {
+        let Ok(_cleanup_slot) = self.cleanup_slots.clone().try_acquire_owned() else {
+            return ColdCleanupResult::default();
+        };
         let settings = self
             .tier_settings
             .lock()
@@ -903,6 +933,11 @@ impl Cache {
             .unwrap_or_else(|e| e.into_inner())
             .clone();
         ensure!(settings.cold_enabled, "cold cache is disabled");
+        let cleanup_slot = self
+            .cleanup_slots
+            .clone()
+            .try_acquire_owned()
+            .map_err(|_| anyhow::anyhow!("cold cache cleanup is already running"))?;
         let id = self.cleanup_sequence.fetch_add(1, Ordering::Relaxed).saturating_add(1);
         let job = ColdCleanupJob {
             id,
@@ -924,6 +959,7 @@ impl Cache {
         }
         let cache = self.clone();
         tokio::spawn(async move {
+            let _cleanup_slot = cleanup_slot;
             {
                 let mut jobs = cache.cleanup_jobs.lock().unwrap_or_else(|e| e.into_inner());
                 if let Some(job) = jobs.get_mut(&id) {

src/cache/collapse.rs

@@ -210,6 +210,19 @@ impl Cache {
             .proactive_scheduled_keys
             .lock()
             .unwrap_or_else(|e| e.into_inner());
+        // Each scheduled key owns a sleeping Tokio task plus refresh context.
+        // Bound the pending set independently from the cache entry count so a
+        // large cache with long TTLs cannot turn into thousands of resident tasks.
+        let pending_limit = settings
+            .proactive_refresh_max_inflight
+            .saturating_mul(64)
+            .clamp(64, 256);
+        if scheduled.len() >= pending_limit && !scheduled.contains(&ticket.key) {
+            self.counters
+                .proactive_skipped
+                .fetch_add(1, Ordering::Relaxed);
+            return None;
+        }
         if !scheduled.insert(ticket.key) {
             return None;
         }

src/cache/dashboard.rs

@@ -38,6 +38,8 @@ impl Cache {
             cold_queue_limit_bytes: settings.cold_queue_max_bytes,
             cold_entries: cold.entries.len(),
             cold_bytes: cold.bytes,
+            cold_limit_bytes: settings.cold_max_bytes,
+            cold_limit_entries: settings.cold_max_entries,
             cold_hits: n(&self.counters.cold_hits),
             cold_promotions: n(&self.counters.cold_promotions),
             hot_demotions: n(&self.counters.hot_demotions),
@@ -121,6 +123,8 @@ impl Cache {
             ),
             ("cold_entries", "gauge", view.cold_entries as u64),
             ("cold_bytes", "gauge", view.cold_bytes),
+            ("cold_limit_bytes", "gauge", view.cold_limit_bytes),
+            ("cold_limit_entries", "gauge", view.cold_limit_entries as u64),
             ("cold_hits_total", "counter", view.cold_hits),
             ("cold_promotions_total", "counter", view.cold_promotions),
             ("hot_demotions_total", "counter", view.hot_demotions),

src/cache/lifecycle.rs

@@ -10,6 +10,8 @@ impl Cache {
             tier_settings: Mutex::new(Settings::default()),
             data_root: None,
             cold_tx: None,
+            cold_delete_fallbacks: Arc::new(Semaphore::new(COLD_DELETE_FALLBACKS)),
+            cleanup_slots: Arc::new(Semaphore::new(COLD_CLEANUP_SLOTS)),
             fills: Arc::new(Semaphore::new(FILL_SLOTS)),
             fill_bytes_active: AtomicU64::new(0),
             transient_bytes_active: Arc::new(AtomicU64::new(0)),
@@ -47,6 +49,8 @@ impl Cache {
             tier_settings: Mutex::new(settings),
             data_root,
             cold_tx: Some(tx),
+            cold_delete_fallbacks: Arc::new(Semaphore::new(COLD_DELETE_FALLBACKS)),
+            cleanup_slots: Arc::new(Semaphore::new(COLD_CLEANUP_SLOTS)),
             fills: Arc::new(Semaphore::new(FILL_SLOTS)),
             fill_bytes_active: AtomicU64::new(0),
             transient_bytes_active: Arc::new(AtomicU64::new(0)),

src/cache/model.rs

@@ -6,7 +6,7 @@ use http_body::{Body as HttpBody, Frame, SizeHint};
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use std::{
-    collections::{BTreeMap, HashMap, HashSet},
+    collections::{BTreeMap, BinaryHeap, HashMap, HashSet},
     path::{Component, Path, PathBuf},
     pin::Pin,
     sync::{
@@ -21,12 +21,28 @@ use tokio_util::sync::CancellationToken;
 
 const MAX_HEADER_BYTES: usize = 32 * 1024;
 const FILL_SLOTS: usize = 8;
+const COLD_DELETE_FALLBACKS: usize = 2;
+const COLD_CLEANUP_SLOTS: usize = 1;
+const BACKGROUND_REVALIDATION_LIMIT: usize = 8;
 // Cold writes can contain complete response bodies. Keep the queue deliberately
 // small so demotion cannot become a second, hidden RAM cache while disk is slow.
 const COLD_QUEUE: usize = 16;
 const COLD_FORMAT_VERSION: u8 = 3;
 type Key = [u8; 32];
 
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+enum CleanupPriority {
+    Automatic(u64, SystemTime, SystemTime),
+    Manual(SystemTime, SystemTime, u64),
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+struct CleanupCandidate {
+    priority: CleanupPriority,
+    sequence: u64,
+    key: Key,
+}
+
 /// `Bytes` releases its allocation when the last reference is dropped, but
 /// glibc can keep large freed arenas mapped. After bulk cache demotions/purges,
 /// return releasable pages to the OS instead of letting RSS stay artificially
@@ -59,10 +75,28 @@ impl Default for CollapseTimeoutFallback {
     }
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum MemoryPreset {
+    Conservative,
+    Recommended,
+    Performance,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum ColdPreset {
+    Recommended,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default, deny_unknown_fields)]
 pub struct Settings {
     pub enabled: bool,
+    /// UI sizing profile last explicitly selected by the administrator. The
+    /// actual numeric limits remain authoritative; this only preserves intent
+    /// across reloads so the dashboard can show which preset was applied.
+    pub memory_preset: Option<MemoryPreset>,
     /// Hard RAM budget for published HOT entries, charged by backing allocation
     /// plus directly owned metadata rather than only logical body length.
     pub max_bytes: usize,
@@ -80,6 +114,9 @@ pub struct Settings {
     pub hot_eviction_max_objects: usize,
     /// Persistent cold tier. Disabled by default to avoid implicit disk writes.
     pub cold_enabled: bool,
+    /// Last explicitly selected COLD policy profile, persisted with the cache
+    /// configuration for the same reason as `memory_preset`.
+    pub cold_preset: Option<ColdPreset>,
     /// Relative to the RustProxy data directory.
     pub cold_path: String,
     pub cold_max_bytes: u64,
@@ -135,6 +172,7 @@ impl Default for Settings {
     fn default() -> Self {
         Self {
             enabled: true,
+            memory_preset: None,
             max_bytes: 256 * 1024 * 1024,
             max_object_bytes: 16 * 1024 * 1024,
             max_entries: 4096,
@@ -142,6 +180,7 @@ impl Default for Settings {
             cold_queue_max_bytes: 32 * 1024 * 1024,
             hot_eviction_max_objects: 64,
             cold_enabled: false,
+            cold_preset: None,
             cold_path: "cache/cold".into(),
             cold_max_bytes: 4 * 1024 * 1024 * 1024,
             cold_max_entries: 50_000,
@@ -572,6 +611,8 @@ pub struct View {
     pub cold_queue_limit_bytes: usize,
     pub cold_entries: usize,
     pub cold_bytes: u64,
+    pub cold_limit_bytes: u64,
+    pub cold_limit_entries: usize,
     pub cold_hits: u64,
     pub cold_promotions: u64,
     pub hot_demotions: u64,
@@ -1067,6 +1108,8 @@ pub struct Cache {
     tier_settings: Mutex<Settings>,
     data_root: Option<PathBuf>,
     cold_tx: Option<mpsc::Sender<ColdCommand>>,
+    cold_delete_fallbacks: Arc<Semaphore>,
+    cleanup_slots: Arc<Semaphore>,
     fills: Arc<Semaphore>,
     fill_bytes_active: AtomicU64,
     transient_bytes_active: Arc<AtomicU64>,

src/cache/tests.rs

@@ -662,6 +662,8 @@ mod tests {
     #[test]
     fn cache_memory_defaults_are_bounded_and_sane() {
         let s = Settings::default();
+        assert_eq!(s.memory_preset, None);
+        assert_eq!(s.cold_preset, None);
         assert_eq!(s.max_bytes, 256 * 1024 * 1024);
         assert_eq!(s.max_object_bytes, 16 * 1024 * 1024);
         assert_eq!(s.transient_max_bytes, 64 * 1024 * 1024);
@@ -672,6 +674,19 @@ mod tests {
         assert!(Cache::fill_headroom_bytes(s.max_bytes) <= 1024 * 1024);
     }
 
+    #[test]
+    fn cache_preset_intent_round_trips_with_configuration() {
+        let mut settings = Settings::default();
+        settings.memory_preset = Some(MemoryPreset::Recommended);
+        settings.cold_preset = Some(ColdPreset::Recommended);
+        let encoded = serde_json::to_value(&settings).unwrap();
+        assert_eq!(encoded["memory_preset"], "recommended");
+        assert_eq!(encoded["cold_preset"], "recommended");
+        let decoded: Settings = serde_json::from_value(encoded).unwrap();
+        assert_eq!(decoded.memory_preset, Some(MemoryPreset::Recommended));
+        assert_eq!(decoded.cold_preset, Some(ColdPreset::Recommended));
+    }
+
     #[test]
     fn transient_budget_is_hard_and_released_with_lease() {
         let cache = Cache::new(1);
@@ -2142,6 +2157,84 @@ mod v022_cache_tests {
         assert_eq!(cache.view().proactive_inflight, 0);
     }
 
+    #[test]
+    fn proactive_refresh_pending_tasks_are_bounded() {
+        let cache = Cache::new(22);
+        let mut settings = active_settings();
+        settings.proactive_refresh_enabled = true;
+        settings.proactive_refresh_percent = 100;
+        settings.proactive_refresh_max_inflight = 1;
+        let mut guards = Vec::new();
+        for index in 0..64 {
+            let ticket = cache
+                .ticket(
+                    22,
+                    "http",
+                    "refresh.example",
+                    "origin",
+                    &format!("/asset-{index}"),
+                    &HeaderMap::new(),
+                )
+                .unwrap();
+            guards.push(
+                cache
+                    .begin_proactive_schedule(&ticket, &settings)
+                    .expect("schedule below pending limit"),
+            );
+        }
+        let overflow = cache
+            .ticket(
+                22,
+                "http",
+                "refresh.example",
+                "origin",
+                "/overflow",
+                &HeaderMap::new(),
+            )
+            .unwrap();
+        assert!(cache.begin_proactive_schedule(&overflow, &settings).is_none());
+        assert!(cache.view().proactive_skipped > 0);
+        drop(guards);
+    }
+
+    #[test]
+    fn background_revalidation_set_has_a_hard_limit() {
+        let cache = Cache::new(22);
+        let base = Revalidation {
+            headers: HeaderMap::new(),
+            body: Bytes::new(),
+            body_allocated: 0,
+            _transient_leases: Vec::new(),
+            hits: 0,
+            transformed: false,
+            storage_key: [0; 32],
+            resident_age_at_lookup: Duration::ZERO,
+            skip_compression: false,
+            reusable_without_validation: false,
+            age: 0,
+            ttl: 0,
+            stale_if_error: 0,
+            must_revalidate: false,
+            proxy_revalidate: false,
+            origin_etag: None,
+            origin_last_modified: None,
+        };
+        let mut guards = Vec::new();
+        for index in 0..BACKGROUND_REVALIDATION_LIMIT {
+            let mut cached = base.clone();
+            cached.storage_key[0..8].copy_from_slice(&(index as u64).to_be_bytes());
+            guards.push(
+                cache
+                    .begin_background_revalidation(&cached)
+                    .expect("revalidation below hard limit"),
+            );
+        }
+        let mut overflow = base;
+        overflow.storage_key = [0xff; 32];
+        assert!(cache.begin_background_revalidation(&overflow).is_none());
+        drop(guards);
+    }
+
     #[tokio::test]
     async fn selective_purge_exact_prefix_tag_cache_key_and_query_are_precise() {
         let cache = Cache::new(22);
@@ -3036,6 +3129,22 @@ mod cold_tier_tests {
         panic!("cold cleanup did not finish");
     }
 
+    #[tokio::test]
+    async fn manual_cold_cleanup_is_single_flight() {
+        let dir = tempfile::tempdir().unwrap();
+        let settings = cold_settings();
+        let shutdown = CancellationToken::new();
+        let cache = Cache::start(11, Some(dir.path().to_path_buf()), settings, shutdown.clone());
+        let _held = cache.cleanup_slots.clone().try_acquire_owned().unwrap();
+        let result = cache.start_cold_cleanup(ColdCleanupRequest {
+            unused_for_seconds: None,
+            oldest: Some(1),
+            hits_zero: false,
+        });
+        assert!(result.is_err());
+        shutdown.cancel();
+    }
+
     #[tokio::test]
     async fn cold_cleanup_skips_an_entry_while_a_read_is_pinned() {
         let dir = tempfile::tempdir().unwrap();

src/cache/tickets.rs

@@ -310,6 +310,9 @@ impl Cache {
         // representation so an SWR refresh for one variant cannot suppress another.
         let key = cached.storage_key;
         let mut active = self.revalidating.lock().unwrap_or_else(|e| e.into_inner());
+        if active.len() >= BACKGROUND_REVALIDATION_LIMIT && !active.contains(&key) {
+            return None;
+        }
         if !active.insert(key) {
             return None;
         }