impl Cache { async fn handle_cold_command(&self, command: ColdCommand) { match command { ColdCommand::Store(draft) => { let bytes = draft.entry.cost as u64; self.cold_queue_items.fetch_sub(1, Ordering::Relaxed); self.cold_queue_bytes.fetch_sub(bytes, Ordering::Relaxed); self.persist_cold(draft).await; } ColdCommand::Delete(paths) => { for path in paths { let _ = tokio::fs::remove_file(path).await; } } ColdCommand::PurgeDirectory(path) => { if let Err(error) = tokio::fs::remove_dir_all(&path).await { if error.kind() != std::io::ErrorKind::NotFound { self.counters.cold_errors.fetch_add(1, Ordering::Relaxed); } } } } } async fn cold_worker( weak: std::sync::Weak, mut rx: mpsc::Receiver, shutdown: CancellationToken, ) { loop { let command = tokio::select! { _ = shutdown.cancelled() => { // The listener shutdown token is shared with the cache. Drain // already accepted demotions before exiting so a restart does // not discard buffers that were already handed to cold storage. rx.close(); while let Ok(command) = rx.try_recv() { let Some(cache) = weak.upgrade() else { return; }; cache.handle_cold_command(command).await; } break; }, command = rx.recv() => command, }; let Some(command) = command else { break; }; let Some(cache) = weak.upgrade() else { break; }; cache.handle_cold_command(command).await; } } async fn persist_cold(&self, draft: ColdDraft) { let bytes = draft.entry.cost as u64; self.cold_write_bytes_active .fetch_add(bytes, Ordering::Relaxed); self.persist_cold_inner(draft).await; self.cold_write_bytes_active .fetch_sub(bytes, Ordering::Relaxed); self.cold_gc_pass().await; // The draft body is out of scope before trimming. `Bytes` has released its // reference and glibc is explicitly asked to return releasable pages. release_allocator_memory(); } async fn persist_cold_inner(&self, draft: ColdDraft) { let current = { let store = self.store.lock().unwrap_or_else(|e| e.into_inner()); store.revision == draft.revision && store.generation == draft.generation && !store.entries.contains_key(&draft.key) && incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary) .is_empty() && store .scope_generations .get(&draft.entry.scope) .copied() .unwrap_or(0) == draft.scope_generation && store .key_generations .get(&draft.entry.base_key) .copied() .unwrap_or(0) == draft.key_generation }; if !current { return; } let Some(headers) = cold_header_pairs(&draft.entry.headers) else { self.counters.cold_errors.fetch_add(1, Ordering::Relaxed); return; }; let settings = self .tier_settings .lock() .unwrap_or_else(|e| e.into_inner()) .clone(); if !settings.cold_enabled || self.cold_root_for(&settings).as_ref() != Some(&draft.root) { return; } let key = key_hex(&draft.key); let body_name = format!("{key}.body"); let (body_path, meta_path, body_tmp, meta_tmp) = cold_paths(&draft.root, &draft.key); let Some(shard) = body_path.parent() else { return; }; if tokio::fs::create_dir_all(shard).await.is_err() { self.counters.cold_errors.fetch_add(1, Ordering::Relaxed); return; } let age_at_store = draft .entry .initial_age .saturating_add(draft.entry.received.elapsed()) .as_secs(); let resident_age_at_store = hot_resident_age(&draft.entry).as_secs(); let file = ColdFile { format: COLD_FORMAT_VERSION, key: key.clone(), base_key: key_hex(&draft.entry.base_key), vary: draft.entry.vary.clone(), revision: draft.revision, scope: draft.entry.scope.clone(), scheme: draft.entry.scheme.clone(), authority: draft.entry.authority.clone(), domain: draft.entry.domain.clone(), path: draft.entry.path.clone(), target: draft.entry.target.clone(), tags: draft.entry.tags.clone(), created_at_ms: draft.entry.created_at.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64, headers, body_file: body_name, body_bytes: draft.entry.body.len(), body_sha256: body_sha256(&draft.entry.body), stored_at_ms: now_ms(), age_at_store_seconds: age_at_store, resident_age_at_store_seconds: resident_age_at_store, ttl: draft.entry.ttl, stale_while_revalidate: draft.entry.stale_while_revalidate, stale_if_error: draft.entry.stale_if_error, must_revalidate: draft.entry.must_revalidate, proxy_revalidate: draft.entry.proxy_revalidate, origin_etag: draft.entry.origin_etag.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned), origin_last_modified: draft.entry.origin_last_modified.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned), last_access_ms: now_ms(), hits: draft.entry.hits, transformed: draft.entry.transformed, skip_compression: draft.entry.skip_compression, requires_revalidation: draft.entry.requires_revalidation, }; let Ok(metadata) = serde_json::to_vec_pretty(&file) else { self.counters.cold_errors.fetch_add(1, Ordering::Relaxed); return; }; let write_ok = tokio::fs::write(&body_tmp, &draft.entry.body).await.is_ok() && tokio::fs::write(&meta_tmp, &metadata).await.is_ok() && tokio::fs::rename(&body_tmp, &body_path).await.is_ok() && tokio::fs::rename(&meta_tmp, &meta_path).await.is_ok(); if !write_ok { let _ = tokio::fs::remove_file(&body_tmp).await; let _ = tokio::fs::remove_file(&meta_tmp).await; self.counters.cold_errors.fetch_add(1, Ordering::Relaxed); return; } let still_current = { let store = self.store.lock().unwrap_or_else(|e| e.into_inner()); store.revision == draft.revision && store.generation == draft.generation && !store.entries.contains_key(&draft.key) && incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary) .is_empty() && store .scope_generations .get(&draft.entry.scope) .copied() .unwrap_or(0) == draft.scope_generation && store .key_generations .get(&draft.entry.base_key) .copied() .unwrap_or(0) == draft.key_generation }; if !still_current { let _ = tokio::fs::remove_file(&body_path).await; let _ = tokio::fs::remove_file(&meta_path).await; return; } let disk_bytes = draft.entry.body.len() as u64 + metadata.len() as u64; let mut delete = Vec::new(); { let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner()); for key in incompatible_cold_variants(&cold, &draft.entry.base_key, &draft.entry.vary) { if let Some(old) = cold.remove(&key) { delete.extend([old.meta_path, old.body_path]); } } if let Some(old) = cold.remove(&draft.key) { if old.meta_path != meta_path { delete.push(old.meta_path); } if old.body_path != body_path { delete.push(old.body_path); } } while cold.entries.len() >= settings.cold_max_entries || cold.bytes.saturating_add(disk_bytes) > settings.cold_max_bytes { let Some((_, key)) = cold.lru.first_key_value().map(|(seq, key)| (*seq, *key)) else { break; }; if let Some(old) = cold.remove(&key) { delete.extend([old.meta_path, old.body_path]); self.counters.cold_evictions.fetch_add(1, Ordering::Relaxed); } } if disk_bytes > settings.cold_max_bytes { delete.extend([meta_path.clone(), body_path.clone()]); } else { let sequence = cold.next_sequence(); let entry = ColdEntry { key: draft.key, base_key: draft.entry.base_key, vary: draft.entry.vary, revision: draft.revision, scope: draft.entry.scope, scheme: draft.entry.scheme, authority: draft.entry.authority, domain: draft.entry.domain, path: draft.entry.path, target: draft.entry.target, tags: draft.entry.tags, created_at: draft.entry.created_at, headers: draft.entry.headers, body_path: body_path.clone(), meta_path: meta_path.clone(), body_bytes: draft.entry.body.len(), body_sha256: file.body_sha256, disk_bytes, stored_at: SystemTime::now(), age_at_store_seconds: age_at_store, resident_age_at_store_seconds: resident_age_at_store, ttl: draft.entry.ttl, stale_while_revalidate: draft.entry.stale_while_revalidate, stale_if_error: draft.entry.stale_if_error, must_revalidate: draft.entry.must_revalidate, proxy_revalidate: draft.entry.proxy_revalidate, origin_etag: draft.entry.origin_etag, origin_last_modified: draft.entry.origin_last_modified, last_access: SystemTime::now(), hits: draft.entry.hits, transformed: draft.entry.transformed, sequence, generation: sequence, skip_compression: draft.entry.skip_compression, requires_revalidation: draft.entry.requires_revalidation, pinned_reads: 0, }; cold.bytes = cold.bytes.saturating_add(disk_bytes); cold.lru.insert(sequence, draft.key); cold.index_variant(entry.base_key, draft.key); cold.entries.insert(draft.key, entry); self.counters.hot_demotions.fetch_add(1, Ordering::Relaxed); } } for path in delete { let _ = tokio::fs::remove_file(path).await; } } }