v0.25.1

src/cache/cold/worker.rs

@@ -0,0 +1,270 @@
+impl Cache {
+    async fn handle_cold_command(&self, command: ColdCommand) {
+        match command {
+            ColdCommand::Store(draft) => {
+                let bytes = draft.entry.cost as u64;
+                self.cold_queue_items.fetch_sub(1, Ordering::Relaxed);
+                self.cold_queue_bytes.fetch_sub(bytes, Ordering::Relaxed);
+                self.persist_cold(draft).await;
+            }
+            ColdCommand::Delete(paths) => {
+                for path in paths {
+                    let _ = tokio::fs::remove_file(path).await;
+                }
+            }
+            ColdCommand::PurgeDirectory(path) => {
+                if let Err(error) = tokio::fs::remove_dir_all(&path).await {
+                    if error.kind() != std::io::ErrorKind::NotFound {
+                        self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
+                    }
+                }
+            }
+        }
+    }
+    async fn cold_worker(
+        weak: std::sync::Weak<Self>,
+        mut rx: mpsc::Receiver<ColdCommand>,
+        shutdown: CancellationToken,
+    ) {
+        loop {
+            let command = tokio::select! {
+                _ = shutdown.cancelled() => {
+                    // The listener shutdown token is shared with the cache. Drain
+                    // already accepted demotions before exiting so a restart does
+                    // not discard buffers that were already handed to cold storage.
+                    rx.close();
+                    while let Ok(command) = rx.try_recv() {
+                        let Some(cache) = weak.upgrade() else { return; };
+                        cache.handle_cold_command(command).await;
+                    }
+                    break;
+                },
+                command = rx.recv() => command,
+            };
+            let Some(command) = command else {
+                break;
+            };
+            let Some(cache) = weak.upgrade() else {
+                break;
+            };
+            cache.handle_cold_command(command).await;
+        }
+    }
+    async fn persist_cold(&self, draft: ColdDraft) {
+        let bytes = draft.entry.cost as u64;
+        self.cold_write_bytes_active
+            .fetch_add(bytes, Ordering::Relaxed);
+        self.persist_cold_inner(draft).await;
+        self.cold_write_bytes_active
+            .fetch_sub(bytes, Ordering::Relaxed);
+        self.cold_gc_pass().await;
+        // The draft body is out of scope before trimming. `Bytes` has released its
+        // reference and glibc is explicitly asked to return releasable pages.
+        release_allocator_memory();
+    }
+    async fn persist_cold_inner(&self, draft: ColdDraft) {
+        let current = {
+            let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
+            store.revision == draft.revision
+                && store.generation == draft.generation
+                && !store.entries.contains_key(&draft.key)
+                && incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary)
+                    .is_empty()
+                && store
+                    .scope_generations
+                    .get(&draft.entry.scope)
+                    .copied()
+                    .unwrap_or(0)
+                    == draft.scope_generation
+                && store
+                    .key_generations
+                    .get(&draft.entry.base_key)
+                    .copied()
+                    .unwrap_or(0)
+                    == draft.key_generation
+        };
+        if !current {
+            return;
+        }
+        let Some(headers) = cold_header_pairs(&draft.entry.headers) else {
+            self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
+            return;
+        };
+        let settings = self
+            .tier_settings
+            .lock()
+            .unwrap_or_else(|e| e.into_inner())
+            .clone();
+        if !settings.cold_enabled || self.cold_root_for(&settings).as_ref() != Some(&draft.root) {
+            return;
+        }
+        let key = key_hex(&draft.key);
+        let body_name = format!("{key}.body");
+        let (body_path, meta_path, body_tmp, meta_tmp) = cold_paths(&draft.root, &draft.key);
+        let Some(shard) = body_path.parent() else { return; };
+        if tokio::fs::create_dir_all(shard).await.is_err() {
+            self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
+            return;
+        }
+        let age_at_store = draft
+            .entry
+            .initial_age
+            .saturating_add(draft.entry.received.elapsed())
+            .as_secs();
+        let resident_age_at_store = hot_resident_age(&draft.entry).as_secs();
+        let file = ColdFile {
+            format: COLD_FORMAT_VERSION,
+            key: key.clone(),
+            base_key: key_hex(&draft.entry.base_key),
+            vary: draft.entry.vary.clone(),
+            revision: draft.revision,
+            scope: draft.entry.scope.clone(),
+            scheme: draft.entry.scheme.clone(),
+            authority: draft.entry.authority.clone(),
+            domain: draft.entry.domain.clone(),
+            path: draft.entry.path.clone(),
+            target: draft.entry.target.clone(),
+            tags: draft.entry.tags.clone(),
+            created_at_ms: draft.entry.created_at.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64,
+            headers,
+            body_file: body_name,
+            body_bytes: draft.entry.body.len(),
+            body_sha256: body_sha256(&draft.entry.body),
+            stored_at_ms: now_ms(),
+            age_at_store_seconds: age_at_store,
+            resident_age_at_store_seconds: resident_age_at_store,
+            ttl: draft.entry.ttl,
+            stale_while_revalidate: draft.entry.stale_while_revalidate,
+            stale_if_error: draft.entry.stale_if_error,
+            must_revalidate: draft.entry.must_revalidate,
+            proxy_revalidate: draft.entry.proxy_revalidate,
+            origin_etag: draft.entry.origin_etag.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned),
+            origin_last_modified: draft.entry.origin_last_modified.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned),
+            last_access_ms: now_ms(),
+            hits: draft.entry.hits,
+            transformed: draft.entry.transformed,
+            skip_compression: draft.entry.skip_compression,
+            requires_revalidation: draft.entry.requires_revalidation,
+        };
+        let Ok(metadata) = serde_json::to_vec_pretty(&file) else {
+            self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
+            return;
+        };
+        let write_ok = tokio::fs::write(&body_tmp, &draft.entry.body).await.is_ok()
+            && tokio::fs::write(&meta_tmp, &metadata).await.is_ok()
+            && tokio::fs::rename(&body_tmp, &body_path).await.is_ok()
+            && tokio::fs::rename(&meta_tmp, &meta_path).await.is_ok();
+        if !write_ok {
+            let _ = tokio::fs::remove_file(&body_tmp).await;
+            let _ = tokio::fs::remove_file(&meta_tmp).await;
+            self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
+            return;
+        }
+        let still_current = {
+            let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
+            store.revision == draft.revision
+                && store.generation == draft.generation
+                && !store.entries.contains_key(&draft.key)
+                && incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary)
+                    .is_empty()
+                && store
+                    .scope_generations
+                    .get(&draft.entry.scope)
+                    .copied()
+                    .unwrap_or(0)
+                    == draft.scope_generation
+                && store
+                    .key_generations
+                    .get(&draft.entry.base_key)
+                    .copied()
+                    .unwrap_or(0)
+                    == draft.key_generation
+        };
+        if !still_current {
+            let _ = tokio::fs::remove_file(&body_path).await;
+            let _ = tokio::fs::remove_file(&meta_path).await;
+            return;
+        }
+        let disk_bytes = draft.entry.body.len() as u64 + metadata.len() as u64;
+        let mut delete = Vec::new();
+        {
+            let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
+            for key in incompatible_cold_variants(&cold, &draft.entry.base_key, &draft.entry.vary) {
+                if let Some(old) = cold.remove(&key) {
+                    delete.extend([old.meta_path, old.body_path]);
+                }
+            }
+            if let Some(old) = cold.remove(&draft.key) {
+                if old.meta_path != meta_path {
+                    delete.push(old.meta_path);
+                }
+                if old.body_path != body_path {
+                    delete.push(old.body_path);
+                }
+            }
+            while cold.entries.len() >= settings.cold_max_entries
+                || cold.bytes.saturating_add(disk_bytes) > settings.cold_max_bytes
+            {
+                let Some((_, key)) = cold.lru.first_key_value().map(|(seq, key)| (*seq, *key))
+                else {
+                    break;
+                };
+                if let Some(old) = cold.remove(&key) {
+                    delete.extend([old.meta_path, old.body_path]);
+                    self.counters.cold_evictions.fetch_add(1, Ordering::Relaxed);
+                }
+            }
+            if disk_bytes > settings.cold_max_bytes {
+                delete.extend([meta_path.clone(), body_path.clone()]);
+            } else {
+                let sequence = cold.next_sequence();
+                let entry = ColdEntry {
+                    key: draft.key,
+                    base_key: draft.entry.base_key,
+                    vary: draft.entry.vary,
+                    revision: draft.revision,
+                    scope: draft.entry.scope,
+                    scheme: draft.entry.scheme,
+                    authority: draft.entry.authority,
+                    domain: draft.entry.domain,
+                    path: draft.entry.path,
+                    target: draft.entry.target,
+                    tags: draft.entry.tags,
+                    created_at: draft.entry.created_at,
+                    headers: draft.entry.headers,
+                    body_path: body_path.clone(),
+                    meta_path: meta_path.clone(),
+                    body_bytes: draft.entry.body.len(),
+                    body_sha256: file.body_sha256,
+                    disk_bytes,
+                    stored_at: SystemTime::now(),
+                    age_at_store_seconds: age_at_store,
+                    resident_age_at_store_seconds: resident_age_at_store,
+                    ttl: draft.entry.ttl,
+                    stale_while_revalidate: draft.entry.stale_while_revalidate,
+                    stale_if_error: draft.entry.stale_if_error,
+                    must_revalidate: draft.entry.must_revalidate,
+                    proxy_revalidate: draft.entry.proxy_revalidate,
+                    origin_etag: draft.entry.origin_etag,
+                    origin_last_modified: draft.entry.origin_last_modified,
+                    last_access: SystemTime::now(),
+                    hits: draft.entry.hits,
+                    transformed: draft.entry.transformed,
+                    sequence,
+                    generation: sequence,
+                    skip_compression: draft.entry.skip_compression,
+                    requires_revalidation: draft.entry.requires_revalidation,
+                    pinned_reads: 0,
+                };
+                cold.bytes = cold.bytes.saturating_add(disk_bytes);
+                cold.lru.insert(sequence, draft.key);
+                cold.index_variant(entry.base_key, draft.key);
+                cold.entries.insert(draft.key, entry);
+                self.counters.hot_demotions.fetch_add(1, Ordering::Relaxed);
+            }
+        }
+        for path in delete {
+            let _ = tokio::fs::remove_file(path).await;
+        }
+    }
+}