v0.24.5
This commit is contained in:
gru committed
commit
5e23e9af02
240 files changed
+99957
No files matched your search
Vendored
+974
@@ -0,0 +1,974 @@
|
||||
impl Cache {
|
||||
fn cold_root_for(&self, settings: &Settings) -> Option<PathBuf> {
|
||||
settings
|
||||
.cold_enabled
|
||||
.then(|| {
|
||||
self.data_root
|
||||
.as_ref()
|
||||
.map(|root| root.join(&settings.cold_path))
|
||||
})
|
||||
.flatten()
|
||||
}
|
||||
fn load_cold_startup(&self) {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let Some(root) = self.cold_root_for(&settings) else {
|
||||
return;
|
||||
};
|
||||
let revision = self
|
||||
.store
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.revision;
|
||||
let Ok(read_dir) = std::fs::read_dir(&root) else {
|
||||
return;
|
||||
};
|
||||
|
||||
// v0.21 intentionally does not migrate the legacy flat cold-cache layout.
|
||||
// Remove only known cache artifacts from the old root and rebuild naturally.
|
||||
let mut shard_dirs = Vec::new();
|
||||
for item in read_dir.flatten() {
|
||||
let path = item.path();
|
||||
if path.is_file() {
|
||||
let extension = path.extension().and_then(|v| v.to_str()).unwrap_or_default();
|
||||
if matches!(extension, "json" | "body" | "tmp") {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let Some(name) = path.file_name().and_then(|v| v.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if name.len() == 2 && name.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
shard_dirs.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
let mut loaded = Vec::new();
|
||||
for shard in shard_dirs {
|
||||
let Some(shard_name) = shard.file_name().and_then(|v| v.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(items) = std::fs::read_dir(&shard) else {
|
||||
continue;
|
||||
};
|
||||
for item in items.flatten() {
|
||||
let path = item.path();
|
||||
if path.extension().and_then(|v| v.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
let Ok(bytes) = std::fs::read(&path) else {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
};
|
||||
let Ok(file) = serde_json::from_slice::<ColdFile>(&bytes) else {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
};
|
||||
let body_path = shard.join(&file.body_file);
|
||||
if file.format != COLD_FORMAT_VERSION || file.revision != revision {
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(&body_path);
|
||||
continue;
|
||||
}
|
||||
let Some(key) = parse_key_hex(&file.key) else {
|
||||
continue;
|
||||
};
|
||||
if !file.key[..2].eq_ignore_ascii_case(shard_name) {
|
||||
continue;
|
||||
}
|
||||
let Some(base_key) = parse_key_hex(&file.base_key) else {
|
||||
continue;
|
||||
};
|
||||
let mut vary = file.vary.clone();
|
||||
if vary.len() > 64
|
||||
|| vary
|
||||
.iter()
|
||||
.any(|name| name.parse::<http::HeaderName>().is_err())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
vary.sort();
|
||||
vary.dedup();
|
||||
if path.file_stem().and_then(|v| v.to_str()) != Some(file.key.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let Some(headers) = pairs_to_headers(&file.headers) else {
|
||||
continue;
|
||||
};
|
||||
if body_path.parent() != Some(shard.as_path()) {
|
||||
continue;
|
||||
}
|
||||
let Ok(body_meta) = std::fs::metadata(&body_path) else {
|
||||
continue;
|
||||
};
|
||||
if body_meta.len() != file.body_bytes as u64
|
||||
|| file.body_sha256.len() != 64
|
||||
|| !file.body_sha256.bytes().all(|b| b.is_ascii_hexdigit())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let origin_etag = file
|
||||
.origin_etag
|
||||
.as_deref()
|
||||
.and_then(|v| HeaderValue::from_str(v).ok());
|
||||
let origin_last_modified = file
|
||||
.origin_last_modified
|
||||
.as_deref()
|
||||
.and_then(|v| HeaderValue::from_str(v).ok());
|
||||
let stored_at = system_time_ms(file.stored_at_ms);
|
||||
let current_age = file
|
||||
.age_at_store_seconds
|
||||
.saturating_add(stored_at.elapsed().unwrap_or_default().as_secs());
|
||||
if current_age
|
||||
>= retention_deadline(
|
||||
file.ttl,
|
||||
file.stale_while_revalidate,
|
||||
file.stale_if_error,
|
||||
origin_etag.is_some() || origin_last_modified.is_some(),
|
||||
)
|
||||
{
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _ = std::fs::remove_file(&body_path);
|
||||
continue;
|
||||
}
|
||||
loaded.push(ColdEntry {
|
||||
key,
|
||||
base_key,
|
||||
vary,
|
||||
revision,
|
||||
scope: file.scope,
|
||||
scheme: file.scheme,
|
||||
authority: if file.authority.is_empty() { file.domain.clone() } else { file.authority },
|
||||
domain: file.domain,
|
||||
path: file.path.clone(),
|
||||
target: if file.target.is_empty() { file.path } else { file.target },
|
||||
tags: file.tags,
|
||||
created_at: if file.created_at_ms == 0 { stored_at } else { system_time_ms(file.created_at_ms) },
|
||||
headers,
|
||||
body_path,
|
||||
meta_path: path,
|
||||
body_bytes: file.body_bytes,
|
||||
body_sha256: file.body_sha256,
|
||||
disk_bytes: body_meta.len().saturating_add(bytes.len() as u64),
|
||||
stored_at,
|
||||
last_access: if file.last_access_ms == 0 {
|
||||
stored_at
|
||||
} else {
|
||||
system_time_ms(file.last_access_ms)
|
||||
},
|
||||
age_at_store_seconds: file.age_at_store_seconds,
|
||||
resident_age_at_store_seconds: file.resident_age_at_store_seconds,
|
||||
ttl: file.ttl,
|
||||
stale_while_revalidate: file.stale_while_revalidate,
|
||||
stale_if_error: file.stale_if_error,
|
||||
must_revalidate: file.must_revalidate,
|
||||
proxy_revalidate: file.proxy_revalidate,
|
||||
origin_etag,
|
||||
origin_last_modified,
|
||||
hits: file.hits,
|
||||
transformed: file.transformed,
|
||||
sequence: 0,
|
||||
generation: 0,
|
||||
skip_compression: file.skip_compression,
|
||||
requires_revalidation: file.requires_revalidation,
|
||||
pinned_reads: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
loaded.sort_by_key(|entry| entry.last_access);
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for mut entry in loaded {
|
||||
for key in incompatible_cold_variants(&cold, &entry.base_key, &entry.vary) {
|
||||
if let Some(old) = cold.remove(&key) {
|
||||
let _ = std::fs::remove_file(old.meta_path);
|
||||
let _ = std::fs::remove_file(old.body_path);
|
||||
}
|
||||
}
|
||||
while cold.entries.len() >= settings.cold_max_entries
|
||||
|| cold.bytes.saturating_add(entry.disk_bytes) > settings.cold_max_bytes
|
||||
{
|
||||
let Some((_, key)) = cold.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if let Some(old) = cold.remove(&key) {
|
||||
let _ = std::fs::remove_file(old.meta_path);
|
||||
let _ = std::fs::remove_file(old.body_path);
|
||||
}
|
||||
}
|
||||
entry.sequence = cold.next_sequence();
|
||||
entry.generation = entry.sequence;
|
||||
cold.bytes = cold.bytes.saturating_add(entry.disk_bytes);
|
||||
cold.lru.insert(entry.sequence, entry.key);
|
||||
cold.index_variant(entry.base_key, entry.key);
|
||||
cold.entries.insert(entry.key, entry);
|
||||
}
|
||||
}
|
||||
async fn handle_cold_command(&self, command: ColdCommand) {
|
||||
match command {
|
||||
ColdCommand::Store(draft) => {
|
||||
let bytes = draft.entry.cost as u64;
|
||||
self.cold_queue_items.fetch_sub(1, Ordering::Relaxed);
|
||||
self.cold_queue_bytes.fetch_sub(bytes, Ordering::Relaxed);
|
||||
self.persist_cold(draft).await;
|
||||
}
|
||||
ColdCommand::Delete(paths) => {
|
||||
for path in paths {
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
}
|
||||
ColdCommand::PurgeDirectory(path) => {
|
||||
if let Err(error) = tokio::fs::remove_dir_all(&path).await {
|
||||
if error.kind() != std::io::ErrorKind::NotFound {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn cold_worker(
|
||||
weak: std::sync::Weak<Self>,
|
||||
mut rx: mpsc::Receiver<ColdCommand>,
|
||||
shutdown: CancellationToken,
|
||||
) {
|
||||
loop {
|
||||
let command = tokio::select! {
|
||||
_ = shutdown.cancelled() => {
|
||||
// The listener shutdown token is shared with the cache. Drain
|
||||
// already accepted demotions before exiting so a restart does
|
||||
// not discard buffers that were already handed to cold storage.
|
||||
rx.close();
|
||||
while let Ok(command) = rx.try_recv() {
|
||||
let Some(cache) = weak.upgrade() else { return; };
|
||||
cache.handle_cold_command(command).await;
|
||||
}
|
||||
break;
|
||||
},
|
||||
command = rx.recv() => command,
|
||||
};
|
||||
let Some(command) = command else {
|
||||
break;
|
||||
};
|
||||
let Some(cache) = weak.upgrade() else {
|
||||
break;
|
||||
};
|
||||
cache.handle_cold_command(command).await;
|
||||
}
|
||||
}
|
||||
async fn persist_cold(&self, draft: ColdDraft) {
|
||||
let bytes = draft.entry.cost as u64;
|
||||
self.cold_write_bytes_active
|
||||
.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.persist_cold_inner(draft).await;
|
||||
self.cold_write_bytes_active
|
||||
.fetch_sub(bytes, Ordering::Relaxed);
|
||||
self.cold_gc_pass().await;
|
||||
// The draft body is out of scope before trimming. `Bytes` has released its
|
||||
// reference and glibc is explicitly asked to return releasable pages.
|
||||
release_allocator_memory();
|
||||
}
|
||||
async fn persist_cold_inner(&self, draft: ColdDraft) {
|
||||
let current = {
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
store.revision == draft.revision
|
||||
&& store.generation == draft.generation
|
||||
&& !store.entries.contains_key(&draft.key)
|
||||
&& incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary)
|
||||
.is_empty()
|
||||
&& store
|
||||
.scope_generations
|
||||
.get(&draft.entry.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== draft.scope_generation
|
||||
&& store
|
||||
.key_generations
|
||||
.get(&draft.entry.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== draft.key_generation
|
||||
};
|
||||
if !current {
|
||||
return;
|
||||
}
|
||||
let Some(headers) = cold_header_pairs(&draft.entry.headers) else {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
};
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
if !settings.cold_enabled || self.cold_root_for(&settings).as_ref() != Some(&draft.root) {
|
||||
return;
|
||||
}
|
||||
let key = key_hex(&draft.key);
|
||||
let body_name = format!("{key}.body");
|
||||
let (body_path, meta_path, body_tmp, meta_tmp) = cold_paths(&draft.root, &draft.key);
|
||||
let Some(shard) = body_path.parent() else { return; };
|
||||
if tokio::fs::create_dir_all(shard).await.is_err() {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
let age_at_store = draft
|
||||
.entry
|
||||
.initial_age
|
||||
.saturating_add(draft.entry.received.elapsed())
|
||||
.as_secs();
|
||||
let resident_age_at_store = hot_resident_age(&draft.entry).as_secs();
|
||||
let file = ColdFile {
|
||||
format: COLD_FORMAT_VERSION,
|
||||
key: key.clone(),
|
||||
base_key: key_hex(&draft.entry.base_key),
|
||||
vary: draft.entry.vary.clone(),
|
||||
revision: draft.revision,
|
||||
scope: draft.entry.scope.clone(),
|
||||
scheme: draft.entry.scheme.clone(),
|
||||
authority: draft.entry.authority.clone(),
|
||||
domain: draft.entry.domain.clone(),
|
||||
path: draft.entry.path.clone(),
|
||||
target: draft.entry.target.clone(),
|
||||
tags: draft.entry.tags.clone(),
|
||||
created_at_ms: draft.entry.created_at.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(u64::MAX as u128) as u64,
|
||||
headers,
|
||||
body_file: body_name,
|
||||
body_bytes: draft.entry.body.len(),
|
||||
body_sha256: body_sha256(&draft.entry.body),
|
||||
stored_at_ms: now_ms(),
|
||||
age_at_store_seconds: age_at_store,
|
||||
resident_age_at_store_seconds: resident_age_at_store,
|
||||
ttl: draft.entry.ttl,
|
||||
stale_while_revalidate: draft.entry.stale_while_revalidate,
|
||||
stale_if_error: draft.entry.stale_if_error,
|
||||
must_revalidate: draft.entry.must_revalidate,
|
||||
proxy_revalidate: draft.entry.proxy_revalidate,
|
||||
origin_etag: draft.entry.origin_etag.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned),
|
||||
origin_last_modified: draft.entry.origin_last_modified.as_ref().and_then(|v| v.to_str().ok()).map(str::to_owned),
|
||||
last_access_ms: now_ms(),
|
||||
hits: draft.entry.hits,
|
||||
transformed: draft.entry.transformed,
|
||||
skip_compression: draft.entry.skip_compression,
|
||||
requires_revalidation: draft.entry.requires_revalidation,
|
||||
};
|
||||
let Ok(metadata) = serde_json::to_vec_pretty(&file) else {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
};
|
||||
let write_ok = tokio::fs::write(&body_tmp, &draft.entry.body).await.is_ok()
|
||||
&& tokio::fs::write(&meta_tmp, &metadata).await.is_ok()
|
||||
&& tokio::fs::rename(&body_tmp, &body_path).await.is_ok()
|
||||
&& tokio::fs::rename(&meta_tmp, &meta_path).await.is_ok();
|
||||
if !write_ok {
|
||||
let _ = tokio::fs::remove_file(&body_tmp).await;
|
||||
let _ = tokio::fs::remove_file(&meta_tmp).await;
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
let still_current = {
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
store.revision == draft.revision
|
||||
&& store.generation == draft.generation
|
||||
&& !store.entries.contains_key(&draft.key)
|
||||
&& incompatible_hot_variants(&store, &draft.entry.base_key, &draft.entry.vary)
|
||||
.is_empty()
|
||||
&& store
|
||||
.scope_generations
|
||||
.get(&draft.entry.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== draft.scope_generation
|
||||
&& store
|
||||
.key_generations
|
||||
.get(&draft.entry.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== draft.key_generation
|
||||
};
|
||||
if !still_current {
|
||||
let _ = tokio::fs::remove_file(&body_path).await;
|
||||
let _ = tokio::fs::remove_file(&meta_path).await;
|
||||
return;
|
||||
}
|
||||
let disk_bytes = draft.entry.body.len() as u64 + metadata.len() as u64;
|
||||
let mut delete = Vec::new();
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for key in incompatible_cold_variants(&cold, &draft.entry.base_key, &draft.entry.vary) {
|
||||
if let Some(old) = cold.remove(&key) {
|
||||
delete.extend([old.meta_path, old.body_path]);
|
||||
}
|
||||
}
|
||||
if let Some(old) = cold.remove(&draft.key) {
|
||||
if old.meta_path != meta_path {
|
||||
delete.push(old.meta_path);
|
||||
}
|
||||
if old.body_path != body_path {
|
||||
delete.push(old.body_path);
|
||||
}
|
||||
}
|
||||
while cold.entries.len() >= settings.cold_max_entries
|
||||
|| cold.bytes.saturating_add(disk_bytes) > settings.cold_max_bytes
|
||||
{
|
||||
let Some((_, key)) = cold.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
if let Some(old) = cold.remove(&key) {
|
||||
delete.extend([old.meta_path, old.body_path]);
|
||||
self.counters.cold_evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
if disk_bytes > settings.cold_max_bytes {
|
||||
delete.extend([meta_path.clone(), body_path.clone()]);
|
||||
} else {
|
||||
let sequence = cold.next_sequence();
|
||||
let entry = ColdEntry {
|
||||
key: draft.key,
|
||||
base_key: draft.entry.base_key,
|
||||
vary: draft.entry.vary,
|
||||
revision: draft.revision,
|
||||
scope: draft.entry.scope,
|
||||
scheme: draft.entry.scheme,
|
||||
authority: draft.entry.authority,
|
||||
domain: draft.entry.domain,
|
||||
path: draft.entry.path,
|
||||
target: draft.entry.target,
|
||||
tags: draft.entry.tags,
|
||||
created_at: draft.entry.created_at,
|
||||
headers: draft.entry.headers,
|
||||
body_path: body_path.clone(),
|
||||
meta_path: meta_path.clone(),
|
||||
body_bytes: draft.entry.body.len(),
|
||||
body_sha256: file.body_sha256,
|
||||
disk_bytes,
|
||||
stored_at: SystemTime::now(),
|
||||
age_at_store_seconds: age_at_store,
|
||||
resident_age_at_store_seconds: resident_age_at_store,
|
||||
ttl: draft.entry.ttl,
|
||||
stale_while_revalidate: draft.entry.stale_while_revalidate,
|
||||
stale_if_error: draft.entry.stale_if_error,
|
||||
must_revalidate: draft.entry.must_revalidate,
|
||||
proxy_revalidate: draft.entry.proxy_revalidate,
|
||||
origin_etag: draft.entry.origin_etag,
|
||||
origin_last_modified: draft.entry.origin_last_modified,
|
||||
last_access: SystemTime::now(),
|
||||
hits: draft.entry.hits,
|
||||
transformed: draft.entry.transformed,
|
||||
sequence,
|
||||
generation: sequence,
|
||||
skip_compression: draft.entry.skip_compression,
|
||||
requires_revalidation: draft.entry.requires_revalidation,
|
||||
pinned_reads: 0,
|
||||
};
|
||||
cold.bytes = cold.bytes.saturating_add(disk_bytes);
|
||||
cold.lru.insert(sequence, draft.key);
|
||||
cold.index_variant(entry.base_key, draft.key);
|
||||
cold.entries.insert(draft.key, entry);
|
||||
self.counters.hot_demotions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
for path in delete {
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
}
|
||||
fn enqueue_demotion(
|
||||
&self,
|
||||
key: Key,
|
||||
entry: Entry,
|
||||
revision: u64,
|
||||
generation: u64,
|
||||
scope_generation: u64,
|
||||
key_generation: u64,
|
||||
) -> bool {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let Some(root) = self.cold_root_for(&settings) else {
|
||||
return false;
|
||||
};
|
||||
let Some(tx) = &self.cold_tx else {
|
||||
return false;
|
||||
};
|
||||
let queued_bytes = entry.cost as u64;
|
||||
let queue_limit = u64::try_from(settings.cold_queue_max_bytes).unwrap_or(u64::MAX);
|
||||
loop {
|
||||
let current = self.cold_queue_bytes.load(Ordering::Relaxed);
|
||||
let Some(next) = current.checked_add(queued_bytes) else {
|
||||
self.counters
|
||||
.cold_queue_dropped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
};
|
||||
if next > queue_limit {
|
||||
self.counters
|
||||
.cold_queue_dropped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.cold_queue_bytes
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
let Some(memory_lease) = self.try_reserve_transient(entry.cost, &settings) else {
|
||||
self.cold_queue_bytes
|
||||
.fetch_sub(queued_bytes, Ordering::Relaxed);
|
||||
self.counters
|
||||
.cold_queue_dropped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
};
|
||||
self.cold_queue_items.fetch_add(1, Ordering::Relaxed);
|
||||
if tx
|
||||
.try_send(ColdCommand::Store(ColdDraft {
|
||||
key,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
entry,
|
||||
root,
|
||||
_memory_lease: memory_lease,
|
||||
}))
|
||||
.is_err()
|
||||
{
|
||||
self.cold_queue_items.fetch_sub(1, Ordering::Relaxed);
|
||||
self.cold_queue_bytes
|
||||
.fetch_sub(queued_bytes, Ordering::Relaxed);
|
||||
self.counters
|
||||
.cold_queue_dropped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
fn demote_idle(&self) {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
if !settings.cold_enabled
|
||||
|| (!settings.cold_on_idle && !settings.cold_on_age)
|
||||
|| self.cold_root_for(&settings).is_none()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let available = self.cold_tx.as_ref().map(|tx| tx.capacity()).unwrap_or(0);
|
||||
if available == 0 {
|
||||
return;
|
||||
}
|
||||
let idle_cutoff = Duration::from_secs(settings.cold_after_seconds);
|
||||
let age_cutoff = Duration::from_secs(settings.cold_max_hot_seconds);
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let keys: Vec<_> = store
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| {
|
||||
(settings.cold_on_idle && entry.last_access.elapsed() >= idle_cutoff)
|
||||
|| (settings.cold_on_age && entry.resident_since.elapsed() >= age_cutoff)
|
||||
})
|
||||
.map(|(key, _)| *key)
|
||||
.take(available)
|
||||
.collect();
|
||||
for key in keys {
|
||||
let Some(entry) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&entry.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store.key_generations.get(&entry.base_key).copied().unwrap_or(0);
|
||||
if !self.enqueue_demotion(key, entry, revision, generation, scope_generation, key_generation) {
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
drop(store);
|
||||
release_allocator_memory();
|
||||
}
|
||||
/// Best-effort graceful shutdown handoff from RAM to the persistent cold
|
||||
/// tier. The cold worker shares the process shutdown token, so this path
|
||||
/// writes directly instead of depending on the background queue.
|
||||
async fn flush_hot_to_cold_inner(&self, force_shutdown_policy: bool) -> usize {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let Some(root) = self.cold_root_for(&settings) else {
|
||||
return 0;
|
||||
};
|
||||
if !force_shutdown_policy && !settings.cold_on_shutdown {
|
||||
return 0;
|
||||
}
|
||||
let mut moved = 0usize;
|
||||
loop {
|
||||
let draft = {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let Some(entry) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&entry.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store.key_generations.get(&entry.base_key).copied().unwrap_or(0);
|
||||
let bytes = entry.cost;
|
||||
let memory_lease = self.try_reserve_transient(bytes, &settings).unwrap_or_else(|| {
|
||||
let bytes = bytes as u64;
|
||||
self.transient_bytes_active.fetch_add(bytes, Ordering::Relaxed);
|
||||
TransientLease(Arc::new(TransientLeaseInner {
|
||||
counter: self.transient_bytes_active.clone(),
|
||||
bytes,
|
||||
}))
|
||||
});
|
||||
ColdDraft {
|
||||
key,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
entry,
|
||||
root: root.clone(),
|
||||
_memory_lease: memory_lease,
|
||||
}
|
||||
};
|
||||
self.persist_cold_inner(draft).await;
|
||||
moved += 1;
|
||||
}
|
||||
release_allocator_memory();
|
||||
moved
|
||||
}
|
||||
pub async fn flush_hot_to_cold(&self) -> usize {
|
||||
self.flush_hot_to_cold_inner(false).await
|
||||
}
|
||||
/// Managed software updates explicitly request a hot-to-cold handoff. The
|
||||
/// cold tier must still be enabled, but the ordinary cold_on_shutdown toggle
|
||||
/// does not disable update preservation.
|
||||
pub async fn flush_hot_to_cold_for_update(&self) -> usize {
|
||||
self.flush_hot_to_cold_inner(true).await
|
||||
}
|
||||
async fn sweep_cold(&self) {
|
||||
const BATCH: usize = 256;
|
||||
let mut after_sequence = 0u64;
|
||||
loop {
|
||||
let batch: Vec<_> = {
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
cold.lru
|
||||
.range((
|
||||
std::ops::Bound::Excluded(after_sequence),
|
||||
std::ops::Bound::Unbounded,
|
||||
))
|
||||
.take(BATCH)
|
||||
.map(|(sequence, key)| (*sequence, *key))
|
||||
.collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
after_sequence = batch.last().map(|value| value.0).unwrap_or(after_sequence);
|
||||
let mut delete = Vec::new();
|
||||
let mut expired = 0u64;
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (_, key) in &batch {
|
||||
let should_remove = cold.entries.get(key).is_some_and(|entry| {
|
||||
entry.pinned_reads == 0
|
||||
&& cold_age(entry)
|
||||
>= retention_deadline(
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.stale_if_error,
|
||||
entry.origin_etag.is_some()
|
||||
|| entry.origin_last_modified.is_some(),
|
||||
)
|
||||
});
|
||||
if should_remove {
|
||||
if let Some(entry) = cold.remove(key) {
|
||||
delete.extend([entry.meta_path, entry.body_path]);
|
||||
expired = expired.saturating_add(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if expired > 0 {
|
||||
self.counters.expired.fetch_add(expired, Ordering::Relaxed);
|
||||
}
|
||||
self.enqueue_cold_delete(delete);
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn enqueue_cold_delete(&self, paths: Vec<PathBuf>) {
|
||||
if paths.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Some(tx) = &self.cold_tx {
|
||||
if tx.try_send(ColdCommand::Delete(paths.clone())).is_ok() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
// Queue saturation must not block request processing. The index is already
|
||||
// updated; a best-effort detached delete keeps physical cleanup asynchronous.
|
||||
tokio::spawn(async move {
|
||||
for path in paths {
|
||||
let _ = tokio::fs::remove_file(path).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn remove_cold_candidates(
|
||||
&self,
|
||||
request: Option<&ColdCleanupRequest>,
|
||||
target_bytes: Option<u64>,
|
||||
target_entries: Option<usize>,
|
||||
max_delete: usize,
|
||||
) -> ColdCleanupResult {
|
||||
const SNAPSHOT_BATCH: usize = 256;
|
||||
const DELETE_BATCH: usize = 32;
|
||||
|
||||
let now = SystemTime::now();
|
||||
let mut candidates = Vec::new();
|
||||
let mut after_sequence = 0u64;
|
||||
|
||||
// Copy candidate metadata in bounded chunks. Request-path lookups only
|
||||
// contend with this mutex for a small snapshot, never for sorting or I/O.
|
||||
loop {
|
||||
let batch: Vec<_> = {
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
cold.lru
|
||||
.range((std::ops::Bound::Excluded(after_sequence), std::ops::Bound::Unbounded))
|
||||
.take(SNAPSHOT_BATCH)
|
||||
.filter_map(|(sequence, key)| {
|
||||
cold.entries.get(key).map(|entry| {
|
||||
(
|
||||
*sequence,
|
||||
*key,
|
||||
entry.hits,
|
||||
entry.last_access,
|
||||
entry.stored_at,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
if batch.is_empty() {
|
||||
break;
|
||||
}
|
||||
after_sequence = batch.last().map(|value| value.0).unwrap_or(after_sequence);
|
||||
for (sequence, key, hits, last_access, stored_at) in batch {
|
||||
let matches = request.is_none_or(|request| {
|
||||
(!request.hits_zero || hits == 0)
|
||||
&& request.unused_for_seconds.is_none_or(|seconds| {
|
||||
now.duration_since(last_access)
|
||||
.unwrap_or_default()
|
||||
.as_secs()
|
||||
>= seconds
|
||||
})
|
||||
});
|
||||
if matches {
|
||||
candidates.push((sequence, key, hits, last_access, stored_at));
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
if request.is_some() {
|
||||
candidates.sort_by_key(|(_, _, hits, last_access, stored_at)| {
|
||||
(*stored_at, *last_access, *hits)
|
||||
});
|
||||
} else {
|
||||
// Automatic GC prefers rarely used, least recently used, oldest data.
|
||||
candidates.sort_by_key(|(_, _, hits, last_access, stored_at)| {
|
||||
(*hits, *last_access, *stored_at)
|
||||
});
|
||||
}
|
||||
|
||||
let requested_limit = request
|
||||
.and_then(|value| value.oldest)
|
||||
.unwrap_or(max_delete)
|
||||
.min(max_delete);
|
||||
let mut result = ColdCleanupResult::default();
|
||||
let mut delete = Vec::new();
|
||||
|
||||
for chunk in candidates.chunks(DELETE_BATCH) {
|
||||
if result.removed_entries >= requested_limit {
|
||||
break;
|
||||
}
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (sequence, key, _, _, _) in chunk {
|
||||
if result.removed_entries >= requested_limit {
|
||||
break;
|
||||
}
|
||||
if let Some(target) = target_bytes {
|
||||
if cold.bytes <= target
|
||||
&& target_entries.is_none_or(|entries| cold.entries.len() <= entries)
|
||||
{
|
||||
break;
|
||||
}
|
||||
} else if let Some(entries) = target_entries {
|
||||
if cold.entries.len() <= entries {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !cold
|
||||
.entries
|
||||
.get(key)
|
||||
.is_some_and(|entry| entry.sequence == *sequence && entry.pinned_reads == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if let Some(entry) = cold.remove(key) {
|
||||
result.removed_entries += 1;
|
||||
result.freed_bytes = result.freed_bytes.saturating_add(entry.disk_bytes);
|
||||
delete.extend([entry.meta_path, entry.body_path]);
|
||||
}
|
||||
}
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
if result.removed_entries > 0 {
|
||||
self.counters
|
||||
.cold_evictions
|
||||
.fetch_add(result.removed_entries as u64, Ordering::Relaxed);
|
||||
}
|
||||
self.enqueue_cold_delete(delete);
|
||||
result
|
||||
}
|
||||
|
||||
async fn cold_gc_pass(&self) -> ColdCleanupResult {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
if !settings.cold_enabled {
|
||||
return ColdCleanupResult::default();
|
||||
}
|
||||
let (bytes, entries) = {
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(cold.bytes, cold.entries.len())
|
||||
};
|
||||
let high_bytes = settings.cold_max_bytes.saturating_mul(90) / 100;
|
||||
let target_bytes = settings.cold_max_bytes.saturating_mul(80) / 100;
|
||||
let high_entries = settings
|
||||
.cold_max_entries
|
||||
.saturating_mul(90)
|
||||
.saturating_add(99)
|
||||
/ 100;
|
||||
let target_entries = settings.cold_max_entries.saturating_mul(80) / 100;
|
||||
if bytes < high_bytes && entries < high_entries {
|
||||
return ColdCleanupResult::default();
|
||||
}
|
||||
self.remove_cold_candidates(
|
||||
None,
|
||||
Some(target_bytes),
|
||||
Some(target_entries),
|
||||
settings.cold_gc_max_delete,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn start_cold_cleanup(
|
||||
self: &Arc<Self>,
|
||||
request: ColdCleanupRequest,
|
||||
) -> Result<ColdCleanupJob> {
|
||||
request.validate()?;
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
ensure!(settings.cold_enabled, "cold cache is disabled");
|
||||
let id = self.cleanup_sequence.fetch_add(1, Ordering::Relaxed).saturating_add(1);
|
||||
let job = ColdCleanupJob {
|
||||
id,
|
||||
status: "queued",
|
||||
removed_entries: 0,
|
||||
freed_bytes: 0,
|
||||
error: None,
|
||||
};
|
||||
{
|
||||
let mut jobs = self.cleanup_jobs.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if jobs.len() >= 64 {
|
||||
let mut ids: Vec<_> = jobs.keys().copied().collect();
|
||||
ids.sort_unstable();
|
||||
for old in ids.into_iter().take(jobs.len().saturating_sub(63)) {
|
||||
jobs.remove(&old);
|
||||
}
|
||||
}
|
||||
jobs.insert(id, job.clone());
|
||||
}
|
||||
let cache = self.clone();
|
||||
tokio::spawn(async move {
|
||||
{
|
||||
let mut jobs = cache.cleanup_jobs.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(job) = jobs.get_mut(&id) {
|
||||
job.status = "running";
|
||||
}
|
||||
}
|
||||
let settings = cache
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let result = cache
|
||||
.remove_cold_candidates(
|
||||
Some(&request),
|
||||
None,
|
||||
None,
|
||||
settings.cold_gc_max_delete,
|
||||
)
|
||||
.await;
|
||||
let mut jobs = cache.cleanup_jobs.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(job) = jobs.get_mut(&id) {
|
||||
job.status = "done";
|
||||
job.removed_entries = result.removed_entries;
|
||||
job.freed_bytes = result.freed_bytes;
|
||||
}
|
||||
});
|
||||
Ok(job)
|
||||
}
|
||||
|
||||
pub fn cold_cleanup_job(&self, id: u64) -> Option<ColdCleanupJob> {
|
||||
self.cleanup_jobs
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.get(&id)
|
||||
.cloned()
|
||||
}
|
||||
fn take_cold(&self, key: &Key) -> Option<ColdEntry> {
|
||||
self.cold
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.remove(key)
|
||||
}
|
||||
fn delete_cold_entry(&self, entry: ColdEntry) {
|
||||
if let Some(tx) = &self.cold_tx {
|
||||
let _ = tx.try_send(ColdCommand::Delete(vec![entry.meta_path, entry.body_path]));
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+257
@@ -0,0 +1,257 @@
|
||||
struct FillFlight {
|
||||
id: u64,
|
||||
scope: String,
|
||||
scheme: String,
|
||||
authority: String,
|
||||
target: String,
|
||||
done: tokio::sync::watch::Sender<bool>,
|
||||
}
|
||||
|
||||
pub enum FillClaim {
|
||||
Disabled,
|
||||
Leader(FillLeaderGuard),
|
||||
Waited,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
pub struct FillLeaderGuard {
|
||||
cache: std::sync::Weak<Cache>,
|
||||
key: Key,
|
||||
id: u64,
|
||||
done: tokio::sync::watch::Sender<bool>,
|
||||
}
|
||||
|
||||
struct CollapseWaiterGuard {
|
||||
cache: std::sync::Weak<Cache>,
|
||||
}
|
||||
|
||||
impl Drop for CollapseWaiterGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(cache) = self.cache.upgrade() {
|
||||
cache.collapse_waiters.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FillLeaderGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(cache) = self.cache.upgrade() {
|
||||
let mut flights = cache
|
||||
.fill_flights
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
let current = flights
|
||||
.get(&self.key)
|
||||
.is_some_and(|flight| flight.id == self.id);
|
||||
if current {
|
||||
flights.remove(&self.key);
|
||||
}
|
||||
}
|
||||
// Publish completion even when a purge already removed this flight from the map.
|
||||
self.done.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub async fn claim_fill(
|
||||
self: &Arc<Self>,
|
||||
ticket: &Ticket,
|
||||
settings: &Settings,
|
||||
) -> FillClaim {
|
||||
if !settings.collapse_enabled {
|
||||
return FillClaim::Disabled;
|
||||
}
|
||||
let existing = {
|
||||
let mut flights = self
|
||||
.fill_flights
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(flight) = flights.get(&ticket.key) {
|
||||
Some(flight.clone())
|
||||
} else {
|
||||
let id = self
|
||||
.fill_flight_sequence
|
||||
.fetch_add(1, Ordering::Relaxed)
|
||||
.wrapping_add(1);
|
||||
let (done, _) = tokio::sync::watch::channel(false);
|
||||
flights.insert(
|
||||
ticket.key,
|
||||
Arc::new(FillFlight {
|
||||
id,
|
||||
scope: ticket.scope.clone(),
|
||||
scheme: ticket.scheme.clone(),
|
||||
authority: ticket.authority.clone(),
|
||||
target: ticket.target.clone(),
|
||||
done: done.clone(),
|
||||
}),
|
||||
);
|
||||
return FillClaim::Leader(FillLeaderGuard {
|
||||
cache: Arc::downgrade(self),
|
||||
key: ticket.key,
|
||||
id,
|
||||
done,
|
||||
});
|
||||
}
|
||||
};
|
||||
let Some(flight) = existing else {
|
||||
unreachable!("existing fill resolved above")
|
||||
};
|
||||
// A watch channel retains the completed state, so a leader finishing between
|
||||
// map lookup and follower polling cannot lose the wakeup.
|
||||
let mut done = flight.done.subscribe();
|
||||
self.collapse_waiters.fetch_add(1, Ordering::Relaxed);
|
||||
let _waiter = CollapseWaiterGuard {
|
||||
cache: Arc::downgrade(self),
|
||||
};
|
||||
let waited = if *done.borrow() {
|
||||
Ok(())
|
||||
} else {
|
||||
tokio::time::timeout(
|
||||
Duration::from_millis(settings.collapse_wait_timeout_ms),
|
||||
done.changed(),
|
||||
)
|
||||
.await
|
||||
.map(|_| ())
|
||||
};
|
||||
match waited {
|
||||
Ok(()) => {
|
||||
self.counters
|
||||
.collapsed_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
FillClaim::Waited
|
||||
}
|
||||
Err(_) => {
|
||||
self.counters
|
||||
.collapse_timeouts
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
FillClaim::Timeout
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel_all_fill_flights(&self) {
|
||||
let flights = {
|
||||
let mut flights = self
|
||||
.fill_flights
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
flights.drain().map(|(_, flight)| flight).collect::<Vec<_>>()
|
||||
};
|
||||
for flight in flights {
|
||||
flight.done.send_replace(true);
|
||||
}
|
||||
}
|
||||
|
||||
fn cancel_scope_fill_flights(&self, scope: &str) {
|
||||
let removed = {
|
||||
let mut flights = self
|
||||
.fill_flights
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
let keys = flights
|
||||
.iter()
|
||||
.filter(|(_, flight)| flight.scope == scope)
|
||||
.map(|(key, _)| *key)
|
||||
.collect::<Vec<_>>();
|
||||
keys.into_iter()
|
||||
.filter_map(|key| flights.remove(&key))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for flight in removed {
|
||||
flight.done.send_replace(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ProactiveScheduleGuard {
|
||||
cache: std::sync::Weak<Cache>,
|
||||
key: Key,
|
||||
}
|
||||
impl Drop for ProactiveScheduleGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(cache) = self.cache.upgrade() {
|
||||
cache
|
||||
.proactive_scheduled_keys
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.remove(&self.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ProactiveInflightGuard {
|
||||
cache: std::sync::Weak<Cache>,
|
||||
}
|
||||
impl Drop for ProactiveInflightGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(cache) = self.cache.upgrade() {
|
||||
cache.proactive_inflight.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub fn begin_proactive_schedule(
|
||||
self: &Arc<Self>,
|
||||
ticket: &Ticket,
|
||||
settings: &Settings,
|
||||
) -> Option<ProactiveScheduleGuard> {
|
||||
if !settings.proactive_refresh_enabled || settings.proactive_refresh_percent == 0 {
|
||||
return None;
|
||||
}
|
||||
let sample = u16::from_be_bytes([ticket.key[0], ticket.key[1]]) % 100;
|
||||
if sample >= settings.proactive_refresh_percent as u16 {
|
||||
self.counters
|
||||
.proactive_skipped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
let mut scheduled = self
|
||||
.proactive_scheduled_keys
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner());
|
||||
if !scheduled.insert(ticket.key) {
|
||||
return None;
|
||||
}
|
||||
self.counters
|
||||
.proactive_scheduled
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
Some(ProactiveScheduleGuard {
|
||||
cache: Arc::downgrade(self),
|
||||
key: ticket.key,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn try_begin_proactive_refresh(
|
||||
self: &Arc<Self>,
|
||||
settings: &Settings,
|
||||
) -> Option<ProactiveInflightGuard> {
|
||||
let limit = settings.proactive_refresh_max_inflight as u64;
|
||||
loop {
|
||||
let current = self.proactive_inflight.load(Ordering::Relaxed);
|
||||
if current >= limit {
|
||||
self.counters
|
||||
.proactive_skipped
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.proactive_inflight
|
||||
.compare_exchange_weak(
|
||||
current,
|
||||
current + 1,
|
||||
Ordering::AcqRel,
|
||||
Ordering::Relaxed,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
self.counters
|
||||
.proactive_started
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return Some(ProactiveInflightGuard {
|
||||
cache: Arc::downgrade(self),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+429
@@ -0,0 +1,429 @@
|
||||
impl Cache {
|
||||
pub fn view(&self) -> View {
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(store.sweep() as u64, Ordering::Relaxed);
|
||||
let n = |value: &AtomicU64| value.load(Ordering::Relaxed);
|
||||
let hits = n(&self.counters.hits);
|
||||
let misses = n(&self.counters.misses);
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let fill_bytes_active = n(&self.fill_bytes_active);
|
||||
let cold_queue_bytes = n(&self.cold_queue_bytes);
|
||||
let cold_write_bytes_active = n(&self.cold_write_bytes_active);
|
||||
let cold_read_bytes_active = n(&self.cold_read_bytes_active);
|
||||
let transient_bytes = n(self.transient_bytes_active.as_ref());
|
||||
View {
|
||||
hits,
|
||||
misses,
|
||||
bypasses: n(&self.counters.bypasses),
|
||||
stored: n(&self.counters.stored),
|
||||
evictions: n(&self.counters.evictions),
|
||||
expired: n(&self.counters.expired),
|
||||
rejected_fills: n(&self.counters.rejected_fills),
|
||||
memory_pressure_rejections: n(&self.counters.memory_pressure_rejections),
|
||||
hot_eviction_limited: n(&self.counters.hot_eviction_limited),
|
||||
purges: n(&self.counters.purges),
|
||||
entries: store.entries.len(),
|
||||
resident_bytes: store.bytes,
|
||||
hot_logical_bytes: store.logical_bytes,
|
||||
hot_allocated_bytes: store.bytes,
|
||||
hot_limit_bytes: settings.max_bytes,
|
||||
transient_limit_bytes: settings.transient_max_bytes,
|
||||
cold_queue_limit_bytes: settings.cold_queue_max_bytes,
|
||||
cold_entries: cold.entries.len(),
|
||||
cold_bytes: cold.bytes,
|
||||
cold_hits: n(&self.counters.cold_hits),
|
||||
cold_promotions: n(&self.counters.cold_promotions),
|
||||
hot_demotions: n(&self.counters.hot_demotions),
|
||||
cold_evictions: n(&self.counters.cold_evictions),
|
||||
cold_errors: n(&self.counters.cold_errors),
|
||||
cold_queue_dropped: n(&self.counters.cold_queue_dropped),
|
||||
collapsed_fills: n(&self.counters.collapsed_fills),
|
||||
collapse_timeouts: n(&self.counters.collapse_timeouts),
|
||||
collapse_waiters: n(&self.collapse_waiters),
|
||||
purge_entries: n(&self.counters.purge_entries),
|
||||
bans_created: n(&self.counters.bans_created),
|
||||
bans_removed: n(&self.counters.bans_removed),
|
||||
ban_matches: n(&self.counters.ban_matches),
|
||||
active_bans: self.active_bans().len(),
|
||||
proactive_scheduled: n(&self.counters.proactive_scheduled),
|
||||
proactive_started: n(&self.counters.proactive_started),
|
||||
proactive_skipped: n(&self.counters.proactive_skipped),
|
||||
proactive_inflight: n(&self.proactive_inflight),
|
||||
fills_active: FILL_SLOTS - self.fills.available_permits(),
|
||||
fill_bytes_active,
|
||||
cold_queue_items: n(&self.cold_queue_items),
|
||||
cold_queue_bytes,
|
||||
cold_write_bytes_active,
|
||||
cold_read_bytes_active,
|
||||
transient_bytes,
|
||||
hit_ratio: if hits + misses > 0 {
|
||||
hits as f64 / (hits + misses) as f64
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
}
|
||||
}
|
||||
pub fn prometheus(&self) -> String {
|
||||
let view = self.view();
|
||||
let mut text = String::new();
|
||||
for (name, kind, value) in [
|
||||
("hits_total", "counter", view.hits),
|
||||
("misses_total", "counter", view.misses),
|
||||
("bypasses_total", "counter", view.bypasses),
|
||||
("stored_total", "counter", view.stored),
|
||||
("evictions_total", "counter", view.evictions),
|
||||
("expired_total", "counter", view.expired),
|
||||
("rejected_fills_total", "counter", view.rejected_fills),
|
||||
(
|
||||
"memory_pressure_rejections_total",
|
||||
"counter",
|
||||
view.memory_pressure_rejections,
|
||||
),
|
||||
(
|
||||
"hot_eviction_limited_total",
|
||||
"counter",
|
||||
view.hot_eviction_limited,
|
||||
),
|
||||
("purges_total", "counter", view.purges),
|
||||
("purge_entries_total", "counter", view.purge_entries),
|
||||
("collapsed_fills_total", "counter", view.collapsed_fills),
|
||||
("collapse_timeouts_total", "counter", view.collapse_timeouts),
|
||||
("collapse_waiters", "gauge", view.collapse_waiters),
|
||||
("bans_created_total", "counter", view.bans_created),
|
||||
("bans_removed_total", "counter", view.bans_removed),
|
||||
("ban_matches_total", "counter", view.ban_matches),
|
||||
("active_bans", "gauge", view.active_bans as u64),
|
||||
("proactive_scheduled_total", "counter", view.proactive_scheduled),
|
||||
("proactive_started_total", "counter", view.proactive_started),
|
||||
("proactive_skipped_total", "counter", view.proactive_skipped),
|
||||
("proactive_inflight", "gauge", view.proactive_inflight),
|
||||
("entries", "gauge", view.entries as u64),
|
||||
("resident_bytes", "gauge", view.resident_bytes as u64),
|
||||
("hot_logical_bytes", "gauge", view.hot_logical_bytes as u64),
|
||||
("hot_allocated_bytes", "gauge", view.hot_allocated_bytes as u64),
|
||||
("hot_limit_bytes", "gauge", view.hot_limit_bytes as u64),
|
||||
(
|
||||
"transient_limit_bytes",
|
||||
"gauge",
|
||||
view.transient_limit_bytes as u64,
|
||||
),
|
||||
(
|
||||
"cold_queue_limit_bytes",
|
||||
"gauge",
|
||||
view.cold_queue_limit_bytes as u64,
|
||||
),
|
||||
("cold_entries", "gauge", view.cold_entries as u64),
|
||||
("cold_bytes", "gauge", view.cold_bytes),
|
||||
("cold_hits_total", "counter", view.cold_hits),
|
||||
("cold_promotions_total", "counter", view.cold_promotions),
|
||||
("hot_demotions_total", "counter", view.hot_demotions),
|
||||
("cold_evictions_total", "counter", view.cold_evictions),
|
||||
("cold_errors_total", "counter", view.cold_errors),
|
||||
(
|
||||
"cold_queue_dropped_total",
|
||||
"counter",
|
||||
view.cold_queue_dropped,
|
||||
),
|
||||
("fills_active", "gauge", view.fills_active as u64),
|
||||
("fill_bytes_active", "gauge", view.fill_bytes_active),
|
||||
("cold_queue_items", "gauge", view.cold_queue_items),
|
||||
("cold_queue_bytes", "gauge", view.cold_queue_bytes),
|
||||
(
|
||||
"cold_write_bytes_active",
|
||||
"gauge",
|
||||
view.cold_write_bytes_active,
|
||||
),
|
||||
(
|
||||
"cold_read_bytes_active",
|
||||
"gauge",
|
||||
view.cold_read_bytes_active,
|
||||
),
|
||||
("transient_bytes", "gauge", view.transient_bytes),
|
||||
] {
|
||||
text.push_str(&format!(
|
||||
"# TYPE rust_proxy_cache_{name} {kind}\nrust_proxy_cache_{name} {value}\n"
|
||||
));
|
||||
}
|
||||
text.push_str(&format!(
|
||||
"# TYPE rust_proxy_cache_hit_ratio gauge\nrust_proxy_cache_hit_ratio {:.6}\n",
|
||||
view.hit_ratio
|
||||
));
|
||||
text
|
||||
}
|
||||
pub fn bypass(&self) {
|
||||
self.counters.bypasses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
/// Clear all keys and cancel publication of older in-flight fills.
|
||||
pub fn purge(&self, revision: Option<u64>) -> usize {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let mut count = store.entries.len();
|
||||
store.entries.clear();
|
||||
store.variants.clear();
|
||||
store.lru.clear();
|
||||
store.bytes = 0;
|
||||
store.logical_bytes = 0;
|
||||
store.generation = store.generation.wrapping_add(1);
|
||||
store.scope_generations.clear();
|
||||
store.key_generations.clear();
|
||||
if let Some(revision) = revision {
|
||||
store.revision = revision;
|
||||
self.revision.store(revision, Ordering::Relaxed);
|
||||
store.domains.clear();
|
||||
}
|
||||
drop(store);
|
||||
self.cancel_all_fill_flights();
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
count += cold.entries.len();
|
||||
cold.entries.clear();
|
||||
cold.variants.clear();
|
||||
cold.lru.clear();
|
||||
cold.bytes = 0;
|
||||
cold.sequence = 0;
|
||||
}
|
||||
if let (Some(root), Some(tx)) = (self.cold_root_for(&settings), &self.cold_tx) {
|
||||
let _ = tx.try_send(ColdCommand::PurgeDirectory(root));
|
||||
}
|
||||
self.counters.purges.fetch_add(1, Ordering::Relaxed);
|
||||
self.counters.purge_entries.fetch_add(count as u64, Ordering::Relaxed);
|
||||
count
|
||||
}
|
||||
/// Purge one vhost without evicting other hosts. Generation bumps also stop
|
||||
/// an already streaming MISS from repopulating the just-purged scope.
|
||||
pub fn purge_scope(&self, scope: &str) -> usize {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys: Vec<_> = store
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|(_, e)| e.scope == scope)
|
||||
.map(|(k, _)| *k)
|
||||
.collect();
|
||||
let mut count = keys.len();
|
||||
for key in keys {
|
||||
store.remove(&key);
|
||||
}
|
||||
let generation = store.scope_generations.entry(scope.to_owned()).or_default();
|
||||
*generation = generation.wrapping_add(1);
|
||||
drop(store);
|
||||
self.cancel_scope_fill_flights(scope);
|
||||
let mut delete = Vec::new();
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys: Vec<_> = cold
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.scope == scope)
|
||||
.map(|(key, _)| *key)
|
||||
.collect();
|
||||
count += keys.len();
|
||||
for key in keys {
|
||||
if let Some(entry) = cold.remove(&key) {
|
||||
delete.extend([entry.meta_path, entry.body_path]);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let (Some(tx), false) = (&self.cold_tx, delete.is_empty()) {
|
||||
let _ = tx.try_send(ColdCommand::Delete(delete));
|
||||
}
|
||||
self.counters.purges.fetch_add(1, Ordering::Relaxed);
|
||||
self.counters.purge_entries.fetch_add(count as u64, Ordering::Relaxed);
|
||||
count
|
||||
}
|
||||
pub fn scope_view(&self, scope: &str) -> ScopeView {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(store.sweep() as u64, Ordering::Relaxed);
|
||||
let mut view = ScopeView {
|
||||
entries: 0,
|
||||
resident_bytes: 0,
|
||||
cold_entries: 0,
|
||||
cold_bytes: 0,
|
||||
};
|
||||
for entry in store.entries.values().filter(|entry| entry.scope == scope) {
|
||||
view.entries += 1;
|
||||
view.resident_bytes += entry.cost;
|
||||
}
|
||||
drop(store);
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for entry in cold.entries.values().filter(|entry| entry.scope == scope) {
|
||||
view.cold_entries += 1;
|
||||
view.cold_bytes = view.cold_bytes.saturating_add(entry.disk_bytes);
|
||||
}
|
||||
view
|
||||
}
|
||||
pub fn record(&self, scope: &str, domain: &str, outcome: &str, reason: &'static str) {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(c) = store.domain(scope, domain) {
|
||||
match outcome {
|
||||
"HIT" => c.hits += 1,
|
||||
"MISS" => c.misses += 1,
|
||||
_ => c.bypasses += 1,
|
||||
}
|
||||
*c.reasons.entry(reason.into()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
pub fn record_reason(&self, scope: &str, domain: &str, reason: &'static str) {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(c) = store.domain(scope, domain) {
|
||||
*c.reasons.entry(reason.into()).or_default() += 1;
|
||||
}
|
||||
}
|
||||
pub fn dashboard(&self, scope: Option<&str>, offset: usize, limit: usize) -> Dashboard {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(store.sweep() as u64, Ordering::Relaxed);
|
||||
let revision = store.revision;
|
||||
let mut groups: BTreeMap<(String, String), DomainView> = store
|
||||
.domains
|
||||
.iter()
|
||||
.filter(|((vhost, _), _)| scope.is_none_or(|s| s == vhost))
|
||||
.map(|((vhost, domain), counters)| {
|
||||
(
|
||||
(vhost.clone(), domain.clone()),
|
||||
DomainView {
|
||||
vhost: vhost.clone(),
|
||||
domain: domain.clone(),
|
||||
entries: 0,
|
||||
logical_bytes: 0,
|
||||
resident_bytes: 0,
|
||||
hot_entries: 0,
|
||||
cold_entries: 0,
|
||||
cold_bytes: 0,
|
||||
hit_ratio: if counters.hits + counters.misses > 0 {
|
||||
counters.hits as f64 / (counters.hits + counters.misses) as f64
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
counters: counters.clone(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let mut objects = Vec::new();
|
||||
for key in store.lru.values().rev() {
|
||||
let Some(e) = store.entries.get(key) else {
|
||||
continue;
|
||||
};
|
||||
let age = e.initial_age.saturating_add(e.received.elapsed()).as_secs();
|
||||
if age >= e.ttl || !scope.is_none_or(|s| s == e.scope) {
|
||||
continue;
|
||||
}
|
||||
let row = groups
|
||||
.entry((e.scope.clone(), e.domain.clone()))
|
||||
.or_insert_with(|| DomainView {
|
||||
vhost: e.scope.clone(),
|
||||
domain: e.domain.clone(),
|
||||
entries: 0,
|
||||
logical_bytes: 0,
|
||||
resident_bytes: 0,
|
||||
hot_entries: 0,
|
||||
cold_entries: 0,
|
||||
cold_bytes: 0,
|
||||
hit_ratio: 0.0,
|
||||
counters: DomainCounters::default(),
|
||||
});
|
||||
row.entries += 1;
|
||||
row.hot_entries += 1;
|
||||
row.logical_bytes += e.body.len();
|
||||
row.resident_bytes += e.cost;
|
||||
objects.push(ObjectView {
|
||||
vhost: e.scope.clone(),
|
||||
scheme: e.scheme.clone(),
|
||||
authority: e.authority.clone(),
|
||||
domain: e.domain.clone(),
|
||||
path: e.path.clone(),
|
||||
target: e.target.clone(),
|
||||
cache_key: key_hex(key),
|
||||
tags: e.tags.clone(),
|
||||
transformed: e.transformed,
|
||||
body_bytes: e.body.len(),
|
||||
resident_bytes: e.cost,
|
||||
storage_bytes: 0,
|
||||
tier: "hot",
|
||||
hits: e.hits,
|
||||
age_seconds: age,
|
||||
remaining_ttl_seconds: e.ttl.saturating_sub(age),
|
||||
});
|
||||
}
|
||||
drop(store);
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for key in cold.lru.values().rev() {
|
||||
let Some(e) = cold.entries.get(key) else {
|
||||
continue;
|
||||
};
|
||||
if e.revision != revision || !scope.is_none_or(|s| s == e.scope) {
|
||||
continue;
|
||||
}
|
||||
let age = cold_age(e);
|
||||
if age >= e.ttl && !e.requires_revalidation {
|
||||
continue;
|
||||
}
|
||||
let row = groups
|
||||
.entry((e.scope.clone(), e.domain.clone()))
|
||||
.or_insert_with(|| DomainView {
|
||||
vhost: e.scope.clone(),
|
||||
domain: e.domain.clone(),
|
||||
entries: 0,
|
||||
logical_bytes: 0,
|
||||
resident_bytes: 0,
|
||||
hot_entries: 0,
|
||||
cold_entries: 0,
|
||||
cold_bytes: 0,
|
||||
hit_ratio: 0.0,
|
||||
counters: DomainCounters::default(),
|
||||
});
|
||||
row.entries += 1;
|
||||
row.cold_entries += 1;
|
||||
row.cold_bytes = row.cold_bytes.saturating_add(e.disk_bytes);
|
||||
objects.push(ObjectView {
|
||||
vhost: e.scope.clone(),
|
||||
scheme: e.scheme.clone(),
|
||||
authority: e.authority.clone(),
|
||||
domain: e.domain.clone(),
|
||||
path: e.path.clone(),
|
||||
target: e.target.clone(),
|
||||
cache_key: key_hex(key),
|
||||
tags: e.tags.clone(),
|
||||
transformed: e.transformed,
|
||||
body_bytes: e.body_bytes,
|
||||
resident_bytes: 0,
|
||||
storage_bytes: e.disk_bytes,
|
||||
tier: "cold",
|
||||
hits: e.hits,
|
||||
age_seconds: age,
|
||||
remaining_ttl_seconds: e.ttl.saturating_sub(age),
|
||||
});
|
||||
}
|
||||
let total = objects.len();
|
||||
let matching_body_bytes = objects.iter().map(|entry| entry.body_bytes).sum();
|
||||
let matching_resident_bytes = objects.iter().map(|entry| entry.resident_bytes).sum();
|
||||
let matching_cold_bytes = objects.iter().map(|entry| entry.storage_bytes).sum();
|
||||
let limit = limit.clamp(1, 200);
|
||||
let objects = objects.into_iter().skip(offset).take(limit).collect();
|
||||
Dashboard {
|
||||
domains: groups.into_values().collect(),
|
||||
objects,
|
||||
matching_objects: total,
|
||||
matching_body_bytes,
|
||||
matching_resident_bytes,
|
||||
matching_cold_bytes,
|
||||
offset,
|
||||
limit,
|
||||
statistics_revision: revision,
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+343
@@ -0,0 +1,343 @@
|
||||
struct ByteGaugeGuard<'a> {
|
||||
counter: &'a AtomicU64,
|
||||
bytes: u64,
|
||||
}
|
||||
impl<'a> ByteGaugeGuard<'a> {
|
||||
fn new(counter: &'a AtomicU64, bytes: u64) -> Self {
|
||||
counter.fetch_add(bytes, Ordering::Relaxed);
|
||||
Self { counter, bytes }
|
||||
}
|
||||
}
|
||||
impl Drop for ByteGaugeGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.counter.fetch_sub(self.bytes, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
struct FillGauge {
|
||||
cache: Arc<Cache>,
|
||||
bytes: u64,
|
||||
leases: Vec<TransientLease>,
|
||||
}
|
||||
impl FillGauge {
|
||||
fn new(cache: Arc<Cache>, bytes: usize, settings: &Settings) -> Option<Self> {
|
||||
let lease = cache.reserve_fill_allocation(bytes, settings)?;
|
||||
Some(Self {
|
||||
cache,
|
||||
bytes: bytes as u64,
|
||||
leases: vec![lease],
|
||||
})
|
||||
}
|
||||
|
||||
fn grow(&mut self, bytes: usize, settings: &Settings) -> bool {
|
||||
if bytes == 0 {
|
||||
return true;
|
||||
}
|
||||
let Some(lease) = self.cache.reserve_fill_allocation(bytes, settings) else {
|
||||
return false;
|
||||
};
|
||||
self.bytes = self.bytes.saturating_add(bytes as u64);
|
||||
self.leases.push(lease);
|
||||
true
|
||||
}
|
||||
|
||||
fn release(&mut self) {
|
||||
if self.bytes != 0 {
|
||||
self.cache
|
||||
.fill_bytes_active
|
||||
.fetch_sub(self.bytes, Ordering::Relaxed);
|
||||
self.bytes = 0;
|
||||
}
|
||||
self.leases.clear();
|
||||
}
|
||||
}
|
||||
impl Drop for FillGauge {
|
||||
fn drop(&mut self) {
|
||||
self.release();
|
||||
}
|
||||
}
|
||||
|
||||
struct Candidate {
|
||||
cache: Arc<Cache>,
|
||||
ticket: Ticket,
|
||||
plan: Plan,
|
||||
headers: HeaderMap,
|
||||
data: Vec<u8>,
|
||||
fill_gauge: FillGauge,
|
||||
settings: Settings,
|
||||
skip_compression: bool,
|
||||
origin_etag: Option<HeaderValue>,
|
||||
origin_last_modified: Option<HeaderValue>,
|
||||
transformed: bool,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
_fill_leader: Option<FillLeaderGuard>,
|
||||
}
|
||||
impl Candidate {
|
||||
fn finish(mut self) {
|
||||
if self
|
||||
.plan
|
||||
.expected_length
|
||||
.is_some_and(|length| self.data.len() != length)
|
||||
{
|
||||
let reason = if self.transformed {
|
||||
"transform_length_mismatch"
|
||||
} else {
|
||||
"fill_incomplete"
|
||||
};
|
||||
self.cache
|
||||
.record_reason(&self.ticket.scope, &self.ticket.domain, reason);
|
||||
self.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
if !self.headers.contains_key("date") {
|
||||
if let Ok(value) = httpdate::fmt_http_date(SystemTime::now()).parse() {
|
||||
self.headers.insert("date", value);
|
||||
}
|
||||
}
|
||||
// Each downstream exchange receives a new ID, never a cached origin ID.
|
||||
for name in [
|
||||
"x-request-id",
|
||||
"x-cache",
|
||||
"x-cache-hits",
|
||||
"x-cache-status",
|
||||
"x-cache-reason",
|
||||
"cache-status",
|
||||
] {
|
||||
self.headers.remove(name);
|
||||
}
|
||||
let vary = std::mem::take(&mut self.plan.vary);
|
||||
// Keep the actual backing capacity as the HOT charge. Vec::reserve_exact
|
||||
// avoids geometric multi-MiB overshoot while Bytes::from(Vec) transfers
|
||||
// the allocation without copying.
|
||||
let body_allocation = self.data.capacity().max(self.data.len());
|
||||
let body = Bytes::from(std::mem::take(&mut self.data));
|
||||
let scope = self.ticket.scope.clone();
|
||||
let scheme = self.ticket.scheme.clone();
|
||||
let authority = self.ticket.authority.clone();
|
||||
let domain = self.ticket.domain.clone();
|
||||
let path = self.ticket.path.clone();
|
||||
let target = self.ticket.target.clone();
|
||||
let tags = cache_tags(&self.headers);
|
||||
let cost = hot_entry_cost(
|
||||
body_allocation,
|
||||
&self.headers,
|
||||
&vary,
|
||||
&scope,
|
||||
&scheme,
|
||||
&authority,
|
||||
&domain,
|
||||
&path,
|
||||
&target,
|
||||
&tags,
|
||||
self.origin_etag.as_ref(),
|
||||
self.origin_last_modified.as_ref(),
|
||||
);
|
||||
let created_at = self.ticket.created_at;
|
||||
let base_key = self.ticket.key;
|
||||
self.cache.insert(
|
||||
self.ticket,
|
||||
Entry {
|
||||
base_key,
|
||||
vary,
|
||||
scope,
|
||||
scheme,
|
||||
authority,
|
||||
domain,
|
||||
path,
|
||||
target,
|
||||
tags,
|
||||
created_at,
|
||||
body,
|
||||
body_allocated: body_allocation,
|
||||
headers: self.headers,
|
||||
received: self.plan.received,
|
||||
last_access: Instant::now(),
|
||||
resident_since: Instant::now(),
|
||||
resident_age_base: Duration::ZERO,
|
||||
initial_age: self.plan.initial_age,
|
||||
ttl: self.plan.ttl,
|
||||
stale_while_revalidate: self.plan.stale_while_revalidate,
|
||||
stale_if_error: self.plan.stale_if_error,
|
||||
must_revalidate: self.plan.must_revalidate,
|
||||
proxy_revalidate: self.plan.proxy_revalidate,
|
||||
origin_etag: self.origin_etag,
|
||||
origin_last_modified: self.origin_last_modified,
|
||||
cost,
|
||||
sequence: 0,
|
||||
hits: 0,
|
||||
transformed: self.transformed,
|
||||
skip_compression: self.skip_compression,
|
||||
requires_revalidation: self.plan.requires_revalidation,
|
||||
},
|
||||
&self.settings,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
struct FillBody {
|
||||
inner: Body,
|
||||
candidate: Option<Candidate>,
|
||||
done: bool,
|
||||
}
|
||||
impl HttpBody for FillBody {
|
||||
type Data = Bytes;
|
||||
type Error = axum::Error;
|
||||
fn poll_frame(
|
||||
self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<Frame<Bytes>, Self::Error>>> {
|
||||
let this = self.get_mut();
|
||||
match Pin::new(&mut this.inner).poll_frame(cx) {
|
||||
Poll::Ready(Some(Ok(frame))) => {
|
||||
if let Some(candidate) = &mut this.candidate {
|
||||
if let Some(data) = frame.data_ref() {
|
||||
let needed = candidate.data.len().saturating_add(data.len());
|
||||
if needed <= candidate.settings.max_object_bytes {
|
||||
let before = candidate.data.capacity();
|
||||
let required_growth = needed.saturating_sub(before);
|
||||
if required_growth > 0
|
||||
&& !candidate
|
||||
.fill_gauge
|
||||
.grow(required_growth, &candidate.settings)
|
||||
{
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"hot_memory_pressure",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
this.candidate = None;
|
||||
} else if required_growth > 0
|
||||
&& candidate
|
||||
.data
|
||||
.try_reserve_exact(needed.saturating_sub(candidate.data.len()))
|
||||
.is_err()
|
||||
{
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"fill_allocation_failed",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
this.candidate = None;
|
||||
} else {
|
||||
let after = candidate.data.capacity();
|
||||
let unbudgeted = after.saturating_sub(before.saturating_add(required_growth));
|
||||
if unbudgeted > 0
|
||||
&& !candidate.fill_gauge.grow(unbudgeted, &candidate.settings)
|
||||
{
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"fill_allocator_overshoot",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
this.candidate = None;
|
||||
} else {
|
||||
candidate.data.extend_from_slice(data);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"object_too_large",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
this.candidate = None;
|
||||
}
|
||||
} else {
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"fill_incomplete",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
this.candidate = None;
|
||||
} // Do not cache responses with trailers.
|
||||
}
|
||||
if this.inner.is_end_stream() {
|
||||
this.done = true;
|
||||
if let Some(candidate) = this.candidate.take() {
|
||||
candidate.finish();
|
||||
}
|
||||
}
|
||||
Poll::Ready(Some(Ok(frame)))
|
||||
}
|
||||
Poll::Ready(None) => {
|
||||
this.done = true;
|
||||
if let Some(candidate) = this.candidate.take() {
|
||||
candidate.finish();
|
||||
}
|
||||
Poll::Ready(None)
|
||||
}
|
||||
Poll::Ready(Some(Err(error))) => {
|
||||
if let Some(candidate) = &this.candidate {
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"fill_incomplete",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
this.candidate = None;
|
||||
this.done = true;
|
||||
Poll::Ready(Some(Err(error)))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.done
|
||||
}
|
||||
fn size_hint(&self) -> SizeHint {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
impl Drop for FillBody {
|
||||
fn drop(&mut self) {
|
||||
if self.done {
|
||||
return;
|
||||
}
|
||||
if let Some(candidate) = self.candidate.take() {
|
||||
candidate.cache.record_reason(
|
||||
&candidate.ticket.scope,
|
||||
&candidate.ticket.domain,
|
||||
"fill_incomplete",
|
||||
);
|
||||
candidate
|
||||
.cache
|
||||
.counters
|
||||
.rejected_fills
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Vendored
+584
@@ -0,0 +1,584 @@
|
||||
use anyhow::Context as _;
|
||||
|
||||
const MAX_BAN_RULES: usize = 1024;
|
||||
const DEFAULT_BAN_TTL_SECONDS: u64 = 3600;
|
||||
const MAX_BAN_TTL_SECONDS: u64 = 604_800;
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InvalidationOperation {
|
||||
Purge,
|
||||
Ban,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum InvalidationSelector {
|
||||
Exact,
|
||||
Prefix,
|
||||
Tag,
|
||||
Regex,
|
||||
CacheKey,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct InvalidationRequest {
|
||||
pub operation: InvalidationOperation,
|
||||
pub selector: InvalidationSelector,
|
||||
pub value: String,
|
||||
#[serde(default)]
|
||||
pub host: Option<String>,
|
||||
#[serde(default)]
|
||||
pub vhost: Option<String>,
|
||||
#[serde(default)]
|
||||
pub ttl_seconds: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub confirm_wide: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct InvalidationResult {
|
||||
pub operation: InvalidationOperation,
|
||||
pub selector: InvalidationSelector,
|
||||
pub value: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub host: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub vhost: Option<String>,
|
||||
pub affected_entries: usize,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ban_id: Option<u64>,
|
||||
pub elapsed_ms: u64,
|
||||
pub status: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct BanView {
|
||||
pub id: u64,
|
||||
pub selector: InvalidationSelector,
|
||||
pub value: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub scheme: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub host: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub vhost: Option<String>,
|
||||
pub created_at_ms: u64,
|
||||
pub expires_at_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SelectorMatcher {
|
||||
selector: InvalidationSelector,
|
||||
value: String,
|
||||
scheme: Option<String>,
|
||||
host: Option<String>,
|
||||
vhost: Option<String>,
|
||||
regex: Option<regex::Regex>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BanRule {
|
||||
id: u64,
|
||||
matcher: SelectorMatcher,
|
||||
created_at: SystemTime,
|
||||
expires_at: SystemTime,
|
||||
}
|
||||
|
||||
fn canonical_url(scheme: &str, authority: &str, target: &str) -> String {
|
||||
format!("{scheme}://{authority}{target}")
|
||||
}
|
||||
|
||||
fn normalized_matcher(request: &InvalidationRequest) -> Result<SelectorMatcher> {
|
||||
ensure!(!request.value.is_empty(), "invalidation value is required");
|
||||
ensure!(request.value.len() <= 8192, "invalidation value is too long");
|
||||
if let Some(vhost) = &request.vhost {
|
||||
ensure!(!vhost.is_empty() && vhost.len() <= 64, "invalid vhost selector");
|
||||
}
|
||||
if let Some(host) = &request.host {
|
||||
ensure!(!host.is_empty() && host.len() <= 253, "invalid host selector");
|
||||
}
|
||||
let mut value = request.value.trim().to_owned();
|
||||
let mut scheme = None;
|
||||
let mut host = request.host.as_ref().map(|value| value.to_ascii_lowercase());
|
||||
let regex = match request.selector {
|
||||
InvalidationSelector::Exact | InvalidationSelector::Prefix => {
|
||||
if value.starts_with("http://") || value.starts_with("https://") {
|
||||
let url = url::Url::parse(&value).context("invalid purge URL")?;
|
||||
ensure!(url.username().is_empty() && url.password().is_none(), "purge URL must not contain credentials");
|
||||
ensure!(url.fragment().is_none(), "purge URL must not contain a fragment");
|
||||
scheme = Some(url.scheme().to_ascii_lowercase());
|
||||
let authority = match url.port() {
|
||||
Some(port) => format!("{}:{port}", url.host_str().context("purge URL host missing")?),
|
||||
None => url.host_str().context("purge URL host missing")?.to_owned(),
|
||||
};
|
||||
host = Some(authority.to_ascii_lowercase());
|
||||
value = url.path().to_owned();
|
||||
if let Some(query) = url.query() {
|
||||
value.push('?');
|
||||
value.push_str(query);
|
||||
}
|
||||
}
|
||||
ensure!(value.starts_with('/'), "exact/prefix invalidation must use an absolute URL or /path target");
|
||||
None
|
||||
}
|
||||
InvalidationSelector::Tag => {
|
||||
ensure!(value.len() <= 128, "cache tag is too long");
|
||||
ensure!(value.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.' | b':' | b'/')), "invalid cache tag");
|
||||
None
|
||||
}
|
||||
InvalidationSelector::Regex => {
|
||||
ensure!(value.len() <= 512, "regex is too long");
|
||||
Some(
|
||||
regex::RegexBuilder::new(&value)
|
||||
.size_limit(1 << 20)
|
||||
.build()
|
||||
.context("invalid or too expensive regex")?,
|
||||
)
|
||||
}
|
||||
InvalidationSelector::CacheKey => {
|
||||
ensure!(parse_key_hex(&value).is_some(), "cache_key must be 64 hexadecimal characters");
|
||||
value.make_ascii_lowercase();
|
||||
None
|
||||
}
|
||||
};
|
||||
Ok(SelectorMatcher {
|
||||
selector: request.selector,
|
||||
value,
|
||||
scheme,
|
||||
host,
|
||||
vhost: request.vhost.clone(),
|
||||
regex,
|
||||
})
|
||||
}
|
||||
|
||||
fn requires_wide_confirmation(matcher: &SelectorMatcher) -> bool {
|
||||
match matcher.selector {
|
||||
InvalidationSelector::Prefix => matcher.value == "/",
|
||||
InvalidationSelector::Regex => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn host_matches(matcher: &SelectorMatcher, scheme: &str, authority: &str, domain: &str) -> bool {
|
||||
if matcher
|
||||
.scheme
|
||||
.as_ref()
|
||||
.is_some_and(|wanted| !wanted.eq_ignore_ascii_case(scheme))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
matcher.host.as_ref().is_none_or(|wanted| {
|
||||
wanted.eq_ignore_ascii_case(authority) || wanted.eq_ignore_ascii_case(domain)
|
||||
})
|
||||
}
|
||||
|
||||
fn matcher_matches(
|
||||
matcher: &SelectorMatcher,
|
||||
storage_key: &Key,
|
||||
base_key: &Key,
|
||||
scope: &str,
|
||||
scheme: &str,
|
||||
authority: &str,
|
||||
domain: &str,
|
||||
target: &str,
|
||||
tags: &[String],
|
||||
) -> bool {
|
||||
if matcher.vhost.as_ref().is_some_and(|wanted| wanted != scope) {
|
||||
return false;
|
||||
}
|
||||
match matcher.selector {
|
||||
InvalidationSelector::Exact => {
|
||||
host_matches(matcher, scheme, authority, domain) && target == matcher.value
|
||||
}
|
||||
InvalidationSelector::Prefix => {
|
||||
host_matches(matcher, scheme, authority, domain) && target.starts_with(&matcher.value)
|
||||
}
|
||||
InvalidationSelector::Tag => {
|
||||
host_matches(matcher, scheme, authority, domain)
|
||||
&& tags.iter().any(|tag| tag == &matcher.value)
|
||||
},
|
||||
InvalidationSelector::Regex => {
|
||||
if !host_matches(matcher, scheme, authority, domain) {
|
||||
return false;
|
||||
}
|
||||
matcher
|
||||
.regex
|
||||
.as_ref()
|
||||
.is_some_and(|regex| regex.is_match(&canonical_url(scheme, authority, target)))
|
||||
}
|
||||
InvalidationSelector::CacheKey => {
|
||||
host_matches(matcher, scheme, authority, domain)
|
||||
&& (key_hex(storage_key) == matcher.value || key_hex(base_key) == matcher.value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BanRule {
|
||||
fn expired(&self, now: SystemTime) -> bool {
|
||||
now >= self.expires_at
|
||||
}
|
||||
|
||||
fn matches_hot(&self, key: &Key, entry: &Entry, now: SystemTime) -> bool {
|
||||
!self.expired(now)
|
||||
&& entry.created_at <= self.created_at
|
||||
&& matcher_matches(
|
||||
&self.matcher,
|
||||
key,
|
||||
&entry.base_key,
|
||||
&entry.scope,
|
||||
&entry.scheme,
|
||||
&entry.authority,
|
||||
&entry.domain,
|
||||
&entry.target,
|
||||
&entry.tags,
|
||||
)
|
||||
}
|
||||
|
||||
fn matches_cold(&self, key: &Key, entry: &ColdEntry, now: SystemTime) -> bool {
|
||||
!self.expired(now)
|
||||
&& entry.created_at <= self.created_at
|
||||
&& matcher_matches(
|
||||
&self.matcher,
|
||||
key,
|
||||
&entry.base_key,
|
||||
&entry.scope,
|
||||
&entry.scheme,
|
||||
&entry.authority,
|
||||
&entry.domain,
|
||||
&entry.target,
|
||||
&entry.tags,
|
||||
)
|
||||
}
|
||||
|
||||
fn view(&self) -> BanView {
|
||||
BanView {
|
||||
id: self.id,
|
||||
selector: self.matcher.selector,
|
||||
value: self.matcher.value.clone(),
|
||||
scheme: self.matcher.scheme.clone(),
|
||||
host: self.matcher.host.clone(),
|
||||
vhost: self.matcher.vhost.clone(),
|
||||
created_at_ms: self
|
||||
.created_at
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
.min(u64::MAX as u128) as u64,
|
||||
expires_at_ms: self
|
||||
.expires_at
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
.min(u64::MAX as u128) as u64,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
fn active_bans(&self) -> Vec<BanRule> {
|
||||
let now = SystemTime::now();
|
||||
let mut bans = self.bans.lock().unwrap_or_else(|e| e.into_inner());
|
||||
bans.retain(|rule| !rule.expired(now));
|
||||
bans.clone()
|
||||
}
|
||||
|
||||
fn hot_entry_banned(&self, key: &Key, entry: &Entry) -> bool {
|
||||
let now = SystemTime::now();
|
||||
let banned = self
|
||||
.active_bans()
|
||||
.iter()
|
||||
.any(|rule| rule.matches_hot(key, entry, now));
|
||||
if banned {
|
||||
self.counters.ban_matches.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
banned
|
||||
}
|
||||
|
||||
fn cold_entry_banned(&self, key: &Key, entry: &ColdEntry) -> bool {
|
||||
let now = SystemTime::now();
|
||||
let banned = self
|
||||
.active_bans()
|
||||
.iter()
|
||||
.any(|rule| rule.matches_cold(key, entry, now));
|
||||
if banned {
|
||||
self.counters.ban_matches.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
banned
|
||||
}
|
||||
|
||||
fn entry_banned_before_insert(&self, key: &Key, entry: &Entry) -> bool {
|
||||
let now = SystemTime::now();
|
||||
self.active_bans()
|
||||
.iter()
|
||||
.any(|rule| rule.matches_hot(key, entry, now))
|
||||
}
|
||||
|
||||
pub fn bans(&self) -> Vec<BanView> {
|
||||
self.active_bans().iter().map(BanRule::view).collect()
|
||||
}
|
||||
|
||||
pub fn remove_ban(&self, id: u64) -> bool {
|
||||
let mut bans = self.bans.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let before = bans.len();
|
||||
bans.retain(|rule| rule.id != id);
|
||||
let removed = bans.len() != before;
|
||||
if removed {
|
||||
self.counters.bans_removed.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
pub fn invalidate(&self, request: InvalidationRequest) -> Result<InvalidationResult> {
|
||||
let started = Instant::now();
|
||||
let matcher = normalized_matcher(&request)?;
|
||||
if requires_wide_confirmation(&matcher) {
|
||||
ensure!(request.confirm_wide, "wide invalidation requires confirm_wide=true");
|
||||
}
|
||||
match request.operation {
|
||||
InvalidationOperation::Ban => {
|
||||
let ttl = request.ttl_seconds.unwrap_or(DEFAULT_BAN_TTL_SECONDS);
|
||||
ensure!((1..=MAX_BAN_TTL_SECONDS).contains(&ttl), "BAN ttl_seconds must be 1..604800");
|
||||
let now = SystemTime::now();
|
||||
let mut bans = self.bans.lock().unwrap_or_else(|e| e.into_inner());
|
||||
bans.retain(|rule| !rule.expired(now));
|
||||
ensure!(bans.len() < MAX_BAN_RULES, "active BAN rule limit reached");
|
||||
let id = self.ban_sequence.fetch_add(1, Ordering::Relaxed).wrapping_add(1);
|
||||
bans.push(BanRule {
|
||||
id,
|
||||
matcher: matcher.clone(),
|
||||
created_at: now,
|
||||
expires_at: now
|
||||
.checked_add(Duration::from_secs(ttl))
|
||||
.unwrap_or(now),
|
||||
});
|
||||
self.counters.bans_created.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(InvalidationResult {
|
||||
operation: request.operation,
|
||||
selector: request.selector,
|
||||
value: matcher.value,
|
||||
host: matcher.host,
|
||||
vhost: matcher.vhost,
|
||||
affected_entries: 0,
|
||||
ban_id: Some(id),
|
||||
elapsed_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64,
|
||||
status: "active",
|
||||
})
|
||||
}
|
||||
InvalidationOperation::Purge => {
|
||||
// A selective purge is also a publication barrier for every fill that
|
||||
// started before this operation. Existing unrelated objects remain
|
||||
// reusable, but an old in-flight response cannot repopulate a target
|
||||
// that was not yet identifiable (for example Cache-Tag before headers).
|
||||
{
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
store.generation = store.generation.wrapping_add(1);
|
||||
}
|
||||
let affected = self.purge_matcher(&matcher);
|
||||
self.counters.purges.fetch_add(1, Ordering::Relaxed);
|
||||
self.counters
|
||||
.purge_entries
|
||||
.fetch_add(affected as u64, Ordering::Relaxed);
|
||||
Ok(InvalidationResult {
|
||||
operation: request.operation,
|
||||
selector: request.selector,
|
||||
value: matcher.value,
|
||||
host: matcher.host,
|
||||
vhost: matcher.vhost,
|
||||
affected_entries: affected,
|
||||
ban_id: None,
|
||||
elapsed_ms: started.elapsed().as_millis().min(u64::MAX as u128) as u64,
|
||||
status: "completed",
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn purge_matcher(&self, matcher: &SelectorMatcher) -> usize {
|
||||
let mut base_keys = HashSet::new();
|
||||
{
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (key, entry) in &store.entries {
|
||||
if matcher_matches(
|
||||
matcher,
|
||||
key,
|
||||
&entry.base_key,
|
||||
&entry.scope,
|
||||
&entry.scheme,
|
||||
&entry.authority,
|
||||
&entry.domain,
|
||||
&entry.target,
|
||||
&entry.tags,
|
||||
) {
|
||||
base_keys.insert(entry.base_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
{
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for (key, entry) in &cold.entries {
|
||||
if matcher_matches(
|
||||
matcher,
|
||||
key,
|
||||
&entry.base_key,
|
||||
&entry.scope,
|
||||
&entry.scheme,
|
||||
&entry.authority,
|
||||
&entry.domain,
|
||||
&entry.target,
|
||||
&entry.tags,
|
||||
) {
|
||||
base_keys.insert(entry.base_key);
|
||||
}
|
||||
}
|
||||
}
|
||||
let matching_flights = {
|
||||
let flights = self.fill_flights.lock().unwrap_or_else(|e| e.into_inner());
|
||||
flights
|
||||
.iter()
|
||||
.filter_map(|(key, flight)| {
|
||||
let empty_tags: &[String] = &[];
|
||||
matcher_matches(
|
||||
matcher,
|
||||
key,
|
||||
key,
|
||||
&flight.scope,
|
||||
&flight.scheme,
|
||||
&flight.authority,
|
||||
&crate::config::request_host(&flight.authority)
|
||||
.unwrap_or_else(|_| flight.authority.clone()),
|
||||
&flight.target,
|
||||
empty_tags,
|
||||
)
|
||||
.then_some((*key, flight.id))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for (key, _) in &matching_flights {
|
||||
base_keys.insert(*key);
|
||||
}
|
||||
if !base_keys.is_empty() {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for key in &base_keys {
|
||||
let generation = store.key_generations.entry(*key).or_default();
|
||||
*generation = generation.wrapping_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
let hot_removed = {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys = store
|
||||
.entries
|
||||
.iter()
|
||||
.filter_map(|(key, entry)| base_keys.contains(&entry.base_key).then_some(*key))
|
||||
.collect::<Vec<_>>();
|
||||
for key in &keys {
|
||||
store.remove(key);
|
||||
}
|
||||
keys.len()
|
||||
};
|
||||
let mut delete = Vec::new();
|
||||
let cold_removed = {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys = cold
|
||||
.entries
|
||||
.iter()
|
||||
.filter_map(|(key, entry)| base_keys.contains(&entry.base_key).then_some(*key))
|
||||
.collect::<Vec<_>>();
|
||||
for key in &keys {
|
||||
if let Some(entry) = cold.remove(key) {
|
||||
delete.extend([entry.meta_path, entry.body_path]);
|
||||
}
|
||||
}
|
||||
keys.len()
|
||||
};
|
||||
if !delete.is_empty() {
|
||||
if let Some(tx) = &self.cold_tx {
|
||||
let _ = tx.try_send(ColdCommand::Delete(delete));
|
||||
}
|
||||
}
|
||||
let flights = {
|
||||
let mut flights = self.fill_flights.lock().unwrap_or_else(|e| e.into_inner());
|
||||
matching_flights
|
||||
.into_iter()
|
||||
.filter_map(|(key, id)| {
|
||||
if flights.get(&key).is_some_and(|flight| flight.id == id) {
|
||||
flights.remove(&key)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for flight in flights {
|
||||
flight.done.send_replace(true);
|
||||
}
|
||||
if hot_removed > 0 {
|
||||
release_allocator_memory();
|
||||
}
|
||||
hot_removed + cold_removed
|
||||
}
|
||||
|
||||
pub fn gc_banned(&self, limit: usize) -> usize {
|
||||
if limit == 0 {
|
||||
return 0;
|
||||
}
|
||||
let rules = self.active_bans();
|
||||
if rules.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let now = SystemTime::now();
|
||||
let mut removed = 0usize;
|
||||
{
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys = store
|
||||
.entries
|
||||
.iter()
|
||||
.filter_map(|(key, entry)| {
|
||||
(removed < limit && rules.iter().any(|rule| rule.matches_hot(key, entry, now)))
|
||||
.then_some(*key)
|
||||
})
|
||||
.take(limit)
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
if store.remove(&key).is_some() {
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut delete = Vec::new();
|
||||
if removed < limit {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let keys = cold
|
||||
.entries
|
||||
.iter()
|
||||
.filter_map(|(key, entry)| {
|
||||
rules
|
||||
.iter()
|
||||
.any(|rule| rule.matches_cold(key, entry, now))
|
||||
.then_some(*key)
|
||||
})
|
||||
.take(limit - removed)
|
||||
.collect::<Vec<_>>();
|
||||
for key in keys {
|
||||
if let Some(entry) = cold.remove(&key) {
|
||||
delete.extend([entry.meta_path, entry.body_path]);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if !delete.is_empty() {
|
||||
if let Some(tx) = &self.cold_tx {
|
||||
let _ = tx.try_send(ColdCommand::Delete(delete));
|
||||
}
|
||||
}
|
||||
if removed > 0 {
|
||||
self.counters
|
||||
.ban_matches
|
||||
.fetch_add(removed as u64, Ordering::Relaxed);
|
||||
release_allocator_memory();
|
||||
}
|
||||
removed
|
||||
}
|
||||
}
|
||||
Vendored
+281
@@ -0,0 +1,281 @@
|
||||
impl Cache {
|
||||
pub fn new(revision: u64) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
store: Arc::new(Mutex::new(Store {
|
||||
revision,
|
||||
..Default::default()
|
||||
})),
|
||||
revision: AtomicU64::new(revision),
|
||||
cold: Arc::new(Mutex::new(ColdStore::default())),
|
||||
tier_settings: Mutex::new(Settings::default()),
|
||||
data_root: None,
|
||||
cold_tx: None,
|
||||
fills: Arc::new(Semaphore::new(FILL_SLOTS)),
|
||||
fill_bytes_active: AtomicU64::new(0),
|
||||
transient_bytes_active: Arc::new(AtomicU64::new(0)),
|
||||
cold_queue_items: AtomicU64::new(0),
|
||||
cold_queue_bytes: AtomicU64::new(0),
|
||||
cold_write_bytes_active: AtomicU64::new(0),
|
||||
cold_read_bytes_active: AtomicU64::new(0),
|
||||
revalidating: Mutex::new(HashSet::new()),
|
||||
fill_flights: Mutex::new(HashMap::new()),
|
||||
fill_flight_sequence: AtomicU64::new(0),
|
||||
collapse_waiters: AtomicU64::new(0),
|
||||
bans: Mutex::new(Vec::new()),
|
||||
ban_sequence: AtomicU64::new(0),
|
||||
proactive_scheduled_keys: Mutex::new(HashSet::new()),
|
||||
proactive_inflight: AtomicU64::new(0),
|
||||
cleanup_jobs: Mutex::new(HashMap::new()),
|
||||
cleanup_sequence: AtomicU64::new(0),
|
||||
counters: Counters::default(),
|
||||
})
|
||||
}
|
||||
pub fn start(
|
||||
revision: u64,
|
||||
data_root: Option<PathBuf>,
|
||||
settings: Settings,
|
||||
shutdown: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
let (tx, rx) = mpsc::channel(COLD_QUEUE);
|
||||
let cache = Arc::new(Self {
|
||||
store: Arc::new(Mutex::new(Store {
|
||||
revision,
|
||||
..Default::default()
|
||||
})),
|
||||
revision: AtomicU64::new(revision),
|
||||
cold: Arc::new(Mutex::new(ColdStore::default())),
|
||||
tier_settings: Mutex::new(settings),
|
||||
data_root,
|
||||
cold_tx: Some(tx),
|
||||
fills: Arc::new(Semaphore::new(FILL_SLOTS)),
|
||||
fill_bytes_active: AtomicU64::new(0),
|
||||
transient_bytes_active: Arc::new(AtomicU64::new(0)),
|
||||
cold_queue_items: AtomicU64::new(0),
|
||||
cold_queue_bytes: AtomicU64::new(0),
|
||||
cold_write_bytes_active: AtomicU64::new(0),
|
||||
cold_read_bytes_active: AtomicU64::new(0),
|
||||
revalidating: Mutex::new(HashSet::new()),
|
||||
fill_flights: Mutex::new(HashMap::new()),
|
||||
fill_flight_sequence: AtomicU64::new(0),
|
||||
collapse_waiters: AtomicU64::new(0),
|
||||
bans: Mutex::new(Vec::new()),
|
||||
ban_sequence: AtomicU64::new(0),
|
||||
proactive_scheduled_keys: Mutex::new(HashSet::new()),
|
||||
proactive_inflight: AtomicU64::new(0),
|
||||
cleanup_jobs: Mutex::new(HashMap::new()),
|
||||
cleanup_sequence: AtomicU64::new(0),
|
||||
counters: Counters::default(),
|
||||
});
|
||||
cache.load_cold_startup();
|
||||
let weak = Arc::downgrade(&cache);
|
||||
let worker_shutdown = shutdown.clone();
|
||||
tokio::spawn(async move { Self::cold_worker(weak, rx, worker_shutdown).await });
|
||||
let weak = Arc::downgrade(&cache);
|
||||
let maintenance_shutdown = shutdown.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(Duration::from_secs(5));
|
||||
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = maintenance_shutdown.cancelled() => break,
|
||||
_ = tick.tick() => {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
cache.demote_idle();
|
||||
cache.sweep_cold().await;
|
||||
cache.gc_banned(256);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
let weak = Arc::downgrade(&cache);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
let interval = cache
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.cold_gc_interval_seconds;
|
||||
drop(cache);
|
||||
tokio::select! {
|
||||
_ = shutdown.cancelled() => break,
|
||||
_ = tokio::time::sleep(Duration::from_secs(interval)) => {
|
||||
let Some(cache) = weak.upgrade() else { break; };
|
||||
cache.cold_gc_pass().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
cache
|
||||
}
|
||||
fn fill_headroom_bytes(max_bytes: usize) -> usize {
|
||||
// Allocation rounding is allowed a small global cushion, never a whole
|
||||
// max_object allocation. Use roughly 1/64 of HOT, cap it at 1 MiB and
|
||||
// never let the cushion itself exceed 1/8 of a very small HOT budget.
|
||||
let proportional = (max_bytes / 64).clamp(4 * 1024, 1024 * 1024);
|
||||
proportional.min((max_bytes / 8).max(1))
|
||||
}
|
||||
|
||||
fn try_reserve_transient(&self, bytes: usize, settings: &Settings) -> Option<TransientLease> {
|
||||
let bytes = u64::try_from(bytes).ok()?;
|
||||
let limit = u64::try_from(settings.transient_max_bytes).unwrap_or(u64::MAX);
|
||||
loop {
|
||||
let current = self.transient_bytes_active.load(Ordering::Relaxed);
|
||||
let next = current.checked_add(bytes)?;
|
||||
if next > limit {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
if self
|
||||
.transient_bytes_active
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return Some(TransientLease(Arc::new(TransientLeaseInner {
|
||||
counter: self.transient_bytes_active.clone(),
|
||||
bytes,
|
||||
})));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve real allocation for an in-progress fill before growing its buffer.
|
||||
/// Existing HOT objects are LRU-evicted/demoted first so HOT + active fill
|
||||
/// allocation cannot race arbitrarily above the configured HOT budget.
|
||||
fn reserve_fill_allocation(&self, bytes: usize, settings: &Settings) -> Option<TransientLease> {
|
||||
if bytes == 0 {
|
||||
return self.try_reserve_transient(0, settings);
|
||||
}
|
||||
let lease = self.try_reserve_transient(bytes, settings)?;
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let fill_now = usize::try_from(self.fill_bytes_active.load(Ordering::Relaxed))
|
||||
.unwrap_or(usize::MAX);
|
||||
let allowed = settings
|
||||
.max_bytes
|
||||
.saturating_add(Self::fill_headroom_bytes(settings.max_bytes));
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let mut removed = 0usize;
|
||||
let mut released_hot = false;
|
||||
while store
|
||||
.bytes
|
||||
.saturating_add(fill_now)
|
||||
.saturating_add(bytes)
|
||||
> allowed
|
||||
{
|
||||
if removed >= settings.hot_eviction_max_objects {
|
||||
self.counters
|
||||
.hot_eviction_limited
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let Some(old) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
removed = removed.saturating_add(1);
|
||||
released_hot = true;
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&old.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store
|
||||
.key_generations
|
||||
.get(&old.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if !(settings.cold_enabled
|
||||
&& settings.cold_on_lru
|
||||
&& self.enqueue_demotion(
|
||||
key,
|
||||
old,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
))
|
||||
{
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let fits = store
|
||||
.bytes
|
||||
.saturating_add(fill_now)
|
||||
.saturating_add(bytes)
|
||||
<= allowed;
|
||||
if fits {
|
||||
self.fill_bytes_active
|
||||
.fetch_add(bytes as u64, Ordering::Relaxed);
|
||||
}
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
if fits {
|
||||
Some(lease)
|
||||
} else {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
drop(lease);
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn configure(&self, settings: Settings) {
|
||||
*self.tier_settings.lock().unwrap_or_else(|e| e.into_inner()) = settings.clone();
|
||||
// Hot-applied limit reductions take effect immediately for published HOT
|
||||
// objects. Demotion is best effort and remains bounded by transient/COLD
|
||||
// queue budgets; otherwise the LRU object is dropped.
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let mut released_hot = false;
|
||||
while store.entries.len() > settings.max_entries || store.bytes > settings.max_bytes {
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let Some(old) = store.remove(&key) else {
|
||||
continue;
|
||||
};
|
||||
released_hot = true;
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&old.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store
|
||||
.key_generations
|
||||
.get(&old.base_key)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
if !(settings.cold_enabled
|
||||
&& settings.cold_on_lru
|
||||
&& self.enqueue_demotion(
|
||||
key,
|
||||
old,
|
||||
revision,
|
||||
generation,
|
||||
scope_generation,
|
||||
key_generation,
|
||||
))
|
||||
{
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
}
|
||||
pub fn revision(&self) -> u64 {
|
||||
self.revision.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
Vendored
+644
@@ -0,0 +1,644 @@
|
||||
impl Cache {
|
||||
fn get_hot(&self, ticket: &Ticket, count_miss: bool) -> Option<Hit> {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let expired = store.sweep();
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(expired as u64, Ordering::Relaxed);
|
||||
if store.revision != ticket.revision
|
||||
|| store.generation != ticket.generation
|
||||
|| store
|
||||
.scope_generations
|
||||
.get(&ticket.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.scope_generation
|
||||
|| store
|
||||
.key_generations
|
||||
.get(&ticket.key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.key_generation
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if let Some(storage_key) = matching_hot_key(&store, ticket) {
|
||||
let Some(mut entry) = store.remove(&storage_key) else {
|
||||
return None;
|
||||
};
|
||||
if self.hot_entry_banned(&storage_key, &entry) {
|
||||
if count_miss {
|
||||
self.counters.misses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
drop(store);
|
||||
release_allocator_memory();
|
||||
return None;
|
||||
}
|
||||
let resident_age_seconds = hot_resident_age(&entry).as_secs();
|
||||
let age = entry
|
||||
.initial_age
|
||||
.saturating_add(entry.received.elapsed())
|
||||
.as_secs();
|
||||
let state = cache_state(
|
||||
age,
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.must_revalidate,
|
||||
entry.proxy_revalidate,
|
||||
entry.requires_revalidation,
|
||||
&ticket.request_headers,
|
||||
);
|
||||
if state.servable() {
|
||||
entry.hits = entry.hits.saturating_add(1);
|
||||
entry.last_access = Instant::now();
|
||||
entry.sequence = store.next_sequence();
|
||||
let mut response = Response::new(Body::from(entry.body.clone()));
|
||||
*response.headers_mut() = entry.headers.clone();
|
||||
if entry.transformed {
|
||||
response
|
||||
.extensions_mut()
|
||||
.insert(crate::transform::CachedTransformed);
|
||||
}
|
||||
if entry.skip_compression {
|
||||
response.extensions_mut().insert(crate::compression::Skip);
|
||||
}
|
||||
response.headers_mut().insert(
|
||||
"age",
|
||||
HeaderValue::from_str(&age.to_string()).expect("integer header"),
|
||||
);
|
||||
let result = Hit {
|
||||
response,
|
||||
body: entry.body.clone(),
|
||||
hits: entry.hits,
|
||||
transformed: entry.transformed,
|
||||
remaining_ttl: entry.ttl.saturating_sub(age),
|
||||
resident_age_seconds,
|
||||
stale: state.stale(),
|
||||
background_revalidate: state.background_revalidate(),
|
||||
};
|
||||
if let Some(c) = store.domain(&entry.scope, &entry.domain) {
|
||||
c.saved_body_bytes = c.saved_body_bytes.saturating_add(entry.body.len() as u64);
|
||||
}
|
||||
store.bytes = store.bytes.saturating_add(entry.cost);
|
||||
store.logical_bytes = store.logical_bytes.saturating_add(entry.body.len());
|
||||
store.lru.insert(entry.sequence, storage_key);
|
||||
store.index_variant(entry.base_key, storage_key);
|
||||
store.entries.insert(storage_key, entry);
|
||||
self.counters.hits.fetch_add(1, Ordering::Relaxed);
|
||||
return Some(result);
|
||||
}
|
||||
store.bytes = store.bytes.saturating_add(entry.cost);
|
||||
store.logical_bytes = store.logical_bytes.saturating_add(entry.body.len());
|
||||
store.lru.insert(entry.sequence, storage_key);
|
||||
store.index_variant(entry.base_key, storage_key);
|
||||
store.entries.insert(storage_key, entry);
|
||||
}
|
||||
if count_miss {
|
||||
self.counters.misses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
None
|
||||
}
|
||||
pub fn get(&self, ticket: &Ticket) -> Option<Hit> {
|
||||
self.get_hot(ticket, true)
|
||||
}
|
||||
async fn get_cold(&self, ticket: &Ticket, settings: &Settings) -> Option<Hit> {
|
||||
if !self.ticket_current(ticket) {
|
||||
return None;
|
||||
}
|
||||
let banned = {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
matching_cold_key(&cold, ticket).and_then(|storage_key| {
|
||||
let banned = cold
|
||||
.entries
|
||||
.get(&storage_key)
|
||||
.is_some_and(|entry| self.cold_entry_banned(&storage_key, entry));
|
||||
banned.then(|| cold.remove(&storage_key)).flatten()
|
||||
})
|
||||
};
|
||||
if let Some(entry) = banned {
|
||||
self.delete_cold_entry(entry);
|
||||
return None;
|
||||
}
|
||||
let (storage_key, entry) = {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let storage_key = matching_cold_key(&cold, ticket)?;
|
||||
let old_sequence = {
|
||||
let entry = cold.entries.get(&storage_key)?;
|
||||
if entry.revision != ticket.revision || entry.scope != ticket.scope {
|
||||
return None;
|
||||
}
|
||||
let state = cache_state(
|
||||
cold_age(entry),
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.must_revalidate,
|
||||
entry.proxy_revalidate,
|
||||
entry.requires_revalidation,
|
||||
&ticket.request_headers,
|
||||
);
|
||||
if !state.servable() {
|
||||
return None;
|
||||
}
|
||||
entry.sequence
|
||||
};
|
||||
let sequence = cold.next_sequence();
|
||||
cold.lru.remove(&old_sequence);
|
||||
cold.lru.insert(sequence, storage_key);
|
||||
let entry = cold.entries.get_mut(&storage_key)?;
|
||||
entry.sequence = sequence;
|
||||
entry.last_access = SystemTime::now();
|
||||
entry.pinned_reads = entry.pinned_reads.saturating_add(1);
|
||||
(storage_key, entry.clone())
|
||||
};
|
||||
let _pin = ColdReadGuard {
|
||||
cache: self,
|
||||
key: storage_key,
|
||||
generation: entry.generation,
|
||||
};
|
||||
if entry.body_bytes > settings.max_object_bytes {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
let Some(read_lease) = self.try_reserve_transient(entry.body_bytes, settings) else {
|
||||
return None;
|
||||
};
|
||||
let _cold_read = ByteGaugeGuard::new(&self.cold_read_bytes_active, entry.body_bytes as u64);
|
||||
let body_vec = match tokio::fs::read(&entry.body_path).await {
|
||||
Ok(body) if cold_body_matches(&entry, &body) => body,
|
||||
_ => {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(entry) = self.take_cold(&storage_key) {
|
||||
self.delete_cold_entry(entry);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let body_allocated = body_vec.capacity().max(body_vec.len());
|
||||
let extra = body_allocated.saturating_sub(entry.body_bytes);
|
||||
let extra_lease = if extra > 0 {
|
||||
self.try_reserve_transient(extra, settings)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if extra > 0 && extra_lease.is_none() {
|
||||
return None;
|
||||
}
|
||||
let _cold_read_extra = extra_lease
|
||||
.as_ref()
|
||||
.map(|_| ByteGaugeGuard::new(&self.cold_read_bytes_active, extra as u64));
|
||||
let body = Bytes::from(body_vec);
|
||||
if !self.ticket_current(ticket) {
|
||||
return None;
|
||||
}
|
||||
let age = cold_age(&entry);
|
||||
let state = cache_state(
|
||||
age,
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.must_revalidate,
|
||||
entry.proxy_revalidate,
|
||||
entry.requires_revalidation,
|
||||
&ticket.request_headers,
|
||||
);
|
||||
if !state.servable() {
|
||||
return None;
|
||||
}
|
||||
let hits = entry.hits.saturating_add(1);
|
||||
let cost = hot_entry_cost(
|
||||
body_allocated,
|
||||
&entry.headers,
|
||||
&entry.vary,
|
||||
&entry.scope,
|
||||
&entry.scheme,
|
||||
&entry.authority,
|
||||
&entry.domain,
|
||||
&entry.path,
|
||||
&entry.target,
|
||||
&entry.tags,
|
||||
entry.origin_etag.as_ref(),
|
||||
entry.origin_last_modified.as_ref(),
|
||||
);
|
||||
let resident_age_seconds = cold_resident_age(&entry);
|
||||
let hot_entry = Entry {
|
||||
base_key: entry.base_key,
|
||||
vary: entry.vary.clone(),
|
||||
scope: entry.scope.clone(),
|
||||
scheme: entry.scheme.clone(),
|
||||
authority: entry.authority.clone(),
|
||||
domain: entry.domain.clone(),
|
||||
path: entry.path.clone(),
|
||||
target: entry.target.clone(),
|
||||
tags: entry.tags.clone(),
|
||||
created_at: entry.created_at,
|
||||
headers: entry.headers.clone(),
|
||||
body: body.clone(),
|
||||
body_allocated,
|
||||
received: Instant::now(),
|
||||
last_access: Instant::now(),
|
||||
resident_since: Instant::now(),
|
||||
resident_age_base: Duration::from_secs(resident_age_seconds),
|
||||
initial_age: Duration::from_secs(age),
|
||||
ttl: entry.ttl,
|
||||
stale_while_revalidate: entry.stale_while_revalidate,
|
||||
stale_if_error: entry.stale_if_error,
|
||||
must_revalidate: entry.must_revalidate,
|
||||
proxy_revalidate: entry.proxy_revalidate,
|
||||
origin_etag: entry.origin_etag.clone(),
|
||||
origin_last_modified: entry.origin_last_modified.clone(),
|
||||
cost,
|
||||
sequence: 0,
|
||||
hits,
|
||||
transformed: entry.transformed,
|
||||
skip_compression: entry.skip_compression,
|
||||
requires_revalidation: entry.requires_revalidation,
|
||||
};
|
||||
if !self.insert_inner(ticket.clone(), hot_entry, settings, false) {
|
||||
return None;
|
||||
}
|
||||
let mut response = Response::new(Body::from(body.clone()));
|
||||
*response.headers_mut() = entry.headers;
|
||||
if entry.transformed {
|
||||
response
|
||||
.extensions_mut()
|
||||
.insert(crate::transform::CachedTransformed);
|
||||
}
|
||||
if entry.skip_compression {
|
||||
response.extensions_mut().insert(crate::compression::Skip);
|
||||
}
|
||||
response.headers_mut().insert(
|
||||
"age",
|
||||
HeaderValue::from_str(&age.to_string()).expect("integer header"),
|
||||
);
|
||||
{
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(c) = store.domain(&ticket.scope, &ticket.domain) {
|
||||
c.saved_body_bytes = c.saved_body_bytes.saturating_add(entry.body_bytes as u64);
|
||||
}
|
||||
}
|
||||
self.counters.hits.fetch_add(1, Ordering::Relaxed);
|
||||
self.counters.cold_hits.fetch_add(1, Ordering::Relaxed);
|
||||
self.counters
|
||||
.cold_promotions
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
Some(Hit {
|
||||
response,
|
||||
body,
|
||||
hits,
|
||||
transformed: entry.transformed,
|
||||
remaining_ttl: entry.ttl.saturating_sub(age),
|
||||
resident_age_seconds,
|
||||
stale: state.stale(),
|
||||
background_revalidate: state.background_revalidate(),
|
||||
})
|
||||
}
|
||||
pub async fn get_tiered(&self, ticket: &Ticket, settings: &Settings) -> Option<Hit> {
|
||||
if let Some(hit) = self.get_hot(ticket, false) {
|
||||
return Some(hit);
|
||||
}
|
||||
if let Some(hit) = self.get_cold(ticket, settings).await {
|
||||
return Some(hit);
|
||||
}
|
||||
self.counters.misses.fetch_add(1, Ordering::Relaxed);
|
||||
None
|
||||
}
|
||||
pub fn publish_revalidated(
|
||||
&self,
|
||||
ticket: Ticket,
|
||||
cached: &Revalidation,
|
||||
mut headers: HeaderMap,
|
||||
validation_headers: &HeaderMap,
|
||||
settings: &Settings,
|
||||
response_delay: Duration,
|
||||
) -> Option<Hit> {
|
||||
let plan = plan(StatusCode::OK, &headers, settings, response_delay)?;
|
||||
if cached.body.len() > settings.max_object_bytes {
|
||||
return None;
|
||||
}
|
||||
if !headers.contains_key("date") {
|
||||
if let Ok(value) = httpdate::fmt_http_date(SystemTime::now()).parse() {
|
||||
headers.insert("date", value);
|
||||
}
|
||||
}
|
||||
for name in [
|
||||
"x-request-id",
|
||||
"x-cache",
|
||||
"x-cache-hits",
|
||||
"x-cache-status",
|
||||
"x-cache-reason",
|
||||
"cache-status",
|
||||
] {
|
||||
headers.remove(name);
|
||||
}
|
||||
let hits = cached.hits.saturating_add(1);
|
||||
let age = plan.initial_age.as_secs();
|
||||
let remaining_ttl = plan.ttl.saturating_sub(age);
|
||||
let vary = plan.vary.clone();
|
||||
let tags = cache_tags(&headers);
|
||||
let origin_etag = validation_headers
|
||||
.get("etag")
|
||||
.cloned()
|
||||
.or_else(|| cached.origin_etag.clone());
|
||||
let origin_last_modified = validation_headers
|
||||
.get("last-modified")
|
||||
.cloned()
|
||||
.or_else(|| cached.origin_last_modified.clone());
|
||||
let cost = hot_entry_cost(
|
||||
cached.body_allocated,
|
||||
&headers,
|
||||
&vary,
|
||||
&ticket.scope,
|
||||
&ticket.scheme,
|
||||
&ticket.authority,
|
||||
&ticket.domain,
|
||||
&ticket.path,
|
||||
&ticket.target,
|
||||
&tags,
|
||||
origin_etag.as_ref(),
|
||||
origin_last_modified.as_ref(),
|
||||
);
|
||||
let storage_key = variant_key(&ticket.key, &vary, &ticket.request_headers);
|
||||
let resident_age = cached.resident_age_at_lookup.saturating_add(response_delay);
|
||||
let entry = Entry {
|
||||
base_key: ticket.key,
|
||||
vary,
|
||||
scope: ticket.scope.clone(),
|
||||
scheme: ticket.scheme.clone(),
|
||||
authority: ticket.authority.clone(),
|
||||
domain: ticket.domain.clone(),
|
||||
path: ticket.path.clone(),
|
||||
target: ticket.target.clone(),
|
||||
tags,
|
||||
created_at: ticket.created_at,
|
||||
headers: headers.clone(),
|
||||
body: cached.body.clone(),
|
||||
body_allocated: cached.body_allocated,
|
||||
received: plan.received,
|
||||
last_access: Instant::now(),
|
||||
resident_since: Instant::now(),
|
||||
resident_age_base: resident_age,
|
||||
initial_age: plan.initial_age,
|
||||
ttl: plan.ttl,
|
||||
stale_while_revalidate: plan.stale_while_revalidate,
|
||||
stale_if_error: plan.stale_if_error,
|
||||
must_revalidate: plan.must_revalidate,
|
||||
proxy_revalidate: plan.proxy_revalidate,
|
||||
origin_etag,
|
||||
origin_last_modified,
|
||||
cost,
|
||||
sequence: 0,
|
||||
hits,
|
||||
transformed: cached.transformed,
|
||||
skip_compression: cached.skip_compression,
|
||||
requires_revalidation: plan.requires_revalidation,
|
||||
};
|
||||
if !self.insert(ticket, entry, settings) {
|
||||
return None;
|
||||
}
|
||||
if cached.storage_key != storage_key {
|
||||
self.discard_storage_key(&cached.storage_key);
|
||||
}
|
||||
self.counters.hits.fetch_add(1, Ordering::Relaxed);
|
||||
let mut response = cached.response(headers);
|
||||
response.headers_mut().insert(
|
||||
"age",
|
||||
HeaderValue::from_str(&age.to_string()).expect("integer header"),
|
||||
);
|
||||
Some(Hit {
|
||||
response,
|
||||
body: cached.body.clone(),
|
||||
hits,
|
||||
transformed: cached.transformed,
|
||||
remaining_ttl,
|
||||
resident_age_seconds: resident_age.as_secs(),
|
||||
stale: false,
|
||||
background_revalidate: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn wrap(
|
||||
self: &Arc<Self>,
|
||||
response: Response,
|
||||
ticket: Ticket,
|
||||
plan: Plan,
|
||||
settings: &Settings,
|
||||
) -> Response {
|
||||
self.wrap_with_fill(response, ticket, plan, settings, None)
|
||||
}
|
||||
|
||||
pub fn wrap_with_fill(
|
||||
self: &Arc<Self>,
|
||||
response: Response,
|
||||
ticket: Ticket,
|
||||
plan: Plan,
|
||||
settings: &Settings,
|
||||
fill_leader: Option<FillLeaderGuard>,
|
||||
) -> Response {
|
||||
let fill_size = plan.expected_length.unwrap_or(settings.max_object_bytes);
|
||||
let permits = if fill_size > 64 * 1024 * 1024 {
|
||||
FILL_SLOTS as u32
|
||||
} else if fill_size > 16 * 1024 * 1024 {
|
||||
4
|
||||
} else if fill_size > 4 * 1024 * 1024 {
|
||||
2
|
||||
} else {
|
||||
1
|
||||
};
|
||||
let permit = self.fills.clone().try_acquire_many_owned(permits);
|
||||
let Ok(permit) = permit else {
|
||||
self.counters.rejected_fills.fetch_add(1, Ordering::Relaxed);
|
||||
return response;
|
||||
};
|
||||
let (parts, body) = response.into_parts();
|
||||
let headers = parts.headers.clone();
|
||||
let origin_validators = parts
|
||||
.extensions
|
||||
.get::<crate::transform::OriginValidators>()
|
||||
.cloned();
|
||||
let origin_etag = match origin_validators.as_ref() {
|
||||
Some(validators) => validators.etag.clone(),
|
||||
None => headers.get("etag").cloned(),
|
||||
};
|
||||
let origin_last_modified = match origin_validators.as_ref() {
|
||||
Some(validators) => validators.last_modified.clone(),
|
||||
None => headers.get("last-modified").cloned(),
|
||||
};
|
||||
let initial_capacity = fill_size.min(64 * 1024);
|
||||
let Some(mut fill_gauge) = FillGauge::new(self.clone(), initial_capacity, settings) else {
|
||||
self.counters.rejected_fills.fetch_add(1, Ordering::Relaxed);
|
||||
self.record_reason(&ticket.scope, &ticket.domain, "hot_memory_pressure");
|
||||
return Response::from_parts(parts, body);
|
||||
};
|
||||
let data = Vec::with_capacity(initial_capacity);
|
||||
let actual_capacity = data.capacity();
|
||||
if actual_capacity > initial_capacity
|
||||
&& !fill_gauge.grow(actual_capacity - initial_capacity, settings)
|
||||
{
|
||||
self.counters.rejected_fills.fetch_add(1, Ordering::Relaxed);
|
||||
self.record_reason(&ticket.scope, &ticket.domain, "fill_allocator_overshoot");
|
||||
return Response::from_parts(parts, body);
|
||||
}
|
||||
let candidate = Candidate {
|
||||
cache: self.clone(),
|
||||
ticket,
|
||||
plan,
|
||||
headers,
|
||||
data,
|
||||
fill_gauge,
|
||||
settings: settings.clone(),
|
||||
skip_compression: parts.extensions.get::<crate::compression::Skip>().is_some(),
|
||||
origin_etag,
|
||||
origin_last_modified,
|
||||
transformed: parts.extensions.get::<crate::transform::Applied>().is_some(),
|
||||
_permit: permit,
|
||||
_fill_leader: fill_leader,
|
||||
};
|
||||
Response::from_parts(
|
||||
parts,
|
||||
Body::new(FillBody {
|
||||
inner: body,
|
||||
candidate: Some(candidate),
|
||||
done: false,
|
||||
}),
|
||||
)
|
||||
}
|
||||
fn insert(&self, ticket: Ticket, entry: Entry, settings: &Settings) -> bool {
|
||||
self.insert_inner(ticket, entry, settings, true)
|
||||
}
|
||||
fn insert_inner(
|
||||
&self,
|
||||
ticket: Ticket,
|
||||
mut entry: Entry,
|
||||
settings: &Settings,
|
||||
count_stored: bool,
|
||||
) -> bool {
|
||||
entry.base_key = ticket.key;
|
||||
let storage_key = variant_key(&ticket.key, &entry.vary, &ticket.request_headers);
|
||||
let vary_schema = entry.vary.clone();
|
||||
if self.entry_banned_before_insert(&storage_key, &entry) {
|
||||
self.counters.ban_matches.fetch_add(1, Ordering::Relaxed);
|
||||
return false;
|
||||
}
|
||||
if entry.cost > settings.max_bytes
|
||||
|| (!entry.requires_revalidation
|
||||
&& entry.initial_age.saturating_add(entry.received.elapsed()).as_secs()
|
||||
>= retention_deadline(
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.stale_if_error,
|
||||
entry.origin_etag.is_some() || entry.origin_last_modified.is_some(),
|
||||
))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if store.revision != ticket.revision
|
||||
|| store.generation != ticket.generation
|
||||
|| store
|
||||
.scope_generations
|
||||
.get(&ticket.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.scope_generation
|
||||
|| store
|
||||
.key_generations
|
||||
.get(&ticket.key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.key_generation
|
||||
{
|
||||
return false;
|
||||
}
|
||||
// Do not keep objects for incompatible Vary schemas under the same
|
||||
// Varnish-style host+URL hash. A newer representation policy replaces
|
||||
// the older schema while preserving sibling values of the same Vary set.
|
||||
for key in incompatible_hot_variants(&store, &ticket.key, &vary_schema) {
|
||||
store.remove(&key);
|
||||
}
|
||||
store.remove(&storage_key);
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(store.sweep() as u64, Ordering::Relaxed);
|
||||
let revision = store.revision;
|
||||
let generation = store.generation;
|
||||
let mut released_hot = false;
|
||||
let mut removed = 0usize;
|
||||
while store.entries.len() >= settings.max_entries
|
||||
|| store.bytes.saturating_add(entry.cost) > settings.max_bytes
|
||||
{
|
||||
if removed >= settings.hot_eviction_max_objects {
|
||||
self.counters
|
||||
.hot_eviction_limited
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
break;
|
||||
}
|
||||
let Some((_, key)) = store.lru.first_key_value().map(|(seq, key)| (*seq, *key)) else {
|
||||
break;
|
||||
};
|
||||
if let Some(old) = store.remove(&key) {
|
||||
removed = removed.saturating_add(1);
|
||||
released_hot = true;
|
||||
let scope_generation = store
|
||||
.scope_generations
|
||||
.get(&old.scope)
|
||||
.copied()
|
||||
.unwrap_or(0);
|
||||
let key_generation = store.key_generations.get(&old.base_key).copied().unwrap_or(0);
|
||||
if !(settings.cold_enabled
|
||||
&& settings.cold_on_lru
|
||||
&& self.enqueue_demotion(key, old, revision, generation, scope_generation, key_generation))
|
||||
{
|
||||
self.counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
if store.entries.len() >= settings.max_entries
|
||||
|| store.bytes.saturating_add(entry.cost) > settings.max_bytes
|
||||
{
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if count_stored {
|
||||
if let Some(c) = store.domain(&entry.scope, &entry.domain) {
|
||||
c.stored += 1;
|
||||
}
|
||||
}
|
||||
entry.last_access = Instant::now();
|
||||
entry.sequence = store.next_sequence();
|
||||
store.bytes = store.bytes.saturating_add(entry.cost);
|
||||
store.logical_bytes = store.logical_bytes.saturating_add(entry.body.len());
|
||||
store.lru.insert(entry.sequence, storage_key);
|
||||
store.index_variant(entry.base_key, storage_key);
|
||||
store.entries.insert(storage_key, entry);
|
||||
drop(store);
|
||||
if released_hot {
|
||||
release_allocator_memory();
|
||||
}
|
||||
let mut cold_delete = Vec::new();
|
||||
{
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
for key in incompatible_cold_variants(&cold, &ticket.key, &vary_schema) {
|
||||
if let Some(old) = cold.remove(&key) {
|
||||
cold_delete.push(old);
|
||||
}
|
||||
}
|
||||
if let Some(old) = cold.remove(&storage_key) {
|
||||
cold_delete.push(old);
|
||||
}
|
||||
}
|
||||
for old in cold_delete {
|
||||
self.delete_cold_entry(old);
|
||||
}
|
||||
if count_stored {
|
||||
self.counters.stored.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
//! Bounded, conservative shared two-tier cache. Fresh objects live in a fast RAM
|
||||
//! hot tier; optionally, idle/LRU objects are persisted in a bounded cold tier.
|
||||
//! Bodies are streamed while a bounded copy is collected; only complete successful
|
||||
//! bodies are published. The implementation is split by responsibility so storage,
|
||||
//! policy and streaming-fill code can be reviewed independently.
|
||||
|
||||
include!("model.rs");
|
||||
include!("lifecycle.rs");
|
||||
include!("cold.rs");
|
||||
include!("dashboard.rs");
|
||||
include!("tickets.rs");
|
||||
include!("collapse.rs");
|
||||
include!("range.rs");
|
||||
include!("invalidation.rs");
|
||||
include!("lookup.rs");
|
||||
include!("policy.rs");
|
||||
include!("fill.rs");
|
||||
include!("tests.rs");
|
||||
Vendored
+1668
File diff suppressed because it is too large.
Load diff
Vendored
+570
@@ -0,0 +1,570 @@
|
||||
/// Reject malformed/duplicate directives instead of guessing their meaning.
|
||||
fn directives(headers: &HeaderMap) -> Option<HashMap<String, Option<String>>> {
|
||||
let mut result = HashMap::new();
|
||||
for header in headers.get_all("cache-control") {
|
||||
for raw in header.to_str().ok()?.split(',') {
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (key, value) = match raw.split_once('=') {
|
||||
Some((k, v)) => {
|
||||
let v = v.trim();
|
||||
let value = if v.starts_with('"') {
|
||||
// Fail closed on truncated/escaped/compound values. Quoted
|
||||
// commas need not be supported by this conservative cache.
|
||||
let inner = v.strip_prefix('"')?.strip_suffix('"')?;
|
||||
if inner.contains(['"', '\\']) {
|
||||
return None;
|
||||
}
|
||||
inner
|
||||
} else {
|
||||
if v.is_empty() || v.bytes().any(|b| b <= 32 || b == b'"' || b == b'\\') {
|
||||
return None;
|
||||
}
|
||||
v
|
||||
};
|
||||
(k.trim().to_ascii_lowercase(), Some(value.to_owned()))
|
||||
}
|
||||
None => (raw.to_ascii_lowercase(), None),
|
||||
};
|
||||
if key.parse::<http::HeaderName>().is_err() {
|
||||
return None;
|
||||
}
|
||||
if matches!(
|
||||
key.as_str(),
|
||||
"public" | "no-store" | "must-revalidate" | "proxy-revalidate" | "only-if-cached"
|
||||
) && value.is_some()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if key == "immutable" {
|
||||
// RFC 8246: immutable has no meaningful argument and duplicate
|
||||
// occurrences are equivalent to one occurrence. Preserve only
|
||||
// the presence of the directive instead of invalidating the
|
||||
// complete Cache-Control field.
|
||||
result.entry(key).or_insert(None);
|
||||
} else if result.insert(key, value).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(result)
|
||||
}
|
||||
pub fn only_if_cached(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get_all("cache-control")
|
||||
.iter()
|
||||
.filter_map(|v| v.to_str().ok())
|
||||
.any(|v| {
|
||||
v.split(',')
|
||||
.any(|v| v.trim().eq_ignore_ascii_case("only-if-cached"))
|
||||
})
|
||||
}
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RequestPolicy {
|
||||
Use,
|
||||
Revalidate(&'static str),
|
||||
Bypass(&'static str),
|
||||
}
|
||||
impl RequestPolicy {
|
||||
pub fn bypass_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Bypass(reason) => Some(reason),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
pub fn revalidation_reason(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Revalidate(reason) => Some(reason),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
fn pragma_no_cache(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get_all("pragma")
|
||||
.iter()
|
||||
.filter_map(|value| value.to_str().ok())
|
||||
.any(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.any(|token| token.trim().eq_ignore_ascii_case("no-cache"))
|
||||
})
|
||||
}
|
||||
pub fn request_policy(
|
||||
method: &Method,
|
||||
headers: &HeaderMap,
|
||||
empty_body: bool,
|
||||
path: &str,
|
||||
has_query: bool,
|
||||
settings: &Settings,
|
||||
) -> RequestPolicy {
|
||||
if method != Method::GET && method != Method::HEAD {
|
||||
return RequestPolicy::Bypass("request_method");
|
||||
}
|
||||
if !empty_body {
|
||||
return RequestPolicy::Bypass("request_body");
|
||||
}
|
||||
if matches!(request_range(method, headers), RangeSelection::Unsupported) {
|
||||
return RequestPolicy::Bypass("request_range_unsupported");
|
||||
}
|
||||
if headers.contains_key("if-range") && !headers.contains_key("range") {
|
||||
return RequestPolicy::Bypass("request_precondition");
|
||||
}
|
||||
if settings.bypass_query && has_query {
|
||||
return RequestPolicy::Bypass("query_excluded");
|
||||
}
|
||||
if settings
|
||||
.bypass_prefixes
|
||||
.iter()
|
||||
.any(|prefix| crate::config::prefix_matches(prefix, path))
|
||||
{
|
||||
return RequestPolicy::Bypass("path_excluded");
|
||||
}
|
||||
for (key, reason) in [
|
||||
("authorization", "request_authorization"),
|
||||
("cookie", "request_cookie"),
|
||||
// Preconditions other than cache validators can produce 412 and must be
|
||||
// evaluated by the origin until RustProxy implements the full precondition set.
|
||||
("if-match", "request_precondition"),
|
||||
("if-unmodified-since", "request_precondition"),
|
||||
("upgrade", "request_upgrade"),
|
||||
] {
|
||||
if headers.contains_key(key) {
|
||||
if key == "cookie" && settings.ignore_cookie_for_path(path) {
|
||||
continue;
|
||||
}
|
||||
return RequestPolicy::Bypass(reason);
|
||||
}
|
||||
}
|
||||
let Some(control) = directives(headers) else {
|
||||
return RequestPolicy::Bypass("request_cache_control_invalid");
|
||||
};
|
||||
// RFC 9111: request no-store forbids using a stored response and storing this
|
||||
// exchange. It is the hard client-side cache bypass.
|
||||
if control.contains_key("no-store") {
|
||||
return RequestPolicy::Bypass("request_no_store");
|
||||
}
|
||||
// no-cache does not mean no-store. It requires successful validation before a
|
||||
// stored response can be reused. max-age/min-fresh are evaluated against the
|
||||
// cached representation during lookup; only max-age=0 is an unconditional
|
||||
// request-side validation signal.
|
||||
if control.contains_key("no-cache") {
|
||||
return RequestPolicy::Revalidate("request_no_cache");
|
||||
}
|
||||
if let Some(value) = control.get("max-age") {
|
||||
let Some(value) = value.as_deref().and_then(|value| value.parse::<u64>().ok()) else {
|
||||
return RequestPolicy::Bypass("request_cache_control_invalid");
|
||||
};
|
||||
if value == 0 {
|
||||
return RequestPolicy::Revalidate("request_max_age");
|
||||
}
|
||||
}
|
||||
if let Some(value) = control.get("min-fresh") {
|
||||
if value
|
||||
.as_deref()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.is_none()
|
||||
{
|
||||
return RequestPolicy::Bypass("request_cache_control_invalid");
|
||||
}
|
||||
}
|
||||
if let Some(Some(value)) = control.get("max-stale") {
|
||||
if value.parse::<u64>().is_err() {
|
||||
return RequestPolicy::Bypass("request_cache_control_invalid");
|
||||
}
|
||||
}
|
||||
// HTTP/1.0 Pragma: no-cache is a revalidation request when Cache-Control did
|
||||
// not already define the policy. Other Pragma values do not disable caching.
|
||||
if !headers.contains_key("cache-control") && pragma_no_cache(headers) {
|
||||
return RequestPolicy::Revalidate("request_pragma_no_cache");
|
||||
}
|
||||
RequestPolicy::Use
|
||||
}
|
||||
pub fn request_reason(
|
||||
method: &Method,
|
||||
headers: &HeaderMap,
|
||||
empty_body: bool,
|
||||
path: &str,
|
||||
has_query: bool,
|
||||
settings: &Settings,
|
||||
) -> Option<&'static str> {
|
||||
request_policy(method, headers, empty_body, path, has_query, settings).bypass_reason()
|
||||
}
|
||||
pub fn request_cacheable(
|
||||
method: &Method,
|
||||
headers: &HeaderMap,
|
||||
empty_body: bool,
|
||||
path: &str,
|
||||
has_query: bool,
|
||||
settings: &Settings,
|
||||
) -> bool {
|
||||
!matches!(
|
||||
request_policy(method, headers, empty_body, path, has_query, settings),
|
||||
RequestPolicy::Bypass(_)
|
||||
)
|
||||
}
|
||||
|
||||
fn weak_etag(value: &str) -> Option<&str> {
|
||||
let value = value.trim();
|
||||
let value = value
|
||||
.strip_prefix("W/")
|
||||
.or_else(|| value.strip_prefix("w/"))
|
||||
.unwrap_or(value)
|
||||
.trim();
|
||||
(value.len() >= 2 && value.starts_with('"') && value.ends_with('"')).then_some(value)
|
||||
}
|
||||
fn if_none_match_matches(headers: &HeaderMap, etag: &str) -> Option<bool> {
|
||||
let target = weak_etag(etag)?;
|
||||
let values: Vec<_> = headers.get_all("if-none-match").iter().collect();
|
||||
if values.is_empty() {
|
||||
return None;
|
||||
}
|
||||
for value in values {
|
||||
let Ok(value) = value.to_str() else {
|
||||
return Some(false);
|
||||
};
|
||||
let mut start = 0usize;
|
||||
let mut quoted = false;
|
||||
for (index, ch) in value.char_indices() {
|
||||
if ch == '"' {
|
||||
quoted = !quoted;
|
||||
} else if ch == ',' && !quoted {
|
||||
let token = value[start..index].trim();
|
||||
if token == "*" || weak_etag(token).is_some_and(|candidate| candidate == target) {
|
||||
return Some(true);
|
||||
}
|
||||
start = index + 1;
|
||||
}
|
||||
}
|
||||
if quoted {
|
||||
return Some(false);
|
||||
}
|
||||
let token = value[start..].trim();
|
||||
if token == "*" || weak_etag(token).is_some_and(|candidate| candidate == target) {
|
||||
return Some(true);
|
||||
}
|
||||
}
|
||||
Some(false)
|
||||
}
|
||||
fn single_if_none_match_matches(headers: &HeaderMap, etag: &str) -> bool {
|
||||
let Some(target) = weak_etag(etag) else {
|
||||
return false;
|
||||
};
|
||||
let values: Vec<_> = headers.get_all("if-none-match").iter().collect();
|
||||
if values.len() != 1 {
|
||||
return false;
|
||||
}
|
||||
let Ok(value) = values[0].to_str() else {
|
||||
return false;
|
||||
};
|
||||
let mut tokens = value
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|token| !token.is_empty());
|
||||
let Some(token) = tokens.next() else {
|
||||
return false;
|
||||
};
|
||||
tokens.next().is_none()
|
||||
&& token != "*"
|
||||
&& weak_etag(token).is_some_and(|candidate| candidate == target)
|
||||
}
|
||||
/// Varnish-like client-side conditional delivery for a selected cached response.
|
||||
/// If-None-Match takes precedence over If-Modified-Since.
|
||||
pub fn conditional_not_modified(request: &HeaderMap, response: &HeaderMap) -> bool {
|
||||
if request.contains_key("if-none-match") {
|
||||
return response
|
||||
.get("etag")
|
||||
.and_then(|etag| etag.to_str().ok())
|
||||
.and_then(|etag| if_none_match_matches(request, etag))
|
||||
.unwrap_or(false);
|
||||
}
|
||||
let Some(since) = request
|
||||
.get("if-modified-since")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| httpdate::parse_http_date(value).ok())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
response
|
||||
.get("last-modified")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| httpdate::parse_http_date(value).ok())
|
||||
.is_some_and(|modified| modified <= since)
|
||||
}
|
||||
pub fn make_not_modified(mut response: Response) -> Response {
|
||||
*response.status_mut() = StatusCode::NOT_MODIFIED;
|
||||
*response.body_mut() = Body::empty();
|
||||
for name in ["content-length", "content-range", "transfer-encoding"] {
|
||||
response.headers_mut().remove(name);
|
||||
}
|
||||
response
|
||||
}
|
||||
/// Merge a successful backend 304 into the stored representation metadata.
|
||||
pub fn merge_revalidated_headers(cached: &HeaderMap, validation: &HeaderMap) -> HeaderMap {
|
||||
let mut merged = cached.clone();
|
||||
// A new validation starts a new age calculation. Preserve origin Age only if
|
||||
// it is explicitly supplied by the 304 response.
|
||||
if !validation.contains_key("age") {
|
||||
merged.remove("age");
|
||||
}
|
||||
for name in validation.keys() {
|
||||
if matches!(
|
||||
name.as_str(),
|
||||
"content-length" | "content-range" | "transfer-encoding"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
merged.remove(name);
|
||||
for value in validation.get_all(name).iter() {
|
||||
merged.append(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
if !validation.contains_key("date") {
|
||||
if let Ok(value) = httpdate::fmt_http_date(SystemTime::now()).parse() {
|
||||
merged.insert("date", value);
|
||||
}
|
||||
}
|
||||
merged
|
||||
}
|
||||
|
||||
pub fn varies_on(headers: &HeaderMap, wanted: &str) -> bool {
|
||||
headers.get_all("vary").iter().any(|value| {
|
||||
value.to_str().ok().is_some_and(|value| {
|
||||
value
|
||||
.split(',')
|
||||
.any(|name| name.trim().eq_ignore_ascii_case(wanted))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Explains rejected storage; it never relaxes origin or transformed policies.
|
||||
pub fn response_reason(status: StatusCode, headers: &HeaderMap) -> &'static str {
|
||||
if status != StatusCode::OK {
|
||||
return "response_status";
|
||||
}
|
||||
if headers.contains_key("set-cookie") {
|
||||
return "response_set_cookie";
|
||||
}
|
||||
if headers.contains_key("content-range") {
|
||||
return "response_range";
|
||||
}
|
||||
if headers.contains_key("trailer") {
|
||||
return "response_trailers";
|
||||
}
|
||||
let Some(control) = directives(headers) else {
|
||||
return "response_cache_control_invalid";
|
||||
};
|
||||
for key in ["no-store", "private", "must-understand"] {
|
||||
if control.contains_key(key) {
|
||||
return "response_cache_control";
|
||||
}
|
||||
}
|
||||
for value in headers.get_all("vary") {
|
||||
if value.to_str().ok().is_none_or(|value| {
|
||||
value.split(',').any(|name| {
|
||||
let name = name.trim();
|
||||
name == "*" || name.parse::<http::HeaderName>().is_err()
|
||||
})
|
||||
}) {
|
||||
return "response_vary";
|
||||
}
|
||||
}
|
||||
"response_ttl_size_or_metadata"
|
||||
}
|
||||
|
||||
pub struct Plan {
|
||||
received: Instant,
|
||||
initial_age: Duration,
|
||||
ttl: u64,
|
||||
stale_while_revalidate: u64,
|
||||
stale_if_error: u64,
|
||||
must_revalidate: bool,
|
||||
proxy_revalidate: bool,
|
||||
expected_length: Option<usize>,
|
||||
requires_revalidation: bool,
|
||||
vary: Vec<String>,
|
||||
}
|
||||
impl Plan {
|
||||
pub fn remaining_ttl(&self) -> u64 {
|
||||
self.ttl.saturating_sub(self.initial_age.as_secs())
|
||||
}
|
||||
|
||||
pub fn without_vary(mut self, name: &str) -> Self {
|
||||
self.vary.retain(|value| !value.eq_ignore_ascii_case(name));
|
||||
self
|
||||
}
|
||||
|
||||
pub fn restrict(mut self, transformed: Self) -> Self {
|
||||
self.ttl = self.ttl.min(transformed.ttl);
|
||||
self.stale_while_revalidate = self
|
||||
.stale_while_revalidate
|
||||
.min(transformed.stale_while_revalidate);
|
||||
self.stale_if_error = self.stale_if_error.min(transformed.stale_if_error);
|
||||
self.must_revalidate |= transformed.must_revalidate;
|
||||
self.proxy_revalidate |= transformed.proxy_revalidate;
|
||||
self.requires_revalidation |= transformed.requires_revalidation;
|
||||
self.vary.extend(transformed.vary);
|
||||
self.vary.sort();
|
||||
self.vary.dedup();
|
||||
// The transformed response is the representation that is actually
|
||||
// written to cache. Its Content-Length therefore replaces the origin
|
||||
// length; comparing transformed bytes with the origin length rejects a
|
||||
// valid fill and causes a permanent MISS -> MISS loop.
|
||||
self.expected_length = transformed.expected_length;
|
||||
// Express both age clocks at the later observation point before taking
|
||||
// the stricter value. This avoids resetting or double-counting Age when
|
||||
// origin headers are transformed inside the proxy.
|
||||
if transformed.received >= self.received {
|
||||
let carried = self
|
||||
.initial_age
|
||||
.saturating_add(transformed.received.duration_since(self.received));
|
||||
self.received = transformed.received;
|
||||
self.initial_age = carried.max(transformed.initial_age);
|
||||
} else {
|
||||
let carried = transformed
|
||||
.initial_age
|
||||
.saturating_add(self.received.duration_since(transformed.received));
|
||||
self.initial_age = self.initial_age.max(carried);
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
pub fn plan(
|
||||
status: StatusCode,
|
||||
headers: &HeaderMap,
|
||||
settings: &Settings,
|
||||
response_delay: Duration,
|
||||
) -> Option<Plan> {
|
||||
if ["age", "date", "expires", "content-length"]
|
||||
.iter()
|
||||
.any(|name| headers.get_all(*name).iter().count() > 1)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
if status != StatusCode::OK
|
||||
|| headers.contains_key("set-cookie")
|
||||
|| headers.contains_key("content-range")
|
||||
|| headers.contains_key("trailer")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let bytes: usize = headers
|
||||
.iter()
|
||||
.map(|(k, v)| k.as_str().len() + v.as_bytes().len() + 32)
|
||||
.sum();
|
||||
if bytes > MAX_HEADER_BYTES {
|
||||
return None;
|
||||
}
|
||||
if headers
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.is_some_and(|v| {
|
||||
let v = v.trim_start().to_ascii_lowercase();
|
||||
v.starts_with("text/event-stream") || v.starts_with("application/grpc")
|
||||
})
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// Cache-policy and validator fields govern reuse rather than representation
|
||||
// identity. Response Vary drives a secondary key, matching Varnish-style
|
||||
// variants without fragmenting on unrelated request headers.
|
||||
let vary = vary_names(headers)?;
|
||||
let control = directives(headers)?;
|
||||
if ["no-store", "private", "must-understand"]
|
||||
.iter()
|
||||
.any(|name| control.contains_key(*name))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let requires_revalidation = control.contains_key("no-cache");
|
||||
let must_revalidate = control.contains_key("must-revalidate");
|
||||
let proxy_revalidate = control.contains_key("proxy-revalidate");
|
||||
let parse_window = |name: &str| -> Option<u64> {
|
||||
match control.get(name) {
|
||||
None => Some(0),
|
||||
Some(Some(value)) => value.parse::<u64>().ok().map(|v| v.min(604_800)),
|
||||
Some(None) => None,
|
||||
}
|
||||
};
|
||||
let stale_while_revalidate = parse_window("stale-while-revalidate")?;
|
||||
let stale_if_error = parse_window("stale-if-error")?;
|
||||
if requires_revalidation
|
||||
&& !headers.contains_key("etag")
|
||||
&& !headers.contains_key("last-modified")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let now = SystemTime::now();
|
||||
let date = match headers.get("date") {
|
||||
Some(v) => httpdate::parse_http_date(v.to_str().ok()?).ok()?,
|
||||
None => now,
|
||||
};
|
||||
let declared_ttl =
|
||||
if let Some(value) = control.get("s-maxage").or_else(|| control.get("max-age")) {
|
||||
Some(value.as_ref()?.parse::<u64>().ok()?)
|
||||
} else if let Some(expires) = headers.get("expires") {
|
||||
Some(
|
||||
httpdate::parse_http_date(expires.to_str().ok()?)
|
||||
.ok()?
|
||||
.duration_since(date)
|
||||
.ok()?
|
||||
.as_secs(),
|
||||
)
|
||||
} else if control.contains_key("public") {
|
||||
Some(settings.default_ttl_seconds)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Keep the freshness lifetime separate from validator/stale retention.
|
||||
// `no-cache` may be stored with a zero freshness lifetime but must validate
|
||||
// every reuse; it is retained by the bounded revalidation horizon.
|
||||
let ttl = match declared_ttl {
|
||||
Some(value) => value.min(settings.max_ttl_seconds),
|
||||
None if requires_revalidation => 0,
|
||||
None => return None,
|
||||
};
|
||||
let age = match headers.get("age") {
|
||||
Some(v) => v.to_str().ok()?.parse::<u64>().ok()?,
|
||||
None => 0,
|
||||
};
|
||||
// Keep sub-second precision internally and floor only when serializing the
|
||||
// Age header. RFC 9111 adds response delay after correcting received age.
|
||||
let apparent_age = now.duration_since(date).unwrap_or_default();
|
||||
// RFC 9111 current_age: corrected_received_age is the greater of Date-based
|
||||
// apparent age and the received Age value; response delay is then added to
|
||||
// that result (not only to the Age branch).
|
||||
let corrected_received_age = apparent_age.max(Duration::from_secs(age));
|
||||
let initial_age = corrected_received_age.saturating_add(response_delay);
|
||||
if !requires_revalidation
|
||||
&& initial_age.as_secs()
|
||||
>= retention_deadline(
|
||||
ttl,
|
||||
stale_while_revalidate,
|
||||
stale_if_error,
|
||||
has_validator(headers),
|
||||
)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let expected_length = match headers.get("content-length") {
|
||||
Some(v) => Some(v.to_str().ok()?.parse::<usize>().ok()?),
|
||||
None => None,
|
||||
};
|
||||
if expected_length.is_some_and(|n| n > settings.max_object_bytes) {
|
||||
return None;
|
||||
}
|
||||
Some(Plan {
|
||||
received: Instant::now(),
|
||||
initial_age,
|
||||
ttl,
|
||||
stale_while_revalidate,
|
||||
stale_if_error,
|
||||
must_revalidate,
|
||||
proxy_revalidate,
|
||||
expected_length,
|
||||
requires_revalidation,
|
||||
vary,
|
||||
})
|
||||
}
|
||||
Vendored
+184
@@ -0,0 +1,184 @@
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ByteRange {
|
||||
start: Option<u64>,
|
||||
end: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RangeSelection {
|
||||
None,
|
||||
Single(ByteRange),
|
||||
Unsupported,
|
||||
}
|
||||
|
||||
pub fn request_range(method: &Method, headers: &HeaderMap) -> RangeSelection {
|
||||
let mut values = headers.get_all("range").iter();
|
||||
let Some(value) = values.next() else {
|
||||
return RangeSelection::None;
|
||||
};
|
||||
// Multiple field lines are equivalent to a combined range-set; RustProxy
|
||||
// intentionally delegates those to the origin with multipart ranges.
|
||||
if values.next().is_some() || headers.get_all("if-range").iter().count() > 1 {
|
||||
return RangeSelection::Unsupported;
|
||||
}
|
||||
// RustProxy serves one byte range itself. HEAD, unknown units, malformed
|
||||
// syntax and multipart requests remain transparent origin pass-through.
|
||||
if method != Method::GET {
|
||||
return RangeSelection::Unsupported;
|
||||
}
|
||||
let Ok(value) = value.to_str() else {
|
||||
return RangeSelection::Unsupported;
|
||||
};
|
||||
let Some(spec) = value.trim().strip_prefix("bytes=") else {
|
||||
return RangeSelection::Unsupported;
|
||||
};
|
||||
if spec.is_empty() || spec.contains(',') {
|
||||
return RangeSelection::Unsupported;
|
||||
}
|
||||
let Some((left, right)) = spec.split_once('-') else {
|
||||
return RangeSelection::Unsupported;
|
||||
};
|
||||
let start = if left.trim().is_empty() {
|
||||
None
|
||||
} else {
|
||||
match left.trim().parse::<u64>() {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => return RangeSelection::Unsupported,
|
||||
}
|
||||
};
|
||||
let end = if right.trim().is_empty() {
|
||||
None
|
||||
} else {
|
||||
match right.trim().parse::<u64>() {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => return RangeSelection::Unsupported,
|
||||
}
|
||||
};
|
||||
if start.is_none() && end.is_none() {
|
||||
return RangeSelection::Unsupported;
|
||||
}
|
||||
if let (Some(start), Some(end)) = (start, end) {
|
||||
if start > end {
|
||||
return RangeSelection::Unsupported;
|
||||
}
|
||||
}
|
||||
RangeSelection::Single(ByteRange { start, end })
|
||||
}
|
||||
|
||||
fn if_range_matches(headers: &HeaderMap, if_range: Option<&HeaderValue>) -> bool {
|
||||
let Some(if_range) = if_range else {
|
||||
return true;
|
||||
};
|
||||
let Ok(candidate) = if_range.to_str() else {
|
||||
return false;
|
||||
};
|
||||
let candidate = candidate.trim();
|
||||
if candidate.starts_with('"') || candidate.starts_with("W/\"") {
|
||||
if candidate.starts_with("W/") {
|
||||
return false;
|
||||
}
|
||||
return headers
|
||||
.get("etag")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.is_some_and(|etag| !etag.trim().starts_with("W/") && etag.trim() == candidate);
|
||||
}
|
||||
let Some(candidate_date) = httpdate::parse_http_date(candidate).ok() else {
|
||||
return false;
|
||||
};
|
||||
headers
|
||||
.get("last-modified")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| httpdate::parse_http_date(value).ok())
|
||||
.is_some_and(|modified| modified <= candidate_date)
|
||||
}
|
||||
|
||||
fn resolve_byte_range(range: ByteRange, len: usize) -> Option<(usize, usize)> {
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
let len_u64 = len as u64;
|
||||
match (range.start, range.end) {
|
||||
(Some(start), Some(end)) => {
|
||||
if start >= len_u64 {
|
||||
return None;
|
||||
}
|
||||
let end = end.min(len_u64.saturating_sub(1));
|
||||
Some((start as usize, end as usize))
|
||||
}
|
||||
(Some(start), None) => {
|
||||
if start >= len_u64 {
|
||||
None
|
||||
} else {
|
||||
Some((start as usize, len - 1))
|
||||
}
|
||||
}
|
||||
(None, Some(suffix)) => {
|
||||
if suffix == 0 {
|
||||
return None;
|
||||
}
|
||||
let take = suffix.min(len_u64) as usize;
|
||||
Some((len - take, len - 1))
|
||||
}
|
||||
(None, None) => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Hit {
|
||||
pub fn apply_range(mut self, range: ByteRange, if_range: Option<&HeaderValue>) -> Self {
|
||||
if !if_range_matches(self.response.headers(), if_range) {
|
||||
return self;
|
||||
}
|
||||
let full_len = self.body.len();
|
||||
self.response
|
||||
.headers_mut()
|
||||
.insert("accept-ranges", HeaderValue::from_static("bytes"));
|
||||
self.response.extensions_mut().insert(crate::compression::Skip);
|
||||
let Some((start, end)) = resolve_byte_range(range, full_len) else {
|
||||
*self.response.status_mut() = StatusCode::RANGE_NOT_SATISFIABLE;
|
||||
*self.response.body_mut() = Body::empty();
|
||||
self.response.headers_mut().insert(
|
||||
"content-range",
|
||||
HeaderValue::from_str(&format!("bytes */{full_len}")).expect("generated range"),
|
||||
);
|
||||
self.response
|
||||
.headers_mut()
|
||||
.insert("content-length", HeaderValue::from_static("0"));
|
||||
return self;
|
||||
};
|
||||
let slice = self.body.slice(start..=end);
|
||||
*self.response.status_mut() = StatusCode::PARTIAL_CONTENT;
|
||||
*self.response.body_mut() = Body::from(slice.clone());
|
||||
self.response.headers_mut().insert(
|
||||
"content-range",
|
||||
HeaderValue::from_str(&format!("bytes {start}-{end}/{full_len}"))
|
||||
.expect("generated range"),
|
||||
);
|
||||
self.response.headers_mut().insert(
|
||||
"content-length",
|
||||
HeaderValue::from_str(&slice.len().to_string()).expect("generated length"),
|
||||
);
|
||||
self.body = slice;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Revalidation {
|
||||
pub fn response_with_range(
|
||||
&self,
|
||||
headers: HeaderMap,
|
||||
range: ByteRange,
|
||||
if_range: Option<&HeaderValue>,
|
||||
) -> Response {
|
||||
let hit = Hit {
|
||||
response: self.response(headers),
|
||||
body: self.body.clone(),
|
||||
hits: self.hits,
|
||||
transformed: self.transformed,
|
||||
remaining_ttl: self.ttl.saturating_sub(self.age),
|
||||
stale: false,
|
||||
background_revalidate: false,
|
||||
resident_age_seconds: self.resident_age_at_lookup.as_secs(),
|
||||
};
|
||||
hit.apply_range(range, if_range).response
|
||||
}
|
||||
}
|
||||
Vendored
+3153
File diff suppressed because it is too large.
Load diff
Vendored
+366
@@ -0,0 +1,366 @@
|
||||
impl Cache {
|
||||
/// Varnish-style primary hash: public URL/host identity first, with response
|
||||
/// `Vary` handled as a secondary variant key. Irrelevant request headers and
|
||||
/// load-balanced backend members therefore do not fragment the cache.
|
||||
pub fn ticket(
|
||||
&self,
|
||||
revision: u64,
|
||||
scheme: &str,
|
||||
authority: &str,
|
||||
route: &str,
|
||||
target: &str,
|
||||
headers: &HeaderMap,
|
||||
) -> Option<Ticket> {
|
||||
self.ticket_scoped(revision, "", scheme, authority, route, target, headers)
|
||||
}
|
||||
pub fn ticket_scoped(
|
||||
&self,
|
||||
revision: u64,
|
||||
scope: &str,
|
||||
scheme: &str,
|
||||
authority: &str,
|
||||
_route: &str,
|
||||
target: &str,
|
||||
headers: &HeaderMap,
|
||||
) -> Option<Ticket> {
|
||||
if target.len() > 8192 || scope.len() > 64 {
|
||||
return None;
|
||||
}
|
||||
let domain = crate::config::request_host(authority)
|
||||
.unwrap_or_else(|_| authority.chars().take(253).collect());
|
||||
// Varnish built-in hashing lowercases Host before hashing, but otherwise
|
||||
// keeps the authority (including an explicit port) distinct. RustProxy has
|
||||
// already validated the authority before this point.
|
||||
let hash_scheme = scheme.to_ascii_lowercase();
|
||||
let hash_authority = authority.to_ascii_lowercase();
|
||||
let mut digest = Sha256::new();
|
||||
let mut field = |value: &[u8]| {
|
||||
digest.update((value.len() as u64).to_be_bytes());
|
||||
digest.update(value);
|
||||
};
|
||||
let revision_text = revision.to_string();
|
||||
for value in [
|
||||
revision_text.as_bytes(),
|
||||
scope.as_bytes(),
|
||||
hash_scheme.as_bytes(),
|
||||
hash_authority.as_bytes(),
|
||||
target.as_bytes(),
|
||||
] {
|
||||
field(value);
|
||||
}
|
||||
let key: Key = digest.finalize().into();
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(store.revision == revision).then_some(Ticket {
|
||||
scheme: hash_scheme,
|
||||
authority: hash_authority,
|
||||
domain,
|
||||
path: target
|
||||
.split('?')
|
||||
.next()
|
||||
.unwrap_or("/")
|
||||
.chars()
|
||||
.take(512)
|
||||
.collect(),
|
||||
target: target.to_owned(),
|
||||
key,
|
||||
request_headers: headers.clone(),
|
||||
generation: store.generation,
|
||||
revision,
|
||||
scope: scope.to_owned(),
|
||||
scope_generation: store.scope_generations.get(scope).copied().unwrap_or(0),
|
||||
key_generation: store.key_generations.get(&key).copied().unwrap_or(0),
|
||||
created_at: SystemTime::now(),
|
||||
})
|
||||
}
|
||||
pub fn revalidation(&self, ticket: &Ticket) -> Option<Revalidation> {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let expired = store.sweep();
|
||||
self.counters
|
||||
.expired
|
||||
.fetch_add(expired as u64, Ordering::Relaxed);
|
||||
if store.revision != ticket.revision
|
||||
|| store.generation != ticket.generation
|
||||
|| store
|
||||
.scope_generations
|
||||
.get(&ticket.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.scope_generation
|
||||
|| store
|
||||
.key_generations
|
||||
.get(&ticket.key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
!= ticket.key_generation
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let storage_key = matching_hot_key(&store, ticket)?;
|
||||
if store
|
||||
.entries
|
||||
.get(&storage_key)
|
||||
.is_some_and(|entry| self.hot_entry_banned(&storage_key, entry))
|
||||
{
|
||||
store.remove(&storage_key);
|
||||
drop(store);
|
||||
release_allocator_memory();
|
||||
return None;
|
||||
}
|
||||
let entry = store.entries.get(&storage_key)?;
|
||||
let age = entry
|
||||
.initial_age
|
||||
.saturating_add(entry.received.elapsed())
|
||||
.as_secs();
|
||||
Some(Revalidation {
|
||||
headers: entry.headers.clone(),
|
||||
body: entry.body.clone(),
|
||||
body_allocated: entry.body_allocated,
|
||||
_transient_leases: Vec::new(),
|
||||
hits: entry.hits,
|
||||
transformed: entry.transformed,
|
||||
storage_key,
|
||||
resident_age_at_lookup: hot_resident_age(entry),
|
||||
skip_compression: entry.skip_compression,
|
||||
reusable_without_validation: age < entry.ttl && !entry.requires_revalidation,
|
||||
age,
|
||||
ttl: entry.ttl,
|
||||
stale_if_error: entry.stale_if_error,
|
||||
must_revalidate: entry.must_revalidate,
|
||||
proxy_revalidate: entry.proxy_revalidate,
|
||||
origin_etag: entry.origin_etag.clone(),
|
||||
origin_last_modified: entry.origin_last_modified.clone(),
|
||||
})
|
||||
}
|
||||
async fn cold_revalidation(&self, ticket: &Ticket) -> Option<Revalidation> {
|
||||
if !self.ticket_current(ticket) {
|
||||
return None;
|
||||
}
|
||||
let banned = {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
matching_cold_key(&cold, ticket).and_then(|storage_key| {
|
||||
let banned = cold
|
||||
.entries
|
||||
.get(&storage_key)
|
||||
.is_some_and(|entry| self.cold_entry_banned(&storage_key, entry));
|
||||
banned.then(|| cold.remove(&storage_key)).flatten()
|
||||
})
|
||||
};
|
||||
if let Some(entry) = banned {
|
||||
self.delete_cold_entry(entry);
|
||||
return None;
|
||||
}
|
||||
let (storage_key, entry) = {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let storage_key = matching_cold_key(&cold, ticket)?;
|
||||
let old_sequence = {
|
||||
let entry = cold.entries.get(&storage_key)?;
|
||||
if entry.revision != ticket.revision || entry.scope != ticket.scope {
|
||||
return None;
|
||||
}
|
||||
let age = cold_age(entry);
|
||||
if age
|
||||
>= retention_deadline(
|
||||
entry.ttl,
|
||||
entry.stale_while_revalidate,
|
||||
entry.stale_if_error,
|
||||
entry.origin_etag.is_some() || entry.origin_last_modified.is_some(),
|
||||
)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
entry.sequence
|
||||
};
|
||||
let sequence = cold.next_sequence();
|
||||
cold.lru.remove(&old_sequence);
|
||||
cold.lru.insert(sequence, storage_key);
|
||||
let entry = cold.entries.get_mut(&storage_key)?;
|
||||
entry.sequence = sequence;
|
||||
entry.last_access = SystemTime::now();
|
||||
entry.pinned_reads = entry.pinned_reads.saturating_add(1);
|
||||
(storage_key, entry.clone())
|
||||
};
|
||||
let _pin = ColdReadGuard {
|
||||
cache: self,
|
||||
key: storage_key,
|
||||
generation: entry.generation,
|
||||
};
|
||||
let settings = self
|
||||
.tier_settings
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.clone();
|
||||
if entry.body_bytes > settings.max_object_bytes {
|
||||
self.counters
|
||||
.memory_pressure_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
return None;
|
||||
}
|
||||
let read_lease = self.try_reserve_transient(entry.body_bytes, &settings)?;
|
||||
let _cold_read = ByteGaugeGuard::new(&self.cold_read_bytes_active, entry.body_bytes as u64);
|
||||
let body_vec = match tokio::fs::read(&entry.body_path).await {
|
||||
Ok(body) if cold_body_matches(&entry, &body) => body,
|
||||
_ => {
|
||||
self.counters.cold_errors.fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(entry) = self.take_cold(&storage_key) {
|
||||
self.delete_cold_entry(entry);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let body_allocated = body_vec.capacity().max(body_vec.len());
|
||||
let extra = body_allocated.saturating_sub(entry.body_bytes);
|
||||
let extra_lease = if extra > 0 {
|
||||
Some(self.try_reserve_transient(extra, &settings)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let _cold_read_extra = extra_lease
|
||||
.as_ref()
|
||||
.map(|_| ByteGaugeGuard::new(&self.cold_read_bytes_active, extra as u64));
|
||||
let body = Bytes::from(body_vec);
|
||||
if !self.ticket_current(ticket) {
|
||||
return None;
|
||||
}
|
||||
let age = cold_age(&entry);
|
||||
let resident_age_at_lookup = Duration::from_secs(cold_resident_age(&entry));
|
||||
Some(Revalidation {
|
||||
headers: entry.headers,
|
||||
body,
|
||||
body_allocated,
|
||||
_transient_leases: extra_lease
|
||||
.into_iter()
|
||||
.chain(std::iter::once(read_lease))
|
||||
.collect(),
|
||||
hits: entry.hits,
|
||||
transformed: entry.transformed,
|
||||
storage_key,
|
||||
resident_age_at_lookup,
|
||||
skip_compression: entry.skip_compression,
|
||||
reusable_without_validation: age < entry.ttl && !entry.requires_revalidation,
|
||||
age,
|
||||
ttl: entry.ttl,
|
||||
stale_if_error: entry.stale_if_error,
|
||||
must_revalidate: entry.must_revalidate,
|
||||
proxy_revalidate: entry.proxy_revalidate,
|
||||
origin_etag: entry.origin_etag,
|
||||
origin_last_modified: entry.origin_last_modified,
|
||||
})
|
||||
}
|
||||
pub async fn revalidation_tiered(&self, ticket: &Ticket) -> Option<Revalidation> {
|
||||
if let Some(cached) = self.revalidation(ticket) {
|
||||
return Some(cached);
|
||||
}
|
||||
self.cold_revalidation(ticket).await
|
||||
}
|
||||
pub fn stale_if_error_hit(&self, cached: &Revalidation) -> Option<Hit> {
|
||||
if !cached.can_stale_if_error() {
|
||||
return None;
|
||||
}
|
||||
let hits = cached.hits.saturating_add(1);
|
||||
let mut headers = cached.headers.clone();
|
||||
headers.insert(
|
||||
"age",
|
||||
HeaderValue::from_str(&cached.age.to_string()).ok()?,
|
||||
);
|
||||
headers.append(
|
||||
"warning",
|
||||
HeaderValue::from_static("110 RustProxy \"Response is stale\""),
|
||||
);
|
||||
let response = cached.response(headers);
|
||||
let touched_hot = {
|
||||
let mut store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(entry) = store.entries.get_mut(&cached.storage_key) {
|
||||
entry.hits = hits;
|
||||
entry.last_access = Instant::now();
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
if !touched_hot {
|
||||
let mut cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if let Some(mut entry) = cold.remove(&cached.storage_key) {
|
||||
entry.hits = hits;
|
||||
entry.last_access = SystemTime::now();
|
||||
entry.sequence = cold.next_sequence();
|
||||
cold.bytes = cold.bytes.saturating_add(entry.disk_bytes);
|
||||
cold.lru.insert(entry.sequence, cached.storage_key);
|
||||
cold.index_variant(entry.base_key, cached.storage_key);
|
||||
cold.entries.insert(cached.storage_key, entry);
|
||||
}
|
||||
}
|
||||
self.counters.hits.fetch_add(1, Ordering::Relaxed);
|
||||
Some(Hit {
|
||||
response,
|
||||
body: cached.body.clone(),
|
||||
hits,
|
||||
transformed: cached.transformed,
|
||||
remaining_ttl: 0,
|
||||
resident_age_seconds: cached.resident_age_at_lookup.as_secs(),
|
||||
stale: true,
|
||||
background_revalidate: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn begin_background_revalidation(
|
||||
self: &Arc<Self>,
|
||||
cached: &Revalidation,
|
||||
) -> Option<BackgroundRevalidationGuard> {
|
||||
// Vary variants share a base Ticket key. Deduplicate only the exact cached
|
||||
// representation so an SWR refresh for one variant cannot suppress another.
|
||||
let key = cached.storage_key;
|
||||
let mut active = self.revalidating.lock().unwrap_or_else(|e| e.into_inner());
|
||||
if !active.insert(key) {
|
||||
return None;
|
||||
}
|
||||
Some(BackgroundRevalidationGuard {
|
||||
cache: Arc::downgrade(self),
|
||||
key,
|
||||
})
|
||||
}
|
||||
pub fn ticket_current(&self, ticket: &Ticket) -> bool {
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
store.revision == ticket.revision
|
||||
&& store.generation == ticket.generation
|
||||
&& store
|
||||
.scope_generations
|
||||
.get(&ticket.scope)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== ticket.scope_generation
|
||||
&& store
|
||||
.key_generations
|
||||
.get(&ticket.key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
== ticket.key_generation
|
||||
}
|
||||
fn discard_storage_key(&self, key: &Key) {
|
||||
self.store
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.remove(key);
|
||||
if let Some(entry) = self.take_cold(key) {
|
||||
self.delete_cold_entry(entry);
|
||||
}
|
||||
}
|
||||
pub fn discard(&self, ticket: &Ticket) {
|
||||
let hot_key = {
|
||||
let store = self.store.lock().unwrap_or_else(|e| e.into_inner());
|
||||
(store.revision == ticket.revision)
|
||||
.then(|| matching_hot_key(&store, ticket))
|
||||
.flatten()
|
||||
};
|
||||
if let Some(key) = hot_key {
|
||||
self.discard_storage_key(&key);
|
||||
return;
|
||||
}
|
||||
let cold_key = {
|
||||
let cold = self.cold.lock().unwrap_or_else(|e| e.into_inner());
|
||||
matching_cold_key(&cold, ticket)
|
||||
};
|
||||
if let Some(key) = cold_key {
|
||||
self.discard_storage_key(&key);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user