v0.32.2

src/admin/openapi.json

@@ -2,7 +2,7 @@
   "openapi": "3.0.3",
   "info": {
     "title": "RustProxy Administration API",
-    "version": "0.32.1",
+    "version": "0.32.2",
     "description": "Administration API for the active panel origin. Sign in through the panel; the same-origin HttpOnly cookie authenticates all private operations. Mutations also require the exact Origin, X-RustProxy-Request: 1 and a session-bound X-CSRF-Token. Loopback HTTP is the default; Settings can enable HTTPS before using a non-loopback administration bind. This document contains no live configuration or credentials. Object schemas marked extensible are summaries: full configuration validation is authoritative in Rust."
   },
   "servers": [

src/proxy/forward/web_source.rs

@@ -68,7 +68,15 @@
     let range = crate::cache::request_range(&request_method, &request_headers);
     let if_range = request_headers.get("if-range").cloned();
     let range_requested = !matches!(range, crate::cache::RangeSelection::None);
-    let needs_validation = policy.revalidation_reason().is_some();
+    // Unlike the original Web Server path, retain stale and client-forced
+    // revalidation candidates. A local 304 can refresh them without replacing
+    // the cached body or resetting X-Cache-Hits (same as Reverse Proxy).
+    let cached_revalidation = if let Some(ticket) = cache_ticket.as_ref() {
+        state.cache.revalidation_tiered(ticket).await
+    } else { None };
+    let needs_validation = policy.revalidation_reason().is_some()
+        || cached_revalidation.as_ref().is_some_and(|cached|
+            !cached.reusable_for_request(&request_headers));
     if let Some(ticket) = &cache_ticket {
         if !needs_validation {
             if let Some(mut hit) = state.cache.get_tiered(ticket, cache_settings).await {
@@ -114,9 +122,12 @@
     // Neither the Request nor the streaming Body is held across a borrowed
     // await: the filesystem/PHP source owns the body for its full lifetime.
     let static_body = std::mem::replace(request.body_mut(), Body::empty());
+    let (origin_request_headers, revalidation_sent) =
+        web_cache_revalidation::web_origin_request_headers(
+            &request_headers, cached_revalidation.as_ref());
     let started = Instant::now();
     let response = crate::web_server::serve(service, request_method.clone(),
-        request_headers.clone(), request_uri, &service_path, static_body,
+        origin_request_headers, request_uri, &service_path, static_body,
         crate::web_server::RequestContext {
             client_ip: context.client_ip,
             client_port: Some(peer.0.port()),
@@ -131,6 +142,45 @@
     ).await { return fallback; }
 
     let mut response = intercept_route_origin_error(response, route, &snapshot);
+    if revalidation_sent && response.status() == StatusCode::NOT_MODIFIED {
+        if let (Some(ticket), Some(cached)) =
+            (cache_ticket.as_ref(), cached_revalidation.as_ref())
+        {
+            let revalidated = web_cache_revalidation::finish_web_revalidation(
+                &state.cache, ticket, cached, response.headers(), cache_settings,
+                started.elapsed(), &request_headers, &request_method,
+                ignore_cookie, force_cache,
+                |headers| apply_response_rules(headers, &snapshot,
+                    route.frontend.as_deref(), Some(route), false),
+            );
+            if let Some(mut hit) = revalidated {
+                state.cache.record(scope, &host, "HIT", hit.reason);
+                if hit.transformed { state.transform_metrics.record_cache_hit(scope); }
+                if let Some(exchange) = request.extensions().get::<Arc<Exchange>>() {
+                    exchange.cache_status.store(1, Ordering::Relaxed);
+                    *exchange.cache_reason.lock().unwrap_or_else(|e| e.into_inner()) = hit.reason;
+                }
+                hit.response.extensions_mut().insert(ResponseRulesApplied);
+                hit.response.extensions_mut().insert(StaticSourceResponse);
+                return intercept_route_origin_error(hit.response, route, &snapshot);
+            }
+            // A simultaneous PURGE/config change invalidated the ticket. Do
+            // not accidentally return an origin 304 without a reusable body.
+            return error(StatusCode::SERVICE_UNAVAILABLE,
+                "RustProxy: Web Server cache entry invalidated during revalidation\n");
+        }
+    }
+    if cached_revalidation.is_some() && response.status() != StatusCode::NOT_MODIFIED
+        && response.status().as_u16() < 500
+    {
+        // A changed response (including a 206, 404 or an uncacheable 200)
+        // supersedes the old object. Also discard when the old entry has no
+        // validators and therefore must be fetched unconditionally. A complete
+        // cacheable GET below can refill the entry.
+        if let Some(ticket) = cache_ticket.as_ref() {
+            state.cache.discard(ticket);
+        }
+    }
     let origin_headers = response.headers().clone();
     let original_plan = crate::cache::plan(response.status(), &origin_headers,
         cache_settings, started.elapsed());

src/proxy/mod.rs

@@ -1,6 +1,7 @@
 include!("types.rs");
 include!("error_fallback.rs");
 include!("cache_helpers.rs");
+mod web_cache_revalidation;
 include!("cache_http_invalidation.rs");
 include!("upstream_helpers.rs");
 include!("redirect_helpers.rs");

src/proxy/web_cache_revalidation.rs

@@ -0,0 +1,129 @@
+//! Conditional validation for a local Web Server origin. The reverse-proxy
+//! source has an equivalent 304 path; both must preserve the stored body and
+//! its hit counter when the origin reports that the representation is unchanged.
+
+use crate::cache::{Cache, RangeSelection, Revalidation, Settings, Ticket};
+use axum::{body::Body, response::Response};
+use http::{HeaderMap, HeaderValue, Method, StatusCode};
+use std::time::Duration;
+
+pub(super) struct WebRevalidated {
+    pub response: Response,
+    pub transformed: bool,
+    pub reason: &'static str,
+}
+
+/// The client's validators describe the *public* representation (possibly
+/// transformed). Only validators belonging to our stored origin representation
+/// may be sent to the Web Server. Never allow an unrelated client ETag to turn
+/// into a 304 that would incorrectly validate our cached body.
+pub(super) fn web_origin_request_headers(
+    client: &HeaderMap,
+    cached: Option<&Revalidation>,
+) -> (HeaderMap, bool) {
+    let mut headers = client.clone();
+    let Some(cached) = cached else { return (headers, false) };
+    headers.remove("if-none-match");
+    headers.remove("if-modified-since");
+    let sent = cached.apply_validators(&mut headers);
+    (headers, sent)
+}
+
+/// A transformed body uses public representation-specific headers. A local
+/// origin's 304 can describe its own encoding/ranges, which must not be copied
+/// onto the cached, already transformed body.
+fn web_merged_validation_headers(cached: &Revalidation, validation: &HeaderMap) -> HeaderMap {
+    let mut merged = cached.merge_validation_headers(validation);
+    if cached.transformed() {
+        for name in [
+            "content-encoding", "vary", "accept-ranges", "content-md5", "digest", "content-digest",
+        ] {
+            merged.remove(name);
+            for value in cached.headers().get_all(name) {
+                merged.append(name, value.clone());
+            }
+        }
+    }
+    merged
+}
+
+/// Handle a real 304 from the local origin, *after* matching it to a stored
+/// object using our own validator. A successful validation increases HITs and
+/// refreshes freshness metadata; an uncacheable 304 is served once and purged.
+/// `apply_rules` keeps vhost response rules identical to the ordinary Web
+/// Server 200/fill path. Invalidated tickets are never resurrected.
+pub(super) fn finish_web_revalidation(
+    cache: &Cache,
+    ticket: &Ticket,
+    cached: &Revalidation,
+    validation: &HeaderMap,
+    settings: &Settings,
+    elapsed: Duration,
+    client: &HeaderMap,
+    method: &Method,
+    ignore_cookie: bool,
+    force_cache: bool,
+    apply_rules: impl FnOnce(&mut HeaderMap),
+) -> Option<WebRevalidated> {
+    let origin_merged = web_merged_validation_headers(cached, validation);
+    // Vhost response rules must never turn a newly private/no-store origin
+    // into a cacheable shared representation.
+    let origin_safe = crate::cache::plan(StatusCode::OK, &origin_merged, settings, elapsed).is_some();
+    let mut merged = origin_merged;
+    apply_rules(&mut merged);
+    if force_cache {
+        if client.contains_key("cookie") {
+            merged.append("vary", HeaderValue::from_static("Cookie"));
+        }
+        if client.contains_key("authorization") {
+            merged.append("vary", HeaderValue::from_static("Authorization"));
+        }
+    }
+    let can_store = origin_safe && !(ignore_cookie && crate::cache::varies_on(&merged, "cookie"));
+    let client_not_modified = crate::cache::conditional_not_modified(client, &merged);
+    let range = crate::cache::request_range(method, client);
+    let if_range = client.get("if-range");
+    let published = if can_store {
+        cache.publish_revalidated(ticket.clone(), cached, merged.clone(), validation, settings, elapsed)
+    } else { None };
+    let (mut response, hits, ttl, resident_age, transformed, reason) =
+        if let Some(mut hit) = published {
+            if !client_not_modified {
+                if let RangeSelection::Single(selection) = range {
+                    hit = hit.apply_range(selection, if_range);
+                }
+            }
+            (
+                hit.response, hit.hits, hit.remaining_ttl, hit.resident_age_seconds,
+                hit.transformed, "cache_revalidated",
+            )
+        } else {
+            if !cache.ticket_current(ticket) {
+                return None;
+            }
+            cache.discard(ticket);
+            let response = if !client_not_modified {
+                if let RangeSelection::Single(selection) = range {
+                    cached.response_with_range(merged, selection, if_range)
+                } else {
+                    cached.response(merged)
+                }
+            } else {
+                cached.response(merged)
+            };
+            (response, 0, 0, 0, cached.transformed(), "cache_revalidated_uncached")
+        };
+    if client_not_modified {
+        response = crate::cache::make_not_modified(response);
+    }
+    if *method == Method::HEAD {
+        *response.body_mut() = Body::empty();
+    }
+    settings.annotate_reason(response.headers_mut(), "HIT", hits, ttl, reason);
+    settings.annotate_transformed(response.headers_mut(), transformed);
+    settings.annotate_resident_age(response.headers_mut(), resident_age);
+    Some(WebRevalidated { response, transformed, reason })
+}
+
+#[cfg(test)]
+mod tests;

src/proxy/web_cache_revalidation/tests.rs

@@ -0,0 +1,166 @@
+use super::*;
+use crate::{cache, web_server};
+use http_body_util::BodyExt;
+use tempfile::TempDir;
+use std::sync::Arc;
+
+fn cache_settings() -> Settings {
+    Settings {
+        enabled: true,
+        expose_status: true,
+        expose_hits: true,
+        expose_cache_status: true,
+        ..Default::default()
+    }
+}
+
+async fn fixture() -> (TempDir, web_server::CompiledService, Arc<Cache>, Ticket, Settings) {
+    let dir = TempDir::new().unwrap();
+    std::fs::write(dir.path().join("shCore.js"), b"before").unwrap();
+    let service = web_server::Service {
+        id: "wordpress".into(),
+        name: "WordPress".into(),
+        document_root: dir.path().to_path_buf(),
+        cache_control: Some("public, max-age=604800".into()),
+        ..Default::default()
+    }.compile().unwrap();
+    let cache = Cache::new(1);
+    let settings = cache_settings();
+    let ticket = cache.ticket(
+        1, "https", "blog.example.test", "wordpress",
+        "/shCore.js?ver=3.0.9b", &HeaderMap::new(),
+    ).unwrap();
+    let initial = serve_file(&service, Method::GET, HeaderMap::new()).await;
+    assert_eq!(initial.status(), StatusCode::OK);
+    let plan = cache::plan(initial.status(), initial.headers(), &settings, Duration::ZERO)
+        .expect("static origin response is cacheable");
+    cache.wrap(initial, ticket.clone(), plan, &settings)
+        .into_body().collect().await.unwrap();
+    (dir, service, cache, ticket, settings)
+}
+
+async fn serve_file(service: &web_server::CompiledService, method: Method, headers: HeaderMap) -> Response {
+    web_server::serve(
+        service, method, headers,
+        "/shCore.js?ver=3.0.9b".parse().unwrap(), "/shCore.js", Body::empty(),
+        web_server::RequestContext {
+            client_ip: "127.0.0.1".parse().unwrap(),
+            client_port: Some(12345),
+            server_name: "blog.example.test".into(),
+            server_port: 443,
+            tls: true,
+            version: http::Version::HTTP_2,
+        },
+    ).await
+}
+
+async fn validate(
+    service: &web_server::CompiledService, cache: &Cache, ticket: &Ticket,
+    settings: &Settings, method: Method, client: HeaderMap,
+) -> Option<WebRevalidated> {
+    let cached = cache.revalidation_tiered(ticket).await.unwrap();
+    let (headers, sent) = web_origin_request_headers(&client, Some(&cached));
+    assert!(sent, "stored validator must be used for local source revalidation");
+    let origin = serve_file(service, method.clone(), headers).await;
+    if origin.status() != StatusCode::NOT_MODIFIED {
+        return None;
+    }
+    finish_web_revalidation(cache, ticket, &cached, origin.headers(), settings,
+        Duration::ZERO, &client, &method, false, false, |_| {})
+}
+
+#[tokio::test]
+async fn browser_refresh_304_preserves_web_server_hit_counter() {
+    let (_dir, service, cache, ticket, settings) = fixture().await;
+    assert_eq!(cache.get_tiered(&ticket, &settings).await.unwrap().hits, 1);
+    for directive in ["max-age=0", "no-cache"] {
+        let mut client = HeaderMap::new();
+        client.insert("cache-control", directive.parse().unwrap());
+        let response = validate(&service, &cache, &ticket, &settings, Method::GET, client)
+            .await.unwrap().response;
+        assert_eq!(response.status(), StatusCode::OK);
+        assert_eq!(response.headers()["x-cache"], "HIT");
+        let hits = response.headers()["x-cache-hits"].to_str().unwrap().parse::<u64>().unwrap();
+        assert!(hits >= 2);
+        assert_eq!(response.headers()["x-cache-reason"], "cache_revalidated");
+        assert_eq!(response.into_body().collect().await.unwrap().to_bytes(), "before");
+    }
+    assert_eq!(cache.get_tiered(&ticket, &settings).await.unwrap().hits, 4);
+}
+
+#[tokio::test]
+async fn browser_conditional_refresh_returns_304_without_resetting_hits() {
+    let (_dir, service, cache, ticket, settings) = fixture().await;
+    let etag = cache.revalidation_tiered(&ticket).await.unwrap().headers()["etag"].clone();
+    let mut client = HeaderMap::new();
+    client.insert("cache-control", HeaderValue::from_static("no-cache"));
+    client.insert("if-none-match", etag);
+    let response = validate(&service, &cache, &ticket, &settings, Method::GET, client)
+        .await.unwrap().response;
+    assert_eq!(response.status(), StatusCode::NOT_MODIFIED);
+    assert!(!response.headers().contains_key("content-length"));
+    assert_eq!(response.headers()["x-cache-hits"], "1");
+    assert!(response.into_body().collect().await.unwrap().to_bytes().is_empty());
+    assert_eq!(cache.get_tiered(&ticket, &settings).await.unwrap().hits, 2);
+}
+
+#[tokio::test]
+async fn mismatched_browser_validator_cannot_validate_different_cached_object() {
+    let (_dir, service, cache, ticket, settings) = fixture().await;
+    let mut client = HeaderMap::new();
+    client.insert("cache-control", HeaderValue::from_static("no-cache"));
+    client.insert("if-none-match", HeaderValue::from_static("\"unrelated\""));
+    let response = validate(&service, &cache, &ticket, &settings, Method::GET, client)
+        .await.unwrap().response;
+    assert_eq!(response.status(), StatusCode::OK);
+    assert_eq!(response.into_body().collect().await.unwrap().to_bytes(), "before");
+}
+
+#[tokio::test]
+async fn range_and_head_requests_use_validated_cached_body() {
+    let (_dir, service, cache, ticket, settings) = fixture().await;
+    let mut range = HeaderMap::new();
+    range.insert("cache-control", HeaderValue::from_static("max-age=0"));
+    range.insert("range", HeaderValue::from_static("bytes=0-2"));
+    let partial = validate(&service, &cache, &ticket, &settings, Method::GET, range)
+        .await.unwrap().response;
+    assert_eq!(partial.status(), StatusCode::PARTIAL_CONTENT);
+    assert_eq!(partial.into_body().collect().await.unwrap().to_bytes(), "bef");
+    let head = validate(&service, &cache, &ticket, &settings, Method::HEAD, HeaderMap::new())
+        .await.unwrap().response;
+    assert_eq!(head.status(), StatusCode::OK);
+    assert!(head.into_body().collect().await.unwrap().to_bytes().is_empty());
+}
+
+#[tokio::test]
+async fn changed_local_file_is_not_falsely_revalidated() {
+    let (dir, service, cache, ticket, settings) = fixture().await;
+    // A size change guarantees different ETag even on filesystems with
+    // coarse modification-time resolution.
+    std::fs::write(dir.path().join("shCore.js"), b"new file content").unwrap();
+    let cached = cache.revalidation_tiered(&ticket).await.unwrap();
+    let (headers, sent) = web_origin_request_headers(&HeaderMap::new(), Some(&cached));
+    assert!(sent);
+    let origin = serve_file(&service, Method::GET, headers).await;
+    assert_eq!(origin.status(), StatusCode::OK);
+    let plan = cache::plan(origin.status(), origin.headers(), &settings, Duration::ZERO).unwrap();
+    cache.discard(&ticket);
+    cache.wrap(origin, ticket.clone(), plan, &settings)
+        .into_body().collect().await.unwrap();
+    assert_eq!(cache.get_tiered(&ticket, &settings).await.unwrap().hits, 1);
+}
+
+#[tokio::test]
+async fn 304_new_private_policy_discards_cached_representation() {
+    let (_dir, _service, cache, ticket, settings) = fixture().await;
+    let cached = cache.revalidation_tiered(&ticket).await.unwrap();
+    let mut headers = HeaderMap::new();
+    headers.insert("cache-control", HeaderValue::from_static("private, max-age=300"));
+    headers.insert("etag", cached.headers()["etag"].clone());
+    let outcome = finish_web_revalidation(&cache, &ticket, &cached, &headers,
+        &settings, Duration::ZERO, &HeaderMap::new(), &Method::GET,
+        false, false, |_| {}).unwrap();
+    assert_eq!(outcome.reason, "cache_revalidated_uncached");
+    assert_eq!(outcome.response.into_body().collect().await.unwrap().to_bytes(), "before");
+    assert!(cache.revalidation_tiered(&ticket).await.is_none());
+}