v0.24.5

src/runtime.rs

@@ -0,0 +1,621 @@
+use crate::{
+    config::{self, Snapshot, Update, ValidationPolicy},
+    db::Store,
+    logging::Logs,
+    metrics::Metrics,
+};
+use arc_swap::ArcSwap;
+use axum::{
+    http::StatusCode,
+    response::{IntoResponse, Response},
+    Json,
+};
+use serde_json::json;
+use sha2::{Digest, Sha256};
+use std::{
+    collections::HashMap,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use tokio::sync::{Mutex, Semaphore};
+
+pub struct App {
+    pub basic_workers: Arc<Semaphore>,
+    pub basic_limits: crate::basic_auth::Limits,
+    pub active_prometheus: crate::prometheus::Settings,
+    pub active_administration: crate::settings::Administration,
+    pub active_administration_tls_policy: crate::tls::ProtocolPolicy,
+    pub active_administration_certificate_files: Option<(String, String)>,
+    pub max_inflight: usize,
+    pub config: ArcSwap<Snapshot>,
+    pub store: Store,
+    pub policy: ValidationPolicy,
+    pub active_listeners: Vec<crate::vhost::Listener>,
+    pub active_streams: Vec<crate::stream::Host>,
+    pub stream_metrics: HashMap<String, Arc<crate::stream::Metrics>>,
+    pub startup_listeners: (std::net::SocketAddr, Option<std::net::SocketAddr>),
+    pub startup_locks: crate::settings::StartupLocks,
+    pub auth: Arc<crate::auth::Auth>,
+    pub viewer_slots: Arc<Semaphore>,
+    pub certificate_slots: Arc<Semaphore>,
+    pub metrics: Arc<Metrics>,
+    pub transform_metrics: Arc<crate::transform::Metrics>,
+    pub cache: Arc<crate::cache::Cache>,
+    pub geoip: Arc<crate::geoip::Database>,
+    pub rate_limiter: Arc<crate::rate_limit::Limiter>,
+    pub acme: Arc<crate::acme::Manager>,
+    pub diagnostics: Arc<crate::diagnostics::Diagnostics>,
+    pub docker: Arc<crate::docker::Manager>,
+    pub logs: Arc<Logs>,
+    pub software_update: Arc<crate::update_check::SharedStatus>,
+    pub shutdown: tokio_util::sync::CancellationToken,
+    pub tunnels: tokio_util::task::TaskTracker,
+    pub tunnel_registry: Arc<crate::proxy::websocket::TunnelRegistry>,
+    pub websocket_idle_timeout: Duration,
+    pub inflight: Arc<Semaphore>,
+    pub started: Instant,
+    pub header_timeout: Duration,
+    pub body_idle_timeout: Duration,
+    update_lock: Mutex<()>,
+    update_slots: Arc<Semaphore>,
+}
+#[derive(Debug)]
+pub struct ApiError(pub StatusCode, pub String);
+impl IntoResponse for ApiError {
+    fn into_response(self) -> Response {
+        let mut response = (self.0, Json(json!({"error": self.1}))).into_response();
+        if self.0 == StatusCode::TOO_MANY_REQUESTS {
+            response
+                .headers_mut()
+                .insert("retry-after", http::HeaderValue::from_static("60"));
+        }
+        response
+    }
+}
+#[derive(Debug)]
+enum CacheInvalidation {
+    None,
+    All(u64),
+}
+
+/// Stable cache compatibility identity. Unlike the configuration revision, this
+/// changes only when a setting can alter an origin request or cached response.
+/// Keeping it stable across logging/TLS/admin saves lets the persistent cold
+/// tier survive a restart after unrelated control-plane edits.
+fn cache_revision(config: &config::Config) -> u64 {
+    let encoded = serde_json::to_vec(&(
+        &config.cache,
+        &config.headers,
+        &config.forwarding,
+        &config.vhosts,
+    ))
+    .expect("serializable cache compatibility view");
+    let digest = Sha256::digest(encoded);
+    u64::from_be_bytes(digest[..8].try_into().expect("sha256 prefix"))
+}
+
+fn cache_invalidation(previous: &config::Config, next: &config::Config) -> CacheInvalidation {
+    let before = cache_revision(previous);
+    let after = cache_revision(next);
+    if before == after {
+        CacheInvalidation::None
+    } else {
+        CacheInvalidation::All(after)
+    }
+}
+
+impl App {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        store: Store,
+        snapshot: Snapshot,
+        policy: ValidationPolicy,
+        auth: Arc<crate::auth::Auth>,
+        max_inflight: usize,
+        header_timeout: Duration,
+        body_idle_timeout: Duration,
+        websocket_idle_timeout: Duration,
+    ) -> Arc<Self> {
+        let defaults = (policy.proxy_addr, policy.tls_addr);
+        Self::new_with_defaults(
+            store,
+            snapshot,
+            policy,
+            auth,
+            max_inflight,
+            header_timeout,
+            body_idle_timeout,
+            websocket_idle_timeout,
+            defaults,
+        )
+    }
+    #[allow(clippy::too_many_arguments)]
+    pub fn new_with_defaults(
+        store: Store,
+        snapshot: Snapshot,
+        policy: ValidationPolicy,
+        auth: Arc<crate::auth::Auth>,
+        max_inflight: usize,
+        header_timeout: Duration,
+        body_idle_timeout: Duration,
+        websocket_idle_timeout: Duration,
+        startup_listeners: (std::net::SocketAddr, Option<std::net::SocketAddr>),
+    ) -> Arc<Self> {
+        Self::new_with_defaults_geoip(
+            store,
+            snapshot,
+            policy,
+            auth,
+            max_inflight,
+            header_timeout,
+            body_idle_timeout,
+            websocket_idle_timeout,
+            startup_listeners,
+            Arc::new(crate::geoip::Database::empty()),
+        )
+    }
+    #[allow(clippy::too_many_arguments)]
+    pub fn new_with_defaults_geoip(
+        store: Store,
+        snapshot: Snapshot,
+        policy: ValidationPolicy,
+        auth: Arc<crate::auth::Auth>,
+        max_inflight: usize,
+        header_timeout: Duration,
+        body_idle_timeout: Duration,
+        websocket_idle_timeout: Duration,
+        startup_listeners: (std::net::SocketAddr, Option<std::net::SocketAddr>),
+        geoip: Arc<crate::geoip::Database>,
+    ) -> Arc<Self> {
+        Self::new_with_startup_locks_geoip(
+            store,
+            snapshot,
+            policy,
+            auth,
+            max_inflight,
+            header_timeout,
+            body_idle_timeout,
+            websocket_idle_timeout,
+            startup_listeners,
+            crate::settings::StartupLocks::default(),
+            geoip,
+        )
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    pub fn new_with_startup_locks_geoip(
+        store: Store,
+        snapshot: Snapshot,
+        policy: ValidationPolicy,
+        auth: Arc<crate::auth::Auth>,
+        max_inflight: usize,
+        header_timeout: Duration,
+        body_idle_timeout: Duration,
+        websocket_idle_timeout: Duration,
+        startup_listeners: (std::net::SocketAddr, Option<std::net::SocketAddr>),
+        startup_locks: crate::settings::StartupLocks,
+        geoip: Arc<crate::geoip::Database>,
+    ) -> Arc<Self> {
+        let shutdown = tokio_util::sync::CancellationToken::new();
+        let diagnostics = crate::diagnostics::Diagnostics::start(shutdown.clone());
+        let docker = crate::docker::Manager::new();
+        let cache_data_root = policy
+            .cert_dir
+            .as_ref()
+            .and_then(|path| path.parent())
+            .map(std::path::Path::to_path_buf);
+        let cache = crate::cache::Cache::start(
+            cache_revision(&snapshot.envelope.config),
+            cache_data_root,
+            snapshot.envelope.config.cache.clone(),
+            shutdown.clone(),
+        );
+        let active_listeners = snapshot.listeners.clone();
+        let active_streams = snapshot.envelope.config.streams.clone();
+        let stream_metrics = active_streams
+            .iter()
+            .map(|stream| {
+                (
+                    stream.id.clone(),
+                    Arc::new(crate::stream::Metrics::default()),
+                )
+            })
+            .collect();
+        let active_prometheus = snapshot.envelope.config.prometheus.clone();
+        let log_settings = snapshot.envelope.config.logging.clone();
+        let mut active_administration = snapshot.envelope.config.administration.clone();
+        let active_administration_tls_policy = snapshot.envelope.config.tls.policy.clone();
+        let active_administration_certificate_files = active_administration
+            .certificate_id
+            .as_deref()
+            .and_then(|id| {
+                snapshot
+                    .envelope
+                    .config
+                    .tls
+                    .certificates
+                    .iter()
+                    .find(|certificate| certificate.id == id)
+            })
+            .map(|certificate| (certificate.cert_file.clone(), certificate.key_file.clone()));
+        active_administration.listen = Some(policy.admin_addr);
+        if let Some(id) = active_administration.certificate_id.as_deref() {
+            if let Some(certificate) = snapshot
+                .envelope
+                .config
+                .tls
+                .certificates
+                .iter()
+                .find(|certificate| certificate.id == id)
+            {
+                active_administration
+                    .allowed_hosts
+                    .extend(certificate.hosts.clone());
+                active_administration.allowed_hosts.sort();
+                active_administration.allowed_hosts.dedup();
+            }
+        }
+        let logs = Logs::start(log_settings, shutdown.clone());
+        let app = Arc::new(Self {
+            basic_workers: Arc::new(Semaphore::new(2)),
+            basic_limits: Default::default(),
+            active_prometheus,
+            active_administration,
+            active_administration_tls_policy,
+            active_administration_certificate_files,
+            max_inflight,
+            active_listeners,
+            active_streams,
+            stream_metrics,
+            cache,
+            geoip,
+            rate_limiter: Arc::new(crate::rate_limit::Limiter::new()),
+            acme: crate::acme::Manager::new(),
+            diagnostics,
+            docker,
+            config: ArcSwap::from_pointee(snapshot),
+            store,
+            policy,
+            startup_listeners,
+            startup_locks,
+            auth,
+            viewer_slots: Arc::new(Semaphore::new(8)),
+            certificate_slots: Arc::new(Semaphore::new(2)),
+            metrics: Arc::new(Metrics::default()),
+            transform_metrics: Arc::new(crate::transform::Metrics::default()),
+            logs,
+            software_update: Arc::new(crate::update_check::SharedStatus::default()),
+            shutdown,
+            tunnels: tokio_util::task::TaskTracker::new(),
+            tunnel_registry: Arc::new(crate::proxy::websocket::TunnelRegistry::default()),
+            websocket_idle_timeout,
+            inflight: Arc::new(Semaphore::new(max_inflight)),
+            started: Instant::now(),
+            header_timeout,
+            body_idle_timeout,
+            update_lock: Mutex::new(()),
+            update_slots: Arc::new(Semaphore::new(8)),
+        });
+        let weak = Arc::downgrade(&app);
+        let shutdown = app.shutdown.clone();
+        app.tunnels
+            .spawn(crate::balance::health_loop(weak, shutdown));
+        app.tunnels.spawn(crate::acme::renewal_loop(
+            Arc::downgrade(&app),
+            app.shutdown.clone(),
+        ));
+        app.tunnels.spawn(crate::docker::poll_loop(
+            Arc::downgrade(&app),
+            app.shutdown.clone(),
+        ));
+        app
+    }
+    pub fn start_update_check(self: &Arc<Self>, config: crate::update_check::Config) {
+        self.software_update.configure(&config);
+        self.tunnels.spawn(crate::update_check::poll_loop(
+            self.software_update.clone(),
+            config,
+            self.shutdown.clone(),
+        ));
+    }
+
+    /// Drain detached writers after the listeners and their handlers have stopped.
+    pub async fn drain_updates(&self) {
+        let _all_slots = self.update_slots.clone().acquire_many_owned(8).await;
+    }
+    /// The detached task completes persistence + publication even if the HTTP caller disconnects.
+    pub async fn update(self: &Arc<Self>, update: Update) -> Result<config::Envelope, ApiError> {
+        self.update_as(update, "local".into()).await
+    }
+    pub async fn update_as(
+        self: &Arc<Self>,
+        update: Update,
+        actor: String,
+    ) -> Result<config::Envelope, ApiError> {
+        let slot = self.update_slots.clone().try_acquire_owned().map_err(|_| {
+            ApiError(
+                StatusCode::TOO_MANY_REQUESTS,
+                "configuration update queue is full".into(),
+            )
+        })?;
+        let expected_revision = update.expected_revision;
+        let revision = expected_revision
+            .checked_add(1)
+            .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "revision overflow".into()))?;
+        let policy = self.policy.clone();
+        let defaults = self.startup_listeners;
+        let previous = self.config.load_full();
+        let previous_config = previous.envelope.config.clone();
+        let active_listeners = self.active_listeners.clone();
+        let observed = self.docker.snapshot().await;
+        let desired_vhost_count = update.config.vhosts.len();
+        let desired_config = update.config;
+        self.startup_locks
+            .validate(&desired_config, self.policy.admin_addr)
+            .map_err(|error| ApiError(StatusCode::UNPROCESSABLE_ENTITY, format!("{error:#}")))?;
+        crate::tls::validate_certificate_removals(&previous_config, &desired_config)
+            .map_err(|error| ApiError(StatusCode::UNPROCESSABLE_ENTITY, format!("{error:#}")))?;
+        let (prepared, invalidation, slot) = tokio::task::spawn_blocking(move || {
+            let effective = crate::docker::effective_config(desired_config, &observed)?;
+            let mut prepared = config::compile_with_active(
+                effective,
+                revision,
+                &policy,
+                defaults,
+                Some(&previous),
+                &active_listeners,
+            )?;
+            prepared
+                .envelope
+                .config
+                .vhosts
+                .truncate(desired_vhost_count);
+            let invalidation = cache_invalidation(&previous_config, &prepared.envelope.config);
+            Ok::<_, anyhow::Error>((prepared, invalidation, slot))
+        })
+        .await
+        .map_err(|_| {
+            ApiError(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "configuration validation task failed".into(),
+            )
+        })?
+        .map_err(|e| ApiError(StatusCode::UNPROCESSABLE_ENTITY, format!("{e:#}")))?;
+        let this = self.clone();
+        tokio::spawn(async move {
+            let _slot = slot;
+            let _lock = this.update_lock.lock().await;
+            if this.config.load().envelope.revision != expected_revision {
+                return Err(ApiError(
+                    StatusCode::CONFLICT,
+                    "configuration changed; reload before saving".into(),
+                ));
+            }
+            match this
+                .store
+                .replace_as(expected_revision, &prepared.envelope.config, &actor)
+                .await
+            {
+                Ok(true) => {}
+                Ok(false) => {
+                    return Err(ApiError(
+                        StatusCode::CONFLICT,
+                        "stored revision differs; external SQLite writes are unsupported".into(),
+                    ))
+                }
+                Err(error) => {
+                    tracing::error!(%error, "configuration persistence failed");
+                    return Err(ApiError(
+                        StatusCode::INTERNAL_SERVER_ERROR,
+                        "configuration was not saved".into(),
+                    ));
+                }
+            }
+            // No await between successful commit and publication. Cache identity is
+            // intentionally decoupled from the configuration revision, so unrelated
+            // control-plane saves do not invalidate hot/cold objects.
+            let envelope = prepared.envelope.clone();
+            match invalidation {
+                CacheInvalidation::All(cache_revision) => {
+                    this.cache.purge(Some(cache_revision));
+                }
+                CacheInvalidation::None => {}
+            }
+            this.cache.configure(envelope.config.cache.clone());
+            this.rate_limiter.reset();
+            this.logs.configure(envelope.config.logging.clone());
+            this.config.store(Arc::new(prepared));
+            tracing::info!(revision, "configuration applied");
+            Ok(envelope)
+        })
+        .await
+        .map_err(|_| {
+            ApiError(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "configuration task failed".into(),
+            )
+        })?
+    }
+
+    /// Re-read configured public certificate/key files and publish a new
+    /// immutable TLS/routing snapshot without changing desired configuration
+    /// or its SQLite revision. Existing connections continue on their current
+    /// TLS session; new H1/H2/H3 handshakes use the refreshed certificates.
+    pub async fn reload_certificates(self: &Arc<Self>) -> Result<u64, ApiError> {
+        self.reload_certificate_snapshot(None).await
+    }
+
+    /// Reload an externally managed/manual certificate by id. The immutable TLS
+    /// resolver is rebuilt as one validated snapshot, so the requested pair is
+    /// re-read atomically with the other configured pairs while desired
+    /// configuration and its revision remain unchanged.
+    pub async fn reload_certificate(self: &Arc<Self>, certificate_id: &str) -> Result<u64, ApiError> {
+        if !crate::config::valid_id(certificate_id) {
+            return Err(ApiError(StatusCode::NOT_FOUND, "certificate not found".into()));
+        }
+        self.reload_certificate_snapshot(Some(certificate_id)).await
+    }
+
+    async fn reload_certificate_snapshot(
+        self: &Arc<Self>,
+        certificate_id: Option<&str>,
+    ) -> Result<u64, ApiError> {
+        let _lock = self.update_lock.lock().await;
+        let previous = self.config.load_full();
+        let desired = previous.envelope.config.clone();
+        if let Some(certificate_id) = certificate_id {
+            let certificate = desired
+                .tls
+                .certificates
+                .iter()
+                .find(|certificate| certificate.id == certificate_id)
+                .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "certificate not found".into()))?;
+            if certificate.issuer.is_some() {
+                return Err(ApiError(
+                    StatusCode::CONFLICT,
+                    "managed certificates are reloaded by the renewal pipeline".into(),
+                ));
+            }
+        }
+        let desired_vhost_count = desired.vhosts.len();
+        let observed = self.docker.snapshot().await;
+        let policy = self.policy.clone();
+        let defaults = self.startup_listeners;
+        let active_listeners = self.active_listeners.clone();
+        let revision = previous.envelope.revision;
+        let prepared = tokio::task::spawn_blocking(move || {
+            let effective = crate::docker::effective_config(desired, &observed)?;
+            let mut prepared = config::compile_with_active(
+                effective,
+                revision,
+                &policy,
+                defaults,
+                Some(&previous),
+                &active_listeners,
+            )?;
+            prepared
+                .envelope
+                .config
+                .vhosts
+                .truncate(desired_vhost_count);
+            Ok::<_, anyhow::Error>(prepared)
+        })
+        .await
+        .map_err(|_| {
+            ApiError(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "certificate reload task failed".into(),
+            )
+        })?
+        .map_err(|error| ApiError(StatusCode::UNPROCESSABLE_ENTITY, format!("{error:#}")))?;
+        self.config.store(Arc::new(prepared));
+        if let Some(certificate_id) = certificate_id {
+            tracing::info!(revision, certificate_id, "configured certificate reloaded from disk");
+        } else {
+            tracing::info!(revision, "configured certificates reloaded from disk");
+        }
+        Ok(revision)
+    }
+
+    /// Rebuild only the effective routing snapshot from the latest Docker observations.
+    /// Desired configuration and its SQLite revision remain unchanged.
+    pub async fn reconcile_docker(self: &Arc<Self>) -> Result<(), ApiError> {
+        let _lock = self.update_lock.lock().await;
+        let previous = self.config.load_full();
+        let desired = previous.envelope.config.clone();
+        let desired_vhost_count = desired.vhosts.len();
+        let observed = self.docker.snapshot().await;
+        let policy = self.policy.clone();
+        let defaults = self.startup_listeners;
+        let active_listeners = self.active_listeners.clone();
+        let revision = previous.envelope.revision;
+        let prepared = tokio::task::spawn_blocking(move || {
+            let effective = crate::docker::effective_config(desired, &observed)?;
+            let mut prepared = config::compile_with_active(
+                effective,
+                revision,
+                &policy,
+                defaults,
+                Some(&previous),
+                &active_listeners,
+            )?;
+            prepared
+                .envelope
+                .config
+                .vhosts
+                .truncate(desired_vhost_count);
+            Ok::<_, anyhow::Error>(prepared)
+        })
+        .await
+        .map_err(|_| {
+            ApiError(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "Docker reconcile task failed".into(),
+            )
+        })?
+        .map_err(|error| ApiError(StatusCode::UNPROCESSABLE_ENTITY, format!("{error:#}")))?;
+        self.config.store(Arc::new(prepared));
+        Ok(())
+    }
+
+    pub async fn refresh_docker(
+        self: &Arc<Self>,
+    ) -> Result<Vec<crate::docker::Observation>, ApiError> {
+        let settings = self.config.load().envelope.config.docker_agents.clone();
+        let (observations, changed) = self.docker.refresh(&settings, true).await;
+        if changed {
+            self.reconcile_docker().await?;
+        }
+        Ok(observations)
+    }
+}
+
+#[cfg(test)]
+mod cache_revision_tests {
+    use super::*;
+
+    #[test]
+    fn operational_saves_keep_cache_identity_but_cache_affecting_saves_rotate_it() {
+        let base = config::demo_config("127.0.0.1:9000".parse().unwrap());
+        let identity = cache_revision(&base);
+
+        let mut logging = base.clone();
+        logging.logging.file.path = "/tmp/rustproxy-test-access.jsonl".into();
+        assert_eq!(cache_revision(&logging), identity);
+        assert!(matches!(
+            cache_invalidation(&base, &logging),
+            CacheInvalidation::None
+        ));
+
+        let mut metrics = base.clone();
+        metrics.prometheus.include_system = !metrics.prometheus.include_system;
+        assert_eq!(cache_revision(&metrics), identity);
+
+        let mut response = base.clone();
+        response.server_header.value = "edge-test".into();
+        response.diagnostic_headers.request_id = !response.diagnostic_headers.request_id;
+        response.compression.gzip = !response.compression.gzip;
+        assert_eq!(cache_revision(&response), identity);
+
+        let mut docker = base.clone();
+        docker.docker_agents.agents.push(crate::docker::Agent {
+            id: "node1".into(),
+            name: "Node 1".into(),
+            ..Default::default()
+        });
+        assert_eq!(cache_revision(&docker), identity);
+
+        let mut cache = base.clone();
+        cache.cache.max_entries += 1;
+        let cache_identity = cache_revision(&cache);
+        assert_ne!(cache_identity, identity);
+        assert!(matches!(
+            cache_invalidation(&base, &cache),
+            CacheInvalidation::All(value) if value == cache_identity
+        ));
+
+        let mut vhost = base.clone();
+        vhost.vhosts[0].backends[0].url = "http://127.0.0.1:9001".into();
+        assert_ne!(cache_revision(&vhost), identity);
+    }
+}