v0.25.1

src/acme/jobs.rs

@@ -0,0 +1,346 @@
+impl Manager {
+    pub fn renewal_view(&self) -> std::collections::BTreeMap<String, RenewalState> {
+        self.renewals
+            .lock()
+            .unwrap_or_else(|e| e.into_inner())
+            .clone()
+    }
+    pub fn restore_renewals(&self, values: std::collections::BTreeMap<String, RenewalState>) {
+        let mut observations = self.renewals.lock().unwrap_or_else(|e| e.into_inner());
+        *observations = values
+            .into_iter()
+            .filter(|(id, _)| crate::config::valid_id(id))
+            .take(64)
+            .collect();
+    }
+    async fn persist(&self, store: &crate::db::Store) {
+        let value = self.renewal_view();
+        if let Err(error) = store.replace_certificate_renewals(&value).await {
+            tracing::warn!(%error, "certificate renewal observation could not be saved to SQLite");
+        }
+    }
+    pub async fn renew(
+        state: Arc<App>,
+        id: &str,
+        actor: String,
+        dry_run: bool,
+        automatic: bool,
+    ) -> Result<Job, ApiError> {
+        let directory = state.policy.cert_dir.clone().ok_or_else(|| {
+            ApiError(StatusCode::CONFLICT, "acme.no_certificate_directory".into())
+        })?;
+        let cert = state
+            .config
+            .load()
+            .envelope
+            .config
+            .tls
+            .certificates
+            .iter()
+            .find(|c| c.id == id)
+            .cloned()
+            .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "acme.certificate_not_found".into()))?;
+        let issuer = cert
+            .issuer
+            .clone()
+            .ok_or_else(|| ApiError(StatusCode::CONFLICT, "acme.not_managed".into()))?;
+        if automatic && !issuer.auto_renew {
+            return Err(ApiError(
+                StatusCode::CONFLICT,
+                "acme.renewal_disabled".into(),
+            ));
+        }
+        let slot = state
+            .acme
+            .slot
+            .clone()
+            .try_acquire_owned()
+            .map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?;
+        let copy = cert.clone();
+        let dir = directory.clone();
+        let domains = tokio::task::spawn_blocking(move || profile_domains(&dir, &copy))
+            .await
+            .map_err(crate::auth::internal)?
+            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.renewal_profile_missing".into()))?;
+        if issuer.provider == "http01" && !state.active_listeners.iter().any(|l| !l.tls) {
+            return Err(ApiError(
+                StatusCode::CONFLICT,
+                "acme.http01_listener_required".into(),
+            ));
+        }
+        let job = Job {
+            id: format!("{:032x}", rand::random::<u128>()),
+            certificate_id: cert.id.clone(),
+            status: "running",
+            kind: if dry_run { "renewal-dry-run" } else { "renewal" },
+            log_directory: certbot_log_directory(&directory, &issuer.provider, &issuer.environment, &cert.id),
+            provider: issuer.provider.clone(),
+            domains,
+            environment: issuer.environment.clone(),
+            message: "acme.renewing".into(),
+            certificate: None,
+        };
+        state
+            .store
+            .audit(
+                &actor,
+                if dry_run {
+                    "acme.renew_dry_run"
+                } else {
+                    "acme.renew"
+                },
+                &cert.id,
+            )
+            .await
+            .map_err(crate::auth::internal)?;
+        *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone());
+        let mut request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider,
+            "environment":issuer.environment,"cert_file":cert.cert_file,"key_file":cert.key_file,"dry_run":dry_run});
+        if issuer.provider != "http01" {
+            let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| {
+                ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into())
+            })?;
+            let credential = crate::secret_store::get(
+                &state.store,
+                data_dir,
+                &crate::secret_store::acme_secret_id(&cert.id, &issuer.provider),
+                Some(&issuer.provider),
+            )
+            .await
+            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.secret_missing".into()))?;
+            request["credential_ini_b64"] = Value::String(
+                base64::engine::general_purpose::STANDARD.encode(credential.as_slice())
+            );
+        }
+        let raw = Zeroizing::new(request.to_string());
+        if let Some(Value::String(secret)) = request.get_mut("credential_ini_b64") {
+            secret.zeroize();
+        }
+        let task_job = job.clone();
+        let tasks = state.tunnels.clone();
+        tasks.spawn(async move {
+            let _slot = slot;
+            let result = async {
+                let output = worker(&state, directory.clone(), raw).await?;
+                if !dry_run && output.get("changed") == Some(&Value::Bool(true)) {
+                    deploy_renewal(&state, &cert, &output, automatic).await?;
+                }
+                Ok::<_, String>(output)
+            }
+            .await;
+            let mut finished = task_job;
+            let now = crate::auth::now();
+            {
+                let mut observations = state
+                    .acme
+                    .renewals
+                    .lock()
+                    .unwrap_or_else(|e| e.into_inner());
+                let entry = observations.entry(cert.id.clone()).or_default();
+                entry.last_checked = now;
+                match result {
+                    Ok(output) => {
+                        let changed = !dry_run && output.get("changed") == Some(&Value::Bool(true));
+                        finished.status = "complete";
+                        finished.message = if dry_run {
+                            "acme.renewal_test_ok"
+                        } else if changed {
+                            "acme.renewed"
+                        } else {
+                            "acme.not_due"
+                        }
+                        .into();
+                        // A dry run must not postpone a due production renewal.
+                        if !dry_run {
+                            entry.failures = 0;
+                            entry.next_check = now + 43200 + (rand::random::<u16>() as i64 % 1800);
+                        }
+                        entry.not_after = output
+                            .get("not_after_unix")
+                            .and_then(Value::as_i64)
+                            .or(entry.not_after);
+                        if changed {
+                            entry.last_renewed = Some(now);
+                        }
+                    }
+                    Err(message) => {
+                        finished.status = "failed";
+                        finished.message = message;
+                        entry.failures = entry.failures.saturating_add(1);
+                        if !dry_run {
+                            entry.next_check = now + 3600 * i64::from(entry.failures.min(6));
+                        }
+                    }
+                }
+                entry.status = finished.status.into();
+                entry.message = finished.message.clone();
+            }
+            *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(finished.clone());
+            state.acme.persist(&state.store).await;
+            let _ = state
+                .store
+                .audit(
+                    &actor,
+                    "acme.renew_result",
+                    &format!("{} {}", cert.id, finished.message),
+                )
+                .await;
+        });
+        Ok(job)
+    }
+}
+async fn deploy_renewal(
+    state: &Arc<App>,
+    original: &Certificate,
+    value: &Value,
+    automatic: bool,
+) -> Result<(), String> {
+    let data: Output = serde_json::from_value(value.clone())
+        .map_err(|_| "acme.invalid_worker_response".to_owned())?;
+    if data.certificate_id != original.id
+        || original
+            .issuer
+            .as_ref()
+            .is_none_or(|i| i.provider != data.provider || i.environment != data.environment)
+    {
+        return Err("acme.renewal_profile_mismatch".into());
+    }
+    let dir = state
+        .policy
+        .cert_dir
+        .clone()
+        .ok_or("acme.no_certificate_directory")?;
+    let mut renewed = tokio::task::spawn_blocking(move || {
+        crate::tls::validate_acme(
+            &dir,
+            data.certificate_id,
+            data.cert_file,
+            data.key_file,
+            &data.domains,
+        )
+    })
+    .await
+    .map_err(|_| "acme.validation_worker_failed".to_owned())?
+    .map_err(|_| "acme.certificate_validation_failed".to_owned())?;
+    // CAS retries preserve edits to other parts of the document. A simultaneous
+    // change of this certificate wins over the worker; never silently overwrite it.
+    for _ in 0..3 {
+        let snapshot = state.config.load_full();
+        let Some(current) = snapshot
+            .envelope
+            .config
+            .tls
+            .certificates
+            .iter()
+            .find(|c| c.id == original.id)
+        else {
+            return Err("acme.configuration_changed".into());
+        };
+        if current.cert_file != original.cert_file
+            || current.key_file != original.key_file
+            || current.issuer.as_ref().is_none_or(|i| {
+                original.issuer.as_ref().is_none_or(|old| {
+                    i.provider != old.provider || i.environment != old.environment
+                }) || (automatic && !i.auto_renew)
+            })
+        {
+            return Err("acme.configuration_changed".into());
+        }
+        renewed.hosts = current.hosts.clone();
+        renewed.issuer = current.issuer.clone();
+        let mut config = snapshot.envelope.config.clone();
+        let slot = config
+            .tls
+            .certificates
+            .iter_mut()
+            .find(|c| c.id == original.id)
+            .ok_or("acme.configuration_changed")?;
+        *slot = renewed.clone();
+        let update = crate::config::Update {
+            expected_revision: snapshot.envelope.revision,
+            config,
+        };
+        match state
+            .update_as(update, "system:certificate-renewal".into())
+            .await
+        {
+            Ok(_) => return Ok(()),
+            Err(ApiError(StatusCode::CONFLICT, _)) => continue,
+            Err(_) => return Err("acme.deployment_failed".into()),
+        }
+    }
+    Err("acme.configuration_changed".into())
+}
+/// One scheduler per process, sharing the issuance semaphore. It does not run
+/// its own shell hooks or stop listeners; validated snapshots rotate TCP and QUIC TLS.
+pub async fn renewal_loop(
+    weak: std::sync::Weak<App>,
+    shutdown: tokio_util::sync::CancellationToken,
+) {
+    // Observations are restored from SQLite before listeners are exposed. If an
+    // older installation has no SQLite observations, the normal scheduler will
+    // safely reconstruct them on its first check; certificate/key state is not
+    // derived from historical observation sidecars.
+    let delay = 300 + (rand::random::<u16>() as u64 % 300);
+    tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(delay)) => {} }
+    loop {
+        let Some(state) = weak.upgrade() else {
+            return;
+        };
+        let certificates = state.config.load().envelope.config.tls.certificates.clone();
+        {
+            let mut observations = state
+                .acme
+                .renewals
+                .lock()
+                .unwrap_or_else(|e| e.into_inner());
+            observations.retain(|id, _| certificates.iter().any(|c| &c.id == id));
+        }
+        let view = state.acme.renewal_view();
+        let now = crate::auth::now();
+        for cert in certificates
+            .iter()
+            .filter(|c| c.issuer.as_ref().is_some_and(|i| i.auto_renew))
+        {
+            if view
+                .get(&cert.id)
+                .is_some_and(|r| r.next_check > now && r.next_check <= now + 86400)
+            {
+                continue;
+            }
+            match Manager::renew(
+                state.clone(),
+                &cert.id,
+                "system:certificate-renewal".into(),
+                false,
+                true,
+            )
+            .await
+            {
+                Ok(_) => break,
+                Err(ApiError(StatusCode::TOO_MANY_REQUESTS, _)) => break,
+                Err(error) => {
+                    state
+                        .acme
+                        .renewals
+                        .lock()
+                        .unwrap_or_else(|e| e.into_inner())
+                        .insert(
+                            cert.id.clone(),
+                            RenewalState {
+                                last_checked: now,
+                                next_check: now + 3600,
+                                failures: 1,
+                                status: "failed".into(),
+                                message: error.1,
+                                ..Default::default()
+                            },
+                        );
+                    state.acme.persist(&state.store).await;
+                }
+            }
+        }
+        drop(state);
+        tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(60)) => {} }
+    }
+}

src/acme/model.rs

@@ -0,0 +1,403 @@
+use crate::{
+    runtime::{ApiError, App},
+    tls::Certificate,
+};
+use base64::Engine as _;
+use http::StatusCode;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::{
+    path::PathBuf,
+    process::Stdio,
+    sync::{Arc, Mutex},
+    time::Duration,
+};
+use tokio::{io::AsyncWriteExt, process::Command, sync::Semaphore};
+use zeroize::{Zeroize, Zeroizing};
+
+#[derive(Debug, Clone, Serialize)]
+pub struct Job {
+    pub id: String,
+    pub certificate_id: String,
+    pub status: &'static str,
+    pub kind: &'static str,
+    pub provider: String,
+    pub domains: Vec<String>,
+    pub environment: String,
+    pub message: String,
+    pub log_directory: String,
+    pub certificate: Option<Certificate>,
+}
+
+pub(crate) fn certbot_log_directory(directory: &std::path::Path, provider: &str, environment: &str, certificate_id: &str) -> String {
+    let suffix = if provider == "rfc2136" { String::new() } else { format!("-{provider}") };
+    directory
+        .join("acme")
+        .join(format!("{environment}-{certificate_id}{suffix}"))
+        .join("logs")
+        .to_string_lossy()
+        .into_owned()
+}
+fn provider_credential_bytes(request: &Value, provider: &str) -> Result<Zeroizing<Vec<u8>>, ApiError> {
+    fn credential<'a>(request: &'a Value, name: &str) -> Result<&'a str, ApiError> {
+        let invalid = || ApiError(StatusCode::BAD_REQUEST, "acme.invalid_api_credential".into());
+        let value = request.get(name).and_then(Value::as_str).ok_or_else(invalid)?;
+        let valid = (8..=4096).contains(&value.len())
+            && value.bytes().all(|byte| {
+                byte.is_ascii_alphanumeric()
+                    || matches!(byte, b'_' | b'.' | b'/' | b'+' | b'-' | b'=')
+            });
+        if valid { Ok(value) } else { Err(invalid()) }
+    }
+    let bytes = match provider {
+        "cloudflare" => format!("dns_cloudflare_api_token = {}\n", credential(request, "api_token")?).into_bytes(),
+        "ovh" => {
+            if request.get("endpoint").is_some_and(|value| !value.is_string()) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_ovh_endpoint".into()));
+            }
+            let endpoint = request.get("endpoint").and_then(Value::as_str).unwrap_or("ovh-eu");
+            if !matches!(endpoint, "ovh-eu" | "ovh-ca") {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_ovh_endpoint".into()));
+            }
+            format!(
+                "dns_ovh_endpoint = {}\ndns_ovh_application_key = {}\ndns_ovh_application_secret = {}\ndns_ovh_consumer_key = {}\n",
+                endpoint,
+                credential(request, "application_key")?,
+                credential(request, "application_secret")?,
+                credential(request, "consumer_key")?
+            )
+            .into_bytes()
+        }
+        "rfc2136" => {
+            let dns_server = request
+                .get("dns_server")
+                .and_then(Value::as_str)
+                .and_then(|value| value.parse::<std::net::IpAddr>().ok())
+                .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_dns_server".into()))?;
+            if request.get("dns_port").is_some_and(|value| value.as_u64().is_none()) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_limits".into()));
+            }
+            let dns_port = request.get("dns_port").and_then(Value::as_u64).unwrap_or(53);
+            if !(1..=65535).contains(&dns_port) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_limits".into()));
+            }
+            let raw_name = request.get("key_name").and_then(Value::as_str).unwrap_or("");
+            let key_name = raw_name.to_ascii_lowercase();
+            let key_name = key_name.trim_end_matches('.');
+            let valid_name = !key_name.is_empty()
+                && key_name.len() <= 253
+                && key_name.split('.').all(|label| {
+                    !label.is_empty()
+                        && label.len() <= 63
+                        && label.bytes().enumerate().all(|(index, byte)| {
+                            (byte.is_ascii_lowercase() || byte.is_ascii_digit())
+                                || (byte == b'-' && index > 0 && index + 1 < label.len())
+                        })
+                });
+            if !valid_name {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_domain".into()));
+            }
+            if request.get("key_algorithm").is_some_and(|value| !value.is_string()) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_algorithm".into()));
+            }
+            let key_algorithm = request
+                .get("key_algorithm")
+                .and_then(Value::as_str)
+                .unwrap_or("HMAC-SHA256");
+            if !matches!(key_algorithm, "HMAC-SHA256" | "HMAC-SHA384" | "HMAC-SHA512") {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_algorithm".into()));
+            }
+            let key_secret = request.get("key_secret").and_then(Value::as_str).ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_api_credential".into()))?;
+            let decoded = Zeroizing::new(
+                base64::engine::general_purpose::STANDARD
+                    .decode(key_secret)
+                    .map_err(|_| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_secret".into()))?,
+            );
+            if key_secret.len() > 1024 || !(16..=512).contains(&decoded.len()) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_secret".into()));
+            }
+            if request.get("sign_query").is_some_and(|value| !value.is_boolean()) {
+                return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_sign_query".into()));
+            }
+            let sign_query = request.get("sign_query").and_then(Value::as_bool).unwrap_or(false);
+            format!(
+                "dns_rfc2136_server = {dns_server}\ndns_rfc2136_port = {dns_port}\ndns_rfc2136_name = {key_name}.\ndns_rfc2136_secret = {key_secret}\ndns_rfc2136_algorithm = {key_algorithm}\ndns_rfc2136_sign_query = {sign_query}\n"
+            )
+            .into_bytes()
+        }
+        "http01" => Vec::new(),
+        _ => return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_provider".into())),
+    };
+    Ok(Zeroizing::new(bytes))
+}
+
+
+pub struct Manager {
+    slot: Arc<Semaphore>,
+    job: Mutex<Option<Job>>,
+    renewals: Mutex<std::collections::BTreeMap<String, RenewalState>>,
+}
+impl Manager {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self {
+            slot: Arc::new(Semaphore::new(1)),
+            job: Mutex::new(None),
+            renewals: Mutex::new(Default::default()),
+        })
+    }
+    pub fn view(&self) -> Option<Job> {
+        self.job.lock().unwrap_or_else(|e| e.into_inner()).clone()
+    }
+    pub async fn start(
+        state: Arc<App>,
+        mut request: Value,
+        actor: String,
+    ) -> Result<Job, ApiError> {
+        let directory = state.policy.cert_dir.clone().ok_or_else(|| {
+            ApiError(StatusCode::CONFLICT, "acme.no_certificate_directory".into())
+        })?;
+        let id = request
+            .get("certificate_id")
+            .and_then(Value::as_str)
+            .filter(|v| crate::config::valid_id(v))
+            .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_id".into()))?
+            .to_owned();
+        let environment = request
+            .get("environment")
+            .and_then(Value::as_str)
+            .unwrap_or("staging")
+            .to_owned();
+        if !matches!(environment.as_str(), "staging" | "production")
+            || request.get("agree_tos") != Some(&Value::Bool(true))
+            || (environment == "production"
+                && request.get("confirm_production") != Some(&Value::Bool(true)))
+        {
+            return Err(ApiError(
+                StatusCode::BAD_REQUEST,
+                "acme.consent_required".into(),
+            ));
+        }
+        let provider = request
+            .get("provider")
+            .and_then(Value::as_str)
+            .unwrap_or("rfc2136")
+            .to_owned();
+        if !matches!(
+            provider.as_str(),
+            "http01" | "cloudflare" | "ovh" | "rfc2136"
+        ) {
+            return Err(ApiError(
+                StatusCode::BAD_REQUEST,
+                "acme.invalid_provider".into(),
+            ));
+        }
+        let names = request
+            .get("domains")
+            .and_then(Value::as_array)
+            .filter(|names| !names.is_empty() && names.len() <= 16)
+            .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_domains".into()))?;
+        let mut domains = Vec::new();
+        for name in names {
+            let raw = name
+                .as_str()
+                .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_domain".into()))?;
+            if provider == "http01" && raw.starts_with("*.") {
+                return Err(ApiError(
+                    StatusCode::BAD_REQUEST,
+                    "acme.http01_no_wildcard".into(),
+                ));
+            }
+            let normalized =
+                crate::config::normalize_host(raw.strip_prefix("*.").unwrap_or(raw))
+                    .map_err(|_| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_domain".into()))?;
+            if !normalized.contains('.') || normalized.parse::<std::net::IpAddr>().is_ok() {
+                return Err(ApiError(
+                    StatusCode::BAD_REQUEST,
+                    "acme.dns_names_only".into(),
+                ));
+            }
+            domains.push(normalized);
+        }
+        if provider == "http01" && !state.active_listeners.iter().any(|l| !l.tls) {
+            return Err(ApiError(
+                StatusCode::CONFLICT,
+                "acme.http01_listener_required".into(),
+            ));
+        }
+        let slot = state
+            .acme
+            .slot
+            .clone()
+            .try_acquire_owned()
+            .map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?;
+        if provider != "http01" {
+            let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| {
+                ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into())
+            })?;
+            let credential = provider_credential_bytes(&request, &provider)?;
+            crate::secret_store::put(
+                &state.store,
+                data_dir,
+                &crate::secret_store::acme_secret_id(&id, &provider),
+                &provider,
+                credential.as_ref(),
+                serde_json::json!({"certificate_id": id.clone(), "purpose": "acme_dns"}),
+            )
+            .await
+            .map_err(crate::auth::internal)?;
+        }
+        let raw = Zeroizing::new(serde_json::to_string(&request).map_err(crate::auth::internal)?);
+        for key in [
+            "key_secret",
+            "api_token",
+            "application_key",
+            "application_secret",
+            "consumer_key",
+        ] {
+            if let Some(Value::String(secret)) = request.get_mut(key) {
+                secret.zeroize();
+            }
+        }
+        if raw.len() > 16384 {
+            return Err(ApiError(
+                StatusCode::PAYLOAD_TOO_LARGE,
+                "acme.request_too_large".into(),
+            ));
+        }
+        let log_directory = certbot_log_directory(&directory, &provider, &environment, &id);
+        let job = Job {
+            id: format!("{:032x}", rand::random::<u128>()),
+            certificate_id: id,
+            status: "running",
+            kind: "issue",
+            provider,
+            domains,
+            environment,
+            message: "acme.working".into(),
+            log_directory,
+            certificate: None,
+        };
+        state
+            .store
+            .audit(&actor, "acme.issue", &job.id)
+            .await
+            .map_err(crate::auth::internal)?;
+        *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone());
+        let task_job = job.clone();
+        let tasks = state.tunnels.clone();
+        tasks.spawn(async move {
+            let _slot = slot;
+            let result = execute(&state, directory, raw).await;
+            let mut finished = task_job;
+            match result {
+                Ok(cert) => {
+                    finished.status = "ready";
+                    finished.message = "acme.ready_to_import".into();
+                    finished.certificate = Some(cert);
+                }
+                Err(message) => {
+                    finished.status = "failed";
+                    finished.message = message;
+                }
+            }
+            // Never write provider stderr, request JSON or TSIG material to audit.
+            let _ = state
+                .store
+                .audit(
+                    &actor,
+                    "acme.result",
+                    &format!("{} {}", finished.id, finished.status),
+                )
+                .await;
+            *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(finished);
+        });
+        Ok(job)
+    }
+}
+#[derive(Deserialize)]
+struct Output {
+    provider: String,
+    environment: String,
+    certificate_id: String,
+    domains: Vec<String>,
+    cert_file: String,
+    key_file: String,
+}
+
+/// Kill the entire private process group if execution is cancelled/abandoned.
+/// Certbot inherits the group, so it cannot outlive a stopped RustProxy worker.
+struct ProcessGroup(Option<u32>);
+impl Drop for ProcessGroup {
+    fn drop(&mut self) {
+        #[cfg(unix)]
+        if let Some(id) = self.0 {
+            // SAFETY: this is the process group created solely for this child.
+            unsafe {
+                libc::kill(-(id as i32), libc::SIGKILL);
+            }
+        }
+    }
+}
+async fn worker(state: &App, directory: PathBuf, raw: Zeroizing<String>) -> Result<Value, String> {
+    let mut command = Command::new("python3");
+    command
+        .arg("-I")
+        .arg("-c")
+        .arg(include_str!("../../scripts/acme_provider.py"))
+        .arg("--cert-dir")
+        .arg(&directory)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .kill_on_drop(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::process::CommandExt;
+        command.as_std_mut().process_group(0);
+    }
+    let mut child = command
+        .spawn()
+        .map_err(|_| "acme.python_not_available".to_owned())?;
+    let mut group = ProcessGroup(child.id());
+    let stdin = child
+        .stdin
+        .take()
+        .ok_or_else(|| "acme.worker_io".to_owned())?;
+    let work = async move {
+        let mut stdin = stdin;
+        stdin
+            .write_all(raw.as_bytes())
+            .await
+            .map_err(|_| "acme.worker_io".to_owned())?;
+        stdin
+            .shutdown()
+            .await
+            .map_err(|_| "acme.worker_io".to_owned())?;
+        drop(stdin);
+        drop(raw);
+        child
+            .wait_with_output()
+            .await
+            .map_err(|_| "acme.worker_io".to_owned())
+    };
+    let output = tokio::select! {
+        _ = state.shutdown.cancelled() => return Err("acme.cancelled".into()),
+        result = tokio::time::timeout(Duration::from_secs(2160), work) => result.map_err(|_| "acme.provider_timeout".to_owned())??,
+    };
+    group.0 = None; // the process has been reaped, never kill a reused PID
+    if output.stdout.len() > 16384 {
+        return Err("acme.invalid_worker_response".into());
+    }
+    let value: Value = serde_json::from_slice(&output.stdout)
+        .map_err(|_| "acme.invalid_worker_response".to_owned())?;
+    if !output.status.success() {
+        let reason = value.get("error").and_then(Value::as_str).filter(|s| {
+            s.starts_with("acme.")
+                && s.len() < 96
+                && s.bytes()
+                    .all(|c| c.is_ascii_lowercase() || c == b'_' || c == b'.')
+        });
+        return Err(reason.unwrap_or("acme.worker_failed").to_owned());
+    }
+    Ok(value)
+}

src/acme/providers.rs

@@ -0,0 +1,228 @@
+async fn execute(
+    state: &App,
+    directory: PathBuf,
+    raw: Zeroizing<String>,
+) -> Result<Certificate, String> {
+    let value = worker(state, directory.clone(), raw).await?;
+    let data: Output =
+        serde_json::from_value(value).map_err(|_| "acme.invalid_worker_response".to_owned())?;
+    tokio::task::spawn_blocking(move || {
+        let result = crate::tls::validate_acme(
+            &directory,
+            data.certificate_id,
+            data.cert_file.clone(),
+            data.key_file.clone(),
+            &data.domains,
+        )
+        .map(|mut cert| {
+            cert.issuer = Some(crate::tls::Issuer {
+                provider: data.provider,
+                environment: data.environment,
+                auto_renew: true,
+            });
+            cert
+        });
+        if result.is_err() {
+            for name in [data.cert_file, data.key_file] {
+                if name.starts_with("acme-") && crate::tls::safe_name(&name).is_ok() {
+                    let _ = std::fs::remove_file(directory.join(name));
+                }
+            }
+        }
+        result.map_err(|_| "acme.certificate_validation_failed".to_owned())
+    })
+    .await
+    .map_err(|_| "acme.validation_worker_failed".to_owned())?
+}
+
+/// A reserved validation path: only an active HTTP-01 job and its exact DNS
+/// names can read bounded tokens. No origin, cache, auth or custom pages here.
+pub async fn challenge(
+    axum::extract::State(state): axum::extract::State<Arc<App>>,
+    request: axum::extract::Request,
+    next: axum::middleware::Next,
+) -> axum::response::Response {
+    use axum::{body::Body, response::IntoResponse};
+    use tokio::io::AsyncReadExt;
+    const PREFIX: &str = "/.well-known/acme-challenge/";
+    let Some(token) = request.uri().path().strip_prefix(PREFIX) else {
+        return next.run(request).await;
+    };
+    let denied = || {
+        (
+            StatusCode::NOT_FOUND,
+            [("cache-control", "no-store")],
+            "ACME challenge not found\n",
+        )
+            .into_response()
+    };
+    if token.is_empty()
+        || token.len() > 256
+        || !token
+            .bytes()
+            .all(|c| c.is_ascii_alphanumeric() || matches!(c, b'_' | b'-'))
+        || request.uri().query().is_some()
+        || request
+            .extensions()
+            .get::<crate::server::Transport>()
+            .is_some_and(|t| t.tls)
+    {
+        return denied();
+    }
+    let host = request
+        .headers()
+        .get(http::header::HOST)
+        .and_then(|h| h.to_str().ok())
+        .and_then(|h| crate::config::request_host(h).ok());
+    let permitted = state.acme.view().is_some_and(|job| {
+        job.status == "running"
+            && job.provider == "http01"
+            && host.is_some_and(|h| job.domains.contains(&h))
+    });
+    if !permitted {
+        return denied();
+    }
+    if request.method() != http::Method::GET && request.method() != http::Method::HEAD {
+        return (
+            StatusCode::METHOD_NOT_ALLOWED,
+            [("allow", "GET, HEAD"), ("cache-control", "no-store")],
+        )
+            .into_response();
+    }
+    let Some(directory) = &state.policy.cert_dir else {
+        return denied();
+    };
+    let path = directory
+        .join("acme-http01/.well-known/acme-challenge")
+        .join(token);
+    let mut options = tokio::fs::OpenOptions::new();
+    options.read(true);
+    #[cfg(unix)]
+    {
+        options.custom_flags(libc::O_NOFOLLOW);
+    }
+    let Ok(file) = options.open(path).await else {
+        return denied();
+    };
+    let Ok(metadata) = file.metadata().await else {
+        return denied();
+    };
+    if !metadata.is_file() || metadata.len() > 4096 {
+        return denied();
+    }
+    let mut data = Vec::new();
+    if file.take(4097).read_to_end(&mut data).await.is_err()
+        || data.len() > 4096
+        || data.is_empty()
+        || !data
+            .iter()
+            .all(|b| b.is_ascii_alphanumeric() || matches!(*b, b'_' | b'-' | b'.'))
+    {
+        return denied();
+    }
+    let length = data.len().to_string();
+    let mut response = (
+        [
+            ("content-type", "application/octet-stream"),
+            ("cache-control", "no-store"),
+            ("x-content-type-options", "nosniff"),
+        ],
+        data,
+    )
+        .into_response();
+    if let Ok(value) = length.parse() {
+        response.headers_mut().insert("content-length", value);
+    }
+    if request.method() == http::Method::HEAD {
+        *response.body_mut() = Body::empty();
+    }
+    response
+}
+
+#[cfg(test)]
+include!("providers/tests.rs");
+
+/// Public diagnostics contain no account/credential material. Desired renewal
+/// permission lives with the certificate; observations never overwrite it.
+#[derive(Clone, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct RenewalState {
+    pub last_checked: i64,
+    pub next_check: i64,
+    pub last_renewed: Option<i64>,
+    pub not_after: Option<i64>,
+    pub failures: u32,
+    pub status: String,
+    pub message: String,
+}
+fn private_json(path: &std::path::Path, limit: u64) -> anyhow::Result<Value> {
+    use std::io::Read;
+    let mut options = std::fs::OpenOptions::new();
+    options.read(true);
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::OpenOptionsExt;
+        options.custom_flags(libc::O_NOFOLLOW);
+    }
+    let file = options.open(path)?;
+    let meta = file.metadata()?;
+    anyhow::ensure!(
+        meta.is_file() && meta.len() <= limit,
+        "invalid private metadata"
+    );
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        anyhow::ensure!(
+            meta.permissions().mode() & 0o077 == 0,
+            "private metadata permissions"
+        );
+    }
+    let mut bytes = Vec::new();
+    file.take(limit + 1).read_to_end(&mut bytes)?;
+    anyhow::ensure!(bytes.len() as u64 <= limit, "private metadata too large");
+    Ok(serde_json::from_slice(&bytes)?)
+}
+fn profile_domains(directory: &std::path::Path, cert: &Certificate) -> anyhow::Result<Vec<String>> {
+    let issuer = cert
+        .issuer
+        .as_ref()
+        .ok_or_else(|| anyhow::anyhow!("no issuer"))?;
+    anyhow::ensure!(
+        crate::config::valid_id(&cert.id)
+            && matches!(
+                issuer.provider.as_str(),
+                "http01" | "rfc2136" | "cloudflare" | "ovh"
+            )
+            && matches!(issuer.environment.as_str(), "production" | "staging"),
+        "invalid profile identity"
+    );
+    let suffix = if issuer.provider == "rfc2136" {
+        String::new()
+    } else {
+        format!("-{}", issuer.provider)
+    };
+    let profile = private_json(
+        &directory
+            .join("acme")
+            .join(format!("{}-{}{}", issuer.environment, cert.id, suffix))
+            .join("profile.json"),
+        16384,
+    )?;
+    anyhow::ensure!(
+        profile.get("certificate_id").and_then(Value::as_str) == Some(cert.id.as_str())
+            && profile.get("provider").and_then(Value::as_str) == Some(issuer.provider.as_str())
+            && profile.get("environment").and_then(Value::as_str)
+                == Some(issuer.environment.as_str()),
+        "profile identity mismatch"
+    );
+    let names: Vec<String> = serde_json::from_value(profile["domains"].clone())?;
+    anyhow::ensure!(
+        !names.is_empty() && names.len() <= 16,
+        "invalid profile names"
+    );
+    for name in &names {
+        crate::config::normalize_host(name.strip_prefix("*.").unwrap_or(name))?;
+    }
+    Ok(names)
+}

src/acme/providers/tests.rs

@@ -0,0 +1,174 @@
+mod challenge_tests {
+    use super::*;
+    use axum::{body::Body, Extension};
+    use http::Request;
+    use http_body_util::BodyExt;
+    use tower::ServiceExt;
+    async fn fixture() -> (Arc<App>, tempfile::TempDir) {
+        let directory = tempfile::tempdir().unwrap();
+        let certs = directory.path().join("certs");
+        let root = certs.join("acme-http01/.well-known/acme-challenge");
+        std::fs::create_dir_all(&root).unwrap();
+        std::fs::write(root.join("sample_token"), b"sample_token.thumbprint").unwrap();
+        let config = crate::config::Config::default();
+        let policy = crate::config::ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: Some(certs),
+            data_dir: None,
+        };
+        let store = crate::db::Store::open(&directory.path().join("db"), &config)
+            .await
+            .unwrap();
+        let auth = crate::auth::Auth::new(store.clone(), directory.path().to_owned())
+            .await
+            .unwrap();
+        let snapshot = crate::config::compile(config, 0, &policy).unwrap();
+        let app = App::new(
+            store,
+            snapshot,
+            policy,
+            auth,
+            8,
+            Duration::from_secs(2),
+            Duration::from_secs(2),
+            Duration::from_secs(30),
+        );
+        *app.acme.job.lock().unwrap() = Some(Job {
+            id: "test".into(),
+            certificate_id: "test".into(),
+            status: "running",
+            kind: "issue",
+            provider: "http01".into(),
+            domains: vec!["example.org".into()],
+            environment: "staging".into(),
+            message: "test".into(),
+            log_directory: "test/logs".into(),
+            certificate: None,
+        });
+        (app, directory)
+    }
+    #[tokio::test]
+    async fn http01_serves_only_current_job_tokens_and_correct_head() {
+        let (app, _directory) = fixture().await;
+        let router = crate::proxy::router(app.clone());
+        let req = |method, host, path| {
+            Request::builder()
+                .method(method)
+                .uri(path)
+                .header("host", host)
+                .body(Body::empty())
+                .unwrap()
+        };
+        let response = router
+            .clone()
+            .oneshot(req(
+                "GET",
+                "example.org",
+                "/.well-known/acme-challenge/sample_token",
+            ))
+            .await
+            .unwrap();
+        assert_eq!(response.status(), 200);
+        assert_eq!(response.headers()["cache-control"], "no-store");
+        assert_eq!(
+            response
+                .into_body()
+                .collect()
+                .await
+                .unwrap()
+                .to_bytes()
+                .as_ref(),
+            b"sample_token.thumbprint"
+        );
+        let response = router
+            .clone()
+            .oneshot(req(
+                "HEAD",
+                "example.org",
+                "/.well-known/acme-challenge/sample_token",
+            ))
+            .await
+            .unwrap();
+        assert_eq!(response.status(), 200);
+        assert_eq!(response.headers()["content-length"], "23");
+        assert!(response
+            .into_body()
+            .collect()
+            .await
+            .unwrap()
+            .to_bytes()
+            .is_empty());
+        for path in [
+            "/.well-known/acme-challenge/../sample_token",
+            "/.well-known/acme-challenge/sample_token?x=1",
+            "/.well-known/acme-challenge/%2e%2e",
+            "/.well-known/acme-challenge/missing",
+        ] {
+            assert_eq!(
+                router
+                    .clone()
+                    .oneshot(req("GET", "example.org", path))
+                    .await
+                    .unwrap()
+                    .status(),
+                404
+            );
+        }
+        assert_eq!(
+            router
+                .clone()
+                .oneshot(req(
+                    "GET",
+                    "unrelated.example.org",
+                    "/.well-known/acme-challenge/sample_token"
+                ))
+                .await
+                .unwrap()
+                .status(),
+            404
+        );
+        assert_eq!(
+            router
+                .clone()
+                .oneshot(req(
+                    "POST",
+                    "example.org",
+                    "/.well-known/acme-challenge/sample_token"
+                ))
+                .await
+                .unwrap()
+                .status(),
+            405
+        );
+        let tls = router
+            .clone()
+            .layer(Extension(crate::server::Transport { tls: true }));
+        assert_eq!(
+            tls.oneshot(req(
+                "GET",
+                "example.org",
+                "/.well-known/acme-challenge/sample_token"
+            ))
+            .await
+            .unwrap()
+            .status(),
+            404
+        );
+        app.acme.job.lock().unwrap().as_mut().unwrap().status = "ready";
+        assert_eq!(
+            router
+                .oneshot(req(
+                    "GET",
+                    "example.org",
+                    "/.well-known/acme-challenge/sample_token"
+                ))
+                .await
+                .unwrap()
+                .status(),
+            404
+        );
+        app.store.close().await;
+    }
+}

src/acme/tests.rs

@@ -0,0 +1,148 @@
+mod renewal_regressions {
+    use super::*;
+    use crate::{
+        auth::Auth,
+        config::{self, Config, ValidationPolicy},
+        db::Store,
+    };
+    async fn configured(automatic: bool) -> (Arc<App>, tempfile::TempDir, Certificate, Value) {
+        let directory = tempfile::tempdir().unwrap();
+        let cert_dir = directory.path().join("certs");
+        crate::tls::initialize_default(&cert_dir).unwrap();
+        for name in ["acme-old", "acme-next"] {
+            crate::tls::generate(&cert_dir, name, vec!["example.org".into()]).unwrap();
+        }
+        let cert = Certificate {
+            id: "site".into(),
+            covered_hosts: Vec::new(),
+            hosts: vec!["example.org".into()],
+            cert_file: "acme-old.pem".into(),
+            key_file: "acme-old.key".into(),
+            issuer: Some(crate::tls::Issuer {
+                provider: "cloudflare".into(),
+                environment: "staging".into(),
+                auto_renew: automatic,
+            }),
+        };
+        let mut config = Config::default();
+        config.tls.certificates.push(cert.clone());
+        config.tls.default_certificate = Some(cert.id.clone());
+        let policy = ValidationPolicy {
+            proxy_addr: "127.0.0.1:8080".parse().unwrap(),
+            admin_addr: "127.0.0.1:9090".parse().unwrap(),
+            tls_addr: None,
+            cert_dir: Some(cert_dir),
+            data_dir: None,
+        };
+        let store = Store::open(&directory.path().join("db"), &config)
+            .await
+            .unwrap();
+        let auth = Auth::new(store.clone(), directory.path().to_owned())
+            .await
+            .unwrap();
+        let snapshot = config::compile(config, 0, &policy).unwrap();
+        let app = App::new(
+            store,
+            snapshot,
+            policy,
+            auth,
+            8,
+            Duration::from_secs(2),
+            Duration::from_secs(2),
+            Duration::from_secs(30),
+        );
+        let output = serde_json::json!({"provider":"cloudflare", "environment":"staging", "certificate_id":"site",
+            "domains":["example.org"], "cert_file":"acme-next.pem", "key_file":"acme-next.key", "changed":true});
+        (app, directory, cert, output)
+    }
+    async fn stop(app: Arc<App>) {
+        app.shutdown.cancel();
+        app.tunnels.close();
+        app.tunnels.wait().await;
+        app.store.close().await;
+    }
+    #[tokio::test]
+    async fn renewed_pair_publishes_one_revision_and_preserves_bindings_and_old_files() {
+        let (app, _directory, original, output) = configured(true).await;
+        deploy_renewal(&app, &original, &output, true)
+            .await
+            .unwrap();
+        let snapshot = app.config.load_full();
+        assert_eq!(snapshot.envelope.revision, 1);
+        let applied = &snapshot.envelope.config.tls.certificates[0];
+        assert_eq!(applied.cert_file, "acme-next.pem");
+        assert_eq!(applied.hosts, original.hosts);
+        assert!(applied.issuer.as_ref().unwrap().auto_renew);
+        assert_eq!(
+            app.store.load().await.unwrap().config.tls.certificates[0].cert_file,
+            "acme-next.pem"
+        );
+        assert!(app
+            .policy
+            .cert_dir
+            .as_ref()
+            .unwrap()
+            .join("acme-old.key")
+            .exists());
+        stop(app).await;
+    }
+    #[tokio::test]
+    async fn invalid_renewed_domain_never_changes_active_configuration() {
+        let (app, _directory, original, mut output) = configured(true).await;
+        output["domains"] = serde_json::json!(["unrelated.example.org"]);
+        assert!(deploy_renewal(&app, &original, &output, true)
+            .await
+            .is_err());
+        assert_eq!(app.config.load().envelope.revision, 0);
+        assert_eq!(
+            app.store.load().await.unwrap().config.tls.certificates[0].cert_file,
+            original.cert_file
+        );
+        stop(app).await;
+    }
+    #[tokio::test]
+    async fn operator_opt_out_wins_over_an_older_automatic_job() {
+        let (app, _directory, mut original, output) = configured(false).await;
+        original.issuer.as_mut().unwrap().auto_renew = true;
+        assert_eq!(
+            deploy_renewal(&app, &original, &output, true)
+                .await
+                .unwrap_err(),
+            "acme.configuration_changed"
+        );
+        assert_eq!(app.config.load().envelope.revision, 0);
+        // Explicit Check now remains allowed when scheduled checks are disabled.
+        deploy_renewal(&app, &original, &output, false)
+            .await
+            .unwrap();
+        assert!(
+            !app.config.load().envelope.config.tls.certificates[0]
+                .issuer
+                .as_ref()
+                .unwrap()
+                .auto_renew
+        );
+        stop(app).await;
+    }
+    #[tokio::test]
+    async fn saved_renewal_observations_are_persisted_in_sqlite_without_provider_credentials() {
+        let (app, _directory, _original, _output) = configured(true).await;
+        app.acme.renewals.lock().unwrap().insert(
+            "site".into(),
+            RenewalState {
+                last_checked: 100,
+                next_check: 43200,
+                status: "complete".into(),
+                message: "acme.not_due".into(),
+                ..Default::default()
+            },
+        );
+        app.acme.persist(&app.store).await;
+        let saved = app.store.load_certificate_renewals().await.unwrap();
+        let state = saved.get("site").unwrap();
+        assert_eq!(state.next_check, 43200);
+        assert_eq!(state.status, "complete");
+        assert_eq!(state.message, "acme.not_due");
+        stop(app).await;
+    }
+}