v0.25.1

src/acme/jobs.rs

@@ -0,0 +1,346 @@
+impl Manager {
+    pub fn renewal_view(&self) -> std::collections::BTreeMap<String, RenewalState> {
+        self.renewals
+            .lock()
+            .unwrap_or_else(|e| e.into_inner())
+            .clone()
+    }
+    pub fn restore_renewals(&self, values: std::collections::BTreeMap<String, RenewalState>) {
+        let mut observations = self.renewals.lock().unwrap_or_else(|e| e.into_inner());
+        *observations = values
+            .into_iter()
+            .filter(|(id, _)| crate::config::valid_id(id))
+            .take(64)
+            .collect();
+    }
+    async fn persist(&self, store: &crate::db::Store) {
+        let value = self.renewal_view();
+        if let Err(error) = store.replace_certificate_renewals(&value).await {
+            tracing::warn!(%error, "certificate renewal observation could not be saved to SQLite");
+        }
+    }
+    pub async fn renew(
+        state: Arc<App>,
+        id: &str,
+        actor: String,
+        dry_run: bool,
+        automatic: bool,
+    ) -> Result<Job, ApiError> {
+        let directory = state.policy.cert_dir.clone().ok_or_else(|| {
+            ApiError(StatusCode::CONFLICT, "acme.no_certificate_directory".into())
+        })?;
+        let cert = state
+            .config
+            .load()
+            .envelope
+            .config
+            .tls
+            .certificates
+            .iter()
+            .find(|c| c.id == id)
+            .cloned()
+            .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "acme.certificate_not_found".into()))?;
+        let issuer = cert
+            .issuer
+            .clone()
+            .ok_or_else(|| ApiError(StatusCode::CONFLICT, "acme.not_managed".into()))?;
+        if automatic && !issuer.auto_renew {
+            return Err(ApiError(
+                StatusCode::CONFLICT,
+                "acme.renewal_disabled".into(),
+            ));
+        }
+        let slot = state
+            .acme
+            .slot
+            .clone()
+            .try_acquire_owned()
+            .map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?;
+        let copy = cert.clone();
+        let dir = directory.clone();
+        let domains = tokio::task::spawn_blocking(move || profile_domains(&dir, &copy))
+            .await
+            .map_err(crate::auth::internal)?
+            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.renewal_profile_missing".into()))?;
+        if issuer.provider == "http01" && !state.active_listeners.iter().any(|l| !l.tls) {
+            return Err(ApiError(
+                StatusCode::CONFLICT,
+                "acme.http01_listener_required".into(),
+            ));
+        }
+        let job = Job {
+            id: format!("{:032x}", rand::random::<u128>()),
+            certificate_id: cert.id.clone(),
+            status: "running",
+            kind: if dry_run { "renewal-dry-run" } else { "renewal" },
+            log_directory: certbot_log_directory(&directory, &issuer.provider, &issuer.environment, &cert.id),
+            provider: issuer.provider.clone(),
+            domains,
+            environment: issuer.environment.clone(),
+            message: "acme.renewing".into(),
+            certificate: None,
+        };
+        state
+            .store
+            .audit(
+                &actor,
+                if dry_run {
+                    "acme.renew_dry_run"
+                } else {
+                    "acme.renew"
+                },
+                &cert.id,
+            )
+            .await
+            .map_err(crate::auth::internal)?;
+        *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone());
+        let mut request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider,
+            "environment":issuer.environment,"cert_file":cert.cert_file,"key_file":cert.key_file,"dry_run":dry_run});
+        if issuer.provider != "http01" {
+            let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| {
+                ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into())
+            })?;
+            let credential = crate::secret_store::get(
+                &state.store,
+                data_dir,
+                &crate::secret_store::acme_secret_id(&cert.id, &issuer.provider),
+                Some(&issuer.provider),
+            )
+            .await
+            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.secret_missing".into()))?;
+            request["credential_ini_b64"] = Value::String(
+                base64::engine::general_purpose::STANDARD.encode(credential.as_slice())
+            );
+        }
+        let raw = Zeroizing::new(request.to_string());
+        if let Some(Value::String(secret)) = request.get_mut("credential_ini_b64") {
+            secret.zeroize();
+        }
+        let task_job = job.clone();
+        let tasks = state.tunnels.clone();
+        tasks.spawn(async move {
+            let _slot = slot;
+            let result = async {
+                let output = worker(&state, directory.clone(), raw).await?;
+                if !dry_run && output.get("changed") == Some(&Value::Bool(true)) {
+                    deploy_renewal(&state, &cert, &output, automatic).await?;
+                }
+                Ok::<_, String>(output)
+            }
+            .await;
+            let mut finished = task_job;
+            let now = crate::auth::now();
+            {
+                let mut observations = state
+                    .acme
+                    .renewals
+                    .lock()
+                    .unwrap_or_else(|e| e.into_inner());
+                let entry = observations.entry(cert.id.clone()).or_default();
+                entry.last_checked = now;
+                match result {
+                    Ok(output) => {
+                        let changed = !dry_run && output.get("changed") == Some(&Value::Bool(true));
+                        finished.status = "complete";
+                        finished.message = if dry_run {
+                            "acme.renewal_test_ok"
+                        } else if changed {
+                            "acme.renewed"
+                        } else {
+                            "acme.not_due"
+                        }
+                        .into();
+                        // A dry run must not postpone a due production renewal.
+                        if !dry_run {
+                            entry.failures = 0;
+                            entry.next_check = now + 43200 + (rand::random::<u16>() as i64 % 1800);
+                        }
+                        entry.not_after = output
+                            .get("not_after_unix")
+                            .and_then(Value::as_i64)
+                            .or(entry.not_after);
+                        if changed {
+                            entry.last_renewed = Some(now);
+                        }
+                    }
+                    Err(message) => {
+                        finished.status = "failed";
+                        finished.message = message;
+                        entry.failures = entry.failures.saturating_add(1);
+                        if !dry_run {
+                            entry.next_check = now + 3600 * i64::from(entry.failures.min(6));
+                        }
+                    }
+                }
+                entry.status = finished.status.into();
+                entry.message = finished.message.clone();
+            }
+            *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(finished.clone());
+            state.acme.persist(&state.store).await;
+            let _ = state
+                .store
+                .audit(
+                    &actor,
+                    "acme.renew_result",
+                    &format!("{} {}", cert.id, finished.message),
+                )
+                .await;
+        });
+        Ok(job)
+    }
+}
+async fn deploy_renewal(
+    state: &Arc<App>,
+    original: &Certificate,
+    value: &Value,
+    automatic: bool,
+) -> Result<(), String> {
+    let data: Output = serde_json::from_value(value.clone())
+        .map_err(|_| "acme.invalid_worker_response".to_owned())?;
+    if data.certificate_id != original.id
+        || original
+            .issuer
+            .as_ref()
+            .is_none_or(|i| i.provider != data.provider || i.environment != data.environment)
+    {
+        return Err("acme.renewal_profile_mismatch".into());
+    }
+    let dir = state
+        .policy
+        .cert_dir
+        .clone()
+        .ok_or("acme.no_certificate_directory")?;
+    let mut renewed = tokio::task::spawn_blocking(move || {
+        crate::tls::validate_acme(
+            &dir,
+            data.certificate_id,
+            data.cert_file,
+            data.key_file,
+            &data.domains,
+        )
+    })
+    .await
+    .map_err(|_| "acme.validation_worker_failed".to_owned())?
+    .map_err(|_| "acme.certificate_validation_failed".to_owned())?;
+    // CAS retries preserve edits to other parts of the document. A simultaneous
+    // change of this certificate wins over the worker; never silently overwrite it.
+    for _ in 0..3 {
+        let snapshot = state.config.load_full();
+        let Some(current) = snapshot
+            .envelope
+            .config
+            .tls
+            .certificates
+            .iter()
+            .find(|c| c.id == original.id)
+        else {
+            return Err("acme.configuration_changed".into());
+        };
+        if current.cert_file != original.cert_file
+            || current.key_file != original.key_file
+            || current.issuer.as_ref().is_none_or(|i| {
+                original.issuer.as_ref().is_none_or(|old| {
+                    i.provider != old.provider || i.environment != old.environment
+                }) || (automatic && !i.auto_renew)
+            })
+        {
+            return Err("acme.configuration_changed".into());
+        }
+        renewed.hosts = current.hosts.clone();
+        renewed.issuer = current.issuer.clone();
+        let mut config = snapshot.envelope.config.clone();
+        let slot = config
+            .tls
+            .certificates
+            .iter_mut()
+            .find(|c| c.id == original.id)
+            .ok_or("acme.configuration_changed")?;
+        *slot = renewed.clone();
+        let update = crate::config::Update {
+            expected_revision: snapshot.envelope.revision,
+            config,
+        };
+        match state
+            .update_as(update, "system:certificate-renewal".into())
+            .await
+        {
+            Ok(_) => return Ok(()),
+            Err(ApiError(StatusCode::CONFLICT, _)) => continue,
+            Err(_) => return Err("acme.deployment_failed".into()),
+        }
+    }
+    Err("acme.configuration_changed".into())
+}
+/// One scheduler per process, sharing the issuance semaphore. It does not run
+/// its own shell hooks or stop listeners; validated snapshots rotate TCP and QUIC TLS.
+pub async fn renewal_loop(
+    weak: std::sync::Weak<App>,
+    shutdown: tokio_util::sync::CancellationToken,
+) {
+    // Observations are restored from SQLite before listeners are exposed. If an
+    // older installation has no SQLite observations, the normal scheduler will
+    // safely reconstruct them on its first check; certificate/key state is not
+    // derived from historical observation sidecars.
+    let delay = 300 + (rand::random::<u16>() as u64 % 300);
+    tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(delay)) => {} }
+    loop {
+        let Some(state) = weak.upgrade() else {
+            return;
+        };
+        let certificates = state.config.load().envelope.config.tls.certificates.clone();
+        {
+            let mut observations = state
+                .acme
+                .renewals
+                .lock()
+                .unwrap_or_else(|e| e.into_inner());
+            observations.retain(|id, _| certificates.iter().any(|c| &c.id == id));
+        }
+        let view = state.acme.renewal_view();
+        let now = crate::auth::now();
+        for cert in certificates
+            .iter()
+            .filter(|c| c.issuer.as_ref().is_some_and(|i| i.auto_renew))
+        {
+            if view
+                .get(&cert.id)
+                .is_some_and(|r| r.next_check > now && r.next_check <= now + 86400)
+            {
+                continue;
+            }
+            match Manager::renew(
+                state.clone(),
+                &cert.id,
+                "system:certificate-renewal".into(),
+                false,
+                true,
+            )
+            .await
+            {
+                Ok(_) => break,
+                Err(ApiError(StatusCode::TOO_MANY_REQUESTS, _)) => break,
+                Err(error) => {
+                    state
+                        .acme
+                        .renewals
+                        .lock()
+                        .unwrap_or_else(|e| e.into_inner())
+                        .insert(
+                            cert.id.clone(),
+                            RenewalState {
+                                last_checked: now,
+                                next_check: now + 3600,
+                                failures: 1,
+                                status: "failed".into(),
+                                message: error.1,
+                                ..Default::default()
+                            },
+                        );
+                    state.acme.persist(&state.store).await;
+                }
+            }
+        }
+        drop(state);
+        tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(60)) => {} }
+    }
+}