impl Manager {
    pub fn renewal_view(&self) -> std::collections::BTreeMap<String, RenewalState> {
        self.renewals
            .lock()
            .unwrap_or_else(|e| e.into_inner())
            .clone()
    }
    pub fn restore_renewals(&self, values: std::collections::BTreeMap<String, RenewalState>) {
        let mut observations = self.renewals.lock().unwrap_or_else(|e| e.into_inner());
        *observations = values
            .into_iter()
            .filter(|(id, _)| crate::config::valid_id(id))
            .take(64)
            .collect();
    }
    async fn persist(&self, store: &crate::db::Store) {
        let value = self.renewal_view();
        if let Err(error) = store.replace_certificate_renewals(&value).await {
            tracing::warn!(%error, "certificate renewal observation could not be saved to SQLite");
        }
    }
    pub async fn renew(
        state: Arc<App>,
        id: &str,
        actor: String,
        dry_run: bool,
        automatic: bool,
    ) -> Result<Job, ApiError> {
        let directory = state.policy.cert_dir.clone().ok_or_else(|| {
            ApiError(StatusCode::CONFLICT, "acme.no_certificate_directory".into())
        })?;
        let cert = state
            .config
            .load()
            .envelope
            .config
            .tls
            .certificates
            .iter()
            .find(|c| c.id == id)
            .cloned()
            .ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "acme.certificate_not_found".into()))?;
        let issuer = cert
            .issuer
            .clone()
            .ok_or_else(|| ApiError(StatusCode::CONFLICT, "acme.not_managed".into()))?;
        if automatic && !issuer.auto_renew {
            return Err(ApiError(
                StatusCode::CONFLICT,
                "acme.renewal_disabled".into(),
            ));
        }
        let slot = state
            .acme
            .slot
            .clone()
            .try_acquire_owned()
            .map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?;
        let copy = cert.clone();
        let dir = directory.clone();
        let domains = tokio::task::spawn_blocking(move || profile_domains(&dir, &copy))
            .await
            .map_err(crate::auth::internal)?
            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.renewal_profile_missing".into()))?;
        if issuer.provider == "http01" && !state.active_listeners.iter().any(|l| !l.tls) {
            return Err(ApiError(
                StatusCode::CONFLICT,
                "acme.http01_listener_required".into(),
            ));
        }
        let job = Job {
            id: format!("{:032x}", rand::random::<u128>()),
            certificate_id: cert.id.clone(),
            status: "running",
            kind: if dry_run { "renewal-dry-run" } else { "renewal" },
            log_directory: certbot_log_directory(&directory, &issuer.provider, &issuer.environment, &cert.id),
            provider: issuer.provider.clone(),
            domains,
            environment: issuer.environment.clone(),
            message: "acme.renewing".into(),
            certificate: None,
        };
        state
            .store
            .audit(
                &actor,
                if dry_run {
                    "acme.renew_dry_run"
                } else {
                    "acme.renew"
                },
                &cert.id,
            )
            .await
            .map_err(crate::auth::internal)?;
        *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone());
        let mut request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider,
            "environment":issuer.environment,"cert_file":cert.cert_file,"key_file":cert.key_file,"dry_run":dry_run});
        if issuer.provider != "http01" {
            let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| {
                ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into())
            })?;
            let credential = crate::secret_store::get(
                &state.store,
                data_dir,
                &crate::secret_store::acme_secret_id(&cert.id, &issuer.provider),
                Some(&issuer.provider),
            )
            .await
            .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.secret_missing".into()))?;
            request["credential_ini_b64"] = Value::String(
                base64::engine::general_purpose::STANDARD.encode(credential.as_slice())
            );
        }
        let raw = Zeroizing::new(request.to_string());
        if let Some(Value::String(secret)) = request.get_mut("credential_ini_b64") {
            secret.zeroize();
        }
        let task_job = job.clone();
        let tasks = state.tunnels.clone();
        tasks.spawn(async move {
            let _slot = slot;
            let result = async {
                let output = worker(&state, directory.clone(), raw).await?;
                if !dry_run && output.get("changed") == Some(&Value::Bool(true)) {
                    deploy_renewal(&state, &cert, &output, automatic).await?;
                }
                Ok::<_, String>(output)
            }
            .await;
            let mut finished = task_job;
            let now = crate::auth::now();
            {
                let mut observations = state
                    .acme
                    .renewals
                    .lock()
                    .unwrap_or_else(|e| e.into_inner());
                let entry = observations.entry(cert.id.clone()).or_default();
                entry.last_checked = now;
                match result {
                    Ok(output) => {
                        let changed = !dry_run && output.get("changed") == Some(&Value::Bool(true));
                        finished.status = "complete";
                        finished.message = if dry_run {
                            "acme.renewal_test_ok"
                        } else if changed {
                            "acme.renewed"
                        } else {
                            "acme.not_due"
                        }
                        .into();
                        // A dry run must not postpone a due production renewal.
                        if !dry_run {
                            entry.failures = 0;
                            entry.next_check = now + 43200 + (rand::random::<u16>() as i64 % 1800);
                        }
                        entry.not_after = output
                            .get("not_after_unix")
                            .and_then(Value::as_i64)
                            .or(entry.not_after);
                        if changed {
                            entry.last_renewed = Some(now);
                        }
                    }
                    Err(message) => {
                        finished.status = "failed";
                        finished.message = message;
                        entry.failures = entry.failures.saturating_add(1);
                        if !dry_run {
                            entry.next_check = now + 3600 * i64::from(entry.failures.min(6));
                        }
                    }
                }
                entry.status = finished.status.into();
                entry.message = finished.message.clone();
            }
            *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(finished.clone());
            state.acme.persist(&state.store).await;
            let _ = state
                .store
                .audit(
                    &actor,
                    "acme.renew_result",
                    &format!("{} {}", cert.id, finished.message),
                )
                .await;
        });
        Ok(job)
    }
}
async fn deploy_renewal(
    state: &Arc<App>,
    original: &Certificate,
    value: &Value,
    automatic: bool,
) -> Result<(), String> {
    let data: Output = serde_json::from_value(value.clone())
        .map_err(|_| "acme.invalid_worker_response".to_owned())?;
    if data.certificate_id != original.id
        || original
            .issuer
            .as_ref()
            .is_none_or(|i| i.provider != data.provider || i.environment != data.environment)
    {
        return Err("acme.renewal_profile_mismatch".into());
    }
    let dir = state
        .policy
        .cert_dir
        .clone()
        .ok_or("acme.no_certificate_directory")?;
    let mut renewed = tokio::task::spawn_blocking(move || {
        crate::tls::validate_acme(
            &dir,
            data.certificate_id,
            data.cert_file,
            data.key_file,
            &data.domains,
        )
    })
    .await
    .map_err(|_| "acme.validation_worker_failed".to_owned())?
    .map_err(|_| "acme.certificate_validation_failed".to_owned())?;
    // CAS retries preserve edits to other parts of the document. A simultaneous
    // change of this certificate wins over the worker; never silently overwrite it.
    for _ in 0..3 {
        let snapshot = state.config.load_full();
        let Some(current) = snapshot
            .envelope
            .config
            .tls
            .certificates
            .iter()
            .find(|c| c.id == original.id)
        else {
            return Err("acme.configuration_changed".into());
        };
        if current.cert_file != original.cert_file
            || current.key_file != original.key_file
            || current.issuer.as_ref().is_none_or(|i| {
                original.issuer.as_ref().is_none_or(|old| {
                    i.provider != old.provider || i.environment != old.environment
                }) || (automatic && !i.auto_renew)
            })
        {
            return Err("acme.configuration_changed".into());
        }
        renewed.hosts = current.hosts.clone();
        renewed.issuer = current.issuer.clone();
        let mut config = snapshot.envelope.config.clone();
        let slot = config
            .tls
            .certificates
            .iter_mut()
            .find(|c| c.id == original.id)
            .ok_or("acme.configuration_changed")?;
        *slot = renewed.clone();
        let update = crate::config::Update {
            expected_revision: snapshot.envelope.revision,
            config,
        };
        match state
            .update_as(update, "system:certificate-renewal".into())
            .await
        {
            Ok(_) => return Ok(()),
            Err(ApiError(StatusCode::CONFLICT, _)) => continue,
            Err(_) => return Err("acme.deployment_failed".into()),
        }
    }
    Err("acme.configuration_changed".into())
}
/// One scheduler per process, sharing the issuance semaphore. It does not run
/// its own shell hooks or stop listeners; validated snapshots rotate TCP and QUIC TLS.
pub async fn renewal_loop(
    weak: std::sync::Weak<App>,
    shutdown: tokio_util::sync::CancellationToken,
) {
    // Observations are restored from SQLite before listeners are exposed. If an
    // older installation has no SQLite observations, the normal scheduler will
    // safely reconstruct them on its first check; certificate/key state is not
    // derived from historical observation sidecars.
    let delay = 300 + (rand::random::<u16>() as u64 % 300);
    tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(delay)) => {} }
    loop {
        let Some(state) = weak.upgrade() else {
            return;
        };
        let certificates = state.config.load().envelope.config.tls.certificates.clone();
        {
            let mut observations = state
                .acme
                .renewals
                .lock()
                .unwrap_or_else(|e| e.into_inner());
            observations.retain(|id, _| certificates.iter().any(|c| &c.id == id));
        }
        let view = state.acme.renewal_view();
        let now = crate::auth::now();
        for cert in certificates
            .iter()
            .filter(|c| c.issuer.as_ref().is_some_and(|i| i.auto_renew))
        {
            if view
                .get(&cert.id)
                .is_some_and(|r| r.next_check > now && r.next_check <= now + 86400)
            {
                continue;
            }
            match Manager::renew(
                state.clone(),
                &cert.id,
                "system:certificate-renewal".into(),
                false,
                true,
            )
            .await
            {
                Ok(_) => break,
                Err(ApiError(StatusCode::TOO_MANY_REQUESTS, _)) => break,
                Err(error) => {
                    state
                        .acme
                        .renewals
                        .lock()
                        .unwrap_or_else(|e| e.into_inner())
                        .insert(
                            cert.id.clone(),
                            RenewalState {
                                last_checked: now,
                                next_check: now + 3600,
                                failures: 1,
                                status: "failed".into(),
                                message: error.1,
                                ..Default::default()
                            },
                        );
                    state.acme.persist(&state.store).await;
                }
            }
        }
        drop(state);
        tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(60)) => {} }
    }
}