347 lines
13 KiB
Rust
347 lines
13 KiB
Rust
impl Manager {
|
|
pub fn renewal_view(&self) -> std::collections::BTreeMap<String, RenewalState> {
|
|
self.renewals
|
|
.lock()
|
|
.unwrap_or_else(|e| e.into_inner())
|
|
.clone()
|
|
}
|
|
pub fn restore_renewals(&self, values: std::collections::BTreeMap<String, RenewalState>) {
|
|
let mut observations = self.renewals.lock().unwrap_or_else(|e| e.into_inner());
|
|
*observations = values
|
|
.into_iter()
|
|
.filter(|(id, _)| crate::config::valid_id(id))
|
|
.take(64)
|
|
.collect();
|
|
}
|
|
async fn persist(&self, store: &crate::db::Store) {
|
|
let value = self.renewal_view();
|
|
if let Err(error) = store.replace_certificate_renewals(&value).await {
|
|
tracing::warn!(%error, "certificate renewal observation could not be saved to SQLite");
|
|
}
|
|
}
|
|
pub async fn renew(
|
|
state: Arc<App>,
|
|
id: &str,
|
|
actor: String,
|
|
dry_run: bool,
|
|
automatic: bool,
|
|
) -> Result<Job, ApiError> {
|
|
let directory = state.policy.cert_dir.clone().ok_or_else(|| {
|
|
ApiError(StatusCode::CONFLICT, "acme.no_certificate_directory".into())
|
|
})?;
|
|
let cert = state
|
|
.config
|
|
.load()
|
|
.envelope
|
|
.config
|
|
.tls
|
|
.certificates
|
|
.iter()
|
|
.find(|c| c.id == id)
|
|
.cloned()
|
|
.ok_or_else(|| ApiError(StatusCode::NOT_FOUND, "acme.certificate_not_found".into()))?;
|
|
let issuer = cert
|
|
.issuer
|
|
.clone()
|
|
.ok_or_else(|| ApiError(StatusCode::CONFLICT, "acme.not_managed".into()))?;
|
|
if automatic && !issuer.auto_renew {
|
|
return Err(ApiError(
|
|
StatusCode::CONFLICT,
|
|
"acme.renewal_disabled".into(),
|
|
));
|
|
}
|
|
let slot = state
|
|
.acme
|
|
.slot
|
|
.clone()
|
|
.try_acquire_owned()
|
|
.map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?;
|
|
let copy = cert.clone();
|
|
let dir = directory.clone();
|
|
let domains = tokio::task::spawn_blocking(move || profile_domains(&dir, ©))
|
|
.await
|
|
.map_err(crate::auth::internal)?
|
|
.map_err(|_| ApiError(StatusCode::CONFLICT, "acme.renewal_profile_missing".into()))?;
|
|
if issuer.provider == "http01" && !state.active_listeners.iter().any(|l| !l.tls) {
|
|
return Err(ApiError(
|
|
StatusCode::CONFLICT,
|
|
"acme.http01_listener_required".into(),
|
|
));
|
|
}
|
|
let job = Job {
|
|
id: format!("{:032x}", rand::random::<u128>()),
|
|
certificate_id: cert.id.clone(),
|
|
status: "running",
|
|
kind: if dry_run { "renewal-dry-run" } else { "renewal" },
|
|
log_directory: certbot_log_directory(&directory, &issuer.provider, &issuer.environment, &cert.id),
|
|
provider: issuer.provider.clone(),
|
|
domains,
|
|
environment: issuer.environment.clone(),
|
|
message: "acme.renewing".into(),
|
|
certificate: None,
|
|
};
|
|
state
|
|
.store
|
|
.audit(
|
|
&actor,
|
|
if dry_run {
|
|
"acme.renew_dry_run"
|
|
} else {
|
|
"acme.renew"
|
|
},
|
|
&cert.id,
|
|
)
|
|
.await
|
|
.map_err(crate::auth::internal)?;
|
|
*state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone());
|
|
let mut request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider,
|
|
"environment":issuer.environment,"cert_file":cert.cert_file,"key_file":cert.key_file,"dry_run":dry_run});
|
|
if issuer.provider != "http01" {
|
|
let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| {
|
|
ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into())
|
|
})?;
|
|
let credential = crate::secret_store::get(
|
|
&state.store,
|
|
data_dir,
|
|
&crate::secret_store::acme_secret_id(&cert.id, &issuer.provider),
|
|
Some(&issuer.provider),
|
|
)
|
|
.await
|
|
.map_err(|_| ApiError(StatusCode::CONFLICT, "acme.secret_missing".into()))?;
|
|
request["credential_ini_b64"] = Value::String(
|
|
base64::engine::general_purpose::STANDARD.encode(credential.as_slice())
|
|
);
|
|
}
|
|
let raw = Zeroizing::new(request.to_string());
|
|
if let Some(Value::String(secret)) = request.get_mut("credential_ini_b64") {
|
|
secret.zeroize();
|
|
}
|
|
let task_job = job.clone();
|
|
let tasks = state.tunnels.clone();
|
|
tasks.spawn(async move {
|
|
let _slot = slot;
|
|
let result = async {
|
|
let output = worker(&state, directory.clone(), raw).await?;
|
|
if !dry_run && output.get("changed") == Some(&Value::Bool(true)) {
|
|
deploy_renewal(&state, &cert, &output, automatic).await?;
|
|
}
|
|
Ok::<_, String>(output)
|
|
}
|
|
.await;
|
|
let mut finished = task_job;
|
|
let now = crate::auth::now();
|
|
{
|
|
let mut observations = state
|
|
.acme
|
|
.renewals
|
|
.lock()
|
|
.unwrap_or_else(|e| e.into_inner());
|
|
let entry = observations.entry(cert.id.clone()).or_default();
|
|
entry.last_checked = now;
|
|
match result {
|
|
Ok(output) => {
|
|
let changed = !dry_run && output.get("changed") == Some(&Value::Bool(true));
|
|
finished.status = "complete";
|
|
finished.message = if dry_run {
|
|
"acme.renewal_test_ok"
|
|
} else if changed {
|
|
"acme.renewed"
|
|
} else {
|
|
"acme.not_due"
|
|
}
|
|
.into();
|
|
// A dry run must not postpone a due production renewal.
|
|
if !dry_run {
|
|
entry.failures = 0;
|
|
entry.next_check = now + 43200 + (rand::random::<u16>() as i64 % 1800);
|
|
}
|
|
entry.not_after = output
|
|
.get("not_after_unix")
|
|
.and_then(Value::as_i64)
|
|
.or(entry.not_after);
|
|
if changed {
|
|
entry.last_renewed = Some(now);
|
|
}
|
|
}
|
|
Err(message) => {
|
|
finished.status = "failed";
|
|
finished.message = message;
|
|
entry.failures = entry.failures.saturating_add(1);
|
|
if !dry_run {
|
|
entry.next_check = now + 3600 * i64::from(entry.failures.min(6));
|
|
}
|
|
}
|
|
}
|
|
entry.status = finished.status.into();
|
|
entry.message = finished.message.clone();
|
|
}
|
|
*state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(finished.clone());
|
|
state.acme.persist(&state.store).await;
|
|
let _ = state
|
|
.store
|
|
.audit(
|
|
&actor,
|
|
"acme.renew_result",
|
|
&format!("{} {}", cert.id, finished.message),
|
|
)
|
|
.await;
|
|
});
|
|
Ok(job)
|
|
}
|
|
}
|
|
async fn deploy_renewal(
|
|
state: &Arc<App>,
|
|
original: &Certificate,
|
|
value: &Value,
|
|
automatic: bool,
|
|
) -> Result<(), String> {
|
|
let data: Output = serde_json::from_value(value.clone())
|
|
.map_err(|_| "acme.invalid_worker_response".to_owned())?;
|
|
if data.certificate_id != original.id
|
|
|| original
|
|
.issuer
|
|
.as_ref()
|
|
.is_none_or(|i| i.provider != data.provider || i.environment != data.environment)
|
|
{
|
|
return Err("acme.renewal_profile_mismatch".into());
|
|
}
|
|
let dir = state
|
|
.policy
|
|
.cert_dir
|
|
.clone()
|
|
.ok_or("acme.no_certificate_directory")?;
|
|
let mut renewed = tokio::task::spawn_blocking(move || {
|
|
crate::tls::validate_acme(
|
|
&dir,
|
|
data.certificate_id,
|
|
data.cert_file,
|
|
data.key_file,
|
|
&data.domains,
|
|
)
|
|
})
|
|
.await
|
|
.map_err(|_| "acme.validation_worker_failed".to_owned())?
|
|
.map_err(|_| "acme.certificate_validation_failed".to_owned())?;
|
|
// CAS retries preserve edits to other parts of the document. A simultaneous
|
|
// change of this certificate wins over the worker; never silently overwrite it.
|
|
for _ in 0..3 {
|
|
let snapshot = state.config.load_full();
|
|
let Some(current) = snapshot
|
|
.envelope
|
|
.config
|
|
.tls
|
|
.certificates
|
|
.iter()
|
|
.find(|c| c.id == original.id)
|
|
else {
|
|
return Err("acme.configuration_changed".into());
|
|
};
|
|
if current.cert_file != original.cert_file
|
|
|| current.key_file != original.key_file
|
|
|| current.issuer.as_ref().is_none_or(|i| {
|
|
original.issuer.as_ref().is_none_or(|old| {
|
|
i.provider != old.provider || i.environment != old.environment
|
|
}) || (automatic && !i.auto_renew)
|
|
})
|
|
{
|
|
return Err("acme.configuration_changed".into());
|
|
}
|
|
renewed.hosts = current.hosts.clone();
|
|
renewed.issuer = current.issuer.clone();
|
|
let mut config = snapshot.envelope.config.clone();
|
|
let slot = config
|
|
.tls
|
|
.certificates
|
|
.iter_mut()
|
|
.find(|c| c.id == original.id)
|
|
.ok_or("acme.configuration_changed")?;
|
|
*slot = renewed.clone();
|
|
let update = crate::config::Update {
|
|
expected_revision: snapshot.envelope.revision,
|
|
config,
|
|
};
|
|
match state
|
|
.update_as(update, "system:certificate-renewal".into())
|
|
.await
|
|
{
|
|
Ok(_) => return Ok(()),
|
|
Err(ApiError(StatusCode::CONFLICT, _)) => continue,
|
|
Err(_) => return Err("acme.deployment_failed".into()),
|
|
}
|
|
}
|
|
Err("acme.configuration_changed".into())
|
|
}
|
|
/// One scheduler per process, sharing the issuance semaphore. It does not run
|
|
/// its own shell hooks or stop listeners; validated snapshots rotate TCP and QUIC TLS.
|
|
pub async fn renewal_loop(
|
|
weak: std::sync::Weak<App>,
|
|
shutdown: tokio_util::sync::CancellationToken,
|
|
) {
|
|
// Observations are restored from SQLite before listeners are exposed. If an
|
|
// older installation has no SQLite observations, the normal scheduler will
|
|
// safely reconstruct them on its first check; certificate/key state is not
|
|
// derived from historical observation sidecars.
|
|
let delay = 300 + (rand::random::<u16>() as u64 % 300);
|
|
tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(delay)) => {} }
|
|
loop {
|
|
let Some(state) = weak.upgrade() else {
|
|
return;
|
|
};
|
|
let certificates = state.config.load().envelope.config.tls.certificates.clone();
|
|
{
|
|
let mut observations = state
|
|
.acme
|
|
.renewals
|
|
.lock()
|
|
.unwrap_or_else(|e| e.into_inner());
|
|
observations.retain(|id, _| certificates.iter().any(|c| &c.id == id));
|
|
}
|
|
let view = state.acme.renewal_view();
|
|
let now = crate::auth::now();
|
|
for cert in certificates
|
|
.iter()
|
|
.filter(|c| c.issuer.as_ref().is_some_and(|i| i.auto_renew))
|
|
{
|
|
if view
|
|
.get(&cert.id)
|
|
.is_some_and(|r| r.next_check > now && r.next_check <= now + 86400)
|
|
{
|
|
continue;
|
|
}
|
|
match Manager::renew(
|
|
state.clone(),
|
|
&cert.id,
|
|
"system:certificate-renewal".into(),
|
|
false,
|
|
true,
|
|
)
|
|
.await
|
|
{
|
|
Ok(_) => break,
|
|
Err(ApiError(StatusCode::TOO_MANY_REQUESTS, _)) => break,
|
|
Err(error) => {
|
|
state
|
|
.acme
|
|
.renewals
|
|
.lock()
|
|
.unwrap_or_else(|e| e.into_inner())
|
|
.insert(
|
|
cert.id.clone(),
|
|
RenewalState {
|
|
last_checked: now,
|
|
next_check: now + 3600,
|
|
failures: 1,
|
|
status: "failed".into(),
|
|
message: error.1,
|
|
..Default::default()
|
|
},
|
|
);
|
|
state.acme.persist(&state.store).await;
|
|
}
|
|
}
|
|
}
|
|
drop(state);
|
|
tokio::select! { _ = shutdown.cancelled() => return, _ = tokio::time::sleep(Duration::from_secs(60)) => {} }
|
|
}
|
|
}
|