0.27.2_hotfix1

scripts/deploy_runtime.py

@@ -3,12 +3,21 @@ from __future__ import annotations
 
 from deploy_core import *  # re-exported internal deployment primitives
 
+ACME_RUNTIME_NAMES = frozenset({'acme-python', 'acme-python.installing', 'acme-python.virtualenv-bootstrap'})
+CERTBOT_PACKAGES = ('certbot==5.8.0', 'certbot-dns-cloudflare==5.8.0', 'certbot-dns-ovh==5.8.0', 'certbot-dns-rfc2136==5.8.0')
+
+def _under_acme_runtime(source: Path, path: Path) -> bool:
+    relative = path.relative_to(source)
+    return bool(relative.parts) and relative.parts[0] in ACME_RUNTIME_NAMES
+
 def checked_tree(source: Path) -> None:
-    """Accept only regular files/directories and relative Certbot-internal symlinks."""
+    """Accept durable data; managed ACME Python environments are rebuildable."""
     boundary = source.resolve(strict=True)
     if source.is_symlink():
         raise ValueError('Data root cannot be a symlink.')
     for p in source.rglob('*'):
+        if _under_acme_runtime(source, p):
+            continue
         st = p.lstat()
         if stat.S_ISLNK(st.st_mode):
             target = os.readlink(p)
@@ -17,12 +26,88 @@ def checked_tree(source: Path) -> None:
         elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)):
             raise ValueError('Data backup contains a special file.')
 
-
 def copy_data(source: Path, target: Path) -> None:
     checked_tree(source)
-    shutil.copytree(source, target, symlinks=True)
+    root = source.resolve(strict=True)
+    def ignore(directory: str, names: list[str]) -> set[str]:
+        return set(names) & ACME_RUNTIME_NAMES if Path(directory).resolve(strict=True) == root else set()
+    shutil.copytree(source, target, symlinks=True, ignore=ignore)
     os.chmod(target, 0o700)
 
+def _run_user(account, data: Path, args: list[str], timeout: int) -> subprocess.CompletedProcess:
+    env = os.environ.copy(); env.update(HOME=str(data), PATH='/usr/local/bin:/usr/bin:/bin')
+    env.pop('PYTHONHOME', None); env.pop('PYTHONPATH', None)
+    return subprocess.run(['runuser', '-u', account.pw_name, '--', *args], check=False, stdin=subprocess.DEVNULL,
+                          stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout, env=env)
+
+def _acme_python(data: Path) -> Path:
+    return data / 'acme-python' / 'bin/python'
+
+def _acme_runtime_valid(data: Path, account) -> bool:
+    python = _acme_python(data)
+    if not python.is_file():
+        return False
+    checks = ([str(python), '-I', '-c', 'import certbot, certbot_dns_cloudflare, certbot_dns_ovh, certbot_dns_rfc2136'],
+              [str(python), '-I', '-m', 'certbot', '--version'])
+    try:
+        return all(_run_user(account, data, command, 30).returncode == 0 for command in checks)
+    except (OSError, subprocess.SubprocessError):
+        return False
+
+def prepare_managed_acme_runtime(data: Path, account) -> None:
+    if _acme_runtime_valid(data, account):
+        return
+    for name in ACME_RUNTIME_NAMES:
+        path = data / name
+        if path.is_symlink() or (path.exists() and not path.is_dir()):
+            path.unlink(missing_ok=True)
+        elif path.is_dir():
+            shutil.rmtree(path, ignore_errors=True)
+    system_python = shutil.which('python3') or shutil.which('python')
+    if not system_python:
+        print('WARNING: managed ACME runtime rebuild skipped: Python is unavailable.', file=sys.stderr); return
+    target = data / 'acme-python'
+    try:
+        created = _run_user(account, data, [system_python, '-m', 'venv', str(target)], 180)
+        if created.returncode != 0:
+            bootstrap = data / 'acme-python.virtualenv-bootstrap'
+            pip_ok = _run_user(account, data, [system_python, '-m', 'pip', '--version'], 30)
+            if pip_ok.returncode != 0:
+                raise RuntimeError(created.stderr.strip()[-1200:])
+            install_ve = _run_user(account, data, [system_python, '-m', 'pip', 'install', '--disable-pip-version-check',
+                '--no-input', '--no-cache-dir', '--retries', '3', '--timeout', '30', '--target', str(bootstrap), 'virtualenv>=20,<21'], 600)
+            if install_ve.returncode != 0:
+                raise RuntimeError(install_ve.stderr.strip()[-1200:])
+            create_ve = _run_user(account, data, [system_python, '-c',
+                "import runpy,sys;sys.path.insert(0,sys.argv[1]);sys.argv=['virtualenv',sys.argv[2]];runpy.run_module('virtualenv',run_name='__main__')",
+                str(bootstrap), str(target)], 180)
+            shutil.rmtree(bootstrap, ignore_errors=True)
+            if create_ve.returncode != 0:
+                raise RuntimeError(create_ve.stderr.strip()[-1200:])
+        python = _acme_python(data)
+        if _run_user(account, data, [str(python), '-m', 'pip', '--version'], 30).returncode != 0:
+            ensured = _run_user(account, data, [str(python), '-m', 'ensurepip', '--upgrade'], 120)
+            if ensured.returncode != 0:
+                raise RuntimeError(ensured.stderr.strip()[-1200:])
+        installed = _run_user(account, data, [str(python), '-m', 'pip', 'install', '--disable-pip-version-check',
+            '--no-input', '--no-cache-dir', '--retries', '3', '--timeout', '30', *CERTBOT_PACKAGES], 600)
+        if installed.returncode != 0 or not _acme_runtime_valid(data, account):
+            raise RuntimeError(installed.stderr.strip()[-1200:])
+        print(f'Managed ACME runtime ready: {target}')
+    except (OSError, RuntimeError, subprocess.SubprocessError) as error:
+        for name in ACME_RUNTIME_NAMES:
+            shutil.rmtree(data / name, ignore_errors=True)
+        print(f'WARNING: managed ACME runtime rebuild failed; system Certbot fallback remains available: {error}', file=sys.stderr)
+
+def restore_managed_acme_runtime(failed: Path, restored: Path) -> None:
+    for name in ACME_RUNTIME_NAMES:
+        source, target = failed / name, restored / name
+        if source.is_dir() and not source.is_symlink() and not target.exists():
+            try:
+                source.rename(target)
+            except OSError:
+                shutil.copytree(source, target, symlinks=True)
+
 
 def chown_tree(path: Path, uid: int, gid: int) -> None:
     os.chown(path, uid, gid)
@@ -288,6 +373,7 @@ def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: boo
         if reset_config:
             reset_saved_config(data)
         apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid)
+        prepare_managed_acme_runtime(data, account)
         switch(prefix, release)
         write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644)
         run(['systemctl', 'daemon-reload'])
@@ -303,7 +389,9 @@ def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: boo
         stop_for_managed_update(data)
         failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12])
         data.rename(failed)  # keep the failed generation for diagnosis; do not delete it
-        copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid)
+        copy_data(backup / 'data', data)
+        restore_managed_acme_runtime(failed, data)
+        chown_tree(data, account.pw_uid, account.pw_gid)
         switch(prefix, record['release'])
         restore_service_unit(backup)
         write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())

src/acme/runtime_env.rs

@@ -75,8 +75,18 @@ fn find_on_path(name: &str) -> Option<PathBuf> {
         .find(|path| path.is_file())
 }
 
-fn system_python() -> Option<PathBuf> {
-    find_on_path("python3").or_else(|| find_on_path("python"))
+fn python_candidates() -> impl Iterator<Item = PathBuf> {
+    [
+        "python3",
+        "python3.14",
+        "python3.13",
+        "python3.12",
+        "python3.11",
+        "python3.10",
+        "python",
+    ]
+    .into_iter()
+    .filter_map(find_on_path)
 }
 
 #[cfg(windows)]
@@ -135,20 +145,27 @@ async fn managed_environment_valid(
     if !command_success(imports, Duration::from_secs(20), shutdown).await {
         return false;
     }
-    // A venv is created under an atomic staging directory and then renamed
-    // into place. pip-generated console scripts contain an absolute shebang,
-    // so the staged `bin/certbot` is not relocatable after that rename. Invoke
-    // Certbot as a module through the final venv interpreter instead. This also
-    // makes existing 0.27.1 environments immediately usable after upgrade.
     let mut version = Command::new(&python);
     version.args(["-I", "-m", "certbot", "--version"]);
     command_success(version, Duration::from_secs(20), shutdown).await
 }
 
-async fn system_has_pip(
-    python: &Path,
-    shutdown: &tokio_util::sync::CancellationToken,
-) -> bool {
+async fn system_python(shutdown: &tokio_util::sync::CancellationToken) -> Option<PathBuf> {
+    for python in python_candidates() {
+        let mut command = Command::new(&python);
+        command.args([
+            "-I",
+            "-c",
+            "import sys;raise SystemExit(0 if sys.version_info >= (3,10) else 1)",
+        ]);
+        if command_success(command, Duration::from_secs(10), shutdown).await {
+            return Some(python);
+        }
+    }
+    None
+}
+
+async fn system_has_pip(python: &Path, shutdown: &tokio_util::sync::CancellationToken) -> bool {
     let mut command = Command::new(python);
     command.args(["-m", "pip", "--version"]);
     command_success(command, Duration::from_secs(20), shutdown).await
@@ -168,10 +185,8 @@ async fn create_virtual_environment(
     if !system_pip {
         return false;
     }
-
-    // Some distributions split the stdlib venv module into a separate package.
-    // Bootstrap virtualenv with the system pip into the RustProxy data directory
-    // instead of modifying site-packages owned by the distribution.
+    let _ = tokio::fs::remove_dir_all(staging).await;
+    // Fall back to an application-local virtualenv bootstrap.
     let bootstrap = staging.with_extension("virtualenv-bootstrap");
     let _ = tokio::fs::remove_dir_all(&bootstrap).await;
     let mut install = Command::new(python);
@@ -183,9 +198,9 @@ async fn create_virtual_environment(
         "--no-input",
         "--no-cache-dir",
         "--retries",
-        "1",
+        "3",
         "--timeout",
-        "15",
+        "30",
         "--target",
     ]);
     install.arg(&bootstrap).arg("virtualenv>=20,<21");
@@ -204,7 +219,7 @@ async fn create_virtual_environment(
     created
 }
 
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 enum BuildFailure {
     Bootstrap,
     Install,
@@ -216,27 +231,26 @@ async fn build_managed_environment(
     system_pip: bool,
     shutdown: &tokio_util::sync::CancellationToken,
 ) -> Result<(), BuildFailure> {
-    let staging = target.with_extension("installing");
-    let _ = tokio::fs::remove_dir_all(&staging).await;
-
-    if !create_virtual_environment(python, &staging, system_pip, shutdown).await {
-        let _ = tokio::fs::remove_dir_all(&staging).await;
+    let legacy_stage = target.with_extension("installing");
+    let _ = tokio::fs::remove_dir_all(&legacy_stage).await;
+    let _ = tokio::fs::remove_dir_all(target).await;
+    let _ = tokio::fs::remove_file(target).await;
+    if !create_virtual_environment(python, target, system_pip, shutdown).await {
+        let _ = tokio::fs::remove_dir_all(target).await;
         return Err(BuildFailure::Bootstrap);
     }
-
-    let staging_python = venv_python(&staging);
-    let mut pip_check = Command::new(&staging_python);
+    let target_python = venv_python(target);
+    let mut pip_check = Command::new(&target_python);
     pip_check.args(["-m", "pip", "--version"]);
     if !command_success(pip_check, Duration::from_secs(30), shutdown).await {
-        let mut ensurepip = Command::new(&staging_python);
+        let mut ensurepip = Command::new(&target_python);
         ensurepip.args(["-m", "ensurepip", "--upgrade"]);
         if !command_success(ensurepip, Duration::from_secs(120), shutdown).await {
-            let _ = tokio::fs::remove_dir_all(&staging).await;
+            let _ = tokio::fs::remove_dir_all(target).await;
             return Err(BuildFailure::Bootstrap);
         }
     }
-
-    let mut pip = Command::new(&staging_python);
+    let mut pip = Command::new(&target_python);
     pip.args([
         "-m",
         "pip",
@@ -245,24 +259,15 @@ async fn build_managed_environment(
         "--no-input",
         "--no-cache-dir",
         "--retries",
-        "1",
+        "3",
         "--timeout",
-        "15",
+        "30",
     ]);
     pip.args(CERTBOT_PACKAGES.iter().copied());
-    if !command_success(pip, Duration::from_secs(300), shutdown).await
-        || !managed_environment_valid(&staging, shutdown).await
+    if !command_success(pip, Duration::from_secs(600), shutdown).await
+        || !managed_environment_valid(target, shutdown).await
     {
-        let _ = tokio::fs::remove_dir_all(&staging).await;
-        return Err(BuildFailure::Install);
-    }
-
-    if target.exists() && tokio::fs::remove_dir_all(target).await.is_err() {
-        let _ = tokio::fs::remove_dir_all(&staging).await;
-        return Err(BuildFailure::Install);
-    }
-    if tokio::fs::rename(&staging, target).await.is_err() {
-        let _ = tokio::fs::remove_dir_all(&staging).await;
+        let _ = tokio::fs::remove_dir_all(target).await;
         return Err(BuildFailure::Install);
     }
     Ok(())
@@ -306,7 +311,6 @@ async fn ensure_runtime(state: &App) -> Result<RuntimeLaunch, String> {
             managed_certbot_python: Some(venv_python(&managed)),
         });
     }
-
     let distro_certbot = find_on_path("certbot");
     let distro_certbot_available = if let Some(certbot) = distro_certbot.as_deref() {
         let mut version = Command::new(certbot);
@@ -315,7 +319,7 @@ async fn ensure_runtime(state: &App) -> Result<RuntimeLaunch, String> {
     } else {
         false
     };
-    let Some(python) = system_python() else {
+    let Some(python) = system_python(&state.shutdown).await else {
         state.acme.set_runtime(RuntimeStatus {
             status: "unavailable",
             mode: "none",
@@ -337,9 +341,9 @@ async fn ensure_runtime(state: &App) -> Result<RuntimeLaunch, String> {
         pip: system_pip,
         certbot: distro_certbot_available,
     });
-
     let build = build_managed_environment(&python, &managed, system_pip, &state.shutdown).await;
     if build.is_ok() {
+        tracing::info!(path = %managed.display(), "managed ACME runtime ready");
         state.acme.set_runtime(RuntimeStatus {
             status: "ready",
             mode: "managed",
@@ -355,8 +359,8 @@ async fn ensure_runtime(state: &App) -> Result<RuntimeLaunch, String> {
             managed_certbot_python: Some(venv_python(&managed)),
         });
     }
-
     if distro_certbot_available {
+        tracing::warn!(python = %python.display(), failure = ?build, "managed ACME runtime setup failed; using system Certbot fallback");
         state.acme.set_runtime(RuntimeStatus {
             status: "ready",
             mode: "system",
@@ -372,7 +376,6 @@ async fn ensure_runtime(state: &App) -> Result<RuntimeLaunch, String> {
             managed_certbot_python: None,
         });
     }
-
     let message = if build == Err(BuildFailure::Bootstrap) && !system_pip {
         "acme.runtime_pip_missing"
     } else {
@@ -397,14 +400,12 @@ pub async fn runtime_setup_once(state: Weak<App>) {
     let _ = ensure_runtime(&state).await;
 }
 
-/// Kill the entire private process group if execution is cancelled/abandoned.
-/// Certbot inherits the group, so it cannot outlive a stopped RustProxy worker.
 struct ProcessGroup(Option<u32>);
 impl Drop for ProcessGroup {
     fn drop(&mut self) {
         #[cfg(unix)]
         if let Some(id) = self.0 {
-            // SAFETY: this is the process group created solely for this child.
+            // SAFETY: this process group belongs only to the managed child.
             unsafe {
                 libc::kill(-(id as i32), libc::SIGKILL);
             }