diff --git a/scripts/deploy_runtime.py b/scripts/deploy_runtime.py index 5422ad5..7548a0c 100755 --- a/scripts/deploy_runtime.py +++ b/scripts/deploy_runtime.py @@ -3,12 +3,21 @@ from __future__ import annotations from deploy_core import * # re-exported internal deployment primitives +ACME_RUNTIME_NAMES = frozenset({'acme-python', 'acme-python.installing', 'acme-python.virtualenv-bootstrap'}) +CERTBOT_PACKAGES = ('certbot==5.8.0', 'certbot-dns-cloudflare==5.8.0', 'certbot-dns-ovh==5.8.0', 'certbot-dns-rfc2136==5.8.0') + +def _under_acme_runtime(source: Path, path: Path) -> bool: + relative = path.relative_to(source) + return bool(relative.parts) and relative.parts[0] in ACME_RUNTIME_NAMES + def checked_tree(source: Path) -> None: - """Accept only regular files/directories and relative Certbot-internal symlinks.""" + """Accept durable data; managed ACME Python environments are rebuildable.""" boundary = source.resolve(strict=True) if source.is_symlink(): raise ValueError('Data root cannot be a symlink.') for p in source.rglob('*'): + if _under_acme_runtime(source, p): + continue st = p.lstat() if stat.S_ISLNK(st.st_mode): target = os.readlink(p) @@ -17,12 +26,88 @@ def checked_tree(source: Path) -> None: elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)): raise ValueError('Data backup contains a special file.') - def copy_data(source: Path, target: Path) -> None: checked_tree(source) - shutil.copytree(source, target, symlinks=True) + root = source.resolve(strict=True) + def ignore(directory: str, names: list[str]) -> set[str]: + return set(names) & ACME_RUNTIME_NAMES if Path(directory).resolve(strict=True) == root else set() + shutil.copytree(source, target, symlinks=True, ignore=ignore) os.chmod(target, 0o700) +def _run_user(account, data: Path, args: list[str], timeout: int) -> subprocess.CompletedProcess: + env = os.environ.copy(); env.update(HOME=str(data), PATH='/usr/local/bin:/usr/bin:/bin') + env.pop('PYTHONHOME', None); env.pop('PYTHONPATH', None) + return subprocess.run(['runuser', '-u', account.pw_name, '--', *args], check=False, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=timeout, env=env) + +def _acme_python(data: Path) -> Path: + return data / 'acme-python' / 'bin/python' + +def _acme_runtime_valid(data: Path, account) -> bool: + python = _acme_python(data) + if not python.is_file(): + return False + checks = ([str(python), '-I', '-c', 'import certbot, certbot_dns_cloudflare, certbot_dns_ovh, certbot_dns_rfc2136'], + [str(python), '-I', '-m', 'certbot', '--version']) + try: + return all(_run_user(account, data, command, 30).returncode == 0 for command in checks) + except (OSError, subprocess.SubprocessError): + return False + +def prepare_managed_acme_runtime(data: Path, account) -> None: + if _acme_runtime_valid(data, account): + return + for name in ACME_RUNTIME_NAMES: + path = data / name + if path.is_symlink() or (path.exists() and not path.is_dir()): + path.unlink(missing_ok=True) + elif path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + system_python = shutil.which('python3') or shutil.which('python') + if not system_python: + print('WARNING: managed ACME runtime rebuild skipped: Python is unavailable.', file=sys.stderr); return + target = data / 'acme-python' + try: + created = _run_user(account, data, [system_python, '-m', 'venv', str(target)], 180) + if created.returncode != 0: + bootstrap = data / 'acme-python.virtualenv-bootstrap' + pip_ok = _run_user(account, data, [system_python, '-m', 'pip', '--version'], 30) + if pip_ok.returncode != 0: + raise RuntimeError(created.stderr.strip()[-1200:]) + install_ve = _run_user(account, data, [system_python, '-m', 'pip', 'install', '--disable-pip-version-check', + '--no-input', '--no-cache-dir', '--retries', '3', '--timeout', '30', '--target', str(bootstrap), 'virtualenv>=20,<21'], 600) + if install_ve.returncode != 0: + raise RuntimeError(install_ve.stderr.strip()[-1200:]) + create_ve = _run_user(account, data, [system_python, '-c', + "import runpy,sys;sys.path.insert(0,sys.argv[1]);sys.argv=['virtualenv',sys.argv[2]];runpy.run_module('virtualenv',run_name='__main__')", + str(bootstrap), str(target)], 180) + shutil.rmtree(bootstrap, ignore_errors=True) + if create_ve.returncode != 0: + raise RuntimeError(create_ve.stderr.strip()[-1200:]) + python = _acme_python(data) + if _run_user(account, data, [str(python), '-m', 'pip', '--version'], 30).returncode != 0: + ensured = _run_user(account, data, [str(python), '-m', 'ensurepip', '--upgrade'], 120) + if ensured.returncode != 0: + raise RuntimeError(ensured.stderr.strip()[-1200:]) + installed = _run_user(account, data, [str(python), '-m', 'pip', 'install', '--disable-pip-version-check', + '--no-input', '--no-cache-dir', '--retries', '3', '--timeout', '30', *CERTBOT_PACKAGES], 600) + if installed.returncode != 0 or not _acme_runtime_valid(data, account): + raise RuntimeError(installed.stderr.strip()[-1200:]) + print(f'Managed ACME runtime ready: {target}') + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + for name in ACME_RUNTIME_NAMES: + shutil.rmtree(data / name, ignore_errors=True) + print(f'WARNING: managed ACME runtime rebuild failed; system Certbot fallback remains available: {error}', file=sys.stderr) + +def restore_managed_acme_runtime(failed: Path, restored: Path) -> None: + for name in ACME_RUNTIME_NAMES: + source, target = failed / name, restored / name + if source.is_dir() and not source.is_symlink() and not target.exists(): + try: + source.rename(target) + except OSError: + shutil.copytree(source, target, symlinks=True) + def chown_tree(path: Path, uid: int, gid: int) -> None: os.chown(path, uid, gid) @@ -288,6 +373,7 @@ def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: boo if reset_config: reset_saved_config(data) apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid) + prepare_managed_acme_runtime(data, account) switch(prefix, release) write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644) run(['systemctl', 'daemon-reload']) @@ -303,7 +389,9 @@ def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: boo stop_for_managed_update(data) failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12]) data.rename(failed) # keep the failed generation for diagnosis; do not delete it - copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid) + copy_data(backup / 'data', data) + restore_managed_acme_runtime(failed, data) + chown_tree(data, account.pw_uid, account.pw_gid) switch(prefix, record['release']) restore_service_unit(backup) write_private(prefix / 'install.json', json.dumps(record, indent=2).encode()) diff --git a/src/acme/runtime_env.rs b/src/acme/runtime_env.rs index 78375bb..bf385f5 100644 --- a/src/acme/runtime_env.rs +++ b/src/acme/runtime_env.rs @@ -75,8 +75,18 @@ fn find_on_path(name: &str) -> Option { .find(|path| path.is_file()) } -fn system_python() -> Option { - find_on_path("python3").or_else(|| find_on_path("python")) +fn python_candidates() -> impl Iterator { + [ + "python3", + "python3.14", + "python3.13", + "python3.12", + "python3.11", + "python3.10", + "python", + ] + .into_iter() + .filter_map(find_on_path) } #[cfg(windows)] @@ -135,20 +145,27 @@ async fn managed_environment_valid( if !command_success(imports, Duration::from_secs(20), shutdown).await { return false; } - // A venv is created under an atomic staging directory and then renamed - // into place. pip-generated console scripts contain an absolute shebang, - // so the staged `bin/certbot` is not relocatable after that rename. Invoke - // Certbot as a module through the final venv interpreter instead. This also - // makes existing 0.27.1 environments immediately usable after upgrade. let mut version = Command::new(&python); version.args(["-I", "-m", "certbot", "--version"]); command_success(version, Duration::from_secs(20), shutdown).await } -async fn system_has_pip( - python: &Path, - shutdown: &tokio_util::sync::CancellationToken, -) -> bool { +async fn system_python(shutdown: &tokio_util::sync::CancellationToken) -> Option { + for python in python_candidates() { + let mut command = Command::new(&python); + command.args([ + "-I", + "-c", + "import sys;raise SystemExit(0 if sys.version_info >= (3,10) else 1)", + ]); + if command_success(command, Duration::from_secs(10), shutdown).await { + return Some(python); + } + } + None +} + +async fn system_has_pip(python: &Path, shutdown: &tokio_util::sync::CancellationToken) -> bool { let mut command = Command::new(python); command.args(["-m", "pip", "--version"]); command_success(command, Duration::from_secs(20), shutdown).await @@ -168,10 +185,8 @@ async fn create_virtual_environment( if !system_pip { return false; } - - // Some distributions split the stdlib venv module into a separate package. - // Bootstrap virtualenv with the system pip into the RustProxy data directory - // instead of modifying site-packages owned by the distribution. + let _ = tokio::fs::remove_dir_all(staging).await; + // Fall back to an application-local virtualenv bootstrap. let bootstrap = staging.with_extension("virtualenv-bootstrap"); let _ = tokio::fs::remove_dir_all(&bootstrap).await; let mut install = Command::new(python); @@ -183,9 +198,9 @@ async fn create_virtual_environment( "--no-input", "--no-cache-dir", "--retries", - "1", + "3", "--timeout", - "15", + "30", "--target", ]); install.arg(&bootstrap).arg("virtualenv>=20,<21"); @@ -204,7 +219,7 @@ async fn create_virtual_environment( created } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] enum BuildFailure { Bootstrap, Install, @@ -216,27 +231,26 @@ async fn build_managed_environment( system_pip: bool, shutdown: &tokio_util::sync::CancellationToken, ) -> Result<(), BuildFailure> { - let staging = target.with_extension("installing"); - let _ = tokio::fs::remove_dir_all(&staging).await; - - if !create_virtual_environment(python, &staging, system_pip, shutdown).await { - let _ = tokio::fs::remove_dir_all(&staging).await; + let legacy_stage = target.with_extension("installing"); + let _ = tokio::fs::remove_dir_all(&legacy_stage).await; + let _ = tokio::fs::remove_dir_all(target).await; + let _ = tokio::fs::remove_file(target).await; + if !create_virtual_environment(python, target, system_pip, shutdown).await { + let _ = tokio::fs::remove_dir_all(target).await; return Err(BuildFailure::Bootstrap); } - - let staging_python = venv_python(&staging); - let mut pip_check = Command::new(&staging_python); + let target_python = venv_python(target); + let mut pip_check = Command::new(&target_python); pip_check.args(["-m", "pip", "--version"]); if !command_success(pip_check, Duration::from_secs(30), shutdown).await { - let mut ensurepip = Command::new(&staging_python); + let mut ensurepip = Command::new(&target_python); ensurepip.args(["-m", "ensurepip", "--upgrade"]); if !command_success(ensurepip, Duration::from_secs(120), shutdown).await { - let _ = tokio::fs::remove_dir_all(&staging).await; + let _ = tokio::fs::remove_dir_all(target).await; return Err(BuildFailure::Bootstrap); } } - - let mut pip = Command::new(&staging_python); + let mut pip = Command::new(&target_python); pip.args([ "-m", "pip", @@ -245,24 +259,15 @@ async fn build_managed_environment( "--no-input", "--no-cache-dir", "--retries", - "1", + "3", "--timeout", - "15", + "30", ]); pip.args(CERTBOT_PACKAGES.iter().copied()); - if !command_success(pip, Duration::from_secs(300), shutdown).await - || !managed_environment_valid(&staging, shutdown).await + if !command_success(pip, Duration::from_secs(600), shutdown).await + || !managed_environment_valid(target, shutdown).await { - let _ = tokio::fs::remove_dir_all(&staging).await; - return Err(BuildFailure::Install); - } - - if target.exists() && tokio::fs::remove_dir_all(target).await.is_err() { - let _ = tokio::fs::remove_dir_all(&staging).await; - return Err(BuildFailure::Install); - } - if tokio::fs::rename(&staging, target).await.is_err() { - let _ = tokio::fs::remove_dir_all(&staging).await; + let _ = tokio::fs::remove_dir_all(target).await; return Err(BuildFailure::Install); } Ok(()) @@ -306,7 +311,6 @@ async fn ensure_runtime(state: &App) -> Result { managed_certbot_python: Some(venv_python(&managed)), }); } - let distro_certbot = find_on_path("certbot"); let distro_certbot_available = if let Some(certbot) = distro_certbot.as_deref() { let mut version = Command::new(certbot); @@ -315,7 +319,7 @@ async fn ensure_runtime(state: &App) -> Result { } else { false }; - let Some(python) = system_python() else { + let Some(python) = system_python(&state.shutdown).await else { state.acme.set_runtime(RuntimeStatus { status: "unavailable", mode: "none", @@ -337,9 +341,9 @@ async fn ensure_runtime(state: &App) -> Result { pip: system_pip, certbot: distro_certbot_available, }); - let build = build_managed_environment(&python, &managed, system_pip, &state.shutdown).await; if build.is_ok() { + tracing::info!(path = %managed.display(), "managed ACME runtime ready"); state.acme.set_runtime(RuntimeStatus { status: "ready", mode: "managed", @@ -355,8 +359,8 @@ async fn ensure_runtime(state: &App) -> Result { managed_certbot_python: Some(venv_python(&managed)), }); } - if distro_certbot_available { + tracing::warn!(python = %python.display(), failure = ?build, "managed ACME runtime setup failed; using system Certbot fallback"); state.acme.set_runtime(RuntimeStatus { status: "ready", mode: "system", @@ -372,7 +376,6 @@ async fn ensure_runtime(state: &App) -> Result { managed_certbot_python: None, }); } - let message = if build == Err(BuildFailure::Bootstrap) && !system_pip { "acme.runtime_pip_missing" } else { @@ -397,14 +400,12 @@ pub async fn runtime_setup_once(state: Weak) { let _ = ensure_runtime(&state).await; } -/// Kill the entire private process group if execution is cancelled/abandoned. -/// Certbot inherits the group, so it cannot outlive a stopped RustProxy worker. struct ProcessGroup(Option); impl Drop for ProcessGroup { fn drop(&mut self) { #[cfg(unix)] if let Some(id) = self.0 { - // SAFETY: this is the process group created solely for this child. + // SAFETY: this process group belongs only to the managed child. unsafe { libc::kill(-(id as i32), libc::SIGKILL); }