From 82afe662b32f4e66c7c840d2e231ee4a4d16f3bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Sat, 6 Jun 2026 00:28:21 +0200 Subject: [PATCH] fix install v2.15.X --- npm_install.py | 300 ++++++++++++++++++++++++++----------------------- 1 file changed, 160 insertions(+), 140 deletions(-) diff --git a/npm_install.py b/npm_install.py index 25ea0d2..810e224 100644 --- a/npm_install.py +++ b/npm_install.py @@ -593,13 +593,23 @@ def github_latest_release_tag(repo: str, override: str = None) -> str: +def _sanitize_angie_log_config(text: str) -> str: + """Make log formats valid in global http/stream context. + + `$server` is only defined inside generated proxy_host server blocks. When it + appears in a global log_format, `angie -t` fails with: unknown "server" + variable. Use built-in `$upstream_addr` instead. + """ + return text.replace("$server", "$upstream_addr") + + def ensure_angie_log_include_files(): """Ensure split log include files required by ANGIE_CONF_TEMPLATE exist. Older installs may only have /etc/angie/conf.d/include/log.conf from the upstream NPM rootfs. The Angie template used by this installer includes - log-proxy.conf in http{} and log-stream.conf in stream{}, so update mode - must create them before running `angie -t`. + log-proxy.conf in http{} and log-stream.conf in stream{}, so fresh/update + mode must create them before running `angie -t`. """ include_dir = Path("/etc/angie/conf.d/include") include_dir.mkdir(parents=True, exist_ok=True) @@ -608,7 +618,7 @@ def ensure_angie_log_include_files(): log_proxy = include_dir / "log-proxy.conf" log_stream = include_dir / "log-stream.conf" - default_proxy = """log_format proxy '[$time_local] $upstream_cache_status $upstream_status $status - $request_method $scheme $host "$request_uri" [Client $remote_addr] [Length $body_bytes_sent] [Gzip $gzip_ratio] [Sent-to $server] "$http_user_agent" "$http_referer"'; + default_proxy = """log_format proxy '[$time_local] $upstream_cache_status $upstream_status $status - $request_method $scheme $host "$request_uri" [Client $remote_addr] [Length $body_bytes_sent] [Gzip $gzip_ratio] [Sent-to $upstream_addr] "$http_user_agent" "$http_referer"'; log_format standard '[$time_local] $status - $request_method $scheme $host "$request_uri" [Client $remote_addr] [Length $body_bytes_sent] [Gzip $gzip_ratio] "$http_user_agent" "$http_referer"'; access_log /data/logs/fallback_access.log proxy; @@ -620,14 +630,22 @@ access_log /data/logs/fallback_stream_access.log stream; """ try: + if log_conf.exists(): + proxy_text = _sanitize_angie_log_config(log_conf.read_text(encoding="utf-8")) + if not proxy_text.strip(): + proxy_text = default_proxy + else: + proxy_text = default_proxy + if not log_proxy.exists(): - if log_conf.exists(): - txt = log_conf.read_text(encoding="utf-8") - # Upstream log.conf is the HTTP/proxy log include in older installs. - write_file(log_proxy, txt if txt.strip() else default_proxy, 0o644) - else: - write_file(log_proxy, default_proxy, 0o644) + write_file(log_proxy, proxy_text, 0o644) print(f" ✓ Created missing {log_proxy.name}") + else: + current = log_proxy.read_text(encoding="utf-8") + sanitized = _sanitize_angie_log_config(current) + if sanitized != current: + write_file(log_proxy, sanitized, 0o644) + print(f" ✓ Fixed invalid $server variable in {log_proxy.name}") if not log_stream.exists(): write_file(log_stream, default_stream, 0o644) @@ -635,8 +653,14 @@ access_log /data/logs/fallback_stream_access.log stream; # Keep legacy log.conf present for compatibility with older/custom configs. if not log_conf.exists(): - write_file(log_conf, default_proxy, 0o644) + write_file(log_conf, proxy_text, 0o644) print(f" ✓ Created missing {log_conf.name}") + else: + current = log_conf.read_text(encoding="utf-8") + sanitized = _sanitize_angie_log_config(current) + if sanitized != current: + write_file(log_conf, sanitized, 0o644) + print(f" ✓ Fixed invalid $server variable in {log_conf.name}") run(["chown", "root:root", str(log_proxy), str(log_stream), str(log_conf)], check=False) except Exception as e: @@ -1050,6 +1074,92 @@ def _detect_certbot_version(certbot_path: Path) -> str: return m.group(1) + +def run_logged(cmd, log_path: Path, timeout=1200, check=True, env=None): + """Run command with stdout/stderr saved to a log file. + + Normal installer output stays concise, but failed builds/installations leave + actionable diagnostics instead of hiding stderr in /dev/null. + """ + log_path = Path(log_path) + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8", errors="replace") as log: + log.write("\n\n$ " + " ".join(map(str, cmd)) + "\n") + log.flush() + result = subprocess.run( + cmd, + timeout=timeout, + check=False, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + ) + if check and result.returncode != 0: + print(f" ✖ Command failed, log: {log_path}") + try: + lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines() + tail = lines[-40:] + if tail: + print(" --- log tail ---") + for line in tail: + print(" " + line[:220]) + print(" --- end log tail ---") + except Exception: + pass + raise subprocess.CalledProcessError(result.returncode, cmd) + return result + + +def _python_version(exe: str) -> tuple[int, int] | None: + try: + out = run_out([exe, "--version"], check=False).strip() + m = re.search(r"Python\s+(\d+)\.(\d+)", out) + if m: + return (int(m.group(1)), int(m.group(2))) + except Exception: + pass + return None + + +def _find_certbot_system_python() -> tuple[str | None, str | None]: + """Prefer python3.11 when installed; otherwise use distro python >= 3.11. + + Debian 13 ships a newer Python (for example 3.13). That is suitable for the + certbot venv and avoids compiling Python 3.11 through pyenv on fresh/update. + """ + candidates = ["python3.11", "python3", "python3.13", "python3.12"] + seen = set() + for exe in candidates: + if exe in seen or not shutil.which(exe): + continue + seen.add(exe) + ver = _python_version(exe) + if ver and ver[0] == 3 and ver[1] >= 11: + out = run_out([exe, "--version"], check=False).strip() + return exe, out + return None, None + + +def _create_certbot_venv_with_python(python_exe: str, python_label: str, venv_dir: Path): + with step(f"Using {python_label} for certbot venv"): + venv_dir.mkdir(parents=True, exist_ok=True) + run([python_exe, "-m", "venv", str(venv_dir)]) + + venv_bin = venv_dir / "bin" + pip_path = venv_bin / "pip" + certbot_path = venv_bin / "certbot" + env_build = os.environ.copy() + env_build["SETUPTOOLS_USE_DISTUTILS"] = "local" + + _install_certbot_stack(pip_path, certbot_path, env_build) + + cb_ver = run_out([str(certbot_path), "--version"], check=False) or "" + pip_ver = run_out([str(pip_path), "--version"], check=False) or "" + print(f" Python: {python_label}") + print(f" Certbot: {cb_ver.strip()}") + print(f" Pip: {pip_ver.strip().split(' from ')[0]}") + def _install_certbot_stack(pip_path: Path, certbot_path: Path, env_build: dict): """Install Certbot and DNS plugins in one venv with matching versions. @@ -1057,11 +1167,13 @@ def _install_certbot_stack(pip_path: Path, certbot_path: Path, env_build: dict): installs this can fail with 'acme==undefined'. We prevent that by making the venv complete before npm.service starts. """ - run( + log_path = Path("/tmp/npm-certbot-venv.log") + run_logged( [str(pip_path), "install", "-U", "pip", "setuptools", "wheel"], + log_path, env=env_build, ) - run( + run_logged( [ str(pip_path), "install", @@ -1071,11 +1183,12 @@ def _install_certbot_stack(pip_path: Path, certbot_path: Path, env_build: dict): "certbot", "tldextract", ], + log_path, env=env_build, ) certbot_ver = _detect_certbot_version(certbot_path) - run( + run_logged( [ str(pip_path), "install", @@ -1084,6 +1197,7 @@ def _install_certbot_stack(pip_path: Path, certbot_path: Path, env_build: dict): f"certbot-dns-cloudflare=={certbot_ver}", f"certbot-dns-rfc2136=={certbot_ver}", ], + log_path, env=env_build, ) @@ -1207,53 +1321,22 @@ def ensure_certbot_venv_ready(venv_dir: Path = Path("/opt/certbot"), force_rebui def setup_certbot_venv(venv_dir: Path = Path("/opt/certbot")): info = os_release() distro_id = (info.get("ID") or "").lower() + version_id = (info.get("VERSION_ID") or "").strip() - # ============================================================ - # STEP 1: Check if Python 3.11 is already available - # ============================================================ - python311_available = False - if shutil.which("python3.11"): - try: - ver_output = run_out(["python3.11", "--version"], check=False).strip() - match = re.search(r"Python (\d+)\.(\d+)", ver_output) - if match: - major, minor = int(match.group(1)), int(match.group(2)) - if major == 3 and minor == 11: - python311_available = True - if DEBUG: - print(f"✔ Found system Python 3.11: {ver_output}") - except Exception: - pass + # Prefer system Python 3.11 if present. On Debian 13, use distro Python + # (for example Python 3.13) instead of compiling Python 3.11 via pyenv. + if distro_id == "debian" and version_id.startswith("13"): + apt_try_install(["python3", "python3-venv", "python3-pip", "python3-dev"]) + else: + apt_try_install(["python3.11-venv", "python3-venv", "python3-pip"]) - # ============================================================ - # STEP 2: Use system Python 3.11 if available - # ============================================================ - if python311_available: - with step(f"Using system Python 3.11 for certbot venv"): - # Ensure python3.11-venv is installed - apt_try_install(["python3.11-venv", "python3-pip"]) - - venv_dir.mkdir(parents=True, exist_ok=True) - run(["python3.11", "-m", "venv", str(venv_dir)]) - - venv_bin = venv_dir / "bin" - pip_path = venv_bin / "pip" - certbot_path = venv_bin / "certbot" - env_build = os.environ.copy() - env_build["SETUPTOOLS_USE_DISTUTILS"] = "local" - - _install_certbot_stack(pip_path, certbot_path, env_build) - - cb_ver = run_out([str(certbot_path), "--version"], check=False) or "" - pip_ver = run_out([str(pip_path), "--version"], check=False) or "" - print(f" Python: {ver_output}") - print(f" Certbot: {cb_ver.strip()}") - print(f" Pip: {pip_ver.strip().split(' from ')[0]}") + python_exe, python_label = _find_certbot_system_python() + if python_exe: + _create_certbot_venv_with_python(python_exe, python_label, venv_dir) return - # ============================================================ - # STEP 3: Ubuntu - install Python 3.11 from deadsnakes PPA - # ============================================================ + # Ubuntu fallback: install Python 3.11 from deadsnakes when no suitable + # system Python is available. if distro_id == "ubuntu": with step( f"Ubuntu detected: {info.get('PRETTY','Ubuntu')}. Install Python 3.11 via deadsnakes" @@ -1262,42 +1345,22 @@ def setup_certbot_venv(venv_dir: Path = Path("/opt/certbot")): run(["apt-get", "update", "-y"], check=False) apt_try_install(["software-properties-common"]) except Exception: - run( - ["apt-get", "install", "-y", "software-properties-common"], - check=False, - ) + run(["apt-get", "install", "-y", "software-properties-common"], check=False) run(["add-apt-repository", "-y", "ppa:deadsnakes/ppa"]) run(["apt-get", "update", "-y"], check=False) run(["apt-get", "install", "-y", "python3.11", "python3.11-venv"]) - with step(f"Create venv at {venv_dir} using python3.11"): - venv_dir.mkdir(parents=True, exist_ok=True) - run(["python3.11", "-m", "venv", str(venv_dir)]) - - venv_bin = venv_dir / "bin" - pip_path = venv_bin / "pip" - certbot_path = venv_bin / "certbot" - env_build = os.environ.copy() - env_build["SETUPTOOLS_USE_DISTUTILS"] = "local" - - _install_certbot_stack(pip_path, certbot_path, env_build) - - cb_ver = run_out([str(certbot_path), "--version"], check=False) or "" - pip_ver = run_out([str(pip_path), "--version"], check=False) or "" - print(f" Python: Python 3.11 (deadsnakes)") - print(f" Certbot: {cb_ver.strip()}") - print(f" Pip: {pip_ver.strip().split(' from ')[0]}") + _create_certbot_venv_with_python("python3.11", "Python 3.11 (deadsnakes)", venv_dir) return - # ============================================================ - # STEP 4: Debian - install Python 3.11 via pyenv - # ============================================================ + # Last resort only: pyenv. Debian 13 should not normally reach this path, + # because it has a suitable distro Python. PYENV_ROOT = Path("/opt/npm/.pyenv") PYENV_OWNER = "npm" PYTHON_VERSION = "3.11.14" + pyenv_log = Path("/tmp/npm-pyenv-python-build.log") - # Build dependencies dla pyenv with step("Installing pyenv build dependencies"): apt_install( [ @@ -1316,9 +1379,9 @@ def setup_certbot_venv(venv_dir: Path = Path("/opt/certbot")): "libffi-dev", "uuid-dev", "liblzma-dev", - # "ca-certificates", - # "curl", - # "git", + "curl", + "git", + "ca-certificates", ] ) @@ -1328,59 +1391,34 @@ def setup_certbot_venv(venv_dir: Path = Path("/opt/certbot")): with step(f"Ensuring pyenv is available at {PYENV_ROOT}"): pyenv_bin_path = PYENV_ROOT / "bin" / "pyenv" - if not pyenv_bin_path.exists(): - run( + run_logged( [ "sudo", "-u", PYENV_OWNER, "bash", "-lc", - 'if [ ! -x "/opt/npm/.pyenv/bin/pyenv" ]; then ' - " command -v git >/dev/null 2>&1 || sudo apt-get install -y git; " - " git clone --depth=1 https://github.com/pyenv/pyenv.git /opt/npm/.pyenv; " - "fi", - ] + 'if [ ! -x "/opt/npm/.pyenv/bin/pyenv" ]; then git clone --depth=1 https://github.com/pyenv/pyenv.git /opt/npm/.pyenv; fi', + ], + pyenv_log, ) - PYENV_BIN_CANDIDATES = [ - str(PYENV_ROOT / "bin" / "pyenv"), - "pyenv", - "/usr/bin/pyenv", - "/usr/lib/pyenv/bin/pyenv", - ] - - pyenv_bin = next( - (c for c in PYENV_BIN_CANDIDATES if shutil.which(c) or Path(c).exists()), None - ) - if not pyenv_bin: + pyenv_bin = PYENV_ROOT / "bin" / "pyenv" + if not pyenv_bin.exists(): raise RuntimeError("No 'pyenv' found even after git clone attempt.") with step(f"Installing Python {PYTHON_VERSION} via pyenv into {PYENV_ROOT}"): run(["mkdir", "-p", str(PYENV_ROOT)]) run(["chown", "-R", f"{PYENV_OWNER}:{PYENV_OWNER}", "/opt/npm"], check=False) - run( - [ - "sudo", - "-u", - PYENV_OWNER, - "bash", - "-lc", - 'if [ ! -x "/opt/npm/.pyenv/bin/pyenv" ]; then ' - " command -v git >/dev/null 2>&1 || sudo apt-get install -y git; " - " git clone --depth=1 https://github.com/pyenv/pyenv.git /opt/npm/.pyenv; " - "fi", - ] - ) install_cmd = ( "export HOME=/opt/npm; " "export PYENV_ROOT=/opt/npm/.pyenv; " 'export PATH="$PYENV_ROOT/bin:/usr/bin:/bin"; ' 'mkdir -p "$PYENV_ROOT"; cd "$HOME"; ' - f"pyenv install -s {PYTHON_VERSION}" + f"pyenv install -v -s {PYTHON_VERSION}" ) - run( + run_logged( [ "sudo", "-u", @@ -1393,7 +1431,9 @@ def setup_certbot_venv(venv_dir: Path = Path("/opt/certbot")): "bash", "-lc", install_cmd, - ] + ], + pyenv_log, + timeout=3600, ) profile_snippet = f"""# Auto-generated by npm-angie-auto-install @@ -1421,29 +1461,9 @@ fi if not python311.exists(): raise RuntimeError(f"No python {PYTHON_VERSION} in {PYENV_ROOT}/versions/.") - venv_bin = venv_dir / "bin" - pip_path = venv_bin / "pip" - certbot_path = venv_bin / "certbot" - - with step(f"Preparing Certbot venv at {venv_dir} (Python {PYTHON_VERSION})"): - venv_dir.mkdir(parents=True, exist_ok=True) - if not venv_dir.exists() or not pip_path.exists(): - run([str(python311), "-m", "venv", str(venv_dir)]) - - env_build = os.environ.copy() - env_build["SETUPTOOLS_USE_DISTUTILS"] = "local" - - _install_certbot_stack(pip_path, certbot_path, env_build) - - cb_ver = run_out([str(certbot_path), "--version"], check=False) or "" - pip_ver = run_out([str(pip_path), "--version"], check=False) or "" - print(f" Python: {PYTHON_VERSION} (pyenv)") - print(f" Certbot: {cb_ver.strip()}") - print(f" Pip: {pip_ver.strip().split(' from ')[0]}") - + _create_certbot_venv_with_python(str(python311), f"Python {PYTHON_VERSION} (pyenv)", venv_dir) run(["chown", "-R", f"{PYENV_OWNER}:{PYENV_OWNER}", str(PYENV_ROOT)], check=False) - def configure_letsencrypt(): with step("configure letsencrypt"): run(["chown", "-R", "npm:npm", "/opt/certbot"], check=False) @@ -3181,7 +3201,7 @@ def update_config_file( if owner: try: - run("chown", owner, str(filepath), check=False) + run(["chown", owner, str(filepath)], check=False) if DEBUG: print(f" Owner set to: {owner}") except Exception as e: