v0.25.1

scripts/deploy_runtime.py

@@ -0,0 +1,356 @@
+"""Runtime install/update operations for deploy.py."""
+from __future__ import annotations
+
+from deploy_core import *  # re-exported internal deployment primitives
+
+def checked_tree(source: Path) -> None:
+    """Accept only regular files/directories and relative Certbot-internal symlinks."""
+    boundary = source.resolve(strict=True)
+    if source.is_symlink():
+        raise ValueError('Data root cannot be a symlink.')
+    for p in source.rglob('*'):
+        st = p.lstat()
+        if stat.S_ISLNK(st.st_mode):
+            target = os.readlink(p)
+            if os.path.isabs(target) or not p.resolve(strict=True).is_relative_to(boundary):
+                raise ValueError('Data backup contains an external or absolute symlink.')
+        elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)):
+            raise ValueError('Data backup contains a special file.')
+
+
+def copy_data(source: Path, target: Path) -> None:
+    checked_tree(source)
+    shutil.copytree(source, target, symlinks=True)
+    os.chmod(target, 0o700)
+
+
+def chown_tree(path: Path, uid: int, gid: int) -> None:
+    os.chown(path, uid, gid)
+    for child in path.rglob('*'):
+        os.chown(child, uid, gid, follow_symlinks=False)
+
+
+def ensure_service_data_access(data: Path, account) -> None:
+    """Repair the managed data-root metadata and verify service-user traversal.
+
+    systemd applies WorkingDirectory after dropping to User=/Group=. A stale
+    root-owned 0700 data directory therefore fails as status=200/CHDIR before
+    RustProxy itself can emit diagnostics. Repair only the managed data root;
+    never chmod/chown parent directories. Then fail before service downtime if a
+    parent directory still prevents traversal.
+    """
+    try:
+        st = data.lstat()
+    except FileNotFoundError as error:
+        raise ValueError(f'Data directory is missing: {data}') from error
+    if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
+        raise ValueError(f'Data directory must be a real directory: {data}')
+
+    os.chown(data, account.pw_uid, account.pw_gid)
+    os.chmod(data, 0o700)
+
+    probe = subprocess.run(
+        ['runuser', '-u', account.pw_name, '--', 'test', '-x', str(data)],
+        check=False, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+    )
+    if probe.returncode != 0:
+        raise ValueError(
+            f'Service user {account.pw_name} cannot traverse data directory {data}; '
+            'check execute/search permissions on its parent directories (for example with namei -l).'
+        )
+
+
+def switch(prefix: Path, release: str) -> None:
+    if not re.fullmatch(r'[A-Za-z0-9_.-]+', release) or '..' in release:
+        raise ValueError('Invalid release name.')
+    temporary = prefix / ('current-' + uuid.uuid4().hex)
+    os.symlink('releases/' + release, temporary)
+    os.replace(temporary, prefix / 'current')
+
+
+def running_release_matches(record: dict) -> bool:
+    """Check the actual systemd process executable without publishing a version API.
+
+    This installer targets Linux/systemd. Fail closed if /proc or MainPID cannot
+    establish that the service runs the exact staged, version-checked binary.
+    """
+    try:
+        result = subprocess.run(['systemctl', 'show', '--property=MainPID', '--value', SERVICE],
+                                check=False, capture_output=True, text=True, timeout=3)
+        if result.returncode != 0 or not result.stdout.strip().isdigit():
+            return False
+        pid = int(result.stdout.strip())
+        if pid <= 1:
+            return False
+        expected = Path(record['prefix']) / 'releases' / record['release'] / 'rust-proxy'
+        return Path(f'/proc/{pid}/exe').samefile(expected)
+    except (OSError, ValueError, KeyError, subprocess.SubprocessError):
+        return False
+
+
+def admin_health_endpoint(record: dict) -> tuple[str, bool, str]:
+    """Resolve the active persisted administration listener and scheme.
+
+    The installer record contains only the startup CLI address. Since 0.9.x the
+    dashboard can persist both a different admin bind and HTTPS in config.sqlite3.
+    Update health checks must follow that active intent or a healthy HTTPS panel is
+    mistaken for a failed release and rollback can fail for the same reason.
+    """
+    raw = record['admin']
+    tls = False
+    allowed_hosts: list[str] = []
+    database = Path(record['data_dir']) / 'config.sqlite3'
+    try:
+        connection = sqlite3.connect(f'file:{database}?mode=ro', uri=True, timeout=2)
+        try:
+            row = connection.execute('SELECT document FROM configuration WHERE id = 1').fetchone()
+        finally:
+            connection.close()
+        if row:
+            document = json.loads(row[0])
+            administration = document.get('administration') or {}
+            if isinstance(administration, dict):
+                if administration.get('listen'):
+                    raw = address(str(administration['listen']))
+                tls = bool(administration.get('tls', False))
+                values = administration.get('allowed_hosts') or []
+                if isinstance(values, list):
+                    allowed_hosts = [str(value).strip() for value in values if str(value).strip()]
+    except (OSError, sqlite3.Error, ValueError, TypeError, json.JSONDecodeError):
+        # Old releases and damaged/uninitialized databases used the startup HTTP
+        # listener. The real startup diagnostics remain authoritative if this
+        # compatibility read cannot resolve persisted settings.
+        pass
+
+    host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
+    ip = ipaddress.ip_address(host)
+    connect_host = ('::1' if ip.version == 6 else '127.0.0.1') if ip.is_unspecified else host
+    authority_host = allowed_hosts[0] if allowed_hosts else connect_host
+    try:
+        authority_ip = ipaddress.ip_address(authority_host)
+        authority = f'[{authority_ip}]:{port}' if authority_ip.version == 6 else f'{authority_ip}:{port}'
+    except ValueError:
+        authority = f'{authority_host}:{port}'
+    connect_raw = f'[{connect_host}]:{port}' if ':' in connect_host else f'{connect_host}:{port}'
+    return connect_raw, tls, authority
+
+
+def healthy(record: dict, version: str, attempts: int = 30) -> bool:
+    raw, tls, health_host = admin_health_endpoint(record)
+    host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
+    for _ in range(attempts):
+        if tls:
+            # Local deployment health is process identity/liveness, not PKI validation.
+            # The configured admin certificate may intentionally be self-signed or for
+            # a DNS name while we connect to the local socket.
+            conn = http.client.HTTPSConnection(host, int(port), timeout=2, context=ssl._create_unverified_context())
+        else:
+            conn = http.client.HTTPConnection(host, int(port), timeout=2)
+        try:
+            conn.request('GET', '/healthz', headers={'Host': health_host})
+            response = conn.getresponse(); content = response.read(4097)
+            active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
+            if (active and response.status == 200 and len(content) <= 4096
+                    and json.loads(content).get('status') == 'ok' and record.get('version') == version
+                    and running_release_matches(record)):
+                return True
+        except (OSError, ValueError, http.client.HTTPException, ssl.SSLError):
+            pass
+        finally:
+            conn.close()
+        time.sleep(1)
+    return False
+
+
+def stage(binary: Path, prefix: Path, user: str, sha: str) -> tuple[str, str]:
+    release = 'build-' + sha[:24]
+    releases = prefix / 'releases'; releases.mkdir(mode=0o755, exist_ok=True); os.chmod(releases, 0o755)
+    directory = releases / release
+    if not directory.exists():
+        directory.mkdir(mode=0o755); os.chmod(directory, 0o755)
+        try:
+            shutil.copyfile(binary, directory / 'rust-proxy')
+            os.chmod(directory / 'rust-proxy', 0o755)
+        except BaseException:
+            shutil.rmtree(directory); raise
+    if directory.is_symlink() or (directory / 'rust-proxy').is_symlink() or digest(directory / 'rust-proxy') != sha:
+        raise ValueError('Staged binary checksum mismatch.')
+    # Never execute the supplied binary as root, even for its version banner.
+    result = run(['runuser', '-u', user, '--', str(directory / 'rust-proxy'), '--version'], capture_output=True, text=True, timeout=15)
+    matched = re.fullmatch(r'(?:rust-proxy|RustProxy) (\d+\.\d+\.\d+(?:[-+][\w.-]+)?)\s*', result.stdout)
+    if not matched:
+        raise ValueError('Binary did not return a RustProxy version.')
+    return release, matched[1]
+
+
+def backup_stopped(record: dict, prefix: Path) -> Path:
+    # Service stop must happen before this call: ACME files and SQLite form one backup.
+    data = Path(record['data_dir'])
+    lock_fd = os.open(data / 'process.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
+    try:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        account = pwd.getpwnam(record['user']); os.fchown(lock_fd, account.pw_uid, account.pw_gid)
+        backups = prefix / 'backups'; backups.mkdir(mode=0o700, exist_ok=True)
+        backup = backups / ('before-' + record['version'] + '-' + uuid.uuid4().hex[:12]); backup.mkdir(mode=0o700)
+        copy_data(data, backup / 'data')
+        write_private(backup / 'install.json', json.dumps(record, indent=2).encode())
+        if UNIT_PATH.exists():
+            shutil.copyfile(UNIT_PATH, backup / 'rustproxy.service'); os.chmod(backup / 'rustproxy.service', 0o600)
+        return backup
+    finally:
+        os.close(lock_fd)
+
+
+def restore_service_unit(backup: Path) -> None:
+    saved = backup / 'rustproxy.service'
+    if saved.exists():
+        copy_private(saved, UNIT_PATH, 0o644)
+    else:
+        UNIT_PATH.unlink(missing_ok=True)
+    run(['systemctl', 'daemon-reload'])
+
+
+def reset_saved_config(data: Path) -> None:
+    """Development-only reset of desired config while preserving auth/certs/audit data."""
+    database = data / 'config.sqlite3'
+    if database.is_symlink() or not database.is_file():
+        raise ValueError('Cannot reset configuration: config.sqlite3 is missing or unsafe.')
+    connection = sqlite3.connect(f'file:{database}?mode=rw', uri=True, timeout=5)
+    try:
+        row = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='configuration'").fetchone()
+        if row is None:
+            raise ValueError('Cannot reset configuration: configuration table is missing.')
+        connection.execute('DELETE FROM configuration WHERE id = 1')
+        connection.commit()
+    finally:
+        connection.close()
+    print('Stored vhost/settings configuration cleared; account, certificates and other private data were preserved.')
+
+
+def service_failure_details() -> str:
+    chunks: list[str] = []
+    commands = (
+        ['systemctl', 'show', SERVICE, '--property=ActiveState,SubState,ExecMainStatus', '--no-pager'],
+        ['journalctl', '-u', SERVICE, '-n', '25', '--no-pager', '--output=cat'],
+    )
+    for command in commands:
+        try:
+            result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=5)
+            output = ((result.stdout or '') + (result.stderr or '')).strip()
+            if output:
+                chunks.append(output)
+        except (OSError, subprocess.SubprocessError):
+            pass
+    return '\n'.join(chunks)[-8000:]
+
+
+def stop_for_managed_update(data: Path) -> None:
+    """Ask RustProxy 0.14.0+ to force hot cache into the enabled cold tier before exit.
+
+    Older installed versions ignore the marker and still follow their ordinary graceful
+    shutdown policy, so the first upgrade to 0.14.0 remains backward compatible.
+    The marker is removed before backup so it never becomes persisted application data.
+    """
+    marker = data / '.managed-update'
+    marker.unlink(missing_ok=True)
+    write_private(marker, b'1\n', 0o644)
+    try:
+        run(['systemctl', 'stop', SERVICE])
+    finally:
+        marker.unlink(missing_ok=True)
+
+
+def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: bool = False, reset_config: bool = False) -> dict:
+    prefix = Path(record['prefix']); data = Path(record['data_dir']); account = pwd.getpwnam(record['user'])
+    # Updates deliberately do not rewrite shell environment. Check the installer-
+    # managed PATH profile early and report the exact remediation target instead.
+    check_shell_path(prefix)
+    # Do this before staging/network work and, critically, before stopping the
+    # currently healthy service. It also repairs stale root:root/0700 metadata
+    # left by manual maintenance while preserving private 0700 semantics.
+    ensure_service_data_access(data, account)
+    release, version = stage(binary, prefix, record['user'], sha)
+    geoip_stage: Path | None = None
+    geoip_names: list[str] = []
+    if refresh_geoip:
+        geoip_stage, geoip_names = prepare_geoip_update(data, prefix)
+    active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
+    stop_for_managed_update(data)
+    try:
+        backup = backup_stopped(record, prefix)
+    except BaseException:
+        if active: run(['systemctl', 'start', SERVICE])
+        if geoip_stage is not None:
+            shutil.rmtree(geoip_stage, ignore_errors=True)
+        raise
+    replacement = dict(record, release=release, version=version, sha256=sha)
+    try:
+        if reset_config:
+            reset_saved_config(data)
+        apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid)
+        switch(prefix, release)
+        write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644)
+        run(['systemctl', 'daemon-reload'])
+        if start:
+            run(['systemctl', 'start', SERVICE])
+            if not healthy(replacement, version):
+                raise RuntimeError('New release failed the local health/version check.')
+        write_private(prefix / 'install.json', json.dumps(replacement, indent=2).encode())
+    except BaseException as error:
+        details = service_failure_details()
+        if details:
+            print('New release diagnostics:\n' + details, file=sys.stderr)
+        stop_for_managed_update(data)
+        failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12])
+        data.rename(failed)  # keep the failed generation for diagnosis; do not delete it
+        copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid)
+        switch(prefix, record['release'])
+        restore_service_unit(backup)
+        write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
+        if active:
+            run(['systemctl', 'start', SERVICE])
+            if not healthy(record, record['version']):
+                raise RuntimeError(f'Update failed: {error}. Rollback restored code/data/unit but previous service health also failed. Backup: {backup}; failed state: {failed}') from error
+        raise RuntimeError(f'Update failed: {error}. Previous code, private data and service unit were restored. Installation is still present; do not run install.sh. Fix the cause and rerun update.sh. Backup: {backup}; failed state: {failed}') from error
+    finally:
+        if geoip_stage is not None:
+            shutil.rmtree(geoip_stage, ignore_errors=True)
+    print(f'Backup (contains private keys): {backup}')
+    return replacement
+
+
+def install(record: dict, binary: Path, sha: str, start: bool, install_geoip: bool = False) -> dict:
+    import grp
+    prefix = Path(record['prefix']); data = Path(record['data_dir'])
+    if (prefix / 'install.json').exists() or (prefix / 'current').exists() or UNIT_PATH.exists():
+        raise ValueError('Installation/service already exists. A failed update rolls back to the previous installed release, so rerun update.sh after fixing the reported cause. install.sh never overwrites an existing installation.')
+    if data.exists() and any(data.iterdir()):
+        raise ValueError('First installation requires an empty data directory; existing data will never be reset.')
+    try:
+        account = pwd.getpwnam(record['user'])
+        if account.pw_uid == 0 or account.pw_shell not in ('/usr/sbin/nologin', '/sbin/nologin', '/bin/false'):
+            raise ValueError('Existing service account must be non-root with a non-login shell.')
+    except KeyError:
+        run(['useradd', '--system', '--user-group', '--home-dir', str(data), '--no-create-home', '--shell', '/usr/sbin/nologin', record['user']])
+        account = pwd.getpwnam(record['user'])
+    record['group'] = grp.getgrgid(account.pw_gid).gr_name
+    prefix.mkdir(parents=True, mode=0o755, exist_ok=True); os.chmod(prefix, 0o755)
+    data.mkdir(parents=True, mode=0o700, exist_ok=True); os.chmod(data, 0o700); os.chown(data, account.pw_uid, account.pw_gid)
+    if install_geoip:
+        download_geoip(data, account.pw_uid, account.pw_gid)
+    release, version = stage(binary, prefix, record['user'], sha)
+    record.update(release=release, version=version, sha256=sha)
+    switch(prefix, release)
+    install_shell_path(prefix)
+    # Record is written even if initial startup fails, so an update can repair it.
+    write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
+    write_private(UNIT_PATH, service_unit(record).encode(), 0o644)
+    run(['systemctl', 'daemon-reload']); run(['systemctl', 'enable', SERVICE])
+    if start:
+        run(['systemctl', 'start', SERVICE])
+        if not healthy(record, version):
+            run(['systemctl', 'stop', SERVICE])
+            raise RuntimeError('Installed but startup failed. Data was retained; inspect journalctl -u rustproxy.')
+    return record
+
+