357 lines
18 KiB
Python
Executable File
357 lines
18 KiB
Python
Executable File
"""Runtime install/update operations for deploy.py."""
|
|
from __future__ import annotations
|
|
|
|
from deploy_core import * # re-exported internal deployment primitives
|
|
|
|
def checked_tree(source: Path) -> None:
|
|
"""Accept only regular files/directories and relative Certbot-internal symlinks."""
|
|
boundary = source.resolve(strict=True)
|
|
if source.is_symlink():
|
|
raise ValueError('Data root cannot be a symlink.')
|
|
for p in source.rglob('*'):
|
|
st = p.lstat()
|
|
if stat.S_ISLNK(st.st_mode):
|
|
target = os.readlink(p)
|
|
if os.path.isabs(target) or not p.resolve(strict=True).is_relative_to(boundary):
|
|
raise ValueError('Data backup contains an external or absolute symlink.')
|
|
elif not (stat.S_ISDIR(st.st_mode) or stat.S_ISREG(st.st_mode)):
|
|
raise ValueError('Data backup contains a special file.')
|
|
|
|
|
|
def copy_data(source: Path, target: Path) -> None:
|
|
checked_tree(source)
|
|
shutil.copytree(source, target, symlinks=True)
|
|
os.chmod(target, 0o700)
|
|
|
|
|
|
def chown_tree(path: Path, uid: int, gid: int) -> None:
|
|
os.chown(path, uid, gid)
|
|
for child in path.rglob('*'):
|
|
os.chown(child, uid, gid, follow_symlinks=False)
|
|
|
|
|
|
def ensure_service_data_access(data: Path, account) -> None:
|
|
"""Repair the managed data-root metadata and verify service-user traversal.
|
|
|
|
systemd applies WorkingDirectory after dropping to User=/Group=. A stale
|
|
root-owned 0700 data directory therefore fails as status=200/CHDIR before
|
|
RustProxy itself can emit diagnostics. Repair only the managed data root;
|
|
never chmod/chown parent directories. Then fail before service downtime if a
|
|
parent directory still prevents traversal.
|
|
"""
|
|
try:
|
|
st = data.lstat()
|
|
except FileNotFoundError as error:
|
|
raise ValueError(f'Data directory is missing: {data}') from error
|
|
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
|
|
raise ValueError(f'Data directory must be a real directory: {data}')
|
|
|
|
os.chown(data, account.pw_uid, account.pw_gid)
|
|
os.chmod(data, 0o700)
|
|
|
|
probe = subprocess.run(
|
|
['runuser', '-u', account.pw_name, '--', 'test', '-x', str(data)],
|
|
check=False, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
)
|
|
if probe.returncode != 0:
|
|
raise ValueError(
|
|
f'Service user {account.pw_name} cannot traverse data directory {data}; '
|
|
'check execute/search permissions on its parent directories (for example with namei -l).'
|
|
)
|
|
|
|
|
|
def switch(prefix: Path, release: str) -> None:
|
|
if not re.fullmatch(r'[A-Za-z0-9_.-]+', release) or '..' in release:
|
|
raise ValueError('Invalid release name.')
|
|
temporary = prefix / ('current-' + uuid.uuid4().hex)
|
|
os.symlink('releases/' + release, temporary)
|
|
os.replace(temporary, prefix / 'current')
|
|
|
|
|
|
def running_release_matches(record: dict) -> bool:
|
|
"""Check the actual systemd process executable without publishing a version API.
|
|
|
|
This installer targets Linux/systemd. Fail closed if /proc or MainPID cannot
|
|
establish that the service runs the exact staged, version-checked binary.
|
|
"""
|
|
try:
|
|
result = subprocess.run(['systemctl', 'show', '--property=MainPID', '--value', SERVICE],
|
|
check=False, capture_output=True, text=True, timeout=3)
|
|
if result.returncode != 0 or not result.stdout.strip().isdigit():
|
|
return False
|
|
pid = int(result.stdout.strip())
|
|
if pid <= 1:
|
|
return False
|
|
expected = Path(record['prefix']) / 'releases' / record['release'] / 'rust-proxy'
|
|
return Path(f'/proc/{pid}/exe').samefile(expected)
|
|
except (OSError, ValueError, KeyError, subprocess.SubprocessError):
|
|
return False
|
|
|
|
|
|
def admin_health_endpoint(record: dict) -> tuple[str, bool, str]:
|
|
"""Resolve the active persisted administration listener and scheme.
|
|
|
|
The installer record contains only the startup CLI address. Since 0.9.x the
|
|
dashboard can persist both a different admin bind and HTTPS in config.sqlite3.
|
|
Update health checks must follow that active intent or a healthy HTTPS panel is
|
|
mistaken for a failed release and rollback can fail for the same reason.
|
|
"""
|
|
raw = record['admin']
|
|
tls = False
|
|
allowed_hosts: list[str] = []
|
|
database = Path(record['data_dir']) / 'config.sqlite3'
|
|
try:
|
|
connection = sqlite3.connect(f'file:{database}?mode=ro', uri=True, timeout=2)
|
|
try:
|
|
row = connection.execute('SELECT document FROM configuration WHERE id = 1').fetchone()
|
|
finally:
|
|
connection.close()
|
|
if row:
|
|
document = json.loads(row[0])
|
|
administration = document.get('administration') or {}
|
|
if isinstance(administration, dict):
|
|
if administration.get('listen'):
|
|
raw = address(str(administration['listen']))
|
|
tls = bool(administration.get('tls', False))
|
|
values = administration.get('allowed_hosts') or []
|
|
if isinstance(values, list):
|
|
allowed_hosts = [str(value).strip() for value in values if str(value).strip()]
|
|
except (OSError, sqlite3.Error, ValueError, TypeError, json.JSONDecodeError):
|
|
# Old releases and damaged/uninitialized databases used the startup HTTP
|
|
# listener. The real startup diagnostics remain authoritative if this
|
|
# compatibility read cannot resolve persisted settings.
|
|
pass
|
|
|
|
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
|
|
ip = ipaddress.ip_address(host)
|
|
connect_host = ('::1' if ip.version == 6 else '127.0.0.1') if ip.is_unspecified else host
|
|
authority_host = allowed_hosts[0] if allowed_hosts else connect_host
|
|
try:
|
|
authority_ip = ipaddress.ip_address(authority_host)
|
|
authority = f'[{authority_ip}]:{port}' if authority_ip.version == 6 else f'{authority_ip}:{port}'
|
|
except ValueError:
|
|
authority = f'{authority_host}:{port}'
|
|
connect_raw = f'[{connect_host}]:{port}' if ':' in connect_host else f'{connect_host}:{port}'
|
|
return connect_raw, tls, authority
|
|
|
|
|
|
def healthy(record: dict, version: str, attempts: int = 30) -> bool:
|
|
raw, tls, health_host = admin_health_endpoint(record)
|
|
host, port = (raw[1:].split(']:') if raw.startswith('[') else raw.rsplit(':', 1))
|
|
for _ in range(attempts):
|
|
if tls:
|
|
# Local deployment health is process identity/liveness, not PKI validation.
|
|
# The configured admin certificate may intentionally be self-signed or for
|
|
# a DNS name while we connect to the local socket.
|
|
conn = http.client.HTTPSConnection(host, int(port), timeout=2, context=ssl._create_unverified_context())
|
|
else:
|
|
conn = http.client.HTTPConnection(host, int(port), timeout=2)
|
|
try:
|
|
conn.request('GET', '/healthz', headers={'Host': health_host})
|
|
response = conn.getresponse(); content = response.read(4097)
|
|
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
|
|
if (active and response.status == 200 and len(content) <= 4096
|
|
and json.loads(content).get('status') == 'ok' and record.get('version') == version
|
|
and running_release_matches(record)):
|
|
return True
|
|
except (OSError, ValueError, http.client.HTTPException, ssl.SSLError):
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
time.sleep(1)
|
|
return False
|
|
|
|
|
|
def stage(binary: Path, prefix: Path, user: str, sha: str) -> tuple[str, str]:
|
|
release = 'build-' + sha[:24]
|
|
releases = prefix / 'releases'; releases.mkdir(mode=0o755, exist_ok=True); os.chmod(releases, 0o755)
|
|
directory = releases / release
|
|
if not directory.exists():
|
|
directory.mkdir(mode=0o755); os.chmod(directory, 0o755)
|
|
try:
|
|
shutil.copyfile(binary, directory / 'rust-proxy')
|
|
os.chmod(directory / 'rust-proxy', 0o755)
|
|
except BaseException:
|
|
shutil.rmtree(directory); raise
|
|
if directory.is_symlink() or (directory / 'rust-proxy').is_symlink() or digest(directory / 'rust-proxy') != sha:
|
|
raise ValueError('Staged binary checksum mismatch.')
|
|
# Never execute the supplied binary as root, even for its version banner.
|
|
result = run(['runuser', '-u', user, '--', str(directory / 'rust-proxy'), '--version'], capture_output=True, text=True, timeout=15)
|
|
matched = re.fullmatch(r'(?:rust-proxy|RustProxy) (\d+\.\d+\.\d+(?:[-+][\w.-]+)?)\s*', result.stdout)
|
|
if not matched:
|
|
raise ValueError('Binary did not return a RustProxy version.')
|
|
return release, matched[1]
|
|
|
|
|
|
def backup_stopped(record: dict, prefix: Path) -> Path:
|
|
# Service stop must happen before this call: ACME files and SQLite form one backup.
|
|
data = Path(record['data_dir'])
|
|
lock_fd = os.open(data / 'process.lock', os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
try:
|
|
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
account = pwd.getpwnam(record['user']); os.fchown(lock_fd, account.pw_uid, account.pw_gid)
|
|
backups = prefix / 'backups'; backups.mkdir(mode=0o700, exist_ok=True)
|
|
backup = backups / ('before-' + record['version'] + '-' + uuid.uuid4().hex[:12]); backup.mkdir(mode=0o700)
|
|
copy_data(data, backup / 'data')
|
|
write_private(backup / 'install.json', json.dumps(record, indent=2).encode())
|
|
if UNIT_PATH.exists():
|
|
shutil.copyfile(UNIT_PATH, backup / 'rustproxy.service'); os.chmod(backup / 'rustproxy.service', 0o600)
|
|
return backup
|
|
finally:
|
|
os.close(lock_fd)
|
|
|
|
|
|
def restore_service_unit(backup: Path) -> None:
|
|
saved = backup / 'rustproxy.service'
|
|
if saved.exists():
|
|
copy_private(saved, UNIT_PATH, 0o644)
|
|
else:
|
|
UNIT_PATH.unlink(missing_ok=True)
|
|
run(['systemctl', 'daemon-reload'])
|
|
|
|
|
|
def reset_saved_config(data: Path) -> None:
|
|
"""Development-only reset of desired config while preserving auth/certs/audit data."""
|
|
database = data / 'config.sqlite3'
|
|
if database.is_symlink() or not database.is_file():
|
|
raise ValueError('Cannot reset configuration: config.sqlite3 is missing or unsafe.')
|
|
connection = sqlite3.connect(f'file:{database}?mode=rw', uri=True, timeout=5)
|
|
try:
|
|
row = connection.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='configuration'").fetchone()
|
|
if row is None:
|
|
raise ValueError('Cannot reset configuration: configuration table is missing.')
|
|
connection.execute('DELETE FROM configuration WHERE id = 1')
|
|
connection.commit()
|
|
finally:
|
|
connection.close()
|
|
print('Stored vhost/settings configuration cleared; account, certificates and other private data were preserved.')
|
|
|
|
|
|
def service_failure_details() -> str:
|
|
chunks: list[str] = []
|
|
commands = (
|
|
['systemctl', 'show', SERVICE, '--property=ActiveState,SubState,ExecMainStatus', '--no-pager'],
|
|
['journalctl', '-u', SERVICE, '-n', '25', '--no-pager', '--output=cat'],
|
|
)
|
|
for command in commands:
|
|
try:
|
|
result = subprocess.run(command, check=False, capture_output=True, text=True, timeout=5)
|
|
output = ((result.stdout or '') + (result.stderr or '')).strip()
|
|
if output:
|
|
chunks.append(output)
|
|
except (OSError, subprocess.SubprocessError):
|
|
pass
|
|
return '\n'.join(chunks)[-8000:]
|
|
|
|
|
|
def stop_for_managed_update(data: Path) -> None:
|
|
"""Ask RustProxy 0.14.0+ to force hot cache into the enabled cold tier before exit.
|
|
|
|
Older installed versions ignore the marker and still follow their ordinary graceful
|
|
shutdown policy, so the first upgrade to 0.14.0 remains backward compatible.
|
|
The marker is removed before backup so it never becomes persisted application data.
|
|
"""
|
|
marker = data / '.managed-update'
|
|
marker.unlink(missing_ok=True)
|
|
write_private(marker, b'1\n', 0o644)
|
|
try:
|
|
run(['systemctl', 'stop', SERVICE])
|
|
finally:
|
|
marker.unlink(missing_ok=True)
|
|
|
|
|
|
def update(record: dict, binary: Path, sha: str, start: bool, refresh_geoip: bool = False, reset_config: bool = False) -> dict:
|
|
prefix = Path(record['prefix']); data = Path(record['data_dir']); account = pwd.getpwnam(record['user'])
|
|
# Updates deliberately do not rewrite shell environment. Check the installer-
|
|
# managed PATH profile early and report the exact remediation target instead.
|
|
check_shell_path(prefix)
|
|
# Do this before staging/network work and, critically, before stopping the
|
|
# currently healthy service. It also repairs stale root:root/0700 metadata
|
|
# left by manual maintenance while preserving private 0700 semantics.
|
|
ensure_service_data_access(data, account)
|
|
release, version = stage(binary, prefix, record['user'], sha)
|
|
geoip_stage: Path | None = None
|
|
geoip_names: list[str] = []
|
|
if refresh_geoip:
|
|
geoip_stage, geoip_names = prepare_geoip_update(data, prefix)
|
|
active = subprocess.run(['systemctl', 'is-active', '--quiet', SERVICE], check=False, stdout=subprocess.DEVNULL).returncode == 0
|
|
stop_for_managed_update(data)
|
|
try:
|
|
backup = backup_stopped(record, prefix)
|
|
except BaseException:
|
|
if active: run(['systemctl', 'start', SERVICE])
|
|
if geoip_stage is not None:
|
|
shutil.rmtree(geoip_stage, ignore_errors=True)
|
|
raise
|
|
replacement = dict(record, release=release, version=version, sha256=sha)
|
|
try:
|
|
if reset_config:
|
|
reset_saved_config(data)
|
|
apply_geoip_update(data, geoip_stage, geoip_names, account.pw_uid, account.pw_gid)
|
|
switch(prefix, release)
|
|
write_private(UNIT_PATH, service_unit(replacement).encode(), 0o644)
|
|
run(['systemctl', 'daemon-reload'])
|
|
if start:
|
|
run(['systemctl', 'start', SERVICE])
|
|
if not healthy(replacement, version):
|
|
raise RuntimeError('New release failed the local health/version check.')
|
|
write_private(prefix / 'install.json', json.dumps(replacement, indent=2).encode())
|
|
except BaseException as error:
|
|
details = service_failure_details()
|
|
if details:
|
|
print('New release diagnostics:\n' + details, file=sys.stderr)
|
|
stop_for_managed_update(data)
|
|
failed = data.with_name(data.name + '.failed-' + uuid.uuid4().hex[:12])
|
|
data.rename(failed) # keep the failed generation for diagnosis; do not delete it
|
|
copy_data(backup / 'data', data); chown_tree(data, account.pw_uid, account.pw_gid)
|
|
switch(prefix, record['release'])
|
|
restore_service_unit(backup)
|
|
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
|
|
if active:
|
|
run(['systemctl', 'start', SERVICE])
|
|
if not healthy(record, record['version']):
|
|
raise RuntimeError(f'Update failed: {error}. Rollback restored code/data/unit but previous service health also failed. Backup: {backup}; failed state: {failed}') from error
|
|
raise RuntimeError(f'Update failed: {error}. Previous code, private data and service unit were restored. Installation is still present; do not run install.sh. Fix the cause and rerun update.sh. Backup: {backup}; failed state: {failed}') from error
|
|
finally:
|
|
if geoip_stage is not None:
|
|
shutil.rmtree(geoip_stage, ignore_errors=True)
|
|
print(f'Backup (contains private keys): {backup}')
|
|
return replacement
|
|
|
|
|
|
def install(record: dict, binary: Path, sha: str, start: bool, install_geoip: bool = False) -> dict:
|
|
import grp
|
|
prefix = Path(record['prefix']); data = Path(record['data_dir'])
|
|
if (prefix / 'install.json').exists() or (prefix / 'current').exists() or UNIT_PATH.exists():
|
|
raise ValueError('Installation/service already exists. A failed update rolls back to the previous installed release, so rerun update.sh after fixing the reported cause. install.sh never overwrites an existing installation.')
|
|
if data.exists() and any(data.iterdir()):
|
|
raise ValueError('First installation requires an empty data directory; existing data will never be reset.')
|
|
try:
|
|
account = pwd.getpwnam(record['user'])
|
|
if account.pw_uid == 0 or account.pw_shell not in ('/usr/sbin/nologin', '/sbin/nologin', '/bin/false'):
|
|
raise ValueError('Existing service account must be non-root with a non-login shell.')
|
|
except KeyError:
|
|
run(['useradd', '--system', '--user-group', '--home-dir', str(data), '--no-create-home', '--shell', '/usr/sbin/nologin', record['user']])
|
|
account = pwd.getpwnam(record['user'])
|
|
record['group'] = grp.getgrgid(account.pw_gid).gr_name
|
|
prefix.mkdir(parents=True, mode=0o755, exist_ok=True); os.chmod(prefix, 0o755)
|
|
data.mkdir(parents=True, mode=0o700, exist_ok=True); os.chmod(data, 0o700); os.chown(data, account.pw_uid, account.pw_gid)
|
|
if install_geoip:
|
|
download_geoip(data, account.pw_uid, account.pw_gid)
|
|
release, version = stage(binary, prefix, record['user'], sha)
|
|
record.update(release=release, version=version, sha256=sha)
|
|
switch(prefix, release)
|
|
install_shell_path(prefix)
|
|
# Record is written even if initial startup fails, so an update can repair it.
|
|
write_private(prefix / 'install.json', json.dumps(record, indent=2).encode())
|
|
write_private(UNIT_PATH, service_unit(record).encode(), 0o644)
|
|
run(['systemctl', 'daemon-reload']); run(['systemctl', 'enable', SERVICE])
|
|
if start:
|
|
run(['systemctl', 'start', SERVICE])
|
|
if not healthy(record, version):
|
|
run(['systemctl', 'stop', SERVICE])
|
|
raise RuntimeError('Installed but startup failed. Data was retained; inspect journalctl -u rustproxy.')
|
|
return record
|
|
|
|
|