v0.28.0

scripts/deploy/deploy_core.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 """Install/update a local release binary on Linux/systemd. No builds.
 
-First install downloads the PDDL user-country GeoIP CSV files from the
-sapics/ip-location-db GitHub releases and verifies publisher SHA-256 files.
-Updates compare the installed GeoIP files with current publisher checksums and
-stage only stale assets before service downtime. Updates stop the service before
+First install downloads the PDDL user-country GeoIP CSV files directly from the
+sapics/ip-location-db GitHub release and validates their structure before publish.
+Updates download/validate current files before downtime, compare content SHA-256
+locally and stage only changed assets. Updates stop the service before
 copying all private data, refresh the managed systemd unit, switch an immutable
 release symlink, check health/version and restore code, data and unit on failure.
 Use --plan to inspect operations without changing the host.
@@ -35,12 +35,10 @@ ROOT = Path(__file__).resolve().parents[2]
 UNIT_PATH = Path('/etc/systemd/system/rustproxy.service')
 SHELL_PATH_PROFILE = Path('/etc/profile.d/rustproxy-path.sh')
 SERVICE = 'rustproxy.service'
-GEOIP_ASSETS = (
-    'user-country-ipv4.csv',
-    'user-country-ipv6.csv',
-)
-GEOIP_RELEASE = 'https://github.com/sapics/ip-location-db/releases/download/latest/'
-GEOIP_CHECKSUM = 'https://github.com/sapics/ip-location-db/releases/download/checksum/'
+GEOIP_SOURCES = {
+    'user-country-ipv4.csv': 'https://github.com/sapics/ip-location-db/releases/download/latest/user-country-ipv4.csv',
+    'user-country-ipv6.csv': 'https://github.com/sapics/ip-location-db/releases/download/latest/user-country-ipv6.csv',
+}
 GEOIP_MAX_BYTES = 256 * 1024 * 1024
 
 
@@ -97,7 +95,7 @@ def digest(path: Path) -> str:
 def fetch_https(url: str, max_bytes: int) -> bytes:
     if urllib.parse.urlparse(url).scheme != 'https':
         raise ValueError('GeoIP downloads require HTTPS.')
-    request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.27.3'})
+    request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.28.0'})
     with urllib.request.urlopen(request, timeout=30) as response:
         if urllib.parse.urlparse(response.geturl()).scheme != 'https':
             raise ValueError('GeoIP download redirected away from HTTPS.')
@@ -115,15 +113,65 @@ def fetch_https(url: str, max_bytes: int) -> bytes:
         return bytes(result)
 
 
-def publisher_geoip_checksums() -> dict[str, str]:
-    expected: dict[str, str] = {}
-    for name in GEOIP_ASSETS:
-        checksum = fetch_https(GEOIP_CHECKSUM + name + '.sha256', 4096).decode('ascii', 'strict').strip()
-        matched = re.search(r'\b([0-9a-fA-F]{64})\b', checksum)
-        if not matched:
-            raise ValueError(f'Invalid publisher checksum for {name}.')
-        expected[name] = matched.group(1).lower()
-    return expected
+def acquire_geoip_update_lock(data: Path, uid: int | None = None, gid: int | None = None) -> int:
+    """Serialize installer/manual GeoIP refresh with the in-process worker."""
+    path = data / '.geoip-update.lock'
+    flags = os.O_RDWR | os.O_CREAT
+    if hasattr(os, 'O_NOFOLLOW'):
+        flags |= os.O_NOFOLLOW
+    fd = os.open(path, flags, 0o600)
+    try:
+        os.fchmod(fd, 0o600)
+        if uid is not None and gid is not None:
+            os.fchown(fd, uid, gid)
+        fcntl.flock(fd, fcntl.LOCK_EX)
+        return fd
+    except BaseException:
+        os.close(fd)
+        raise
+
+
+def release_geoip_update_lock(fd: int | None) -> None:
+    if fd is None:
+        return
+    try:
+        fcntl.flock(fd, fcntl.LOCK_UN)
+    finally:
+        os.close(fd)
+
+
+def validate_geoip_csv(name: str, payload: bytes) -> None:
+    """Reject truncated/error payloads before a GeoIP CSV is published."""
+    family = 4 if name == 'user-country-ipv4.csv' else 6
+    try:
+        text = payload.decode('utf-8', 'strict')
+    except UnicodeDecodeError as error:
+        raise ValueError(f'GeoIP {name} is not valid UTF-8.') from error
+    previous_end: int | None = None
+    rows = 0
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.strip()
+        if not line or line.startswith('#'):
+            continue
+        fields = [value.strip().strip('"') for value in line.split(',')]
+        if len(fields) != 3:
+            raise ValueError(f'GeoIP {name} row {number} must contain exactly 3 columns.')
+        try:
+            start = ipaddress.ip_address(fields[0])
+            end = ipaddress.ip_address(fields[1])
+        except ValueError as error:
+            raise ValueError(f'GeoIP {name} row {number} contains an invalid IP address.') from error
+        if start.version != family or end.version != family or int(start) > int(end):
+            raise ValueError(f'GeoIP {name} row {number} contains an invalid address range.')
+        if previous_end is not None and int(start) <= previous_end:
+            raise ValueError(f'GeoIP {name} ranges overlap or are unsorted at row {number}.')
+        country = fields[2]
+        if country and not re.fullmatch(r'[A-Za-z]{2}', country):
+            raise ValueError(f'GeoIP {name} row {number} contains an invalid country code.')
+        previous_end = int(end)
+        rows += 1
+    if rows == 0:
+        raise ValueError(f'GeoIP {name} does not contain any address ranges.')
 
 
 def geoip_file_digest(path: Path) -> str | None:
@@ -137,29 +185,31 @@ def geoip_file_digest(path: Path) -> str | None:
 
 
 def prepare_geoip_update(data: Path, prefix: Path) -> tuple[Path | None, list[str]]:
-    """Verify local GeoIP files against current publisher checksums.
+    """Download and validate current GeoIP CSVs before service downtime.
 
-    Network/download work happens before the service is stopped. Changed files are
-    staged under the code prefix and are copied into the private data transaction
-    only after its backup exists.
+    Publisher checksums are intentionally not required here: the upstream checksum
+    release can move independently from the `latest` data release. Local SHA-256 is
+    used only to decide whether the validated payload changes an installed file.
     """
-    expected = publisher_geoip_checksums()
     directory = data / 'geoip'
     if directory.exists() and directory.is_symlink():
         raise ValueError('GeoIP directory cannot be a symlink.')
-    stale = [name for name in GEOIP_ASSETS if geoip_file_digest(directory / name) != expected[name]]
+    payloads: dict[str, bytes] = {}
+    stale: list[str] = []
+    for name, url in GEOIP_SOURCES.items():
+        payload = fetch_https(url, GEOIP_MAX_BYTES)
+        validate_geoip_csv(name, payload)
+        payloads[name] = payload
+        if geoip_file_digest(directory / name) != hashlib.sha256(payload).hexdigest():
+            stale.append(name)
     if not stale:
-        print('GeoIP database is current (publisher SHA-256 matches).')
+        print('GeoIP database is current (validated content is unchanged).')
         return None, []
     staging = prefix / ('.geoip-stage-' + uuid.uuid4().hex)
     staging.mkdir(mode=0o700)
     try:
         for name in stale:
-            payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
-            actual = hashlib.sha256(payload).hexdigest()
-            if actual != expected[name]:
-                raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
-            write_private(staging / name, payload, 0o600)
+            write_private(staging / name, payloads[name], 0o600)
         print('GeoIP update prepared: ' + ', '.join(stale))
         return staging, stale
     except BaseException:
@@ -204,20 +254,17 @@ def apply_geoip_update(data: Path, staging: Path | None, names: list[str], uid:
 
 
 def download_geoip(data: Path, uid: int, gid: int) -> None:
-    # First install has no code-prefix staging transaction, so download and verify
-    # each asset directly before publishing it atomically into the private data dir.
-    expected = publisher_geoip_checksums()
+    # First install has no code-prefix staging transaction. Validate each direct
+    # payload before publishing it atomically into the private data directory.
     directory = data / 'geoip'
     if directory.exists() and directory.is_symlink():
         raise ValueError('GeoIP directory cannot be a symlink.')
     directory.mkdir(mode=0o700, exist_ok=True)
     os.chmod(directory, 0o700)
     os.chown(directory, uid, gid)
-    for name in GEOIP_ASSETS:
-        payload = fetch_https(GEOIP_RELEASE + name, GEOIP_MAX_BYTES)
-        actual = hashlib.sha256(payload).hexdigest()
-        if actual != expected[name]:
-            raise ValueError(f'GeoIP SHA-256 mismatch for {name}.')
+    for name, url in GEOIP_SOURCES.items():
+        payload = fetch_https(url, GEOIP_MAX_BYTES)
+        validate_geoip_csv(name, payload)
         target = directory / name
         write_private(target, payload, 0o600)
         os.chown(target, uid, gid)