From 8bef2e68bb96f76e2a2e667ba6e022ecaa33b3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Thu, 1 Oct 2026 21:33:32 +0200 Subject: [PATCH] first commit --- .gitignore | 2 + Dockerfile | 18 +++ README.md | 133 ++++++++++++++++++++++ docker-compose.yml | 15 +++ repo_daemon.py | 273 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 441 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 repo_daemon.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9ce6588 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +repo/* +.venv \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ff3fb69 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.14-alpine + +RUN apk add --no-cache libarchive-tools ca-certificates + +WORKDIR /app +COPY repo_daemon.py /app/repo_daemon.py + +ENV REPO_DIR=.//repo \ + PORT=8478 \ + POLL_INTERVAL=3600 \ + STABLE_SECONDS=60 \ + RETENTION=10 \ + LATEST_NAMES=os-rustproxy-latest.pkg + +VOLUME ["/repo"] +EXPOSE 8478 + +CMD ["python3", "/app/repo_daemon.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..e8572c3 --- /dev/null +++ b/README.md @@ -0,0 +1,133 @@ +# OPNsense / FreeBSD pkg repo daemon + +Lekki daemon na Alpine. Obserwuje `/repo`, odczytuje manifest z paczek FreeBSD `.pkg`, +automatycznie wersjonuje `latest.pkg`, pilnuje retencji i wystawia repo przez HTTP. + +## Start + +```sh +cp .env.example .env +docker compose up -d --build +docker compose logs -f +``` + +Repo: + +```text +http://HOST:8478/ +``` + +## Upload z buildu + +Mozesz zawsze wysylac ten sam plik: + +```sh +scp latest.pkg user@repo-host:/opt/opnsense-pkg-repo/repo/latest.pkg +``` + +Daemon czyta z paczki: + +```text ++COMPACT_MANIFEST +``` + +i pola: + +```text +name +version +``` + +Przyklad: + +```text +latest.pkg +``` + +z manifestem: + +```json +{"name":"myapp","version":"1.4.7"} +``` + +zostanie automatycznie przemianowany na: + +```text +myapp-1.4.7.pkg +``` + +Nie musisz znac wersji po stronie CI. + +## Retencja + +Domyslnie daemon zachowuje 10 najnowszych plikow dla kazdej nazwy pakietu: + +```env +RETENTION=10 +``` + +Np.: + +```text +myapp-1.4.10.pkg +... +myapp-1.4.1.pkg +``` + +Starsze sa automatycznie usuwane. Kolejnosc retencji jest wedlug czasu pojawienia sie +pliku w repo, a nie leksykograficznego sortowania numeru wersji. + +Bez retencji: + +```env +RETENTION=0 +``` + +## Pozostale ENV + +```env +PORT=8080 +POLL_INTERVAL=2 +STABLE_SECONDS=2 +LATEST_NAMES=latest.pkg +``` + +Mozesz miec kilka nazw uploadowych: + +```env +LATEST_NAMES=latest.pkg,myapp-latest.pkg +``` + +## OPNsense + +`/usr/local/etc/pkg/repos/myrepo.conf`: + +```ucl +MyRepo: { + url: "http://HOST:8478", + enabled: yes, + signature_type: "none" +} +``` + +Potem: + +```sh +pkg update -f +pkg search myapp +pkg install -r MyRepo myapp +``` + +## Logi + +```sh +docker compose logs -f +``` + +Przy nowym buildzie zobaczysz np.: + +```text +[ingest] latest.pkg -> myapp-1.4.7.pkg +[retention] usunieto myapp-1.3.2.pkg (myapp, limit=10) +[repo] przebudowano katalog: 10 pakiet(ow) +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4cc2cee --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + repo: + build: . + container_name: opnsense-pkg-repo + restart: unless-stopped + ports: + - "${PORT:-8478}:8478" + volumes: + - ./repo:/repo + environment: + PORT: "8478" + POLL_INTERVAL: "${POLL_INTERVAL:-3600}" + STABLE_SECONDS: "${STABLE_SECONDS:-60}" + RETENTION: "${RETENTION:-10}" + LATEST_NAMES: "${LATEST_NAMES:-os-rustproxy-latest.pkg}" diff --git a/repo_daemon.py b/repo_daemon.py new file mode 100644 index 0000000..202bbb9 --- /dev/null +++ b/repo_daemon.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +import hashlib +import io +import json +import os +import re +import subprocess +import tarfile +import threading +import time +from collections import defaultdict +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +REPO = Path(os.getenv("REPO_DIR", "./repo")).resolve() +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", "8478")) +POLL_INTERVAL = float(os.getenv("POLL_INTERVAL", "3600")) +STABLE_SECONDS = float(os.getenv("STABLE_SECONDS", "60")) +RETENTION = int(os.getenv("RETENTION", "10")) +LATEST_NAMES = { + x.strip() for x in os.getenv("LATEST_NAMES", "latest.pkg").split(",") if x.strip() +} + +CATALOG_FILES = {"meta.conf", "data.pkg", "packagesite.pkg"} +LOCK = threading.Lock() +_seen = {} + + +def log(msg): + print(time.strftime("%Y-%m-%d %H:%M:%S"), msg, flush=True) + + +def sha256(path): + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def read_manifest(pkg): + for member in ("+COMPACT_MANIFEST", "+MANIFEST"): + p = subprocess.run( + ["bsdtar", "-xOf", str(pkg), member], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + if p.returncode == 0 and p.stdout: + return json.loads(p.stdout.decode("utf-8")) + raise RuntimeError(f"{pkg.name}: brak +COMPACT_MANIFEST / +MANIFEST") + + +def safe_filename(value): + # Zachowuje typowe znaki wersji FreeBSD, usuwa tylko znaki niebezpieczne dla sciezki. + value = str(value).strip().replace("/", "_").replace("\\", "_") + value = re.sub(r"[\x00-\x1f]", "_", value) + return value + + +def is_payload_pkg(path): + return ( + path.is_file() + and path.suffix == ".pkg" + and path.name not in CATALOG_FILES + and not path.name.endswith(".tmp.pkg") + ) + + +def stable(path): + try: + st = path.stat() + except FileNotFoundError: + return False + + key = str(path) + state = (st.st_size, st.st_mtime_ns) + now = time.monotonic() + old = _seen.get(key) + + if old is None or old[:2] != state: + _seen[key] = (*state, now) + return False + + return now - old[2] >= STABLE_SECONDS + + +def normalize_latest(): + changed = False + for path in sorted(REPO.rglob("*.pkg")): + if path.name not in LATEST_NAMES or not stable(path): + continue + + try: + manifest = read_manifest(path) + name = safe_filename(manifest["name"]) + version = safe_filename(manifest["version"]) + target = path.with_name(f"{name}-{version}.pkg") + + if target == path: + continue + + if target.exists(): + if sha256(target) == sha256(path): + path.unlink() + log(f"[ingest] {path.name}: wersja {name}-{version} juz istnieje, usunieto duplikat") + else: + os.replace(path, target) + log(f"[ingest] {path.name} -> {target.name} (nadpisano ta sama wersje)") + else: + os.replace(path, target) + log(f"[ingest] {path.name} -> {target.name}") + + _seen.pop(str(path), None) + changed = True + except Exception as e: + log(f"[ingest] BLAD {path.name}: {e}") + + return changed + + +def collect_packages(): + packages = [] + for path in sorted(REPO.rglob("*.pkg")): + if not is_payload_pkg(path): + continue + if path.name in LATEST_NAMES: + # latest.pkg nie trafia do indeksu przed normalizacja. + continue + if not stable(path): + continue + + manifest = read_manifest(path) + manifest["path"] = path.relative_to(REPO).as_posix() + manifest["pkgsize"] = path.stat().st_size + manifest["sum"] = sha256(path) + packages.append((path, manifest)) + return packages + + +def enforce_retention(packages): + if RETENTION <= 0: + return False + + groups = defaultdict(list) + for path, manifest in packages: + groups[manifest.get("name", path.stem)].append((path, manifest)) + + removed = False + for name, entries in groups.items(): + entries.sort(key=lambda item: item[0].stat().st_mtime_ns, reverse=True) + for path, manifest in entries[RETENTION:]: + try: + path.unlink() + _seen.pop(str(path), None) + log(f"[retention] usunieto {path.name} ({name}, limit={RETENTION})") + removed = True + except FileNotFoundError: + pass + + return removed + + +def make_tar(dst, member_name, payload): + tmp = dst.with_name(dst.name + ".tmp") + with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf: + info = tarfile.TarInfo(member_name) + info.size = len(payload) + info.mode = 0o644 + info.mtime = int(time.time()) + tf.addfile(info, io.BytesIO(payload)) + os.replace(tmp, dst) + + +def rebuild(): + with LOCK: + packages = collect_packages() + + if enforce_retention(packages): + packages = collect_packages() + + manifests = [m for _, m in packages] + + packagesite = b"".join( + json.dumps(m, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + b"\n" + for m in manifests + ) + + # Repo v2. packagesite.pkg pozostaje dla zgodnosci ze starszymi klientami. + data = json.dumps( + {"groups": [], "expired_packages": [], "packages": manifests}, + separators=(",", ":"), + ensure_ascii=False, + ).encode("utf-8") + + make_tar(REPO / "packagesite.pkg", "packagesite.yaml", packagesite) + make_tar(REPO / "data.pkg", "data", data) + + meta = ( + 'version: 1;\n' + 'packing_format: "tar";\n' + 'manifests: "packagesite.yaml";\n' + 'manifests_archive: "packagesite";\n' + 'data: "data";\n' + 'data_archive: "data";\n' + 'filesite: "files";\n' + 'filesite_archive: "files";\n' + ) + tmp = REPO / "meta.conf.tmp" + tmp.write_text(meta, encoding="utf-8") + os.replace(tmp, REPO / "meta.conf") + + log(f"[repo] przebudowano katalog: {len(manifests)} pakiet(ow)") + + +def payload_fingerprint(): + items = [] + for path in sorted(REPO.rglob("*.pkg")): + if not is_payload_pkg(path): + continue + try: + st = path.stat() + items.append((path.relative_to(REPO).as_posix(), st.st_size, st.st_mtime_ns)) + except FileNotFoundError: + pass + return tuple(items) + + +def watcher(): + previous = None + while True: + try: + normalize_latest() + current = payload_fingerprint() + + if current != previous: + # collect_packages() wpusci tylko pliki stabilne przez STABLE_SECONDS. + time.sleep(STABLE_SECONDS) + normalize_latest() + rebuild() + previous = payload_fingerprint() + except Exception as e: + log(f"[repo] BLAD przebudowy: {e}") + + time.sleep(POLL_INTERVAL) + + +class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(REPO), **kwargs) + + def end_headers(self): + self.send_header("Cache-Control", "no-cache, no-store, must-revalidate") + super().end_headers() + + def log_message(self, fmt, *args): + log("[http] " + (fmt % args)) + + +def main(): + REPO.mkdir(parents=True, exist_ok=True) + rebuild() + + threading.Thread(target=watcher, daemon=True).start() + + server = ThreadingHTTPServer((HOST, PORT), Handler) + log(f"[http] repo: http://{HOST}:{PORT}/ katalog={REPO}") + server.serve_forever() + + +if __name__ == "__main__": + main()