diff --git a/Dockerfile b/Dockerfile index 2be1b7f..0893956 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ COPY --from=go-toolchain /usr/local/go /usr/local/go ENV PATH=/usr/local/go/bin:${PATH} \ GOPATH=/var/lib/package-builder/go \ - GOCACHE=/var/lib/package-builder/go/cache \ + GOCACHE=/tmp/rust-package-builder/go-cache \ FREEBSD_RUST_TOOLCHAIN=${FREEBSD_RUST_TOOLCHAIN} \ CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_LINKER=clang \ CC_x86_64_unknown_freebsd=clang \ @@ -85,8 +85,10 @@ ENV PATH=/usr/local/go/bin:${PATH} \ CXXFLAGS_x86_64_unknown_freebsd="--target=x86_64-unknown-freebsd15.1 --sysroot=/opt/freebsd-sysroot" \ CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_RUSTFLAGS="-C link-arg=--target=x86_64-unknown-freebsd15.1 -C link-arg=--sysroot=/opt/freebsd-sysroot -C link-arg=-fuse-ld=lld" -# Registry/git/target cache is persistent, while rustup/toolchains stay in the image. -ENV CARGO_HOME=/var/lib/package-builder/cargo +# Cargo registry/git cache stays persistent; build target is disposable and +# must not grow on the /data volume between package builds. +ENV CARGO_HOME=/var/lib/package-builder/cargo \ + CARGO_TARGET_DIR=/tmp/rust-package-builder/cargo-target COPY scripts/ /opt/rust-package-builder/scripts/ COPY packaging/ /opt/rust-package-builder/packaging/ diff --git a/README.md b/README.md index 8fb2e9b..91cc491 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,9 @@ POLL_INTERVAL=300 PACKAGE_RELEASE=auto MAX_GLIBC_VERSION=2.34 RUN_LINUX_TESTS=1 +RELEASE_RETENTION=10 +PACKAGE_RETENTION=10 +CLEAN_BUILD_CACHE=1 OPNSENSE_ABI=26.7 OPNSENSE_RETENTION=10 @@ -231,6 +234,12 @@ APPLY_FREEBSD_PATCHES=1 `RUN_LINUX_TESTS=1` jest domyślne. Testy są bramką tylko dla DEB/RPM/SUSE i nie wpływają na gałąź OPNsense. Ustaw `0`, jeśli chcesz całkowicie pominąć testy Linux. +`PACKAGE_RETENTION=10` zachowuje ostatnie 10 wersji **osobno dla RustProxy i Docker Agenta** w każdym kanale Linux: DEB, RHEL/DNF RPM i SUSE RPM. Oznacza to do 10 kompletnych wersji obu produktów, a nie 10 plików łącznie. + +`RELEASE_RETENTION=10` zachowuje 10 najnowszych atomowych snapshotów `data/repository/releases/`. Aktualnie serwowany snapshot jest zawsze chroniony przed usunięciem. + +`CLEAN_BUILD_CACHE=1` powoduje usunięcie `cargo target` oraz cache kompilacji Go po każdym pełnym cyklu budowania. `CARGO_TARGET_DIR` i `GOCACHE` znajdują się pod `/tmp/rust-package-builder`, więc duże artefakty kompilacji nie trafiają już do trwałego `data/`. Ustaw `0`, jeśli świadomie chcesz zachować te cache pomiędzy buildami w działającym kontenerze. + `OPNSENSE_RETENTION=10` zachowuje ostatnie 10 wersji pakietu dla danej serii/architektury OPNsense. `0` wyłącza usuwanie starych wersji. `FREEBSD_NATIVE_ENABLED=1` (domyślnie) włącza natywny pakiet FreeBSD i oddzielny kanał publikacji. `0` wyłącza budowanie natywnego FreeBSD. **Uwaga przy aktualizacji:** istniejący plik `.env` z `FREEBSD_NATIVE_ENABLED=0` ma pierwszeństwo przed nową wartością domyślną — zmień go ręcznie na `1`, inaczej pakiet FreeBSD nadal nie powstanie. `FREEBSD_NATIVE_RETENTION=10` ogranicza liczbę zachowanych wersji. Repo używa tego samego publicznego hosta/portu co OPNsense, ale pod oddzielnym prefiksem `OPNSENSE_REPOSITORY_URL/freebsd/`. @@ -365,7 +374,7 @@ Testy integracyjne są dostępne w pełnym repozytorium developerskim. ```text data/ ├── aptly/ -├── cargo/ +├── cargo/ # registry/git cache; bez target/ ├── packages/ │ ├── deb/ │ └── rpm/x86_64/ @@ -417,17 +426,17 @@ Jeżeli istnieje starsze `last_commit` oraz poprawne APT/RPM dla tego SHA, zosta The OPNsense branch reuses the Cargo package `VERSION` already detected by the main build pipeline. If `apply-freebsd-patches.sh` is run manually, it falls back to `RUST_PROXY_PYTHON` (`/usr/bin/python3.11`) so Rocky Linux 9's default Python 3.9 does not cause a missing `tomllib` error. -### Czyszczenie cache Cargo +### Czyszczenie danych po kompilacji i retencja -Po **udanym opublikowaniu pakietów** każdej gałęzi (Linux/OPNsense/FreeBSD native) builder -usuwa `target/**/incremental` i `target/debug`, a także pliki starsze -niż 30 dni z `target/release/deps` oraz -`target/x86_64-unknown-freebsd/release/deps`. Zachowuje aktualne artefakty -release, fingerprinty oraz cache zależności, więc kolejne buildy nie startują -od zera. Starsze usunięte zależności mogą wymagać pojedynczej rekompilacji. +Builder nie zapisuje już `cargo target` w `data/cargo/target` ani `GOCACHE` w `data/go/cache`. Rejestr Cargo/cache Git oraz pobrane moduły Go mogą pozostać w `data/`, ale duże artefakty kompilacji trafiają pod `/tmp/rust-package-builder/`. Przy `CLEAN_BUILD_CACHE=1` (domyślnie) są usuwane po każdym cyklu, także gdy jedna z gałęzi zakończy się błędem. Workspace źródła i pliki pośrednie również są tworzone pod `/tmp/rust-package-builder/work` i standardowo usuwane po cyklu. -Ustawienia środowiskowe: `CLEAN_TARGET_AFTER_BUILD=0` wyłącza czyszczenie; -`TARGET_CACHE_MAX_AGE_DAYS=30` określa wiek nieużywanych plików. +Przy starcie builder automatycznie usuwa pozostałości po starszych wydaniach: `data/cargo/target`, `data/go/cache` oraz stare katalogi w `data/work/`. Dzięki temu aktualizacja buildera odzyskuje miejsce bez ręcznego czyszczenia. + +Retencja Linux jest kontrolowana przez `PACKAGE_RETENTION=10`: osobno zachowywane jest 10 DEB RustProxy, 10 DEB Docker Agenta, 10 RHEL RPM każdego produktu i 10 SUSE RPM każdego produktu. Po zmianie zestawu DEB lokalne repo Aptly jest synchronizowane z zachowanymi paczkami, stare snapshoty Aptly są usuwane, a `aptly db cleanup` zwalnia nieużywane obiekty. + +`RELEASE_RETENTION=10` ogranicza liczbę atomowych katalogów `repository/releases`. Kolejne snapshoty korzystają z hardlinków do niezmienionych plików poprzedniego wydania, więc te same RPM-y i dane repozytorium nie są fizycznie kopiowane przy każdym buildzie. + +Retencja OPNsense (`OPNSENSE_RETENTION=10`) i natywnego FreeBSD (`FREEBSD_NATIVE_RETENTION=10`) pozostaje niezależna. ## RustProxy 0.34.x: plik `--config` i miękki reload diff --git a/SHA256SUMS b/SHA256SUMS index 29bc7ae..d98432f 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,6 +1,8 @@ -2bfb200c5ddfa603746de1d5cb70ad43ca9c041bbc5ed361a9ec244c764bd4cf .env.example -17298e7669ed82b4c98abfce803e1a3f0a265b0abdc2567bf1316aa15e98aad3 Dockerfile -bf55ac0d7435b5152c0f5af491d12977c795924a4fb0094569c66e49f7f961d3 README.md +efd2628f1ff160f030259b32d96cf16093ff5f02ec24854643082749b5f5e578 .dockerignore +f5b89608eb69ad40a5dbf8e4f4ab00f4d62b1f3cd6ab30dabab9f3862d59e27a .env.example +c6d0d850c2c2a7491c239228982844e3c583d1fd71f8c75c0be999c306795374 .gitignore +15125941350152641991ef172d0dd63375e8cb7dae55a36abe0e05c6fe3c9c34 Dockerfile +fe94b8e7fbbda93ecffffcb7621d39a299d48726c6c63249ee2c633072d1f6d4 README.md 862263fa1f46c20f0d1e4dac5ffcc75abd55c08211b2c3864c5f8764b9d87793 dashboard/.dockerignore ced0339cb93478c2faf7b7b74215fe03375086e11ca957bd8892b52802fbbd3a dashboard/Dockerfile bc13583e4dc80d4914c7a82a06c1390b5118d4821c7d081f99d68a38e1156bac dashboard/app.py @@ -8,8 +10,9 @@ bc13583e4dc80d4914c7a82a06c1390b5118d4821c7d081f99d68a38e1156bac dashboard/app. f1ac0fb5c4da97db09ff328cb86f4bc763fbdd84d7310d6e8e713be1c8707bb9 dashboard/web/favicon.svg 05821c98841c783014496d88192639609408e7907c3ac15687e6aafd966c5687 dashboard/web/index.html 94010e2be028d8153d1ffe9142e0a5aeb6d484c1edf5cb7318d70c5e32b5c985 dashboard/web/style.css -ebb5e6eefe375ca1357d96f6a7344424c43df0bc51bc01ee66a459e54ef4b3c7 docker-compose.yml -7da761300aef6cf22ce28c1c7a55414b692814023e4d540a3f3ec4111c62926c http/nginx.conf +f0bb19d1668e8779072500ff722074a6b5cc31e04ab712a6084e27c117480f72 docker-compose.yml +88206715d29efce6b2b377d4f04e6ca9dfccf3aa4ee89668a1317358307e40b6 http/nginx.conf +d8459024f04ca514bd8e9d6bd3af872fb942fd85a7cdd5583f4a9d28aab6faba make_zip.py 77e898aebb54f8acf39d3eb1fb348e7f41532da1b0a9e1820a717fdcde406f71 opnsense/patches/0.20.0/0001-opnsense-build.patch 80bb419b980793f3a57b4fda0dc132a872186b3a07e0e6db2585c6057ab4f882 opnsense/plugin/+POST_INSTALL.pre af4bc0ab259df3a0681fea0fcc7d84e08c8b1a1a1eb34300c4a01eccb49fa537 opnsense/plugin/+PRE_DEINSTALL.pre @@ -43,23 +46,24 @@ c8dde46217c008bfc31420c19ecbd31e3db6d71ee48f99be856ad40ca3c7f490 packaging/free b5845f5510adcc3a63ccc710d8e893b33336051651917fa533e8b70df2e2991d packaging/suse-python3 eee3cb8959cb90cd1b5b320ec4674892e5d1a987049d1028dae39c3bdc9f3dc7 scripts/apply-freebsd-patches.sh d1f2ac24d6742afed8d46fe827373cd4b3febebfcccf15f7265bed0cf8ec4c9c scripts/build-freebsd-release.sh -345377ae905dd33e96ee1c7d92728fdd28bd7fab4b9ef448fbe2d105ec89771f scripts/build-linux-release.sh +4e56d41be31d29d387e4441cfb2a771382e700d90abb3980055127a77636102c scripts/build-linux-release.sh 43a51a2734b5f2629598c3d84ca656480a46c2d09bc5b0b858e92aa6c89151ed scripts/build-opnsense-release.sh 16bafbba1eb2000b42c754bae92fee43301e57e0859e2cef17be2550006ab111 scripts/build-opnsense.sh -ed581639a606323d4e91e632bc40a9be7037570f91cdef4264e4db0ca130ee8e scripts/build-release.sh +662037310a19cea7ca11d1f59141e40ba6ca1aa64710c117c6ff7bee3c8f7a2f scripts/build-release.sh f5e2be2a0128e0d768c3ed4b151b21dfd39d35a688a4c3f6fc35877c3e2ee434 scripts/check-and-build.sh cba164bde8967045bfb76de7eff46d4407dfb7ed2bc861fbc600782551d1629e scripts/ensure-rust-toolchain.sh -a26a5ade31962030a63bad36479c153116a5a6c87e518fbe4cfad1746eb5f9cd scripts/entrypoint.sh +82656b00eb17798f35df61d84f338e22e6b0603ca9b48c011f9d398f6ef50d0d scripts/entrypoint.sh c8402314cbcf87c30eb521a273a13628eccb78112acb85e0e85c9a7679550cca scripts/generate-freebsd-index.sh d3193e4bbfe3d853db003610318815fdfc295fa501affffd3e18db4c7074a5a2 scripts/generate-index.sh 80f49932346268f2cd63cbfe4814c666cd665661e82c25f2e99883929ecae0c0 scripts/generate-opnsense-index.sh 094c603f30f6a41bddeedfefa9e2c260dc97f849f0a690475ebe128ce4b49475 scripts/generate-opnsense-repo.py 5aae7f35cc09576c07f8c57ce59cb6153840be15f187e0cf538d6987780f670a scripts/init-storage.sh -13d66803d2125156ff64cb548cc5982210cd17161375eaebf5d0716a58a8ffcd scripts/lib.sh +22593936f05004a299bdc589b5e8553c8537244acf9640f1c745e367d9940537 scripts/lib.sh de4fbbb973d9cddb170ef1e9f4f6f7896018ed5e6184594c9636c0215adf5233 scripts/normalize-systemd-unit.py 0eca8ef60f15dce0c5651fc8a55de10e0d9a11f3816f04153608abcbe2e2e1d8 scripts/package-freebsd.sh b098fa629f9fcc1a54baab3d46b28721eb582c314e084b3210259bf157c124a6 scripts/package-opnsense.sh 37233bdc0e14cdf32a18095fffc5fb219795ec7e838a112eb38fa29195a5d25c scripts/prepare-agent-package.sh fc8c6ae8e35fb68f4851344ddf014759b7bac4f2d07b6b8bb383b3289e3ca23b scripts/prepare-package.sh 4bd555061e9a50faa05e2f8c095da2a6581e19b8d4c7fe01d791e59f5dd2f597 start.sh +bd765bbbf852f5d6e13aff9dbe8e743691b4c3e49168f8f32e77304191ff035c tests/test_cleanup_retention.py bc54e4b93202454c298d1e67aa68f126e25b8c2b20e62db46a89932fdacd2ba9 tests/test_suse.py diff --git a/docker-compose.yml b/docker-compose.yml index 51ba4db..ad44e1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,10 @@ services: PACKAGE_RELEASE: "${PACKAGE_RELEASE:-auto}" MAX_GLIBC_VERSION: "${MAX_GLIBC_VERSION:-2.34}" KEEP_WORKSPACE: "${KEEP_WORKSPACE:-0}" + RELEASE_RETENTION: "${RELEASE_RETENTION:-10}" + PACKAGE_RETENTION: "${PACKAGE_RETENTION:-10}" + CLEAN_BUILD_CACHE: "${CLEAN_BUILD_CACHE:-1}" + BUILD_TMP_DIR: "${BUILD_TMP_DIR:-/tmp/rust-package-builder}" OPNSENSE_ABI: "${OPNSENSE_ABI:-26.7}" OPNSENSE_RETENTION: "${OPNSENSE_RETENTION:-10}" FREEBSD_RELEASE: "${FREEBSD_RELEASE:-15.1-RELEASE}" diff --git a/make_zip.py b/make_zip.py new file mode 100644 index 0000000..892fab3 --- /dev/null +++ b/make_zip.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import os +import sys +import zipfile +import subprocess +from pathlib import Path + + +def run_git_command(args, repo_path: Path) -> bytes: + result = subprocess.run( + ["git", *args], + cwd=repo_path, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + return result.stdout + + +def get_files_to_archive(repo_path: Path) -> list[str]: + output = run_git_command( + ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], + repo_path, + ) + files = output.decode("utf-8", errors="surrogateescape").split("\0") + return [f for f in files if f] + + +def make_zip(repo_path: Path, output_zip: Path) -> None: + files = get_files_to_archive(repo_path) + + output_zip = output_zip.resolve() + if output_zip.exists(): + output_zip.unlink() + + with zipfile.ZipFile(output_zip, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for rel_path in files: + abs_path = repo_path / rel_path + + if not abs_path.exists(): + continue + + if abs_path.resolve() == output_zip: + continue + + zf.write(abs_path, arcname=rel_path) + + print(f"Created: {output_zip}") + print(f"Added files: {len(files)}") + + +def main(): + repo_path = Path.cwd() + + if len(sys.argv) > 1: + output_zip = Path(sys.argv[1]) + else: + output_zip = repo_path / f"{repo_path.name}.zip" + + try: + run_git_command(["rev-parse", "--show-toplevel"], repo_path) + except subprocess.CalledProcessError: + print("Error: this directory is not a Git repository.", file=sys.stderr) + sys.exit(1) + + make_zip(repo_path, output_zip) + + +if __name__ == "__main__": + main() diff --git a/scripts/build-linux-release.sh b/scripts/build-linux-release.sh index e1ac5c7..1964716 100755 --- a/scripts/build-linux-release.sh +++ b/scripts/build-linux-release.sh @@ -151,8 +151,22 @@ cp -f "$DEB_FILE" "$AGENT_DEB_FILE" "$DATA_DIR/packages/deb/" cp -f "$RPM_FILE" "$AGENT_RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/" cp -f "$SUSE_RPM_FILE" "$SUSE_AGENT_RPM_FILE" "$DATA_DIR/packages/suse/x86_64/" -log "updating aptly internal repository" -aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" "$AGENT_DEB_FILE" >/dev/null +# Retention is per product, not per file pool: keep ten RustProxy versions and +# ten Docker Agent versions in each Linux package family by default. +package_retention=${PACKAGE_RETENTION:-10} +prune_package_files "$DATA_DIR/packages/deb" "${PACKAGE_NAME}_*.deb" "$package_retention" +prune_package_files "$DATA_DIR/packages/deb" 'rustproxy-docker-agent_*.deb' "$package_retention" +prune_package_files "$DATA_DIR/packages/rpm/x86_64" "${PACKAGE_NAME}-*.x86_64.rpm" "$package_retention" 'rustproxy-docker-agent-*.x86_64.rpm' +prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$package_retention" +prune_package_files "$DATA_DIR/packages/suse/x86_64" "${PACKAGE_NAME}-*.x86_64.rpm" "$package_retention" 'rustproxy-docker-agent-*.x86_64.rpm' +prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$package_retention" + +log "updating aptly internal repository from retained DEB packages" +# Make the mutable local repo match the retained raw DEB pool exactly. Existing +# snapshots are immutable and remain valid until the new snapshot is published. +aptly -config="$APTLY_CONF" repo remove package-builder "$PACKAGE_NAME" >/dev/null 2>&1 || true +aptly -config="$APTLY_CONF" repo remove package-builder rustproxy-docker-agent >/dev/null 2>&1 || true +aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DATA_DIR/packages/deb" >/dev/null SNAPSHOT="linux-${VERSION}-${SHORT_SHA}-$(date +%s)" aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null @@ -174,6 +188,18 @@ APT_PUBLIC="$DATA_DIR/aptly/public" gpg --batch --verify "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 \ || fatal "APT Release signature verification failed" +# The new snapshot is now live. Older Linux snapshots are no longer needed; +# removing them lets aptly reclaim unreferenced pool objects. +while IFS= read -r old_snapshot; do + [[ -n "$old_snapshot" && "$old_snapshot" != "$SNAPSHOT" ]] || continue + if ! aptly -config="$APTLY_CONF" snapshot drop "$old_snapshot" >/dev/null 2>&1; then + log "WARN: could not drop old aptly snapshot $old_snapshot" + fi +done < <(aptly -config="$APTLY_CONF" snapshot list -raw | grep '^linux-' || true) +if ! aptly -config="$APTLY_CONF" db cleanup >/dev/null 2>&1; then + log "WARN: aptly database cleanup failed" +fi + mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64" "$RELEASE_DIR/opnsense" "$RELEASE_DIR/freebsd" CURRENT_TARGET="" if [[ -L "$REPOSITORY/current" ]]; then @@ -190,9 +216,21 @@ if [[ -n "$CURRENT_TARGET" ]]; then done fi -rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ -rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ -rsync -a --delete "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/ +if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/apt" ]]; then + rsync -a --delete --link-dest="$CURRENT_TARGET/apt/" "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ +else + rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ +fi +if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/rpm" ]]; then + rsync -a --delete --link-dest="$CURRENT_TARGET/rpm/" "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ +else + rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ +fi +if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/suse" ]]; then + rsync -a --delete --link-dest="$CURRENT_TARGET/suse/" "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/ +else + rsync -a --delete "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/ +fi for rpm_root in "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64"; do rm -rf "$rpm_root/repodata" createrepo_c --update "$rpm_root" >/dev/null diff --git a/scripts/build-release.sh b/scripts/build-release.sh index b08af1b..dea8b91 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -15,21 +15,26 @@ BUILD_FREEBSD=${5:-0} SHORT_SHA=${SHA:0:12} BUILD_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}" -WORK="$DATA_DIR/work/$BUILD_ID" +BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rust-package-builder} +export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$BUILD_TMP_DIR/cargo-target} +WORK="$BUILD_TMP_DIR/work/$BUILD_ID" SRC="$WORK/source" PKG="$WORK/package" OUT="$WORK/out" cleanup() { if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then - rm -rf "$WORK" + rm -rf -- "$WORK" else log "workspace kept: $WORK" fi + if is_true "${CLEAN_BUILD_CACHE:-1}"; then + rm -rf -- "$CARGO_TARGET_DIR" "${GOCACHE:-$BUILD_TMP_DIR/go-cache}" + fi } trap cleanup EXIT -mkdir -p "$SRC" "$PKG" "$OUT" +mkdir -p "$SRC" "$PKG" "$OUT" "$CARGO_TARGET_DIR" log "cloning source" git clone --quiet --no-tags --single-branch --branch "$SOURCE_BRANCH" "$SOURCE_REPO" "$SRC" @@ -113,7 +118,6 @@ FREEBSD_OK=1 if [[ "$BUILD_LINUX" == "1" ]]; then if /opt/rust-package-builder/scripts/build-linux-release.sh; then log "Linux branch completed for $SHA" - cleanup_cargo_target else LINUX_OK=0 log "ERROR: Linux branch failed for $SHA; OPNsense branch will still run" @@ -123,7 +127,6 @@ fi if [[ "$BUILD_OPNSENSE" == "1" ]]; then if /opt/rust-package-builder/scripts/build-opnsense-release.sh; then log "OPNsense branch completed for $SHA" - cleanup_cargo_target else OPNSENSE_OK=0 log "ERROR: OPNsense branch failed for $SHA" @@ -133,7 +136,6 @@ fi if [[ "$BUILD_FREEBSD" == "1" ]]; then if /opt/rust-package-builder/scripts/build-freebsd-release.sh; then log "FreeBSD native branch completed for $SHA" - cleanup_cargo_target else FREEBSD_OK=0 log "ERROR: FreeBSD native branch failed for $SHA" @@ -151,6 +153,11 @@ if [[ "$LINUX_STATE" == "$SHA" && "$OPNSENSE_STATE" == "$SHA" && ( "${FREEBSD_NA log "all branches complete for $SHA" fi +# Every branch publication creates an atomic repository snapshot. Keep only +# the newest configured set while always protecting the snapshot currently +# served through repository/current. +prune_release_dirs "$DATA_DIR/repository/releases" "${RELEASE_RETENTION:-10}" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)" + if [[ "$LINUX_OK" != "1" || "$OPNSENSE_OK" != "1" || "$FREEBSD_OK" != "1" ]]; then exit 1 fi diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index e8351be..dd451d7 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -18,6 +18,10 @@ APPLY_FREEBSD_PATCHES=${APPLY_FREEBSD_PATCHES:-1} RUN_LINUX_TESTS=${RUN_LINUX_TESTS:-1} FREEBSD_NATIVE_ENABLED=${FREEBSD_NATIVE_ENABLED:-1} FREEBSD_NATIVE_RETENTION=${FREEBSD_NATIVE_RETENTION:-10} +RELEASE_RETENTION=${RELEASE_RETENTION:-10} +PACKAGE_RETENTION=${PACKAGE_RETENTION:-10} +CLEAN_BUILD_CACHE=${CLEAN_BUILD_CACHE:-1} +BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rust-package-builder} [[ "$POLL_INTERVAL" =~ ^[0-9]+$ ]] || fatal "POLL_INTERVAL must be an integer" (( POLL_INTERVAL >= 1 )) || fatal "POLL_INTERVAL must be >= 1" @@ -27,6 +31,8 @@ FREEBSD_NATIVE_RETENTION=${FREEBSD_NATIVE_RETENTION:-10} [[ "$RUN_LINUX_TESTS" =~ ^[01]$ ]] || fatal "RUN_LINUX_TESTS must be 0 or 1" [[ "$FREEBSD_NATIVE_ENABLED" =~ ^[01]$ ]] || fatal "FREEBSD_NATIVE_ENABLED must be 0 or 1" [[ "$FREEBSD_NATIVE_RETENTION" =~ ^[0-9]+$ ]] || fatal "FREEBSD_NATIVE_RETENTION must be a nonnegative integer" +require_positive_integer RELEASE_RETENTION "$RELEASE_RETENTION" +require_positive_integer PACKAGE_RETENTION "$PACKAGE_RETENTION" if [[ "$FREEBSD_NATIVE_ENABLED" != "1" ]]; then log 'WARNING: native FreeBSD build is disabled (FREEBSD_NATIVE_ENABLED=0); no FreeBSD packages will be published' fi @@ -39,12 +45,15 @@ export PACKAGE_RELEASE export OPNSENSE_ABI OPNSENSE_RETENTION export FREEBSD_RELEASE FREEBSD_ABI FREEBSD_ARCH FREEBSD_RUST_TOOLCHAIN export APPLY_FREEBSD_PATCHES RUN_LINUX_TESTS FREEBSD_NATIVE_ENABLED FREEBSD_NATIVE_RETENTION +export RELEASE_RETENTION PACKAGE_RETENTION CLEAN_BUILD_CACHE BUILD_TMP_DIR export MAX_GLIBC_VERSION=${MAX_GLIBC_VERSION:-2.34} export GPG_PRIVATE_KEY_FILE=${GPG_PRIVATE_KEY_FILE:-/run/secrets/gpg/private-key.asc} export GPG_PASSPHRASE_FILE=${GPG_PASSPHRASE_FILE:-/run/secrets/gpg/passphrase} export GNUPGHOME=${GNUPGHOME:-/run/package-builder/gnupg} export CARGO_HOME="$DATA_DIR/cargo" -export CARGO_TARGET_DIR="$DATA_DIR/cargo/target" +export CARGO_TARGET_DIR="$BUILD_TMP_DIR/cargo-target" +export GOPATH="$DATA_DIR/go" +export GOCACHE="$BUILD_TMP_DIR/go-cache" for cmd in git go cargo rustc rustup nfpm aptly createrepo_c gpg rsync python3 python3.11 \ objdump ar rpm clang llvm-readelf llvm-ar llvm-ranlib file tar zstd bsdtar sha256sum; do @@ -64,9 +73,29 @@ mkdir -p \ "$DATA_DIR/packages/rpm/x86_64" \ "$DATA_DIR/packages/suse/x86_64" \ "$DATA_DIR/packages/deb" -mkdir -p "$GNUPGHOME" +mkdir -p "$BUILD_TMP_DIR/work" "$CARGO_TARGET_DIR" "$GOCACHE" "$GNUPGHOME" chmod 0700 "$GNUPGHOME" +# Migration/maintenance cleanup for older builder versions. Large build +# products must not accumulate on the persistent /data volume. +rm -rf -- "$DATA_DIR/cargo/target" "$DATA_DIR/go/cache" +find "$DATA_DIR/work" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null || true +find "$BUILD_TMP_DIR/work" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null || true +if is_true "$CLEAN_BUILD_CACHE"; then + rm -rf -- "$CARGO_TARGET_DIR" "$GOCACHE" + mkdir -p "$CARGO_TARGET_DIR" "$GOCACHE" +fi + +# Apply retention immediately on startup, before waiting for another source +# change. Main RustProxy and Docker Agent packages are counted separately. +prune_package_files "$DATA_DIR/packages/deb" 'rustproxy_*.deb' "$PACKAGE_RETENTION" +prune_package_files "$DATA_DIR/packages/deb" 'rustproxy-docker-agent_*.deb' "$PACKAGE_RETENTION" +prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-*.x86_64.rpm' "$PACKAGE_RETENTION" 'rustproxy-docker-agent-*.x86_64.rpm' +prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$PACKAGE_RETENTION" +prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-*.x86_64.rpm' "$PACKAGE_RETENTION" 'rustproxy-docker-agent-*.x86_64.rpm' +prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$PACKAGE_RETENTION" +prune_release_dirs "$DATA_DIR/repository/releases" "$RELEASE_RETENTION" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)" + [[ -r "$GPG_PRIVATE_KEY_FILE" ]] || fatal "GPG private key not found: $GPG_PRIVATE_KEY_FILE" log "importing GPG signing key" gpg --batch --import "$GPG_PRIVATE_KEY_FILE" >/dev/null 2>&1 || fatal "cannot import GPG key" diff --git a/scripts/lib.sh b/scripts/lib.sh index 9b9b205..ba94520 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -2,7 +2,7 @@ set -Eeuo pipefail # Increment when package payload/installation semantics change without a source commit. -PACKAGING_REVISION=6 +PACKAGING_REVISION=7 OPNSENSE_PACKAGING_REVISION=4 FREEBSD_NATIVE_PACKAGING_REVISION=2 @@ -44,40 +44,60 @@ atomic_write() { mv -f "$tmp" "$path" } -# Keep Cargo's release dependency cache so subsequent builds remain incremental. -# Run only after a branch has successfully published its packages. -cleanup_cargo_target() { - [[ "${CLEAN_TARGET_AFTER_BUILD:-1}" == "1" ]] || return 0 - - local target="${CARGO_TARGET_DIR:-}" - local expected="${DATA_DIR:-}/cargo/target" - # Never recursively delete an arbitrary path provided via environment. - if [[ -z "$target" || "$target" != "$expected" || ! -d "$target" || -L "$target" ]]; then - log "Cargo target cleanup skipped: unexpected target directory" - return 0 - fi - - local days="${TARGET_CACHE_MAX_AGE_DAYS:-30}" - if [[ ! "$days" =~ ^[0-9]+$ ]] || (( days < 1 )); then - log "Cargo target cleanup skipped: invalid TARGET_CACHE_MAX_AGE_DAYS=$days" - return 0 - fi - - local before after - before=$(du -sk "$target" 2>/dev/null | awk '{print $1}') - - # Release dependencies, fingerprints and build scripts remain cached. - # Rust incremental data is disposable; debug output is unused by release builds. - find "$target" -type d -name incremental -prune -exec rm -rf -- {} + 2>/dev/null || true - if [[ -d "$target/debug" && ! -L "$target/debug" ]]; then - rm -rf -- "$target/debug" || true - fi - - # Expire old compiled dependency files gradually, not the entire target. - # Fresh build outputs stay available for subsequent Cargo invocations. - find "$target/release/deps" "$target/x86_64-unknown-freebsd/release/deps" \ - -maxdepth 1 -type f -mtime "+$days" -delete 2>/dev/null || true - - after=$(du -sk "$target" 2>/dev/null | awk '{print $1}') - log "Cargo target cleanup: reclaimed $(( ${before:-0} - ${after:-0} )) KiB (cache retained)" +is_true() { + case "${1:-}" in + 1|true|TRUE|yes|YES|on|ON) return 0 ;; + *) return 1 ;; + esac +} + +require_positive_integer() { + local name=$1 value=$2 + [[ "$value" =~ ^[1-9][0-9]*$ ]] || fatal "$name must be a positive integer" +} + +# Keep the newest N artifacts matching a package family. An optional exclude +# glob is useful for repositories that contain both rustproxy-* and +# rustproxy-docker-agent-* files in the same directory. +prune_package_files() { + local directory=$1 pattern=$2 keep=$3 exclude_pattern=${4:-} + require_positive_integer PACKAGE_RETENTION "$keep" + [[ -d "$directory" ]] || return 0 + + local -a find_args=("$directory" -maxdepth 1 -type f -name "$pattern") + if [[ -n "$exclude_pattern" ]]; then + find_args+=( ! -name "$exclude_pattern" ) + fi + + mapfile -t stale < <( + find "${find_args[@]}" -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}' + ) + + local path + for path in "${stale[@]}"; do + [[ -n "$path" ]] || continue + log "package retention: removing $path" + rm -f -- "$path" + done +} + +prune_release_dirs() { + local releases_dir=$1 keep=$2 protected=${3:-} + require_positive_integer RELEASE_RETENTION "$keep" + [[ -d "$releases_dir" ]] || return 0 + + mapfile -t stale < <( + find "$releases_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}' + ) + + local path + for path in "${stale[@]}"; do + [[ -n "$path" && "$path" != "$protected" ]] || continue + log "release retention: removing $path" + rm -rf -- "$path" + done } diff --git a/tests/test_cleanup_retention.py b/tests/test_cleanup_retention.py new file mode 100644 index 0000000..9d34d70 --- /dev/null +++ b/tests/test_cleanup_retention.py @@ -0,0 +1,115 @@ +from pathlib import Path +import os +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class CleanupRetentionTests(unittest.TestCase): + def test_runtime_target_is_not_persistent_data(self): + dockerfile = (ROOT / "Dockerfile").read_text() + entrypoint = (ROOT / "scripts/entrypoint.sh").read_text() + build = (ROOT / "scripts/build-release.sh").read_text() + + runtime = dockerfile.split("# Cargo registry/git cache stays persistent", 1)[1] + self.assertIn("CARGO_HOME=/var/lib/package-builder/cargo", runtime) + self.assertIn("CARGO_TARGET_DIR=/tmp/rust-package-builder/cargo-target", runtime) + self.assertIn("GOCACHE=/tmp/rust-package-builder/go-cache", dockerfile) + self.assertNotIn("CARGO_TARGET_DIR=/var/lib/package-builder/cargo/target", runtime) + self.assertNotIn("GOCACHE=/var/lib/package-builder/go/cache", dockerfile) + self.assertIn('export CARGO_TARGET_DIR="$BUILD_TMP_DIR/cargo-target"', entrypoint) + self.assertIn('rm -rf -- "$DATA_DIR/cargo/target" "$DATA_DIR/go/cache"', entrypoint) + self.assertIn('WORK="$BUILD_TMP_DIR/work/$BUILD_ID"', build) + self.assertIn('if is_true "${CLEAN_BUILD_CACHE:-1}"; then', build) + self.assertIn('rm -rf -- "$CARGO_TARGET_DIR" "${GOCACHE:-$BUILD_TMP_DIR/go-cache}"', build) + + def test_default_retention_is_ten_and_exported_to_compose(self): + env = (ROOT / ".env.example").read_text() + compose = (ROOT / "docker-compose.yml").read_text() + entrypoint = (ROOT / "scripts/entrypoint.sh").read_text() + + self.assertIn("RELEASE_RETENTION=10", env) + self.assertIn("PACKAGE_RETENTION=10", env) + self.assertIn("CLEAN_BUILD_CACHE=1", env) + self.assertIn('RELEASE_RETENTION: "${RELEASE_RETENTION:-10}"', compose) + self.assertIn('PACKAGE_RETENTION: "${PACKAGE_RETENTION:-10}"', compose) + self.assertIn('CLEAN_BUILD_CACHE: "${CLEAN_BUILD_CACHE:-1}"', compose) + self.assertIn('RELEASE_RETENTION=${RELEASE_RETENTION:-10}', entrypoint) + self.assertIn('PACKAGE_RETENTION=${PACKAGE_RETENTION:-10}', entrypoint) + + def test_linux_retention_and_aptly_cleanup_are_enabled(self): + linux = (ROOT / "scripts/build-linux-release.sh").read_text() + self.assertIn('package_retention=${PACKAGE_RETENTION:-10}', linux) + self.assertIn("'rustproxy-docker-agent_*.deb'", linux) + self.assertIn("'rustproxy-docker-agent-*.x86_64.rpm'", linux) + self.assertIn('repo add -force-replace package-builder "$DATA_DIR/packages/deb"', linux) + self.assertIn('snapshot list -raw', linux) + self.assertIn('aptly -config="$APTLY_CONF" db cleanup', linux) + self.assertIn('--link-dest="$CURRENT_TARGET/apt/"', linux) + self.assertIn('--link-dest="$CURRENT_TARGET/rpm/"', linux) + self.assertIn('--link-dest="$CURRENT_TARGET/suse/"', linux) + + def test_package_retention_keeps_ten_versions_per_product(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + rpm = root / "rpm" + rpm.mkdir() + for idx in range(12): + main = rpm / f"rustproxy-0.2.{idx}-1.x86_64.rpm" + agent = rpm / f"rustproxy-docker-agent-0.2.{idx}-1.x86_64.rpm" + main.write_bytes(b"main") + agent.write_bytes(b"agent") + ts = 1_700_000_000 + idx + os.utime(main, (ts, ts)) + os.utime(agent, (ts, ts)) + + script = "\n".join([ + "set -Eeuo pipefail", + f"source {ROOT / 'scripts/lib.sh'}", + f"prune_package_files {rpm} 'rustproxy-*.x86_64.rpm' 10 'rustproxy-docker-agent-*.x86_64.rpm'", + f"prune_package_files {rpm} 'rustproxy-docker-agent-*.x86_64.rpm' 10", + ]) + subprocess.run(["bash", "-lc", script], check=True) + + main = [p for p in rpm.glob("rustproxy-*.x86_64.rpm") if not p.name.startswith("rustproxy-docker-agent-")] + agent = list(rpm.glob("rustproxy-docker-agent-*.x86_64.rpm")) + self.assertEqual(len(main), 10) + self.assertEqual(len(agent), 10) + self.assertFalse((rpm / "rustproxy-0.2.0-1.x86_64.rpm").exists()) + self.assertFalse((rpm / "rustproxy-docker-agent-0.2.0-1.x86_64.rpm").exists()) + self.assertTrue((rpm / "rustproxy-0.2.11-1.x86_64.rpm").exists()) + self.assertTrue((rpm / "rustproxy-docker-agent-0.2.11-1.x86_64.rpm").exists()) + + def test_release_retention_keeps_newest_ten(self): + with tempfile.TemporaryDirectory() as tmp: + releases = Path(tmp) / "releases" + releases.mkdir() + for idx in range(12): + path = releases / f"release-{idx:02d}" + path.mkdir() + ts = 1_700_000_000 + idx + os.utime(path, (ts, ts)) + + protected = releases / "release-11" + script = "\n".join([ + "set -Eeuo pipefail", + f"source {ROOT / 'scripts/lib.sh'}", + f"prune_release_dirs {releases} 10 {protected}", + ]) + subprocess.run(["bash", "-lc", script], check=True) + + remaining = sorted(p.name for p in releases.iterdir() if p.is_dir()) + self.assertEqual(len(remaining), 10) + self.assertNotIn("release-00", remaining) + self.assertNotIn("release-01", remaining) + self.assertIn("release-11", remaining) + + def test_packaging_revision_forces_one_new_linux_build(self): + lib = (ROOT / "scripts/lib.sh").read_text() + self.assertIn("PACKAGING_REVISION=7", lib) + + +if __name__ == "__main__": + unittest.main()