cleanups

scripts/build-linux-release.sh

@@ -151,8 +151,22 @@ cp -f "$DEB_FILE" "$AGENT_DEB_FILE" "$DATA_DIR/packages/deb/"
 cp -f "$RPM_FILE" "$AGENT_RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/"
 cp -f "$SUSE_RPM_FILE" "$SUSE_AGENT_RPM_FILE" "$DATA_DIR/packages/suse/x86_64/"
 
-log "updating aptly internal repository"
-aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" "$AGENT_DEB_FILE" >/dev/null
+# Retention is per product, not per file pool: keep ten RustProxy versions and
+# ten Docker Agent versions in each Linux package family by default.
+package_retention=${PACKAGE_RETENTION:-10}
+prune_package_files "$DATA_DIR/packages/deb" "${PACKAGE_NAME}_*.deb" "$package_retention"
+prune_package_files "$DATA_DIR/packages/deb" 'rustproxy-docker-agent_*.deb' "$package_retention"
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" "${PACKAGE_NAME}-*.x86_64.rpm" "$package_retention" 'rustproxy-docker-agent-*.x86_64.rpm'
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$package_retention"
+prune_package_files "$DATA_DIR/packages/suse/x86_64" "${PACKAGE_NAME}-*.x86_64.rpm" "$package_retention" 'rustproxy-docker-agent-*.x86_64.rpm'
+prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$package_retention"
+
+log "updating aptly internal repository from retained DEB packages"
+# Make the mutable local repo match the retained raw DEB pool exactly.  Existing
+# snapshots are immutable and remain valid until the new snapshot is published.
+aptly -config="$APTLY_CONF" repo remove package-builder "$PACKAGE_NAME" >/dev/null 2>&1 || true
+aptly -config="$APTLY_CONF" repo remove package-builder rustproxy-docker-agent >/dev/null 2>&1 || true
+aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DATA_DIR/packages/deb" >/dev/null
 SNAPSHOT="linux-${VERSION}-${SHORT_SHA}-$(date +%s)"
 aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null
 
@@ -174,6 +188,18 @@ APT_PUBLIC="$DATA_DIR/aptly/public"
 gpg --batch --verify "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 \
     || fatal "APT Release signature verification failed"
 
+# The new snapshot is now live.  Older Linux snapshots are no longer needed;
+# removing them lets aptly reclaim unreferenced pool objects.
+while IFS= read -r old_snapshot; do
+    [[ -n "$old_snapshot" && "$old_snapshot" != "$SNAPSHOT" ]] || continue
+    if ! aptly -config="$APTLY_CONF" snapshot drop "$old_snapshot" >/dev/null 2>&1; then
+        log "WARN: could not drop old aptly snapshot $old_snapshot"
+    fi
+done < <(aptly -config="$APTLY_CONF" snapshot list -raw | grep '^linux-' || true)
+if ! aptly -config="$APTLY_CONF" db cleanup >/dev/null 2>&1; then
+    log "WARN: aptly database cleanup failed"
+fi
+
 mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64" "$RELEASE_DIR/opnsense" "$RELEASE_DIR/freebsd"
 CURRENT_TARGET=""
 if [[ -L "$REPOSITORY/current" ]]; then
@@ -190,9 +216,21 @@ if [[ -n "$CURRENT_TARGET" ]]; then
     done
 fi
 
-rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
-rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
-rsync -a --delete "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/
+if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/apt" ]]; then
+    rsync -a --delete --link-dest="$CURRENT_TARGET/apt/" "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
+else
+    rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
+fi
+if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/rpm" ]]; then
+    rsync -a --delete --link-dest="$CURRENT_TARGET/rpm/" "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
+else
+    rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
+fi
+if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/suse" ]]; then
+    rsync -a --delete --link-dest="$CURRENT_TARGET/suse/" "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/
+else
+    rsync -a --delete "$DATA_DIR/packages/suse"/ "$RELEASE_DIR/suse"/
+fi
 for rpm_root in "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64"; do
     rm -rf "$rpm_root/repodata"
     createrepo_c --update "$rpm_root" >/dev/null

scripts/build-release.sh

@@ -15,21 +15,26 @@ BUILD_FREEBSD=${5:-0}
 
 SHORT_SHA=${SHA:0:12}
 BUILD_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}"
-WORK="$DATA_DIR/work/$BUILD_ID"
+BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rust-package-builder}
+export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$BUILD_TMP_DIR/cargo-target}
+WORK="$BUILD_TMP_DIR/work/$BUILD_ID"
 SRC="$WORK/source"
 PKG="$WORK/package"
 OUT="$WORK/out"
 
 cleanup() {
     if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then
-        rm -rf "$WORK"
+        rm -rf -- "$WORK"
     else
         log "workspace kept: $WORK"
     fi
+    if is_true "${CLEAN_BUILD_CACHE:-1}"; then
+        rm -rf -- "$CARGO_TARGET_DIR" "${GOCACHE:-$BUILD_TMP_DIR/go-cache}"
+    fi
 }
 trap cleanup EXIT
 
-mkdir -p "$SRC" "$PKG" "$OUT"
+mkdir -p "$SRC" "$PKG" "$OUT" "$CARGO_TARGET_DIR"
 
 log "cloning source"
 git clone --quiet --no-tags --single-branch --branch "$SOURCE_BRANCH" "$SOURCE_REPO" "$SRC"
@@ -113,7 +118,6 @@ FREEBSD_OK=1
 if [[ "$BUILD_LINUX" == "1" ]]; then
     if /opt/rust-package-builder/scripts/build-linux-release.sh; then
         log "Linux branch completed for $SHA"
-        cleanup_cargo_target
     else
         LINUX_OK=0
         log "ERROR: Linux branch failed for $SHA; OPNsense branch will still run"
@@ -123,7 +127,6 @@ fi
 if [[ "$BUILD_OPNSENSE" == "1" ]]; then
     if /opt/rust-package-builder/scripts/build-opnsense-release.sh; then
         log "OPNsense branch completed for $SHA"
-        cleanup_cargo_target
     else
         OPNSENSE_OK=0
         log "ERROR: OPNsense branch failed for $SHA"
@@ -133,7 +136,6 @@ fi
 if [[ "$BUILD_FREEBSD" == "1" ]]; then
     if /opt/rust-package-builder/scripts/build-freebsd-release.sh; then
         log "FreeBSD native branch completed for $SHA"
-        cleanup_cargo_target
     else
         FREEBSD_OK=0
         log "ERROR: FreeBSD native branch failed for $SHA"
@@ -151,6 +153,11 @@ if [[ "$LINUX_STATE" == "$SHA" && "$OPNSENSE_STATE" == "$SHA" && ( "${FREEBSD_NA
     log "all branches complete for $SHA"
 fi
 
+# Every branch publication creates an atomic repository snapshot.  Keep only
+# the newest configured set while always protecting the snapshot currently
+# served through repository/current.
+prune_release_dirs "$DATA_DIR/repository/releases" "${RELEASE_RETENTION:-10}" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)"
+
 if [[ "$LINUX_OK" != "1" || "$OPNSENSE_OK" != "1" || "$FREEBSD_OK" != "1" ]]; then
     exit 1
 fi

scripts/entrypoint.sh

@@ -18,6 +18,10 @@ APPLY_FREEBSD_PATCHES=${APPLY_FREEBSD_PATCHES:-1}
 RUN_LINUX_TESTS=${RUN_LINUX_TESTS:-1}
 FREEBSD_NATIVE_ENABLED=${FREEBSD_NATIVE_ENABLED:-1}
 FREEBSD_NATIVE_RETENTION=${FREEBSD_NATIVE_RETENTION:-10}
+RELEASE_RETENTION=${RELEASE_RETENTION:-10}
+PACKAGE_RETENTION=${PACKAGE_RETENTION:-10}
+CLEAN_BUILD_CACHE=${CLEAN_BUILD_CACHE:-1}
+BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rust-package-builder}
 
 [[ "$POLL_INTERVAL" =~ ^[0-9]+$ ]] || fatal "POLL_INTERVAL must be an integer"
 (( POLL_INTERVAL >= 1 )) || fatal "POLL_INTERVAL must be >= 1"
@@ -27,6 +31,8 @@ FREEBSD_NATIVE_RETENTION=${FREEBSD_NATIVE_RETENTION:-10}
 [[ "$RUN_LINUX_TESTS" =~ ^[01]$ ]] || fatal "RUN_LINUX_TESTS must be 0 or 1"
 [[ "$FREEBSD_NATIVE_ENABLED" =~ ^[01]$ ]] || fatal "FREEBSD_NATIVE_ENABLED must be 0 or 1"
 [[ "$FREEBSD_NATIVE_RETENTION" =~ ^[0-9]+$ ]] || fatal "FREEBSD_NATIVE_RETENTION must be a nonnegative integer"
+require_positive_integer RELEASE_RETENTION "$RELEASE_RETENTION"
+require_positive_integer PACKAGE_RETENTION "$PACKAGE_RETENTION"
 if [[ "$FREEBSD_NATIVE_ENABLED" != "1" ]]; then
     log 'WARNING: native FreeBSD build is disabled (FREEBSD_NATIVE_ENABLED=0); no FreeBSD packages will be published'
 fi
@@ -39,12 +45,15 @@ export PACKAGE_RELEASE
 export OPNSENSE_ABI OPNSENSE_RETENTION
 export FREEBSD_RELEASE FREEBSD_ABI FREEBSD_ARCH FREEBSD_RUST_TOOLCHAIN
 export APPLY_FREEBSD_PATCHES RUN_LINUX_TESTS FREEBSD_NATIVE_ENABLED FREEBSD_NATIVE_RETENTION
+export RELEASE_RETENTION PACKAGE_RETENTION CLEAN_BUILD_CACHE BUILD_TMP_DIR
 export MAX_GLIBC_VERSION=${MAX_GLIBC_VERSION:-2.34}
 export GPG_PRIVATE_KEY_FILE=${GPG_PRIVATE_KEY_FILE:-/run/secrets/gpg/private-key.asc}
 export GPG_PASSPHRASE_FILE=${GPG_PASSPHRASE_FILE:-/run/secrets/gpg/passphrase}
 export GNUPGHOME=${GNUPGHOME:-/run/package-builder/gnupg}
 export CARGO_HOME="$DATA_DIR/cargo"
-export CARGO_TARGET_DIR="$DATA_DIR/cargo/target"
+export CARGO_TARGET_DIR="$BUILD_TMP_DIR/cargo-target"
+export GOPATH="$DATA_DIR/go"
+export GOCACHE="$BUILD_TMP_DIR/go-cache"
 
 for cmd in git go cargo rustc rustup nfpm aptly createrepo_c gpg rsync python3 python3.11 \
     objdump ar rpm clang llvm-readelf llvm-ar llvm-ranlib file tar zstd bsdtar sha256sum; do
@@ -64,9 +73,29 @@ mkdir -p \
     "$DATA_DIR/packages/rpm/x86_64" \
     "$DATA_DIR/packages/suse/x86_64" \
     "$DATA_DIR/packages/deb"
-mkdir -p "$GNUPGHOME"
+mkdir -p "$BUILD_TMP_DIR/work" "$CARGO_TARGET_DIR" "$GOCACHE" "$GNUPGHOME"
 chmod 0700 "$GNUPGHOME"
 
+# Migration/maintenance cleanup for older builder versions.  Large build
+# products must not accumulate on the persistent /data volume.
+rm -rf -- "$DATA_DIR/cargo/target" "$DATA_DIR/go/cache"
+find "$DATA_DIR/work" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null || true
+find "$BUILD_TMP_DIR/work" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + 2>/dev/null || true
+if is_true "$CLEAN_BUILD_CACHE"; then
+    rm -rf -- "$CARGO_TARGET_DIR" "$GOCACHE"
+    mkdir -p "$CARGO_TARGET_DIR" "$GOCACHE"
+fi
+
+# Apply retention immediately on startup, before waiting for another source
+# change.  Main RustProxy and Docker Agent packages are counted separately.
+prune_package_files "$DATA_DIR/packages/deb" 'rustproxy_*.deb' "$PACKAGE_RETENTION"
+prune_package_files "$DATA_DIR/packages/deb" 'rustproxy-docker-agent_*.deb' "$PACKAGE_RETENTION"
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-*.x86_64.rpm' "$PACKAGE_RETENTION" 'rustproxy-docker-agent-*.x86_64.rpm'
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$PACKAGE_RETENTION"
+prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-*.x86_64.rpm' "$PACKAGE_RETENTION" 'rustproxy-docker-agent-*.x86_64.rpm'
+prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustproxy-docker-agent-*.x86_64.rpm' "$PACKAGE_RETENTION"
+prune_release_dirs "$DATA_DIR/repository/releases" "$RELEASE_RETENTION" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)"
+
 [[ -r "$GPG_PRIVATE_KEY_FILE" ]] || fatal "GPG private key not found: $GPG_PRIVATE_KEY_FILE"
 log "importing GPG signing key"
 gpg --batch --import "$GPG_PRIVATE_KEY_FILE" >/dev/null 2>&1 || fatal "cannot import GPG key"

scripts/lib.sh

@@ -2,7 +2,7 @@
 set -Eeuo pipefail
 
 # Increment when package payload/installation semantics change without a source commit.
-PACKAGING_REVISION=6
+PACKAGING_REVISION=7
 OPNSENSE_PACKAGING_REVISION=4
 FREEBSD_NATIVE_PACKAGING_REVISION=2
 
@@ -44,40 +44,60 @@ atomic_write() {
     mv -f "$tmp" "$path"
 }
 
-# Keep Cargo's release dependency cache so subsequent builds remain incremental.
-# Run only after a branch has successfully published its packages.
-cleanup_cargo_target() {
-    [[ "${CLEAN_TARGET_AFTER_BUILD:-1}" == "1" ]] || return 0
-
-    local target="${CARGO_TARGET_DIR:-}"
-    local expected="${DATA_DIR:-}/cargo/target"
-    # Never recursively delete an arbitrary path provided via environment.
-    if [[ -z "$target" || "$target" != "$expected" || ! -d "$target" || -L "$target" ]]; then
-        log "Cargo target cleanup skipped: unexpected target directory"
-        return 0
-    fi
-
-    local days="${TARGET_CACHE_MAX_AGE_DAYS:-30}"
-    if [[ ! "$days" =~ ^[0-9]+$ ]] || (( days < 1 )); then
-        log "Cargo target cleanup skipped: invalid TARGET_CACHE_MAX_AGE_DAYS=$days"
-        return 0
-    fi
-
-    local before after
-    before=$(du -sk "$target" 2>/dev/null | awk '{print $1}')
-
-    # Release dependencies, fingerprints and build scripts remain cached.
-    # Rust incremental data is disposable; debug output is unused by release builds.
-    find "$target" -type d -name incremental -prune -exec rm -rf -- {} + 2>/dev/null || true
-    if [[ -d "$target/debug" && ! -L "$target/debug" ]]; then
-        rm -rf -- "$target/debug" || true
-    fi
-
-    # Expire old compiled dependency files gradually, not the entire target.
-    # Fresh build outputs stay available for subsequent Cargo invocations.
-    find "$target/release/deps" "$target/x86_64-unknown-freebsd/release/deps" \
-        -maxdepth 1 -type f -mtime "+$days" -delete 2>/dev/null || true
-
-    after=$(du -sk "$target" 2>/dev/null | awk '{print $1}')
-    log "Cargo target cleanup: reclaimed $(( ${before:-0} - ${after:-0} )) KiB (cache retained)"
+is_true() {
+    case "${1:-}" in
+        1|true|TRUE|yes|YES|on|ON) return 0 ;;
+        *) return 1 ;;
+    esac
+}
+
+require_positive_integer() {
+    local name=$1 value=$2
+    [[ "$value" =~ ^[1-9][0-9]*$ ]] || fatal "$name must be a positive integer"
+}
+
+# Keep the newest N artifacts matching a package family.  An optional exclude
+# glob is useful for repositories that contain both rustproxy-* and
+# rustproxy-docker-agent-* files in the same directory.
+prune_package_files() {
+    local directory=$1 pattern=$2 keep=$3 exclude_pattern=${4:-}
+    require_positive_integer PACKAGE_RETENTION "$keep"
+    [[ -d "$directory" ]] || return 0
+
+    local -a find_args=("$directory" -maxdepth 1 -type f -name "$pattern")
+    if [[ -n "$exclude_pattern" ]]; then
+        find_args+=( ! -name "$exclude_pattern" )
+    fi
+
+    mapfile -t stale < <(
+        find "${find_args[@]}" -printf '%T@ %p\n' \
+            | sort -nr \
+            | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}'
+    )
+
+    local path
+    for path in "${stale[@]}"; do
+        [[ -n "$path" ]] || continue
+        log "package retention: removing $path"
+        rm -f -- "$path"
+    done
+}
+
+prune_release_dirs() {
+    local releases_dir=$1 keep=$2 protected=${3:-}
+    require_positive_integer RELEASE_RETENTION "$keep"
+    [[ -d "$releases_dir" ]] || return 0
+
+    mapfile -t stale < <(
+        find "$releases_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \
+            | sort -nr \
+            | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}'
+    )
+
+    local path
+    for path in "${stale[@]}"; do
+        [[ -n "$path" && "$path" != "$protected" ]] || continue
+        log "release retention: removing $path"
+        rm -rf -- "$path"
+    done
 }