cleanups

2026-11-03 10:29:41 +01:00
1 parent 60ec470f9f
commit fe2ad1590d
@@ -5,10 +5,21 @@ source /opt/rustbox-package-builder/scripts/lib.sh
 : "${SRC:?SRC required}"
 REPOSITORY="$DATA_DIR/repository"
 APTLY_CONF="$DATA_DIR/state/aptly.conf"
-OUT="$DATA_DIR/work/out"
-PKG="$DATA_DIR/work/package"
-rm -rf "$OUT" "$PKG"
-mkdir -p "$OUT" "$DATA_DIR/packages/deb" "$DATA_DIR/packages/rpm/x86_64" "$DATA_DIR/packages/suse/x86_64"
+BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rustbox-package-builder}
+OUT="$BUILD_TMP_DIR/out"
+PKG="$BUILD_TMP_DIR/package"
+export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$BUILD_TMP_DIR/cargo-target}
+
+cleanup_build_artifacts() {
+  rm -rf -- "$OUT" "$PKG"
+  if is_true "${CLEAN_BUILD_CACHE:-1}"; then
+    rm -rf -- "$CARGO_TARGET_DIR"
+  fi
+}
+trap cleanup_build_artifacts EXIT
+
+rm -rf -- "$OUT" "$PKG"
+mkdir -p "$OUT" "$DATA_DIR/packages/deb" "$DATA_DIR/packages/rpm/x86_64" "$DATA_DIR/packages/suse/x86_64" "$CARGO_TARGET_DIR"

 VERSION=$(python3.11 - "$SRC/Cargo.toml" <<'PY'
 import sys, tomllib
@@ -78,7 +89,16 @@ cp -f "$DEB" "$DATA_DIR/packages/deb/"
 cp -f "$RPM" "$DATA_DIR/packages/rpm/x86_64/"
 cp -f "$SUSE" "$DATA_DIR/packages/suse/x86_64/"

-aptly -config="$APTLY_CONF" repo add -force-replace rustbox "$DEB" >/dev/null
+package_retention=${PACKAGE_RETENTION:-10}
+prune_package_files "$DATA_DIR/packages/deb" 'rustbox_*.deb' "$package_retention"
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustbox-*.x86_64.rpm' "$package_retention"
+prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustbox-*.suse.x86_64.rpm' "$package_retention"
+
+# Keep the mutable aptly repository in exact sync with the retained DEB files.
+# Published snapshots are immutable, so rebuilding this repo does not disturb the
+# currently served repository until the new snapshot is switched in below.
+aptly -config="$APTLY_CONF" repo remove rustbox rustbox >/dev/null 2>&1 || true
+aptly -config="$APTLY_CONF" repo add -force-replace rustbox "$DATA_DIR/packages/deb" >/dev/null
 SNAPSHOT="rustbox-${VERSION}-${SHORT_SHA}-$(date +%s)"
 aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo rustbox >/dev/null
 APT_SIGN=(-batch -gpg-key="$GPG_KEY_ID")
@@ -90,12 +110,28 @@ fi
 APT_PUBLIC="$DATA_DIR/aptly/public"
 [[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "APT InRelease missing"

+# Once the new snapshot is live, old aptly snapshots no longer serve traffic.
+# Drop them and let aptly remove unreferenced pool objects.
+while IFS= read -r old_snapshot; do
+  [[ -n "$old_snapshot" && "$old_snapshot" != "$SNAPSHOT" ]] || continue
+  if ! aptly -config="$APTLY_CONF" snapshot drop "$old_snapshot" >/dev/null 2>&1; then
+    log "WARN: could not drop old aptly snapshot $old_snapshot"
+  fi
+done < <(aptly -config="$APTLY_CONF" snapshot list -raw | grep '^rustbox-' || true)
+if ! aptly -config="$APTLY_CONF" db cleanup >/dev/null 2>&1; then
+  log "WARN: aptly database cleanup failed"
+fi
+
 RELEASE_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}"
 RELEASE_DIR="$REPOSITORY/releases/$RELEASE_ID"
 mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64"
-rsync -a --delete "$APT_PUBLIC/" "$RELEASE_DIR/apt/"
-rsync -a --delete "$DATA_DIR/packages/rpm/x86_64/" "$RELEASE_DIR/rpm/x86_64/"
-rsync -a --delete "$DATA_DIR/packages/suse/x86_64/" "$RELEASE_DIR/suse/x86_64/"
+
+# All paths are on the same persistent data volume. Hard-linking package and
+# repository files avoids multiplying identical package payloads for every
+# atomic release directory while preserving independent directory trees.
+cp -al "$APT_PUBLIC/." "$RELEASE_DIR/apt/"
+cp -al "$DATA_DIR/packages/rpm/x86_64/." "$RELEASE_DIR/rpm/x86_64/"
+cp -al "$DATA_DIR/packages/suse/x86_64/." "$RELEASE_DIR/suse/x86_64/"

 for rpm_root in "$RELEASE_DIR/rpm/x86_64"; do
  rm -rf "$rpm_root/repodata"; createrepo_c --update "$rpm_root" >/dev/null
@@ -135,9 +171,6 @@ printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit"
 printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version"
 printf '%s\n' "$RELEASE" | atomic_write "$DATA_DIR/state/last_package_release"

-retention=${RELEASE_RETENTION:-10}
-if [[ "$retention" =~ ^[1-9][0-9]*$ ]]; then
-  mapfile -t old < <(find "$REPOSITORY/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk -v keep="$retention" 'NR>keep {$1=""; sub(/^ /,""); print}')
-  for dir in "${old[@]}"; do [[ "$dir" == "$RELEASE_DIR" ]] || rm -rf -- "$dir"; done
-fi
+release_retention=${RELEASE_RETENTION:-10}
+prune_release_dirs "$REPOSITORY/releases" "$release_retention" "$RELEASE_DIR"
 log "published RustBox $VERSION-$RELEASE: $REPOSITORY_URL"
@@ -11,9 +11,31 @@ source /opt/rustbox-package-builder/scripts/lib.sh
 : "${RUN_LINUX_TESTS:=compile}"
 : "${MAX_GLIBC_VERSION:=2.34}"
 : "${RELEASE_RETENTION:=10}"
-export SOURCE_BRANCH POLL_INTERVAL REPO_DISTRIBUTION REPO_COMPONENT PACKAGE_RELEASE RUN_LINUX_TESTS MAX_GLIBC_VERSION RELEASE_RETENTION
-mkdir -p "$DATA_DIR"/{state,repository,work,cargo,packages} /tmp/rustbox-gnupg
+: "${PACKAGE_RETENTION:=10}"
+: "${CLEAN_BUILD_CACHE:=1}"
+: "${BUILD_TMP_DIR:=/tmp/rustbox-package-builder}"
+require_positive_integer RELEASE_RETENTION "$RELEASE_RETENTION"
+require_positive_integer PACKAGE_RETENTION "$PACKAGE_RETENTION"
+export SOURCE_BRANCH POLL_INTERVAL REPO_DISTRIBUTION REPO_COMPONENT PACKAGE_RELEASE RUN_LINUX_TESTS MAX_GLIBC_VERSION RELEASE_RETENTION PACKAGE_RETENTION CLEAN_BUILD_CACHE BUILD_TMP_DIR
+mkdir -p "$DATA_DIR"/{state,repository,work,packages} "$BUILD_TMP_DIR" /tmp/rustbox-gnupg
 chmod 0700 /tmp/rustbox-gnupg
+
+# Migration/maintenance cleanup for older builder versions which stored large
+# cargo targets and package work directories on the persistent data volume.
+rm -rf -- "$DATA_DIR/work/out" "$DATA_DIR/work/package" "$DATA_DIR/cargo/target"
+rmdir "$DATA_DIR/cargo" 2>/dev/null || true
+rm -rf -- "$BUILD_TMP_DIR/out" "$BUILD_TMP_DIR/package"
+if is_true "$CLEAN_BUILD_CACHE"; then
+  rm -rf -- "$BUILD_TMP_DIR/cargo-target"
+fi
+
+# Apply file-level retention at startup as well, so stale raw package pools and
+# old atomic release directories are reduced even before the next source build.
+prune_package_files "$DATA_DIR/packages/deb" 'rustbox_*.deb' "$PACKAGE_RETENTION"
+prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustbox-*.x86_64.rpm' "$PACKAGE_RETENTION"
+prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustbox-*.suse.x86_64.rpm' "$PACKAGE_RETENTION"
+prune_release_dirs "$DATA_DIR/repository/releases" "$RELEASE_RETENTION" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)"
+
 export GNUPGHOME=/tmp/rustbox-gnupg
 GPG_PRIVATE_KEY_FILE=/run/secrets/gpg/private-key.asc
 [[ -s "$GPG_PRIVATE_KEY_FILE" ]] || fatal "missing GPG private key: $GPG_PRIVATE_KEY_FILE"
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail

-PACKAGING_REVISION=2
+PACKAGING_REVISION=3
 DATA_DIR=${DATA_DIR:-/var/lib/rustbox-package-builder}

 log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
@@ -13,3 +13,53 @@ print(json.dumps(sys.argv[1]))
 PY
 }
 atomic_write() { local path=$1 tmp="${path}.tmp.$$"; cat > "$tmp"; mv -f "$tmp" "$path"; }
+
+is_true() {
+  case "${1:-}" in
+    1|true|TRUE|yes|YES|on|ON) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+require_positive_integer() {
+  local name=$1 value=$2
+  [[ "$value" =~ ^[1-9][0-9]*$ ]] || fatal "$name must be a positive integer"
+}
+
+prune_package_files() {
+  local directory=$1 pattern=$2 keep=$3
+  require_positive_integer PACKAGE_RETENTION "$keep"
+  [[ -d "$directory" ]] || return 0
+
+  mapfile -t stale < <(
+    find "$directory" -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' \
+      | sort -nr \
+      | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}'
+  )
+
+  local path
+  for path in "${stale[@]}"; do
+    [[ -n "$path" ]] || continue
+    log "package retention: removing $path"
+    rm -f -- "$path"
+  done
+}
+
+prune_release_dirs() {
+  local releases_dir=$1 keep=$2 protected=${3:-}
+  require_positive_integer RELEASE_RETENTION "$keep"
+  [[ -d "$releases_dir" ]] || return 0
+
+  mapfile -t stale < <(
+    find "$releases_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \
+      | sort -nr \
+      | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}'
+  )
+
+  local path
+  for path in "${stale[@]}"; do
+    [[ -n "$path" && "$path" != "$protected" ]] || continue
+    log "release retention: removing $path"
+    rm -rf -- "$path"
+  done
+}