diff --git a/Dockerfile b/Dockerfile index a807981..0b941cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ RUN set -eux; \ cargo --version ENV DATA_DIR=/var/lib/rustbox-package-builder \ - CARGO_TARGET_DIR=/var/lib/rustbox-package-builder/cargo/target + CARGO_TARGET_DIR=/tmp/rustbox-package-builder/cargo-target COPY scripts/ /opt/rustbox-package-builder/scripts/ COPY packaging/ /opt/rustbox-package-builder/packaging/ diff --git a/README.md b/README.md index a7af650..7eb49eb 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,18 @@ PACKAGE_RELEASE=auto RUN_LINUX_TESTS=compile MAX_GLIBC_VERSION=2.34 RELEASE_RETENTION=10 +PACKAGE_RETENTION=10 +CLEAN_BUILD_CACHE=1 ``` `RUN_LINUX_TESTS=compile` runs `cargo test --release --locked --no-run`. Use `full` to run the test suite or `0` to skip Rust tests. +`PACKAGE_RETENTION=10` keeps at most the 10 newest DEB packages, 10 newest RHEL-family RPM packages, and 10 newest SUSE RPM packages in the published package pools. `RELEASE_RETENTION=10` separately controls atomic repository release directories. + +`CLEAN_BUILD_CACHE=1` removes the Cargo target directory after every build (successful or failed). Build output and package staging directories always live under `/tmp/rustbox-package-builder` and are removed after each build, so repeated compilations do not accumulate large compiler artifacts in `data/`. Set `CLEAN_BUILD_CACHE=0` only if you deliberately want to trade disk space for faster subsequent builds. + +On startup the builder also removes legacy `data/cargo/target`, `data/work/out`, and `data/work/package` directories created by older releases. + All three supported Linux package families are built on every successful release. SUSE is not treated as a separate non-Linux target. ## Installed RustBox layout diff --git a/docker-compose.yml b/docker-compose.yml index ff416f3..55b6caa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: RUN_LINUX_TESTS: "${RUN_LINUX_TESTS:-compile}" MAX_GLIBC_VERSION: "${MAX_GLIBC_VERSION:-2.34}" RELEASE_RETENTION: "${RELEASE_RETENTION:-10}" + PACKAGE_RETENTION: "${PACKAGE_RETENTION:-10}" + CLEAN_BUILD_CACHE: "${CLEAN_BUILD_CACHE:-1}" volumes: - ./data:/var/lib/rustbox-package-builder - ./gpg:/run/secrets/gpg:ro diff --git a/packaging/rustbox.toml b/packaging/rustbox.toml index 1f374e5..69e3480 100644 --- a/packaging/rustbox.toml +++ b/packaging/rustbox.toml @@ -39,6 +39,12 @@ default_interface_skin = "apple-mail" sieve_enabled = true diagnostics_enabled = true about_enabled = true + +# Allow multiple IMAP accounts in one browser session. +# When disabled, RustBox hides multi-account UI, rejects adding another account, +# and realtime notifications/watchers are limited to the active account. +multi_account_enabled = true + # Security headers added by RustBox. Off by default for compatibility. # # When enabled: diff --git a/scripts/build-release.sh b/scripts/build-release.sh index e586241..0823d60 100755 --- a/scripts/build-release.sh +++ b/scripts/build-release.sh @@ -5,10 +5,21 @@ source /opt/rustbox-package-builder/scripts/lib.sh : "${SRC:?SRC required}" REPOSITORY="$DATA_DIR/repository" APTLY_CONF="$DATA_DIR/state/aptly.conf" -OUT="$DATA_DIR/work/out" -PKG="$DATA_DIR/work/package" -rm -rf "$OUT" "$PKG" -mkdir -p "$OUT" "$DATA_DIR/packages/deb" "$DATA_DIR/packages/rpm/x86_64" "$DATA_DIR/packages/suse/x86_64" +BUILD_TMP_DIR=${BUILD_TMP_DIR:-/tmp/rustbox-package-builder} +OUT="$BUILD_TMP_DIR/out" +PKG="$BUILD_TMP_DIR/package" +export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-$BUILD_TMP_DIR/cargo-target} + +cleanup_build_artifacts() { + rm -rf -- "$OUT" "$PKG" + if is_true "${CLEAN_BUILD_CACHE:-1}"; then + rm -rf -- "$CARGO_TARGET_DIR" + fi +} +trap cleanup_build_artifacts EXIT + +rm -rf -- "$OUT" "$PKG" +mkdir -p "$OUT" "$DATA_DIR/packages/deb" "$DATA_DIR/packages/rpm/x86_64" "$DATA_DIR/packages/suse/x86_64" "$CARGO_TARGET_DIR" VERSION=$(python3.11 - "$SRC/Cargo.toml" <<'PY' import sys, tomllib @@ -78,7 +89,16 @@ cp -f "$DEB" "$DATA_DIR/packages/deb/" cp -f "$RPM" "$DATA_DIR/packages/rpm/x86_64/" cp -f "$SUSE" "$DATA_DIR/packages/suse/x86_64/" -aptly -config="$APTLY_CONF" repo add -force-replace rustbox "$DEB" >/dev/null +package_retention=${PACKAGE_RETENTION:-10} +prune_package_files "$DATA_DIR/packages/deb" 'rustbox_*.deb' "$package_retention" +prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustbox-*.x86_64.rpm' "$package_retention" +prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustbox-*.suse.x86_64.rpm' "$package_retention" + +# Keep the mutable aptly repository in exact sync with the retained DEB files. +# Published snapshots are immutable, so rebuilding this repo does not disturb the +# currently served repository until the new snapshot is switched in below. +aptly -config="$APTLY_CONF" repo remove rustbox rustbox >/dev/null 2>&1 || true +aptly -config="$APTLY_CONF" repo add -force-replace rustbox "$DATA_DIR/packages/deb" >/dev/null SNAPSHOT="rustbox-${VERSION}-${SHORT_SHA}-$(date +%s)" aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo rustbox >/dev/null APT_SIGN=(-batch -gpg-key="$GPG_KEY_ID") @@ -90,12 +110,28 @@ fi APT_PUBLIC="$DATA_DIR/aptly/public" [[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "APT InRelease missing" +# Once the new snapshot is live, old aptly snapshots no longer serve traffic. +# Drop them and let aptly remove unreferenced pool objects. +while IFS= read -r old_snapshot; do + [[ -n "$old_snapshot" && "$old_snapshot" != "$SNAPSHOT" ]] || continue + if ! aptly -config="$APTLY_CONF" snapshot drop "$old_snapshot" >/dev/null 2>&1; then + log "WARN: could not drop old aptly snapshot $old_snapshot" + fi +done < <(aptly -config="$APTLY_CONF" snapshot list -raw | grep '^rustbox-' || true) +if ! aptly -config="$APTLY_CONF" db cleanup >/dev/null 2>&1; then + log "WARN: aptly database cleanup failed" +fi + RELEASE_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}" RELEASE_DIR="$REPOSITORY/releases/$RELEASE_ID" mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/suse/x86_64" -rsync -a --delete "$APT_PUBLIC/" "$RELEASE_DIR/apt/" -rsync -a --delete "$DATA_DIR/packages/rpm/x86_64/" "$RELEASE_DIR/rpm/x86_64/" -rsync -a --delete "$DATA_DIR/packages/suse/x86_64/" "$RELEASE_DIR/suse/x86_64/" + +# All paths are on the same persistent data volume. Hard-linking package and +# repository files avoids multiplying identical package payloads for every +# atomic release directory while preserving independent directory trees. +cp -al "$APT_PUBLIC/." "$RELEASE_DIR/apt/" +cp -al "$DATA_DIR/packages/rpm/x86_64/." "$RELEASE_DIR/rpm/x86_64/" +cp -al "$DATA_DIR/packages/suse/x86_64/." "$RELEASE_DIR/suse/x86_64/" for rpm_root in "$RELEASE_DIR/rpm/x86_64"; do rm -rf "$rpm_root/repodata"; createrepo_c --update "$rpm_root" >/dev/null @@ -135,9 +171,6 @@ printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit" printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version" printf '%s\n' "$RELEASE" | atomic_write "$DATA_DIR/state/last_package_release" -retention=${RELEASE_RETENTION:-10} -if [[ "$retention" =~ ^[1-9][0-9]*$ ]]; then - mapfile -t old < <(find "$REPOSITORY/releases" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk -v keep="$retention" 'NR>keep {$1=""; sub(/^ /,""); print}') - for dir in "${old[@]}"; do [[ "$dir" == "$RELEASE_DIR" ]] || rm -rf -- "$dir"; done -fi +release_retention=${RELEASE_RETENTION:-10} +prune_release_dirs "$REPOSITORY/releases" "$release_retention" "$RELEASE_DIR" log "published RustBox $VERSION-$RELEASE: $REPOSITORY_URL" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index e46cd58..374c999 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -11,9 +11,31 @@ source /opt/rustbox-package-builder/scripts/lib.sh : "${RUN_LINUX_TESTS:=compile}" : "${MAX_GLIBC_VERSION:=2.34}" : "${RELEASE_RETENTION:=10}" -export SOURCE_BRANCH POLL_INTERVAL REPO_DISTRIBUTION REPO_COMPONENT PACKAGE_RELEASE RUN_LINUX_TESTS MAX_GLIBC_VERSION RELEASE_RETENTION -mkdir -p "$DATA_DIR"/{state,repository,work,cargo,packages} /tmp/rustbox-gnupg +: "${PACKAGE_RETENTION:=10}" +: "${CLEAN_BUILD_CACHE:=1}" +: "${BUILD_TMP_DIR:=/tmp/rustbox-package-builder}" +require_positive_integer RELEASE_RETENTION "$RELEASE_RETENTION" +require_positive_integer PACKAGE_RETENTION "$PACKAGE_RETENTION" +export SOURCE_BRANCH POLL_INTERVAL REPO_DISTRIBUTION REPO_COMPONENT PACKAGE_RELEASE RUN_LINUX_TESTS MAX_GLIBC_VERSION RELEASE_RETENTION PACKAGE_RETENTION CLEAN_BUILD_CACHE BUILD_TMP_DIR +mkdir -p "$DATA_DIR"/{state,repository,work,packages} "$BUILD_TMP_DIR" /tmp/rustbox-gnupg chmod 0700 /tmp/rustbox-gnupg + +# Migration/maintenance cleanup for older builder versions which stored large +# cargo targets and package work directories on the persistent data volume. +rm -rf -- "$DATA_DIR/work/out" "$DATA_DIR/work/package" "$DATA_DIR/cargo/target" +rmdir "$DATA_DIR/cargo" 2>/dev/null || true +rm -rf -- "$BUILD_TMP_DIR/out" "$BUILD_TMP_DIR/package" +if is_true "$CLEAN_BUILD_CACHE"; then + rm -rf -- "$BUILD_TMP_DIR/cargo-target" +fi + +# Apply file-level retention at startup as well, so stale raw package pools and +# old atomic release directories are reduced even before the next source build. +prune_package_files "$DATA_DIR/packages/deb" 'rustbox_*.deb' "$PACKAGE_RETENTION" +prune_package_files "$DATA_DIR/packages/rpm/x86_64" 'rustbox-*.x86_64.rpm' "$PACKAGE_RETENTION" +prune_package_files "$DATA_DIR/packages/suse/x86_64" 'rustbox-*.suse.x86_64.rpm' "$PACKAGE_RETENTION" +prune_release_dirs "$DATA_DIR/repository/releases" "$RELEASE_RETENTION" "$(readlink -f "$DATA_DIR/repository/current" 2>/dev/null || true)" + export GNUPGHOME=/tmp/rustbox-gnupg GPG_PRIVATE_KEY_FILE=/run/secrets/gpg/private-key.asc [[ -s "$GPG_PRIVATE_KEY_FILE" ]] || fatal "missing GPG private key: $GPG_PRIVATE_KEY_FILE" diff --git a/scripts/lib.sh b/scripts/lib.sh index 6cdfdce..38632fb 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -PACKAGING_REVISION=2 +PACKAGING_REVISION=3 DATA_DIR=${DATA_DIR:-/var/lib/rustbox-package-builder} log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } @@ -13,3 +13,53 @@ print(json.dumps(sys.argv[1])) PY } atomic_write() { local path=$1 tmp="${path}.tmp.$$"; cat > "$tmp"; mv -f "$tmp" "$path"; } + +is_true() { + case "${1:-}" in + 1|true|TRUE|yes|YES|on|ON) return 0 ;; + *) return 1 ;; + esac +} + +require_positive_integer() { + local name=$1 value=$2 + [[ "$value" =~ ^[1-9][0-9]*$ ]] || fatal "$name must be a positive integer" +} + +prune_package_files() { + local directory=$1 pattern=$2 keep=$3 + require_positive_integer PACKAGE_RETENTION "$keep" + [[ -d "$directory" ]] || return 0 + + mapfile -t stale < <( + find "$directory" -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}' + ) + + local path + for path in "${stale[@]}"; do + [[ -n "$path" ]] || continue + log "package retention: removing $path" + rm -f -- "$path" + done +} + +prune_release_dirs() { + local releases_dir=$1 keep=$2 protected=${3:-} + require_positive_integer RELEASE_RETENTION "$keep" + [[ -d "$releases_dir" ]] || return 0 + + mapfile -t stale < <( + find "$releases_dir" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \ + | sort -nr \ + | awk -v keep="$keep" 'NR>keep {$1=""; sub(/^ /,""); print}' + ) + + local path + for path in "${stale[@]}"; do + [[ -n "$path" && "$path" != "$protected" ]] || continue + log "release retention: removing $path" + rm -rf -- "$path" + done +} diff --git a/start.sh b/start.sh index eba2e52..e84994f 100755 --- a/start.sh +++ b/start.sh @@ -53,7 +53,7 @@ if [[ "$REPOSITORY_URL" == http://* && ! "$REPOSITORY_URL" =~ :[0-9]+$ ]]; then log "REPOSITORY_URL updated to $REPOSITORY_URL" fi -mkdir -p "$DATA_DIR"/{repository,state,cargo,work,packages} "$GPG_DIR" +mkdir -p "$DATA_DIR"/{repository,state,work,packages} "$GPG_DIR" chmod 0700 "$GPG_DIR" 2>/dev/null || true log "building builder and repository dashboard" diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 9544271..942285a 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,4 +1,7 @@ from pathlib import Path +import os +import subprocess +import tempfile import tomllib ROOT = Path(__file__).resolve().parents[1] @@ -85,4 +88,47 @@ def test_rustup_installation_and_runtime_cargo_home_are_consistent(): assert "CARGO_HOME=/opt/cargo" in dockerfile runtime_env = dockerfile.split("ENV DATA_DIR=/var/lib/rustbox-package-builder", 1)[1] assert "CARGO_HOME=/var/lib/rustbox-package-builder/cargo" not in runtime_env - assert "CARGO_TARGET_DIR=/var/lib/rustbox-package-builder/cargo/target" in runtime_env + assert "CARGO_TARGET_DIR=/tmp/rustbox-package-builder/cargo-target" in runtime_env + assert "CARGO_TARGET_DIR=/var/lib/rustbox-package-builder/cargo/target" not in runtime_env + + + +def test_build_cache_cleanup_and_package_retention_defaults(): + env = (ROOT / ".env.example").read_text() + compose = (ROOT / "docker-compose.yml").read_text() + entrypoint = (ROOT / "scripts/entrypoint.sh").read_text() + build = (ROOT / "scripts/build-release.sh").read_text() + + assert "PACKAGE_RETENTION=10" in env + assert "CLEAN_BUILD_CACHE=1" in env + assert 'PACKAGE_RETENTION: "${PACKAGE_RETENTION:-10}"' in compose + assert 'CLEAN_BUILD_CACHE: "${CLEAN_BUILD_CACHE:-1}"' in compose + assert 'rm -rf -- "$DATA_DIR/work/out" "$DATA_DIR/work/package" "$DATA_DIR/cargo/target"' in entrypoint + assert 'trap cleanup_build_artifacts EXIT' in build + assert 'aptly -config="$APTLY_CONF" db cleanup' in build + assert 'cp -al "$DATA_DIR/packages/rpm/x86_64/." "$RELEASE_DIR/rpm/x86_64/"' in build + + +def test_prune_package_files_keeps_only_newest_ten(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + pkg = root / "deb" + pkg.mkdir() + for idx in range(12): + path = pkg / f"rustbox_0.2.{idx}-1_amd64.deb" + path.write_bytes(str(idx).encode()) + ts = 1_700_000_000 + idx + os.utime(path, (ts, ts)) + + script = "\n".join([ + "set -Eeuo pipefail", + f"source {ROOT / 'scripts/lib.sh'}", + f"prune_package_files {pkg} 'rustbox_*.deb' 10", + ]) + subprocess.run(["bash", "-lc", script], check=True) + + remaining = sorted(p.name for p in pkg.glob("*.deb")) + assert len(remaining) == 10 + assert "rustbox_0.2.0-1_amd64.deb" not in remaining + assert "rustbox_0.2.1-1_amd64.deb" not in remaining + assert "rustbox_0.2.11-1_amd64.deb" in remaining