From 3d6c49df65818413616eefbbea38b6d57818bf58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Tue, 6 Oct 2026 23:03:38 +0200 Subject: [PATCH] first commit --- .dockerignore | 5 + .env.example | 4 + .gitignore | 4 + Dockerfile | 53 +++++++ README.md | 175 ++++++++++++++++++++++++ docker-compose.yml | 14 ++ scripts/build-release.sh | 235 ++++++++++++++++++++++++++++++++ scripts/check-and-build.sh | 20 +++ scripts/entrypoint.sh | 66 +++++++++ scripts/generate-index.sh | 51 +++++++ scripts/init-storage.sh | 46 +++++++ scripts/lib.sh | 40 ++++++ scripts/prepare-package.sh | 227 ++++++++++++++++++++++++++++++ start.sh | 149 ++++++++++++++++++++ tests/check-rustproxy-source.sh | 21 +++ tests/static.sh | 27 ++++ 16 files changed, 1137 insertions(+) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100755 scripts/build-release.sh create mode 100755 scripts/check-and-build.sh create mode 100755 scripts/entrypoint.sh create mode 100755 scripts/generate-index.sh create mode 100755 scripts/init-storage.sh create mode 100755 scripts/lib.sh create mode 100755 scripts/prepare-package.sh create mode 100755 start.sh create mode 100755 tests/check-rustproxy-source.sh create mode 100755 tests/static.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7bcde68 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +.gitignore +data +gpg +*.zip diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..1b5aed8 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +SOURCE_REPO=https://git.linuxiarz.pl/gru/RustProxy.git +SOURCE_BRANCH=master +REPOSITORY_URL=https://packages.linuxiarz.pl +POLL_INTERVAL=300 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80bdfb0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +data/ +gpg/ +*.zip diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ecb626f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +FROM rockylinux:9 + +ARG RUST_TOOLCHAIN=stable +ARG NFPM_VERSION=2.47.0 +ARG NFPM_RPM_SHA256=5ad3287a0b1d0c068e556bd84965a9866ab6451ef861506a3c09b8971ca5326f +ARG APTLY_VERSION=1.6.3 +ARG APTLY_ZIP_SHA256=535ed24eb706bc25a316b59d709a7e74e10165f2ffab8552097ba7e4816a3e7e + +RUN dnf -y install --allowerasing \ + bash binutils ca-certificates cmake coreutils createrepo_c curl findutils \ + gcc gcc-c++ git gnupg2 gzip make openssl pkgconf-pkg-config python3 python3.11 \ + rpm rsync shadow-utils sqlite-devel tar unzip xz \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +RUN set -eux; \ + curl -fsSL -o /tmp/nfpm.rpm \ + "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm-${NFPM_VERSION}-1.x86_64.rpm"; \ + echo "${NFPM_RPM_SHA256} /tmp/nfpm.rpm" | sha256sum -c -; \ + rpm -Uvh /tmp/nfpm.rpm; \ + rm -f /tmp/nfpm.rpm; \ + nfpm --version + +RUN set -eux; \ + curl -fsSL -o /tmp/aptly.zip \ + "https://github.com/aptly-dev/aptly/releases/download/v${APTLY_VERSION}/aptly_${APTLY_VERSION}_linux_amd64.zip"; \ + echo "${APTLY_ZIP_SHA256} /tmp/aptly.zip" | sha256sum -c -; \ + mkdir -p /tmp/aptly; \ + unzip -q /tmp/aptly.zip -d /tmp/aptly; \ + install -m 0755 "$(find /tmp/aptly -type f -name aptly | head -n1)" /usr/local/bin/aptly; \ + rm -rf /tmp/aptly /tmp/aptly.zip; \ + aptly version + +ENV RUSTUP_HOME=/opt/rustup \ + CARGO_HOME=/opt/cargo \ + RUST_PROXY_PYTHON=/usr/bin/python3.11 \ + PATH=/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN set -eux; \ + curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup-init.sh; \ + sh /tmp/rustup-init.sh -y --no-modify-path --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" --component rustfmt --component clippy; \ + rm -f /tmp/rustup-init.sh; \ + rustc --version; \ + cargo --version + +# Registry/git/target cache is persistent, while rustup/toolchains stay in the image. +ENV CARGO_HOME=/var/lib/package-builder/cargo + +COPY scripts/ /opt/rust-package-builder/scripts/ +RUN chmod 0755 /opt/rust-package-builder/scripts/*.sh + +VOLUME ["/var/lib/package-builder"] +ENTRYPOINT ["/opt/rust-package-builder/scripts/entrypoint.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..00c684d --- /dev/null +++ b/README.md @@ -0,0 +1,175 @@ +# Rust Package Builder + +Minimalny kontener, który cyklicznie sprawdza branch Git, buduje projekt Rust i publikuje podpisane repozytoria APT oraz RPM/DNF. + +## Obsługiwane cele + +- architektura: amd64 / x86_64 +- Debian 12, Debian 13 +- Ubuntu 24.04, Ubuntu 26.04 +- RHEL/Rocky/Alma 9 i 10 + +Obraz buildera bazuje na Rocky Linux 9. Dzięki temu binarka jest budowana na glibc 2.34; builder dodatkowo odrzuca wynik wymagający nowszego symbolu glibc. Ta sama binarka może być pakowana do DEB i RPM dla wymienionych, nowszych systemów x86_64. + +## RustProxy + +Dla dostarczonego RustProxy builder automatycznie używa plików znajdujących się w repozytorium źródłowym: + +- `debian/rustproxy.service` +- `debian/rustproxy.environment` +- `debian/rustproxy.logrotate` +- `debian/rustproxy.tmpfiles` +- logiki `debian/rustproxy.postinst` +- `grafana/rustproxy-overview.json` + +Nie utrzymuje drugiej kopii konfiguracji systemd RustProxy. Wersja i nazwa binarki pochodzą z metadanych Cargo, a nazwa pakietu i maintainer są przejmowane z `debian/control`, jeśli plik istnieje. + +## Uruchomienie + +Najprostszy wariant wymaga ustawienia tylko dwóch wartości: + +```bash +cp .env.example .env +# edytuj .env i ustaw SOURCE_REPO oraz REPOSITORY_URL +./start.sh +``` + +`start.sh` automatycznie: + +- sprawdza Docker i Docker Compose, +- tworzy katalogi `data/` i `gpg/`, +- buduje obraz buildera, +- przy pierwszym uruchomieniu generuje trwały klucz GPG RSA 3072 bez hasła, +- zapisuje klucz prywatny jako `gpg/private-key.asc`, publiczny jako `gpg/public-key.asc` oraz fingerprint jako `gpg/fingerprint`, +- przy kolejnych uruchomieniach ponownie używa tego samego klucza, +- uruchamia usługę przez `docker compose up -d`, +- builder natychmiast sprawdza repozytorium Git i po udanym buildzie publikuje APT oraz RPM/DNF. + +Można też uruchomić bez wcześniejszego tworzenia `.env`: + +```bash +SOURCE_REPO=https://git.linuxiarz.pl/gru/RustProxy.git \ +REPOSITORY_URL=https://packages.linuxiarz.pl \ +./start.sh +``` + +W takim przypadku `start.sh` zapisze te dwie wartości do `.env`. + +Publiczny katalog repozytorium: + +```text +./data/repository/ +``` + +Wystaw ten katalog przez nginx/RustProxy/Apache pod adresem z `REPOSITORY_URL`. Builder celowo nie uruchamia własnego HTTP ani TLS. + +RustProxy 0.26.6 wymaga podczas builda Python 3.10+; obraz zawiera Python 3.11 i ustawia `RUST_PROXY_PYTHON=/usr/bin/python3.11`. + +### Klucz GPG + +Klucz jest generowany tylko wtedy, gdy `gpg/private-key.asc` nie istnieje. Nie usuwaj katalogu `gpg/` po publikacji repozytorium: zmiana klucza spowoduje, że klienci z wcześniej zainstalowanym kluczem przestaną ufać nowym podpisom. + +Do backupu wystarczy zachować co najmniej: + +```text +gpg/private-key.asc +gpg/public-key.asc +gpg/fingerprint +data/ +``` + +## Konfiguracja + +Wymagane: + +```text +SOURCE_REPO +REPOSITORY_URL +``` + +Domyślne: + +```text +SOURCE_BRANCH=master +POLL_INTERVAL=300 +REPO_DISTRIBUTION=stable +REPO_COMPONENT=main +PACKAGE_RELEASE=auto +MAX_GLIBC_VERSION=2.34 +GPG_PRIVATE_KEY_FILE=/run/secrets/gpg/private-key.asc +GPG_PASSPHRASE_FILE=/run/secrets/gpg/passphrase +``` + +`GPG_PASSPHRASE_FILE` nie musi istnieć dla klucza bez hasła. `GPG_KEY_ID` może być podany przez ENV; jeśli go brak, builder wybiera pierwszy zaimportowany klucz prywatny. + +## Dane trwałe + +```text +/var/lib/package-builder/ +├── aptly/ # baza/pool aptly +├── cargo/ # registry + target cache +├── packages/ +│ ├── deb/ # historia zbudowanych DEB +│ └── rpm/x86_64/ # historia zbudowanych RPM +├── repository/ +│ ├── current -> releases/ +│ ├── apt -> current/apt +│ ├── rpm -> current/rpm +│ ├── repo-key.gpg -> current/repo-key.gpg +│ ├── index.html -> current/index.html +│ └── releases/ +├── state/ +│ ├── last_commit +│ ├── last_version +│ └── last_package_release +└── work/ +``` + +Poprzednie wersje pakietów nie są usuwane. Aktualne repozytoria APT i RPM zawierają historię pakietów. Przy `PACKAGE_RELEASE=auto` nowy commit z niezmienionym `Cargo.toml version` dostaje kolejną rewizję paczki (`-2`, `-3`, ...), więc nie nadpisuje wcześniej opublikowanego artefaktu. + +## Publikacja bez uszkodzenia działającego repo + +Nowy commit przechodzi kolejno: + +```text +clone -> cargo build --release -> nFPM DEB/RPM -> podpisy -> aptly -> createrepo_c -> walidacja -> atomic switch +``` + +Publiczny symlink `repository/current` jest podmieniany dopiero na końcu. Błąd kompilacji, pakowania, podpisu lub generowania metadata nie przełącza publicznego repozytorium. `last_commit` jest zapisywany dopiero po udanej publikacji, więc nieudany commit zostanie ponowiony w następnym cyklu. + +Kolejne snapshoty w `repository/releases/` używają hardlinków do niezmienionych plików poprzedniego snapshotu, więc zachowanie historii nie oznacza pełnej kopii wszystkich paczek przy każdym buildzie. + +## Podpisy + +- DEB: podpis GPG nFPM (`_gpgorigin`) oraz podpisane `Release.gpg`/`InRelease` przez aptly. +- RPM: podpis GPG pakietu nFPM. +- RPM metadata: `repodata/repomd.xml.asc`; instrukcja używa `repo_gpgcheck=1` i `gpgcheck=1`. +- publiczny klucz: `https://.../repo-key.gpg` jako ASCII-armored OpenPGP, zgodny z `gpg --dearmor` i `rpm --import`. + +## Test plików dostarczonego RustProxy + +```bash +./tests/static.sh +./tests/check-rustproxy-source.sh /sciezka/do/RustProxy +``` + +## HTTP + +Builder nie uruchamia serwera HTTP i nie obsługuje TLS. Generuje katalog `repository/` oraz prosty `index.html` z instrukcją instalacji dopasowaną do `REPOSITORY_URL` i nazwy pakietu. + +## Własny klucz GPG (opcjonalnie) + +Jeżeli zamiast automatycznie generowanego klucza chcesz użyć własnego, umieść go przed pierwszym `./start.sh` jako: + +```text +gpg/private-key.asc +``` + +Przykładowy eksport: + +```bash +gpg --armor --export-secret-keys YOUR_KEY_ID > gpg/private-key.asc +chmod 600 gpg/private-key.asc +``` + +`start.sh` wykryje istniejący klucz i nie wygeneruje nowego. Klucz prywatny nie jest kopiowany do obrazu. Kontener otrzymuje katalog `gpg/` wyłącznie jako read-only mount `/run/secrets/gpg`. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ca64598 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + rust-package-builder: + build: + context: . + image: linuxiarz/rust-package-builder:latest + environment: + SOURCE_REPO: "${SOURCE_REPO:?SOURCE_REPO is required}" + SOURCE_BRANCH: "${SOURCE_BRANCH:-master}" + REPOSITORY_URL: "${REPOSITORY_URL:?REPOSITORY_URL is required}" + POLL_INTERVAL: "${POLL_INTERVAL:-300}" + volumes: + - ./data:/var/lib/package-builder + - ./gpg:/run/secrets/gpg:ro + restart: unless-stopped diff --git a/scripts/build-release.sh b/scripts/build-release.sh new file mode 100755 index 0000000..031c4ef --- /dev/null +++ b/scripts/build-release.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +SHA=${1:?commit SHA required} +SHORT_SHA=${SHA:0:12} +BUILD_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}" +WORK="$DATA_DIR/work/$BUILD_ID" +SRC="$WORK/source" +PKG="$WORK/package" +OUT="$WORK/out" +REPOSITORY="$DATA_DIR/repository" +APTLY_CONF="$DATA_DIR/state/aptly.conf" + +cleanup() { + if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then + rm -rf "$WORK" + else + log "workspace kept: $WORK" + fi +} +trap cleanup EXIT + +mkdir -p "$SRC" "$PKG" "$OUT" + +log "cloning source" +git clone --quiet --no-tags --single-branch --branch "$SOURCE_BRANCH" "$SOURCE_REPO" "$SRC" +git -C "$SRC" checkout --quiet --detach "$SHA" +ACTUAL_SHA=$(git -C "$SRC" rev-parse HEAD) +[[ "$ACTUAL_SHA" == "$SHA" ]] || fatal "checkout SHA mismatch" + +cd "$SRC" +META_JSON=$(cargo metadata --format-version 1 --no-deps) +export META_JSON SRC + +readarray -t META < <(python3 <<'PY' +import json, os +m=json.loads(os.environ['META_JSON']) +src=os.path.realpath(os.environ['SRC']) +packages=m.get('packages', []) +chosen=None +for p in packages: + if os.path.realpath(os.path.dirname(p['manifest_path'])) == src: + chosen=p; break +if chosen is None: + defaults=set(m.get('workspace_default_members') or []) + for p in packages: + if p['id'] in defaults: + chosen=p; break +if chosen is None and packages: + chosen=packages[0] +if chosen is None: + raise SystemExit('no Cargo package found') +bin_targets=[t for t in chosen.get('targets', []) if 'bin' in t.get('kind', [])] +if not bin_targets: + raise SystemExit('no binary target found') +values=[ + chosen.get('name') or '', + chosen.get('version') or '', + bin_targets[0].get('name') or '', + chosen.get('description') or '', + chosen.get('license') or '', + chosen.get('homepage') or '', + chosen.get('repository') or '', +] +for v in values: + print(str(v).replace('\n',' ')) +PY +) + +CARGO_NAME=${META[0]} +VERSION=${META[1]} +BINARY_NAME=${META[2]} +DESCRIPTION=${META[3]} +LICENSE=${META[4]} +CARGO_HOMEPAGE=${META[5]} +CARGO_REPOSITORY=${META[6]} + +[[ -n "$VERSION" && -n "$BINARY_NAME" ]] || fatal "invalid Cargo metadata" + +PACKAGE_NAME=${PACKAGE_NAME:-} +MAINTAINER=${PACKAGE_MAINTAINER:-} +CONTROL_HOMEPAGE="" +if [[ -r "$SRC/debian/control" ]]; then + [[ -n "$PACKAGE_NAME" ]] || PACKAGE_NAME=$(read_first_field "$SRC/debian/control" Package || true) + [[ -n "$MAINTAINER" ]] || MAINTAINER=$(read_first_field "$SRC/debian/control" Maintainer || true) + CONTROL_HOMEPAGE=$(read_first_field "$SRC/debian/control" Homepage || true) +fi +PACKAGE_NAME=${PACKAGE_NAME:-${CARGO_NAME//-/}} +MAINTAINER=${MAINTAINER:-Rust Package Builder } +DESCRIPTION=${DESCRIPTION:-$PACKAGE_NAME} +LICENSE=${LICENSE:-Proprietary} +HOMEPAGE=${PACKAGE_HOMEPAGE:-${CARGO_HOMEPAGE:-${CARGO_REPOSITORY:-${CONTROL_HOMEPAGE:-$SOURCE_REPO}}}} + +REQUESTED_PACKAGE_RELEASE=$PACKAGE_RELEASE +if [[ "$REQUESTED_PACKAGE_RELEASE" == "auto" ]]; then + LAST_VERSION="" + LAST_PACKAGE_RELEASE=0 + [[ -r "$DATA_DIR/state/last_version" ]] && LAST_VERSION=$(cat "$DATA_DIR/state/last_version") + [[ -r "$DATA_DIR/state/last_package_release" ]] && LAST_PACKAGE_RELEASE=$(cat "$DATA_DIR/state/last_package_release") + if [[ "$LAST_VERSION" == "$VERSION" && "$LAST_PACKAGE_RELEASE" =~ ^[0-9]+$ ]]; then + PACKAGE_RELEASE=$((LAST_PACKAGE_RELEASE + 1)) + else + PACKAGE_RELEASE=1 + fi +else + PACKAGE_RELEASE=$REQUESTED_PACKAGE_RELEASE +fi + +export PACKAGE_NAME VERSION BINARY_NAME DESCRIPTION LICENSE HOMEPAGE MAINTAINER PACKAGE_RELEASE +log "building version $VERSION ($PACKAGE_NAME / $BINARY_NAME)" +log "cargo build started" +if [[ -f Cargo.lock ]]; then + cargo build --release --locked +else + cargo build --release +fi +log "cargo build completed" + +BINARY_PATH="$CARGO_TARGET_DIR/release/$BINARY_NAME" +[[ -x "$BINARY_PATH" ]] || fatal "built binary not found: $BINARY_PATH" + +MAX_GLIBC_FOUND=$(objdump -T "$BINARY_PATH" 2>/dev/null | grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/^GLIBC_//' | sort -V | tail -n1 || true) +if [[ -n "$MAX_GLIBC_FOUND" ]]; then + if [[ "$(printf '%s\n%s\n' "$MAX_GLIBC_VERSION" "$MAX_GLIBC_FOUND" | sort -V | tail -n1)" != "$MAX_GLIBC_VERSION" ]]; then + fatal "binary requires GLIBC_$MAX_GLIBC_FOUND, target maximum is GLIBC_$MAX_GLIBC_VERSION" + fi + log "glibc compatibility check: <= $MAX_GLIBC_FOUND" +fi + +/opt/rust-package-builder/scripts/prepare-package.sh "$SRC" "$PKG" "$BINARY_PATH" + +DEB_FILE="$OUT/${PACKAGE_NAME}_${VERSION}-${PACKAGE_RELEASE}_amd64.deb" +RPM_FILE="$OUT/${PACKAGE_NAME}-${VERSION}-${PACKAGE_RELEASE}.x86_64.rpm" + +log "creating deb package" +nfpm package --config "$PKG/nfpm-deb.yaml" --packager deb --target "$DEB_FILE" +log "creating rpm package" +nfpm package --config "$PKG/nfpm-rpm.yaml" --packager rpm --target "$RPM_FILE" + +ar t "$DEB_FILE" | grep '^_gpgorigin$' >/dev/null || fatal "DEB package does not contain nFPM GPG origin signature" +rpm -qpi "$RPM_FILE" | awk -F: '/^Signature/ && $0 !~ /\(none\)/ {ok=1} END{exit !ok}' || fatal "RPM package is not signed" +log "package signatures verified" + +cp -f "$DEB_FILE" "$DATA_DIR/packages/deb/" +cp -f "$RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/" + +log "updating aptly internal repository" +aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" >/dev/null +SNAPSHOT="release-${VERSION}-${SHORT_SHA}-$(date +%s)" +aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null + +APTLY_SIGN_ARGS=(-batch -gpg-key="$GPG_KEY_ID") +if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then + APTLY_SIGN_ARGS+=(-passphrase-file="$GPG_PASSPHRASE_FILE") +fi + +if aptly -config="$APTLY_CONF" publish list -raw | awk -v d="$REPO_DISTRIBUTION" '$2 == d {found=1} END{exit !found}'; then + aptly -config="$APTLY_CONF" publish switch \ + "${APTLY_SIGN_ARGS[@]}" \ + "$REPO_DISTRIBUTION" . "$SNAPSHOT" >/dev/null +else + aptly -config="$APTLY_CONF" publish snapshot \ + "${APTLY_SIGN_ARGS[@]}" \ + -distribution="$REPO_DISTRIBUTION" \ + -component="$REPO_COMPONENT" \ + -architectures=amd64 \ + "$SNAPSHOT" >/dev/null +fi + +APT_PUBLIC="$DATA_DIR/aptly/public" +[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "aptly did not create InRelease" +[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" ]] || fatal "aptly did not create Release.gpg" +gpg --batch --verify \ + "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" \ + "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 || fatal "APT Release signature verification failed" + +RELEASE_DIR="$REPOSITORY/releases/$BUILD_ID" +mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" + +CURRENT_TARGET="" +if [[ -L "$REPOSITORY/current" ]]; then + CURRENT_LINK=$(readlink "$REPOSITORY/current") + CURRENT_TARGET="$REPOSITORY/$CURRENT_LINK" +fi + +log "staging apt repository" +if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/apt" ]]; then + rsync -a --delete --link-dest="$CURRENT_TARGET/apt"/ "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ +else + rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ +fi + +log "updating rpm repository" +if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/rpm" ]]; then + rsync -a --delete --link-dest="$CURRENT_TARGET/rpm"/ "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ +else + rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ +fi +rm -rf "$RELEASE_DIR/rpm/x86_64/repodata" +createrepo_c --update "$RELEASE_DIR/rpm/x86_64" >/dev/null + +GPG_SIGN_ARGS=(--batch --yes --pinentry-mode loopback) +if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then + GPG_SIGN_ARGS+=(--passphrase-file "$GPG_PASSPHRASE_FILE") +fi +gpg "${GPG_SIGN_ARGS[@]}" --armor --detach-sign \ + --local-user "$GPG_KEY_ID" \ + --output "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \ + "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" +gpg --batch --verify \ + "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \ + "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" >/dev/null 2>&1 || fatal "RPM metadata signature verification failed" + +gpg --batch --armor --export "$GPG_KEY_ID" > "$RELEASE_DIR/repo-key.gpg" +[[ -s "$RELEASE_DIR/repo-key.gpg" ]] || fatal "public GPG key export failed" + +/opt/rust-package-builder/scripts/generate-index.sh "$RELEASE_DIR/index.html" "$PACKAGE_NAME" + +# Validate all externally visible endpoints before the atomic switch. +[[ -s "$RELEASE_DIR/index.html" ]] +[[ -f "$RELEASE_DIR/apt/dists/$REPO_DISTRIBUTION/InRelease" ]] +[[ -f "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" ]] + +log "publishing repository atomically" +rm -f "$REPOSITORY/current.new" +ln -s "releases/$BUILD_ID" "$REPOSITORY/current.new" +mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current" + +printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit" +printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version" +printf '%s\n' "$PACKAGE_RELEASE" | atomic_write "$DATA_DIR/state/last_package_release" +printf '%s\n' "$BUILD_ID" | atomic_write "$DATA_DIR/state/last_release" + +log "build completed: version=$VERSION package_release=$PACKAGE_RELEASE commit=$SHA release=$BUILD_ID" diff --git a/scripts/check-and-build.sh b/scripts/check-and-build.sh new file mode 100755 index 0000000..d21f0fc --- /dev/null +++ b/scripts/check-and-build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +STATE_FILE="$DATA_DIR/state/last_commit" + +log "checking repository" +REMOTE_SHA=$(git ls-remote "$SOURCE_REPO" "refs/heads/$SOURCE_BRANCH" | awk 'NR == 1 {print $1}') +[[ "$REMOTE_SHA" =~ ^[0-9a-fA-F]{40}$ ]] || fatal "cannot resolve branch $SOURCE_BRANCH from $SOURCE_REPO" + +LAST_SHA="" +[[ -r "$STATE_FILE" ]] && LAST_SHA=$(cat "$STATE_FILE") + +if [[ "$REMOTE_SHA" == "$LAST_SHA" ]]; then + log "no changes: $REMOTE_SHA" + exit 0 +fi + +log "new commit detected: $REMOTE_SHA" +/opt/rust-package-builder/scripts/build-release.sh "$REMOTE_SHA" diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100755 index 0000000..30fdf67 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +: "${SOURCE_REPO:?SOURCE_REPO is required}" +SOURCE_BRANCH=${SOURCE_BRANCH:-master} +: "${REPOSITORY_URL:?REPOSITORY_URL is required}" +POLL_INTERVAL=${POLL_INTERVAL:-300} +PACKAGE_RELEASE=${PACKAGE_RELEASE:-auto} + +[[ "$POLL_INTERVAL" =~ ^[0-9]+$ ]] || fatal "POLL_INTERVAL must be an integer" +(( POLL_INTERVAL >= 1 )) || fatal "POLL_INTERVAL must be >= 1" +[[ "$PACKAGE_RELEASE" == "auto" || "$PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]] || fatal "PACKAGE_RELEASE must be auto or a positive integer" + +export DATA_DIR=${DATA_DIR:-/var/lib/package-builder} +export SOURCE_REPO SOURCE_BRANCH REPOSITORY_URL POLL_INTERVAL +export REPO_DISTRIBUTION=${REPO_DISTRIBUTION:-stable} +export REPO_COMPONENT=${REPO_COMPONENT:-main} +export PACKAGE_RELEASE +export MAX_GLIBC_VERSION=${MAX_GLIBC_VERSION:-2.34} +export GPG_PRIVATE_KEY_FILE=${GPG_PRIVATE_KEY_FILE:-/run/secrets/gpg/private-key.asc} +export GPG_PASSPHRASE_FILE=${GPG_PASSPHRASE_FILE:-/run/secrets/gpg/passphrase} +export GNUPGHOME=${GNUPGHOME:-/run/package-builder/gnupg} +export CARGO_HOME="$DATA_DIR/cargo" +export CARGO_TARGET_DIR="$DATA_DIR/cargo/target" + +for cmd in git cargo rustc nfpm aptly createrepo_c gpg rsync python3 python3.11 objdump ar rpm; do + require_cmd "$cmd" +done + +mkdir -p \ + "$DATA_DIR/repository/releases" \ + "$DATA_DIR/state" \ + "$DATA_DIR/cargo" \ + "$DATA_DIR/work" \ + "$DATA_DIR/packages/rpm/x86_64" \ + "$DATA_DIR/packages/deb" +mkdir -p "$GNUPGHOME" +chmod 0700 "$GNUPGHOME" + +[[ -r "$GPG_PRIVATE_KEY_FILE" ]] || fatal "GPG private key not found: $GPG_PRIVATE_KEY_FILE" +log "importing GPG signing key" +gpg --batch --import "$GPG_PRIVATE_KEY_FILE" >/dev/null 2>&1 || fatal "cannot import GPG key" + +if [[ -z "${GPG_KEY_ID:-}" ]]; then + GPG_KEY_ID=$(gpg --batch --with-colons --list-secret-keys | awk -F: '/^sec:/ && !seen_sec {seen_sec=1; next} seen_sec && /^fpr:/ && first=="" {first=$10} END {print first}') +fi +[[ -n "$GPG_KEY_ID" ]] || fatal "cannot determine GPG key fingerprint" +export GPG_KEY_ID + +if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then + NFPM_PASSPHRASE=$(cat "$GPG_PASSPHRASE_FILE") + export NFPM_PASSPHRASE +else + unset NFPM_PASSPHRASE || true +fi + +/opt/rust-package-builder/scripts/init-storage.sh + +log "builder started: branch=$SOURCE_BRANCH interval=${POLL_INTERVAL}s" +while true; do + if ! /opt/rust-package-builder/scripts/check-and-build.sh; then + log "ERROR: build cycle failed; published repository was not switched" + fi + sleep "$POLL_INTERVAL" +done diff --git a/scripts/generate-index.sh b/scripts/generate-index.sh new file mode 100755 index 0000000..5668c99 --- /dev/null +++ b/scripts/generate-index.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +OUT=${1:?output file required} +PACKAGE=${2:?package name required} +URL=$(trim_trailing_slash "$REPOSITORY_URL") + +export OUT PACKAGE URL REPO_DISTRIBUTION REPO_COMPONENT +python3 <<'PY_END' +import html, os +out=os.environ['OUT'] +package=os.environ['PACKAGE'] +url=os.environ['URL'] +dist=os.environ['REPO_DISTRIBUTION'] +component=os.environ['REPO_COMPONENT'] +h=html.escape +content=f''' + +{h(package)} repository + +

{h(package)}

+

Debian / Ubuntu

+
curl -fsSL {h(url)}/repo-key.gpg \\
+  | sudo gpg --dearmor -o /usr/share/keyrings/{h(package)}.gpg
+
+echo "deb [signed-by=/usr/share/keyrings/{h(package)}.gpg] {h(url)}/apt {h(dist)} {h(component)}" \\
+  | sudo tee /etc/apt/sources.list.d/{h(package)}.list
+
+sudo apt update
+sudo apt install {h(package)}
+

RHEL / Rocky / Alma / Fedora

+
sudo rpm --import {h(url)}/repo-key.gpg
+
+sudo tee /etc/yum.repos.d/{h(package)}.repo <<'REPO'
+[{h(package)}]
+name={h(package)} Repository
+baseurl={h(url)}/rpm/$basearch
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey={h(url)}/repo-key.gpg
+REPO
+
+sudo dnf install {h(package)}
+ + +''' +with open(out, 'w', encoding='utf-8') as f: + f.write(content) +PY_END diff --git a/scripts/init-storage.sh b/scripts/init-storage.sh new file mode 100755 index 0000000..90a1cfb --- /dev/null +++ b/scripts/init-storage.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +APTLY_CONF="$DATA_DIR/state/aptly.conf" +APTLY_ROOT="$DATA_DIR/aptly" +REPOSITORY="$DATA_DIR/repository" + +mkdir -p "$APTLY_ROOT" "$REPOSITORY/releases" + +cat > "$APTLY_CONF" </dev/null; then + log "creating aptly local repository" + aptly -config="$APTLY_CONF" repo create \ + -distribution="$REPO_DISTRIBUTION" \ + -component="$REPO_COMPONENT" \ + package-builder >/dev/null +fi + +for item in apt rpm repo-key.gpg index.html; do + if [[ -e "$REPOSITORY/$item" && ! -L "$REPOSITORY/$item" ]]; then + fatal "$REPOSITORY/$item exists and is not a symlink; refusing to overwrite" + fi +done + +ln -sfn current/apt "$REPOSITORY/apt" +ln -sfn current/rpm "$REPOSITORY/rpm" +ln -sfn current/repo-key.gpg "$REPOSITORY/repo-key.gpg" +ln -sfn current/index.html "$REPOSITORY/index.html" diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100755 index 0000000..098621a --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +log() { + printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" +} + +fatal() { + log "ERROR: $*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || fatal "required command not found: $1" +} + +trim_trailing_slash() { + local value=$1 + while [[ "$value" == */ ]]; do value=${value%/}; done + printf '%s' "$value" +} + +yaml_quote() { + python3 - "$1" <<'PY' +import json, sys +print(json.dumps(sys.argv[1])) +PY +} + +read_first_field() { + local file=$1 key=$2 + awk -v key="$key" 'BEGIN{IGNORECASE=1} $0 ~ "^" key ":[[:space:]]*" {sub(/^[^:]+:[[:space:]]*/, ""); print; exit}' "$file" +} + +atomic_write() { + local path=$1 + local tmp="${path}.tmp.$$" + cat > "$tmp" + mv -f "$tmp" "$path" +} diff --git a/scripts/prepare-package.sh b/scripts/prepare-package.sh new file mode 100755 index 0000000..c52876d --- /dev/null +++ b/scripts/prepare-package.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source /opt/rust-package-builder/scripts/lib.sh + +SRC=${1:?source path required} +PKG=${2:?package workspace required} +BINARY_PATH=${3:?binary path required} + +rm -rf "$PKG" +mkdir -p "$PKG/payload" "$PKG/scripts" +install -D -m 0755 "$BINARY_PATH" "$PKG/payload/usr/bin/$BINARY_NAME" + +SERVICE_SRC="$SRC/debian/${PACKAGE_NAME}.service" +ENV_SRC="$SRC/debian/${PACKAGE_NAME}.environment" +LOGROTATE_SRC="$SRC/debian/${PACKAGE_NAME}.logrotate" +TMPFILES_SRC="$SRC/debian/${PACKAGE_NAME}.tmpfiles" +POSTINST_SRC="$SRC/debian/${PACKAGE_NAME}.postinst" + +if [[ -r "$SERVICE_SRC" ]]; then + install -D -m 0644 "$SERVICE_SRC" "$PKG/payload/usr/lib/systemd/system/${PACKAGE_NAME}.service" +else + mkdir -p "$PKG/payload/usr/lib/systemd/system" + cat > "$PKG/payload/usr/lib/systemd/system/${PACKAGE_NAME}.service" < "$PKG/scripts/deb-postinstall.sh" <<'SCRIPT' +#!/bin/sh +set -e +SCRIPT +if [[ -r "$POSTINST_SRC" ]]; then + sed -e '/^#![[:space:]]*\/bin\/sh/d' \ + -e '/^[[:space:]]*#DEBHELPER#[[:space:]]*$/d' \ + -e '/^[[:space:]]*exit[[:space:]]\+0[[:space:]]*$/d' \ + "$POSTINST_SRC" >> "$PKG/scripts/deb-postinstall.sh" +else + cat >> "$PKG/scripts/deb-postinstall.sh" <