#!/usr/bin/env bash set -Eeuo pipefail source /opt/rust-package-builder/scripts/lib.sh SHA=${1:?commit SHA required} BUILD_REASON=${2:-new-commit} SHORT_SHA=${SHA:0:12} BUILD_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}" WORK="$DATA_DIR/work/$BUILD_ID" SRC="$WORK/source" PKG="$WORK/package" OUT="$WORK/out" REPOSITORY="$DATA_DIR/repository" APTLY_CONF="$DATA_DIR/state/aptly.conf" RELEASE_DIR="" PUBLISHED=0 cleanup() { if [[ "$PUBLISHED" != "1" && -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then rm -rf "$RELEASE_DIR" fi if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then rm -rf "$WORK" else log "workspace kept: $WORK" fi } trap cleanup EXIT mkdir -p "$SRC" "$PKG" "$OUT" log "cloning source" git clone --quiet --no-tags --single-branch --branch "$SOURCE_BRANCH" "$SOURCE_REPO" "$SRC" git -C "$SRC" checkout --quiet --detach "$SHA" ACTUAL_SHA=$(git -C "$SRC" rev-parse HEAD) [[ "$ACTUAL_SHA" == "$SHA" ]] || fatal "checkout SHA mismatch" SOURCE_REVISION=$(git -C "$SRC" rev-list --count "$SHA") cd "$SRC" META_JSON=$(cargo metadata --format-version 1 --no-deps) export META_JSON SRC readarray -t META < <(python3 <<'PY' import json, os m=json.loads(os.environ['META_JSON']) src=os.path.realpath(os.environ['SRC']) packages=m.get('packages', []) chosen=None for p in packages: if os.path.realpath(os.path.dirname(p['manifest_path'])) == src: chosen=p; break if chosen is None: defaults=set(m.get('workspace_default_members') or []) for p in packages: if p['id'] in defaults: chosen=p; break if chosen is None and packages: chosen=packages[0] if chosen is None: raise SystemExit('no Cargo package found') bin_targets=[t for t in chosen.get('targets', []) if 'bin' in t.get('kind', [])] if not bin_targets: raise SystemExit('no binary target found') values=[ chosen.get('name') or '', chosen.get('version') or '', bin_targets[0].get('name') or '', chosen.get('description') or '', chosen.get('license') or '', chosen.get('homepage') or '', chosen.get('repository') or '', ] for v in values: print(str(v).replace('\n',' ')) PY ) CARGO_NAME=${META[0]} VERSION=${META[1]} BINARY_NAME=${META[2]} DESCRIPTION=${META[3]} LICENSE=${META[4]} CARGO_HOMEPAGE=${META[5]} CARGO_REPOSITORY=${META[6]} [[ -n "$VERSION" && -n "$BINARY_NAME" ]] || fatal "invalid Cargo metadata" PACKAGE_NAME=${PACKAGE_NAME:-} MAINTAINER=${PACKAGE_MAINTAINER:-} CONTROL_HOMEPAGE="" if [[ -r "$SRC/debian/control" ]]; then [[ -n "$PACKAGE_NAME" ]] || PACKAGE_NAME=$(read_first_field "$SRC/debian/control" Package || true) [[ -n "$MAINTAINER" ]] || MAINTAINER=$(read_first_field "$SRC/debian/control" Maintainer || true) CONTROL_HOMEPAGE=$(read_first_field "$SRC/debian/control" Homepage || true) fi PACKAGE_NAME=${PACKAGE_NAME:-${CARGO_NAME//-/}} MAINTAINER=${MAINTAINER:-Rust Package Builder } DESCRIPTION=${DESCRIPTION:-$PACKAGE_NAME} LICENSE=${LICENSE:-Proprietary} HOMEPAGE=${PACKAGE_HOMEPAGE:-${CARGO_HOMEPAGE:-${CARGO_REPOSITORY:-${CONTROL_HOMEPAGE:-$SOURCE_REPO}}}} REQUESTED_PACKAGE_RELEASE=$PACKAGE_RELEASE if [[ "$REQUESTED_PACKAGE_RELEASE" == "auto" ]]; then LAST_COMMIT="" LAST_VERSION="" LAST_PACKAGE_RELEASE=0 [[ -r "$DATA_DIR/state/last_commit" ]] && LAST_COMMIT=$(cat "$DATA_DIR/state/last_commit") [[ -r "$DATA_DIR/state/last_version" ]] && LAST_VERSION=$(cat "$DATA_DIR/state/last_version") [[ -r "$DATA_DIR/state/last_package_release" ]] && LAST_PACKAGE_RELEASE=$(cat "$DATA_DIR/state/last_package_release") # Repairing an incomplete unified release for the exact same source commit # must not create an artificial new DEB/RPM release number. This is the # migration path from the older Linux-only builder: Linux packages already # exist, while the new OPNsense artifacts still need to be produced. if [[ "$BUILD_REASON" == "repair-missing-artifacts" \ && "$LAST_COMMIT" == "$SHA" \ && "$LAST_VERSION" == "$VERSION" \ && "$LAST_PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]]; then PACKAGE_RELEASE=$LAST_PACKAGE_RELEASE log "repair build: reusing Linux package release $PACKAGE_RELEASE for unchanged commit" elif [[ "$LAST_VERSION" == "$VERSION" && "$LAST_PACKAGE_RELEASE" =~ ^[0-9]+$ ]]; then PACKAGE_RELEASE=$((LAST_PACKAGE_RELEASE + 1)) else PACKAGE_RELEASE=1 fi else PACKAGE_RELEASE=$REQUESTED_PACKAGE_RELEASE fi export PACKAGE_NAME VERSION BINARY_NAME DESCRIPTION LICENSE HOMEPAGE MAINTAINER PACKAGE_RELEASE log "building version $VERSION ($PACKAGE_NAME / $BINARY_NAME)" if [[ "${RUN_LINUX_TESTS:-1}" == "1" ]]; then log "cargo tests started" if [[ -f Cargo.lock ]]; then cargo test --release --locked else cargo test --release fi log "cargo tests completed" fi log "Linux cargo build started" if [[ -f Cargo.lock ]]; then cargo build --release --locked else cargo build --release fi log "Linux cargo build completed" BINARY_PATH="$CARGO_TARGET_DIR/release/$BINARY_NAME" [[ -x "$BINARY_PATH" ]] || fatal "built binary not found: $BINARY_PATH" MAX_GLIBC_FOUND=$(objdump -T "$BINARY_PATH" 2>/dev/null | grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/^GLIBC_//' | sort -V | tail -n1 || true) if [[ -n "$MAX_GLIBC_FOUND" ]]; then if [[ "$(printf '%s\n%s\n' "$MAX_GLIBC_VERSION" "$MAX_GLIBC_FOUND" | sort -V | tail -n1)" != "$MAX_GLIBC_VERSION" ]]; then fatal "binary requires GLIBC_$MAX_GLIBC_FOUND, target maximum is GLIBC_$MAX_GLIBC_VERSION" fi log "glibc compatibility check: <= $MAX_GLIBC_FOUND" fi /opt/rust-package-builder/scripts/prepare-package.sh "$SRC" "$PKG" "$BINARY_PATH" DEB_FILE="$OUT/${PACKAGE_NAME}_${VERSION}-${PACKAGE_RELEASE}_amd64.deb" RPM_FILE="$OUT/${PACKAGE_NAME}-${VERSION}-${PACKAGE_RELEASE}.x86_64.rpm" log "creating deb package" nfpm package --config "$PKG/nfpm-deb.yaml" --packager deb --target "$DEB_FILE" log "creating rpm package" nfpm package --config "$PKG/nfpm-rpm.yaml" --packager rpm --target "$RPM_FILE" ar t "$DEB_FILE" | grep '^_gpgorigin$' >/dev/null || fatal "DEB package does not contain nFPM GPG origin signature" rpm -qpi "$RPM_FILE" | awk -F: '/^Signature/ && $0 !~ /\(none\)/ {ok=1} END{exit !ok}' || fatal "RPM package is not signed" log "Linux package signatures verified" # OPNsense is built from an isolated source copy. FreeBSD-specific patches never # touch the Linux source checkout used above. FREEBSD_BINARY="$OUT/rust-proxy-freebsd" /opt/rust-package-builder/scripts/build-opnsense.sh "$SRC" "$FREEBSD_BINARY" "$BINARY_NAME" "$WORK" if [[ "$SOURCE_REVISION" == "0" ]]; then OPNSENSE_PACKAGE_VERSION="$VERSION" else OPNSENSE_PACKAGE_VERSION="${VERSION}_${SOURCE_REVISION}" fi OPNSENSE_PACKAGE_FILE="$OUT/os-rustproxy-${OPNSENSE_PACKAGE_VERSION}.pkg" /opt/rust-package-builder/scripts/package-opnsense.sh \ "$SRC" "$FREEBSD_BINARY" "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_VERSION" # Only after all package formats have been built successfully do we update the # persistent Linux package stores and the internal APT repository. cp -f "$DEB_FILE" "$DATA_DIR/packages/deb/" cp -f "$RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/" log "updating aptly internal repository" aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" >/dev/null SNAPSHOT="release-${VERSION}-${SHORT_SHA}-$(date +%s)" aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null APTLY_SIGN_ARGS=(-batch -gpg-key="$GPG_KEY_ID") if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then APTLY_SIGN_ARGS+=(-passphrase-file="$GPG_PASSPHRASE_FILE") fi if aptly -config="$APTLY_CONF" publish list -raw | awk -v d="$REPO_DISTRIBUTION" '$2 == d {found=1} END{exit !found}'; then aptly -config="$APTLY_CONF" publish switch \ -force-overwrite \ "${APTLY_SIGN_ARGS[@]}" \ "$REPO_DISTRIBUTION" . "$SNAPSHOT" >/dev/null else aptly -config="$APTLY_CONF" publish snapshot \ -force-overwrite \ "${APTLY_SIGN_ARGS[@]}" \ -distribution="$REPO_DISTRIBUTION" \ -component="$REPO_COMPONENT" \ -architectures=amd64 \ "$SNAPSHOT" >/dev/null fi APT_PUBLIC="$DATA_DIR/aptly/public" [[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "aptly did not create InRelease" [[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" ]] || fatal "aptly did not create Release.gpg" gpg --batch --verify \ "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" \ "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 || fatal "APT Release signature verification failed" RELEASE_DIR="$REPOSITORY/releases/$BUILD_ID" mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/opnsense" CURRENT_TARGET="" if [[ -L "$REPOSITORY/current" ]]; then CURRENT_LINK=$(readlink "$REPOSITORY/current") CURRENT_TARGET="$REPOSITORY/$CURRENT_LINK" fi log "staging apt repository" if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/apt" ]]; then rsync -a --delete --link-dest="$CURRENT_TARGET/apt"/ "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ else rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/ fi log "updating rpm repository" if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/rpm" ]]; then rsync -a --delete --link-dest="$CURRENT_TARGET/rpm"/ "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ else rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/ fi rm -rf "$RELEASE_DIR/rpm/x86_64/repodata" createrepo_c --update "$RELEASE_DIR/rpm/x86_64" >/dev/null GPG_SIGN_ARGS=(--batch --yes --pinentry-mode loopback) if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then GPG_SIGN_ARGS+=(--passphrase-file "$GPG_PASSPHRASE_FILE") fi gpg "${GPG_SIGN_ARGS[@]}" --armor --detach-sign \ --local-user "$GPG_KEY_ID" \ --output "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \ "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" gpg --batch --verify \ "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \ "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" >/dev/null 2>&1 || fatal "RPM metadata signature verification failed" log "staging OPNsense pkg repository" if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/opnsense" ]]; then rsync -a --delete --link-dest="$CURRENT_TARGET/opnsense"/ "$CURRENT_TARGET/opnsense"/ "$RELEASE_DIR/opnsense"/ fi OPNSENSE_PACKAGE_DIR="$RELEASE_DIR/opnsense/$OPNSENSE_ABI/amd64" mkdir -p "$OPNSENSE_PACKAGE_DIR" # The previous OPNsense snapshot is hard-linked into this one. Unlink every # file we are about to replace so the currently published snapshot stays immutable. rm -f \ "$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" \ "$RELEASE_DIR/opnsense/RustProxy.conf" \ "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" \ "$RELEASE_DIR/opnsense/SHA256" \ "$RELEASE_DIR/opnsense/BUILD-INFO.txt" \ "$RELEASE_DIR/opnsense/index.html" cp -f "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_DIR/" /opt/rust-package-builder/scripts/generate-opnsense-repo.py \ "$RELEASE_DIR/opnsense" --retention "$OPNSENSE_RETENTION" cat > "$RELEASE_DIR/opnsense/RustProxy.conf" < "$RELEASE_DIR/opnsense/SHA256" cat > "$RELEASE_DIR/opnsense/BUILD-INFO.txt" < "$RELEASE_DIR/repo-key.gpg" [[ -s "$RELEASE_DIR/repo-key.gpg" ]] || fatal "public GPG key export failed" /opt/rust-package-builder/scripts/generate-index.sh "$RELEASE_DIR/index.html" "$PACKAGE_NAME" # Validate both public repositories before one atomic switch exposes either one. [[ -s "$RELEASE_DIR/index.html" ]] [[ -f "$RELEASE_DIR/apt/dists/$REPO_DISTRIBUTION/InRelease" ]] [[ -f "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" ]] [[ -s "$RELEASE_DIR/opnsense/index.html" ]] [[ -f "$RELEASE_DIR/opnsense/meta.conf" ]] [[ -f "$RELEASE_DIR/opnsense/packagesite.pkg" ]] [[ -f "$RELEASE_DIR/opnsense/data.pkg" ]] [[ -f "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" ]] bsdtar -tf "$RELEASE_DIR/opnsense/packagesite.pkg" | grep -Fx 'packagesite.yaml' >/dev/null || fatal "invalid OPNsense packagesite.pkg" bsdtar -xOf "$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" +COMPACT_MANIFEST >/dev/null || fatal "invalid OPNsense package manifest" log "publishing Linux and OPNsense repositories atomically" rm -f "$REPOSITORY/current.new" ln -s "releases/$BUILD_ID" "$REPOSITORY/current.new" mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current" PUBLISHED=1 printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit" printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version" printf '%s\n' "$PACKAGE_RELEASE" | atomic_write "$DATA_DIR/state/last_package_release" printf '%s\n' "$OPNSENSE_PACKAGE_VERSION" | atomic_write "$DATA_DIR/state/last_opnsense_package_version" printf '%s\n' "$BUILD_ID" | atomic_write "$DATA_DIR/state/last_release" log "build completed: reason=$BUILD_REASON version=$VERSION linux_release=$PACKAGE_RELEASE opnsense=$OPNSENSE_PACKAGE_VERSION commit=$SHA release=$BUILD_ID"