This commit is contained in:
Mateusz Gruszczyński committed 2026-10-07 09:57:52 +02:00
1 parent e2398a4379
commit 92aa0b676d
9 files changed
+425 -310

No files matched your search

+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env bash
set -Eeuo pipefail
source /opt/rust-package-builder/scripts/lib.sh
: "${SHA:?SHA required}"
: "${SRC:?SRC required}"
: "${PKG:?PKG required}"
: "${OUT:?OUT required}"
: "${VERSION:?VERSION required}"
: "${BINARY_NAME:?BINARY_NAME required}"
: "${PACKAGE_NAME:?PACKAGE_NAME required}"
REPOSITORY="$DATA_DIR/repository"
APTLY_CONF="$DATA_DIR/state/aptly.conf"
BRANCH_RELEASE_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}-linux"
RELEASE_DIR="$REPOSITORY/releases/$BRANCH_RELEASE_ID"
PUBLISHED=0
cleanup_linux_release() {
if [[ "$PUBLISHED" != "1" && -d "$RELEASE_DIR" ]]; then
rm -rf "$RELEASE_DIR"
fi
}
trap cleanup_linux_release EXIT
REQUESTED_PACKAGE_RELEASE=$PACKAGE_RELEASE
if [[ "$REQUESTED_PACKAGE_RELEASE" == "auto" ]]; then
LAST_LINUX_COMMIT=""
LAST_VERSION=""
LAST_PACKAGE_RELEASE=0
[[ -r "$DATA_DIR/state/last_linux_commit" ]] && LAST_LINUX_COMMIT=$(cat "$DATA_DIR/state/last_linux_commit")
[[ -r "$DATA_DIR/state/last_version" ]] && LAST_VERSION=$(cat "$DATA_DIR/state/last_version")
[[ -r "$DATA_DIR/state/last_package_release" ]] && LAST_PACKAGE_RELEASE=$(cat "$DATA_DIR/state/last_package_release")
if [[ "$LAST_LINUX_COMMIT" == "$SHA" \
&& "$LAST_VERSION" == "$VERSION" \
&& "$LAST_PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]]; then
PACKAGE_RELEASE=$LAST_PACKAGE_RELEASE
log "Linux repair build: reusing package release $PACKAGE_RELEASE"
elif [[ "$LAST_VERSION" == "$VERSION" && "$LAST_PACKAGE_RELEASE" =~ ^[0-9]+$ ]]; then
PACKAGE_RELEASE=$((LAST_PACKAGE_RELEASE + 1))
else
PACKAGE_RELEASE=1
fi
else
PACKAGE_RELEASE=$REQUESTED_PACKAGE_RELEASE
fi
export PACKAGE_RELEASE
cd "$SRC"
log "Linux branch: version $VERSION release $PACKAGE_RELEASE"
if [[ "${RUN_LINUX_TESTS:-1}" == "1" ]]; then
log "Linux cargo tests started"
if [[ -f Cargo.lock ]]; then
cargo test --release --locked
else
cargo test --release
fi
log "Linux cargo tests completed"
fi
log "Linux cargo build started"
if [[ -f Cargo.lock ]]; then
cargo build --release --locked
else
cargo build --release
fi
log "Linux cargo build completed"
BINARY_PATH="$CARGO_TARGET_DIR/release/$BINARY_NAME"
[[ -x "$BINARY_PATH" ]] || fatal "built binary not found: $BINARY_PATH"
MAX_GLIBC_FOUND=$(objdump -T "$BINARY_PATH" 2>/dev/null | grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/^GLIBC_//' | sort -V | tail -n1 || true)
if [[ -n "$MAX_GLIBC_FOUND" ]]; then
if [[ "$(printf '%s\n%s\n' "$MAX_GLIBC_VERSION" "$MAX_GLIBC_FOUND" | sort -V | tail -n1)" != "$MAX_GLIBC_VERSION" ]]; then
fatal "binary requires GLIBC_$MAX_GLIBC_FOUND, target maximum is GLIBC_$MAX_GLIBC_VERSION"
fi
log "glibc compatibility check: <= $MAX_GLIBC_FOUND"
fi
/opt/rust-package-builder/scripts/prepare-package.sh "$SRC" "$PKG" "$BINARY_PATH"
DEB_FILE="$OUT/${PACKAGE_NAME}_${VERSION}-${PACKAGE_RELEASE}_amd64.deb"
RPM_FILE="$OUT/${PACKAGE_NAME}-${VERSION}-${PACKAGE_RELEASE}.x86_64.rpm"
log "creating deb package"
nfpm package --config "$PKG/nfpm-deb.yaml" --packager deb --target "$DEB_FILE"
log "creating rpm package"
nfpm package --config "$PKG/nfpm-rpm.yaml" --packager rpm --target "$RPM_FILE"
ar t "$DEB_FILE" | grep '^_gpgorigin$' >/dev/null || fatal "DEB package does not contain nFPM GPG origin signature"
rpm -qpi "$RPM_FILE" | awk -F: '/^Signature/ && $0 !~ /\(none\)/ {ok=1} END{exit !ok}' || fatal "RPM package is not signed"
log "Linux package signatures verified"
cp -f "$DEB_FILE" "$DATA_DIR/packages/deb/"
cp -f "$RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/"
log "updating aptly internal repository"
aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" >/dev/null
SNAPSHOT="linux-${VERSION}-${SHORT_SHA}-$(date +%s)"
aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null
APTLY_SIGN_ARGS=(-batch -gpg-key="$GPG_KEY_ID")
if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then
APTLY_SIGN_ARGS+=(-passphrase-file="$GPG_PASSPHRASE_FILE")
fi
if aptly -config="$APTLY_CONF" publish list -raw | awk -v d="$REPO_DISTRIBUTION" '$2 == d {found=1} END{exit !found}'; then
aptly -config="$APTLY_CONF" publish switch -force-overwrite "${APTLY_SIGN_ARGS[@]}" "$REPO_DISTRIBUTION" . "$SNAPSHOT" >/dev/null
else
aptly -config="$APTLY_CONF" publish snapshot -force-overwrite "${APTLY_SIGN_ARGS[@]}" \
-distribution="$REPO_DISTRIBUTION" -component="$REPO_COMPONENT" -architectures=amd64 "$SNAPSHOT" >/dev/null
fi
APT_PUBLIC="$DATA_DIR/aptly/public"
[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "aptly did not create InRelease"
[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" ]] || fatal "aptly did not create Release.gpg"
gpg --batch --verify "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 \
|| fatal "APT Release signature verification failed"
mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/opnsense"
CURRENT_TARGET=""
if [[ -L "$REPOSITORY/current" ]]; then
CURRENT_LINK=$(readlink "$REPOSITORY/current")
CURRENT_TARGET="$REPOSITORY/$CURRENT_LINK"
fi
log "staging Linux repository snapshot"
if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/opnsense" ]]; then
rsync -a --delete --link-dest="$CURRENT_TARGET/opnsense"/ "$CURRENT_TARGET/opnsense"/ "$RELEASE_DIR/opnsense"/
fi
rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
rm -rf "$RELEASE_DIR/rpm/x86_64/repodata"
createrepo_c --update "$RELEASE_DIR/rpm/x86_64" >/dev/null
GPG_SIGN_ARGS=(--batch --yes --pinentry-mode loopback)
if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then
GPG_SIGN_ARGS+=(--passphrase-file "$GPG_PASSPHRASE_FILE")
fi
gpg "${GPG_SIGN_ARGS[@]}" --armor --detach-sign --local-user "$GPG_KEY_ID" \
--output "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml"
gpg --batch --verify "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" >/dev/null 2>&1 \
|| fatal "RPM metadata signature verification failed"
gpg --batch --armor --export "$GPG_KEY_ID" > "$RELEASE_DIR/repo-key.gpg"
[[ -s "$RELEASE_DIR/repo-key.gpg" ]] || fatal "public GPG key export failed"
/opt/rust-package-builder/scripts/generate-index.sh "$RELEASE_DIR/index.html" "$PACKAGE_NAME"
cat > "$RELEASE_DIR/LINUX-BUILD-INFO.txt" <<INFO
app_version=$VERSION
package_release=$PACKAGE_RELEASE
source_url=$SOURCE_REPO
source_ref=$SOURCE_BRANCH
source_commit=$SHA
build_reason=$BUILD_REASON
INFO
[[ -s "$RELEASE_DIR/index.html" ]]
[[ -f "$RELEASE_DIR/apt/dists/$REPO_DISTRIBUTION/InRelease" ]]
[[ -f "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" ]]
log "publishing Linux repository branch"
rm -f "$REPOSITORY/current.new"
ln -s "releases/$BRANCH_RELEASE_ID" "$REPOSITORY/current.new"
mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current"
PUBLISHED=1
printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_linux_commit"
printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version"
printf '%s\n' "$PACKAGE_RELEASE" | atomic_write "$DATA_DIR/state/last_package_release"
printf '%s\n' "$BRANCH_RELEASE_ID" | atomic_write "$DATA_DIR/state/last_linux_release"
printf '%s\n' "$BRANCH_RELEASE_ID" | atomic_write "$DATA_DIR/state/last_release"
log "Linux publication completed: version=$VERSION release=$PACKAGE_RELEASE commit=$SHA"
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
set -Eeuo pipefail
source /opt/rust-package-builder/scripts/lib.sh
: "${SHA:?SHA required}"
: "${SRC:?SRC required}"
: "${OUT:?OUT required}"
: "${VERSION:?VERSION required}"
: "${BINARY_NAME:?BINARY_NAME required}"
: "${SOURCE_REVISION:?SOURCE_REVISION required}"
REPOSITORY="$DATA_DIR/repository"
BRANCH_RELEASE_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}-opnsense"
RELEASE_DIR="$REPOSITORY/releases/$BRANCH_RELEASE_ID"
PUBLISHED=0
cleanup_opnsense_release() {
if [[ "$PUBLISHED" != "1" && -d "$RELEASE_DIR" ]]; then
rm -rf "$RELEASE_DIR"
fi
}
trap cleanup_opnsense_release EXIT
FREEBSD_BINARY="$OUT/rust-proxy-freebsd"
/opt/rust-package-builder/scripts/build-opnsense.sh "$SRC" "$FREEBSD_BINARY" "$BINARY_NAME" "$WORK"
if [[ "$SOURCE_REVISION" == "0" ]]; then
OPNSENSE_PACKAGE_VERSION="$VERSION"
else
OPNSENSE_PACKAGE_VERSION="${VERSION}_${SOURCE_REVISION}"
fi
OPNSENSE_PACKAGE_FILE="$OUT/os-rustproxy-${OPNSENSE_PACKAGE_VERSION}.pkg"
/opt/rust-package-builder/scripts/package-opnsense.sh \
"$SRC" "$FREEBSD_BINARY" "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_VERSION"
mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/opnsense"
CURRENT_TARGET=""
if [[ -L "$REPOSITORY/current" ]]; then
CURRENT_LINK=$(readlink "$REPOSITORY/current")
CURRENT_TARGET="$REPOSITORY/$CURRENT_LINK"
fi
log "staging OPNsense repository snapshot"
if [[ -n "$CURRENT_TARGET" ]]; then
for component in apt rpm opnsense; do
if [[ -d "$CURRENT_TARGET/$component" ]]; then
rsync -a --delete --link-dest="$CURRENT_TARGET/$component"/ "$CURRENT_TARGET/$component"/ "$RELEASE_DIR/$component"/
fi
done
for file in repo-key.gpg index.html LINUX-BUILD-INFO.txt; do
if [[ -f "$CURRENT_TARGET/$file" ]]; then
cp -a "$CURRENT_TARGET/$file" "$RELEASE_DIR/$file"
fi
done
fi
OPNSENSE_PACKAGE_DIR="$RELEASE_DIR/opnsense/$OPNSENSE_ABI/amd64"
mkdir -p "$OPNSENSE_PACKAGE_DIR"
rm -f \
"$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" \
"$RELEASE_DIR/opnsense/RustProxy.conf" \
"$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" \
"$RELEASE_DIR/opnsense/SHA256" \
"$RELEASE_DIR/opnsense/BUILD-INFO.txt" \
"$RELEASE_DIR/opnsense/index.html"
cp -f "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_DIR/"
/opt/rust-package-builder/scripts/generate-opnsense-repo.py "$RELEASE_DIR/opnsense" --retention "$OPNSENSE_RETENTION"
cat > "$RELEASE_DIR/opnsense/RustProxy.conf" <<REPOCONF
RustProxy: {
url: "${OPNSENSE_REPOSITORY_URL%/}",
enabled: yes,
mirror_type: "none",
signature_type: "none"
}
REPOCONF
cp -f "$OPNSENSE_PACKAGE_FILE" "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg"
(cd "$RELEASE_DIR/opnsense" && sha256sum "$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")") > "$RELEASE_DIR/opnsense/SHA256"
cat > "$RELEASE_DIR/opnsense/BUILD-INFO.txt" <<INFO
app_version=$VERSION
package_version=$OPNSENSE_PACKAGE_VERSION
source_url=$SOURCE_REPO
source_ref=$SOURCE_BRANCH
source_commit=$SHA
build_reason=$BUILD_REASON
source_revision=$SOURCE_REVISION
opnsense_series=$OPNSENSE_ABI
freebsd_release=$FREEBSD_RELEASE
freebsd_abi=$FREEBSD_ABI
freebsd_arch=$FREEBSD_ARCH
package=$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")
INFO
/opt/rust-package-builder/scripts/generate-opnsense-index.sh "$RELEASE_DIR/opnsense/index.html"
[[ -s "$RELEASE_DIR/opnsense/index.html" ]]
[[ -f "$RELEASE_DIR/opnsense/meta.conf" ]]
[[ -f "$RELEASE_DIR/opnsense/packagesite.pkg" ]]
[[ -f "$RELEASE_DIR/opnsense/data.pkg" ]]
[[ -f "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" ]]
bsdtar -tf "$RELEASE_DIR/opnsense/packagesite.pkg" | grep -Fx 'packagesite.yaml' >/dev/null || fatal "invalid OPNsense packagesite.pkg"
bsdtar -xOf "$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" +COMPACT_MANIFEST >/dev/null || fatal "invalid OPNsense package manifest"
log "publishing OPNsense repository branch"
rm -f "$REPOSITORY/current.new"
ln -s "releases/$BRANCH_RELEASE_ID" "$REPOSITORY/current.new"
mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current"
PUBLISHED=1
printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_opnsense_commit"
printf '%s\n' "$OPNSENSE_PACKAGE_VERSION" | atomic_write "$DATA_DIR/state/last_opnsense_package_version"
printf '%s\n' "$BRANCH_RELEASE_ID" | atomic_write "$DATA_DIR/state/last_opnsense_release"
printf '%s\n' "$BRANCH_RELEASE_ID" | atomic_write "$DATA_DIR/state/last_release"
log "OPNsense publication completed: package=$OPNSENSE_PACKAGE_VERSION commit=$SHA"
+33 -231
View File
@@ -4,21 +4,21 @@ source /opt/rust-package-builder/scripts/lib.sh
SHA=${1:?commit SHA required}
BUILD_REASON=${2:-new-commit}
BUILD_LINUX=${3:-1}
BUILD_OPNSENSE=${4:-1}
[[ "$BUILD_LINUX" =~ ^[01]$ ]] || fatal "BUILD_LINUX must be 0 or 1"
[[ "$BUILD_OPNSENSE" =~ ^[01]$ ]] || fatal "BUILD_OPNSENSE must be 0 or 1"
(( BUILD_LINUX == 1 || BUILD_OPNSENSE == 1 )) || { log "nothing to build for $SHA"; exit 0; }
SHORT_SHA=${SHA:0:12}
BUILD_ID="$(date -u '+%Y%m%dT%H%M%SZ')-${SHORT_SHA}"
WORK="$DATA_DIR/work/$BUILD_ID"
SRC="$WORK/source"
PKG="$WORK/package"
OUT="$WORK/out"
REPOSITORY="$DATA_DIR/repository"
APTLY_CONF="$DATA_DIR/state/aptly.conf"
RELEASE_DIR=""
PUBLISHED=0
cleanup() {
if [[ "$PUBLISHED" != "1" && -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then
rm -rf "$RELEASE_DIR"
fi
if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then
rm -rf "$WORK"
else
@@ -99,239 +99,41 @@ DESCRIPTION=${DESCRIPTION:-$PACKAGE_NAME}
LICENSE=${LICENSE:-Proprietary}
HOMEPAGE=${PACKAGE_HOMEPAGE:-${CARGO_HOMEPAGE:-${CARGO_REPOSITORY:-${CONTROL_HOMEPAGE:-$SOURCE_REPO}}}}
REQUESTED_PACKAGE_RELEASE=$PACKAGE_RELEASE
if [[ "$REQUESTED_PACKAGE_RELEASE" == "auto" ]]; then
LAST_COMMIT=""
LAST_VERSION=""
LAST_PACKAGE_RELEASE=0
[[ -r "$DATA_DIR/state/last_commit" ]] && LAST_COMMIT=$(cat "$DATA_DIR/state/last_commit")
[[ -r "$DATA_DIR/state/last_version" ]] && LAST_VERSION=$(cat "$DATA_DIR/state/last_version")
[[ -r "$DATA_DIR/state/last_package_release" ]] && LAST_PACKAGE_RELEASE=$(cat "$DATA_DIR/state/last_package_release")
export SHA SHORT_SHA BUILD_ID BUILD_REASON SOURCE_REVISION WORK SRC PKG OUT
export PACKAGE_NAME VERSION BINARY_NAME DESCRIPTION LICENSE HOMEPAGE MAINTAINER
# Repairing an incomplete unified release for the exact same source commit
# must not create an artificial new DEB/RPM release number. This is the
# migration path from the older Linux-only builder: Linux packages already
# exist, while the new OPNsense artifacts still need to be produced.
if [[ "$BUILD_REASON" == "repair-missing-artifacts" \
&& "$LAST_COMMIT" == "$SHA" \
&& "$LAST_VERSION" == "$VERSION" \
&& "$LAST_PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]]; then
PACKAGE_RELEASE=$LAST_PACKAGE_RELEASE
log "repair build: reusing Linux package release $PACKAGE_RELEASE for unchanged commit"
elif [[ "$LAST_VERSION" == "$VERSION" && "$LAST_PACKAGE_RELEASE" =~ ^[0-9]+$ ]]; then
PACKAGE_RELEASE=$((LAST_PACKAGE_RELEASE + 1))
log "build plan: commit=$SHA linux=$BUILD_LINUX opnsense=$BUILD_OPNSENSE version=$VERSION"
LINUX_OK=1
OPNSENSE_OK=1
if [[ "$BUILD_LINUX" == "1" ]]; then
if /opt/rust-package-builder/scripts/build-linux-release.sh; then
log "Linux branch completed for $SHA"
else
PACKAGE_RELEASE=1
LINUX_OK=0
log "ERROR: Linux branch failed for $SHA; OPNsense branch will still run"
fi
else
PACKAGE_RELEASE=$REQUESTED_PACKAGE_RELEASE
fi
export PACKAGE_NAME VERSION BINARY_NAME DESCRIPTION LICENSE HOMEPAGE MAINTAINER PACKAGE_RELEASE
log "building version $VERSION ($PACKAGE_NAME / $BINARY_NAME)"
if [[ "${RUN_LINUX_TESTS:-1}" == "1" ]]; then
log "cargo tests started"
if [[ -f Cargo.lock ]]; then
cargo test --release --locked
if [[ "$BUILD_OPNSENSE" == "1" ]]; then
if /opt/rust-package-builder/scripts/build-opnsense-release.sh; then
log "OPNsense branch completed for $SHA"
else
cargo test --release
OPNSENSE_OK=0
log "ERROR: OPNsense branch failed for $SHA"
fi
log "cargo tests completed"
fi
log "Linux cargo build started"
if [[ -f Cargo.lock ]]; then
cargo build --release --locked
else
cargo build --release
fi
log "Linux cargo build completed"
BINARY_PATH="$CARGO_TARGET_DIR/release/$BINARY_NAME"
[[ -x "$BINARY_PATH" ]] || fatal "built binary not found: $BINARY_PATH"
MAX_GLIBC_FOUND=$(objdump -T "$BINARY_PATH" 2>/dev/null | grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/^GLIBC_//' | sort -V | tail -n1 || true)
if [[ -n "$MAX_GLIBC_FOUND" ]]; then
if [[ "$(printf '%s\n%s\n' "$MAX_GLIBC_VERSION" "$MAX_GLIBC_FOUND" | sort -V | tail -n1)" != "$MAX_GLIBC_VERSION" ]]; then
fatal "binary requires GLIBC_$MAX_GLIBC_FOUND, target maximum is GLIBC_$MAX_GLIBC_VERSION"
fi
log "glibc compatibility check: <= $MAX_GLIBC_FOUND"
LINUX_STATE=""
OPNSENSE_STATE=""
[[ -r "$DATA_DIR/state/last_linux_commit" ]] && LINUX_STATE=$(cat "$DATA_DIR/state/last_linux_commit")
[[ -r "$DATA_DIR/state/last_opnsense_commit" ]] && OPNSENSE_STATE=$(cat "$DATA_DIR/state/last_opnsense_commit")
if [[ "$LINUX_STATE" == "$SHA" && "$OPNSENSE_STATE" == "$SHA" ]]; then
printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit"
log "all branches complete for $SHA"
fi
/opt/rust-package-builder/scripts/prepare-package.sh "$SRC" "$PKG" "$BINARY_PATH"
DEB_FILE="$OUT/${PACKAGE_NAME}_${VERSION}-${PACKAGE_RELEASE}_amd64.deb"
RPM_FILE="$OUT/${PACKAGE_NAME}-${VERSION}-${PACKAGE_RELEASE}.x86_64.rpm"
log "creating deb package"
nfpm package --config "$PKG/nfpm-deb.yaml" --packager deb --target "$DEB_FILE"
log "creating rpm package"
nfpm package --config "$PKG/nfpm-rpm.yaml" --packager rpm --target "$RPM_FILE"
ar t "$DEB_FILE" | grep '^_gpgorigin$' >/dev/null || fatal "DEB package does not contain nFPM GPG origin signature"
rpm -qpi "$RPM_FILE" | awk -F: '/^Signature/ && $0 !~ /\(none\)/ {ok=1} END{exit !ok}' || fatal "RPM package is not signed"
log "Linux package signatures verified"
# OPNsense is built from an isolated source copy. FreeBSD-specific patches never
# touch the Linux source checkout used above.
FREEBSD_BINARY="$OUT/rust-proxy-freebsd"
/opt/rust-package-builder/scripts/build-opnsense.sh "$SRC" "$FREEBSD_BINARY" "$BINARY_NAME" "$WORK"
if [[ "$SOURCE_REVISION" == "0" ]]; then
OPNSENSE_PACKAGE_VERSION="$VERSION"
else
OPNSENSE_PACKAGE_VERSION="${VERSION}_${SOURCE_REVISION}"
if [[ "$LINUX_OK" != "1" || "$OPNSENSE_OK" != "1" ]]; then
exit 1
fi
OPNSENSE_PACKAGE_FILE="$OUT/os-rustproxy-${OPNSENSE_PACKAGE_VERSION}.pkg"
/opt/rust-package-builder/scripts/package-opnsense.sh \
"$SRC" "$FREEBSD_BINARY" "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_VERSION"
# Only after all package formats have been built successfully do we update the
# persistent Linux package stores and the internal APT repository.
cp -f "$DEB_FILE" "$DATA_DIR/packages/deb/"
cp -f "$RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/"
log "updating aptly internal repository"
aptly -config="$APTLY_CONF" repo add -force-replace package-builder "$DEB_FILE" >/dev/null
SNAPSHOT="release-${VERSION}-${SHORT_SHA}-$(date +%s)"
aptly -config="$APTLY_CONF" snapshot create "$SNAPSHOT" from repo package-builder >/dev/null
APTLY_SIGN_ARGS=(-batch -gpg-key="$GPG_KEY_ID")
if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then
APTLY_SIGN_ARGS+=(-passphrase-file="$GPG_PASSPHRASE_FILE")
fi
if aptly -config="$APTLY_CONF" publish list -raw | awk -v d="$REPO_DISTRIBUTION" '$2 == d {found=1} END{exit !found}'; then
aptly -config="$APTLY_CONF" publish switch \
-force-overwrite \
"${APTLY_SIGN_ARGS[@]}" \
"$REPO_DISTRIBUTION" . "$SNAPSHOT" >/dev/null
else
aptly -config="$APTLY_CONF" publish snapshot \
-force-overwrite \
"${APTLY_SIGN_ARGS[@]}" \
-distribution="$REPO_DISTRIBUTION" \
-component="$REPO_COMPONENT" \
-architectures=amd64 \
"$SNAPSHOT" >/dev/null
fi
APT_PUBLIC="$DATA_DIR/aptly/public"
[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/InRelease" ]] || fatal "aptly did not create InRelease"
[[ -f "$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" ]] || fatal "aptly did not create Release.gpg"
gpg --batch --verify \
"$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release.gpg" \
"$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 || fatal "APT Release signature verification failed"
RELEASE_DIR="$REPOSITORY/releases/$BUILD_ID"
mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/opnsense"
CURRENT_TARGET=""
if [[ -L "$REPOSITORY/current" ]]; then
CURRENT_LINK=$(readlink "$REPOSITORY/current")
CURRENT_TARGET="$REPOSITORY/$CURRENT_LINK"
fi
log "staging apt repository"
if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/apt" ]]; then
rsync -a --delete --link-dest="$CURRENT_TARGET/apt"/ "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
else
rsync -a --delete "$APT_PUBLIC"/ "$RELEASE_DIR/apt"/
fi
log "updating rpm repository"
if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/rpm" ]]; then
rsync -a --delete --link-dest="$CURRENT_TARGET/rpm"/ "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
else
rsync -a --delete "$DATA_DIR/packages/rpm"/ "$RELEASE_DIR/rpm"/
fi
rm -rf "$RELEASE_DIR/rpm/x86_64/repodata"
createrepo_c --update "$RELEASE_DIR/rpm/x86_64" >/dev/null
GPG_SIGN_ARGS=(--batch --yes --pinentry-mode loopback)
if [[ -r "$GPG_PASSPHRASE_FILE" ]]; then
GPG_SIGN_ARGS+=(--passphrase-file "$GPG_PASSPHRASE_FILE")
fi
gpg "${GPG_SIGN_ARGS[@]}" --armor --detach-sign \
--local-user "$GPG_KEY_ID" \
--output "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \
"$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml"
gpg --batch --verify \
"$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \
"$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" >/dev/null 2>&1 || fatal "RPM metadata signature verification failed"
log "staging OPNsense pkg repository"
if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/opnsense" ]]; then
rsync -a --delete --link-dest="$CURRENT_TARGET/opnsense"/ "$CURRENT_TARGET/opnsense"/ "$RELEASE_DIR/opnsense"/
fi
OPNSENSE_PACKAGE_DIR="$RELEASE_DIR/opnsense/$OPNSENSE_ABI/amd64"
mkdir -p "$OPNSENSE_PACKAGE_DIR"
# The previous OPNsense snapshot is hard-linked into this one. Unlink every
# file we are about to replace so the currently published snapshot stays immutable.
rm -f \
"$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" \
"$RELEASE_DIR/opnsense/RustProxy.conf" \
"$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" \
"$RELEASE_DIR/opnsense/SHA256" \
"$RELEASE_DIR/opnsense/BUILD-INFO.txt" \
"$RELEASE_DIR/opnsense/index.html"
cp -f "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_DIR/"
/opt/rust-package-builder/scripts/generate-opnsense-repo.py \
"$RELEASE_DIR/opnsense" --retention "$OPNSENSE_RETENTION"
cat > "$RELEASE_DIR/opnsense/RustProxy.conf" <<REPOCONF
RustProxy: {
url: "${OPNSENSE_REPOSITORY_URL%/}",
enabled: yes,
mirror_type: "none",
signature_type: "none"
}
REPOCONF
cp -f "$OPNSENSE_PACKAGE_FILE" "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg"
(cd "$RELEASE_DIR/opnsense" && sha256sum "$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")") > "$RELEASE_DIR/opnsense/SHA256"
cat > "$RELEASE_DIR/opnsense/BUILD-INFO.txt" <<INFO
app_version=$VERSION
package_version=$OPNSENSE_PACKAGE_VERSION
source_url=$SOURCE_REPO
source_ref=$SOURCE_BRANCH
source_commit=$SHA
build_reason=$BUILD_REASON
source_revision=$SOURCE_REVISION
opnsense_series=$OPNSENSE_ABI
freebsd_release=$FREEBSD_RELEASE
freebsd_abi=$FREEBSD_ABI
freebsd_arch=$FREEBSD_ARCH
package=$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")
INFO
/opt/rust-package-builder/scripts/generate-opnsense-index.sh "$RELEASE_DIR/opnsense/index.html"
gpg --batch --armor --export "$GPG_KEY_ID" > "$RELEASE_DIR/repo-key.gpg"
[[ -s "$RELEASE_DIR/repo-key.gpg" ]] || fatal "public GPG key export failed"
/opt/rust-package-builder/scripts/generate-index.sh "$RELEASE_DIR/index.html" "$PACKAGE_NAME"
# Validate both public repositories before one atomic switch exposes either one.
[[ -s "$RELEASE_DIR/index.html" ]]
[[ -f "$RELEASE_DIR/apt/dists/$REPO_DISTRIBUTION/InRelease" ]]
[[ -f "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" ]]
[[ -s "$RELEASE_DIR/opnsense/index.html" ]]
[[ -f "$RELEASE_DIR/opnsense/meta.conf" ]]
[[ -f "$RELEASE_DIR/opnsense/packagesite.pkg" ]]
[[ -f "$RELEASE_DIR/opnsense/data.pkg" ]]
[[ -f "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" ]]
bsdtar -tf "$RELEASE_DIR/opnsense/packagesite.pkg" | grep -Fx 'packagesite.yaml' >/dev/null || fatal "invalid OPNsense packagesite.pkg"
bsdtar -xOf "$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" +COMPACT_MANIFEST >/dev/null || fatal "invalid OPNsense package manifest"
log "publishing Linux and OPNsense repositories atomically"
rm -f "$REPOSITORY/current.new"
ln -s "releases/$BUILD_ID" "$REPOSITORY/current.new"
mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current"
PUBLISHED=1
printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit"
printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version"
printf '%s\n' "$PACKAGE_RELEASE" | atomic_write "$DATA_DIR/state/last_package_release"
printf '%s\n' "$OPNSENSE_PACKAGE_VERSION" | atomic_write "$DATA_DIR/state/last_opnsense_package_version"
printf '%s\n' "$BUILD_ID" | atomic_write "$DATA_DIR/state/last_release"
log "build completed: reason=$BUILD_REASON version=$VERSION linux_release=$PACKAGE_RELEASE opnsense=$OPNSENSE_PACKAGE_VERSION commit=$SHA release=$BUILD_ID"
+51 -26
View File
@@ -2,24 +2,37 @@
set -Eeuo pipefail
source /opt/rust-package-builder/scripts/lib.sh
STATE_FILE="$DATA_DIR/state/last_commit"
REPOSITORY="$DATA_DIR/repository"
CURRENT="$REPOSITORY/current"
published_release_complete_for_commit() {
linux_release_complete_for_commit() {
local sha=$1
local current="$REPOSITORY/current"
local opnsense="$current/opnsense"
[[ -d "$CURRENT" ]] || return 1
[[ -f "$CURRENT/apt/dists/$REPO_DISTRIBUTION/InRelease" ]] || return 1
[[ -f "$CURRENT/rpm/x86_64/repodata/repomd.xml" ]] || return 1
if [[ -f "$CURRENT/LINUX-BUILD-INFO.txt" ]] && grep -Fqx "source_commit=$sha" "$CURRENT/LINUX-BUILD-INFO.txt"; then
return 0
fi
local state_sha=""
[[ -r "$DATA_DIR/state/last_linux_commit" ]] && state_sha=$(cat "$DATA_DIR/state/last_linux_commit")
[[ "$state_sha" == "$sha" ]] && return 0
# Migration from the older unified/Linux-only builder. Its last_commit
# represented the Linux publication before per-branch state existed.
local legacy_sha=""
[[ -r "$DATA_DIR/state/last_commit" ]] && legacy_sha=$(cat "$DATA_DIR/state/last_commit")
[[ "$legacy_sha" == "$sha" ]] || return 1
return 0
}
opnsense_release_complete_for_commit() {
local sha=$1
local opnsense="$CURRENT/opnsense"
local package_dir="$opnsense/$OPNSENSE_ABI/amd64"
[[ -d "$current" ]] || return 1
# Linux side of the atomic release.
[[ -f "$current/apt/dists/$REPO_DISTRIBUTION/InRelease" ]] || return 1
[[ -f "$current/rpm/x86_64/repodata/repomd.xml" ]] || return 1
# OPNsense side. BUILD-INFO ties the published pkg repository to the same
# source commit so an old Linux-only release cannot be mistaken for a
# complete unified release after migration.
[[ -d "$CURRENT" ]] || return 1
[[ -f "$opnsense/meta.conf" ]] || return 1
[[ -f "$opnsense/packagesite.pkg" ]] || return 1
[[ -f "$opnsense/data.pkg" ]] || return 1
@@ -30,7 +43,6 @@ published_release_complete_for_commit() {
grep -Fqx "opnsense_series=$OPNSENSE_ABI" "$opnsense/BUILD-INFO.txt" || return 1
[[ -d "$package_dir" ]] || return 1
find "$package_dir" -maxdepth 1 -type f -name 'os-rustproxy-*.pkg' -print -quit | grep -q . || return 1
return 0
}
@@ -38,19 +50,32 @@ log "checking repository"
REMOTE_SHA=$(git ls-remote "$SOURCE_REPO" "refs/heads/$SOURCE_BRANCH" | awk 'NR == 1 {print $1}')
[[ "$REMOTE_SHA" =~ ^[0-9a-fA-F]{40}$ ]] || fatal "cannot resolve branch $SOURCE_BRANCH from $SOURCE_REPO"
LAST_SHA=""
[[ -r "$STATE_FILE" ]] && LAST_SHA=$(cat "$STATE_FILE")
BUILD_LINUX=1
BUILD_OPNSENSE=1
if linux_release_complete_for_commit "$REMOTE_SHA"; then
BUILD_LINUX=0
# Persist migrated per-branch state when an older installation already has
# a valid Linux repository for this exact commit.
printf '%s\n' "$REMOTE_SHA" | atomic_write "$DATA_DIR/state/last_linux_commit"
fi
if opnsense_release_complete_for_commit "$REMOTE_SHA"; then
BUILD_OPNSENSE=0
printf '%s\n' "$REMOTE_SHA" | atomic_write "$DATA_DIR/state/last_opnsense_commit"
fi
if [[ "$REMOTE_SHA" == "$LAST_SHA" ]]; then
if published_release_complete_for_commit "$REMOTE_SHA"; then
log "no changes and unified release is complete: $REMOTE_SHA"
exit 0
fi
log "commit unchanged, but unified release is incomplete; rebuilding missing artifacts for $REMOTE_SHA"
/opt/rust-package-builder/scripts/build-release.sh "$REMOTE_SHA" repair-missing-artifacts
if [[ "$BUILD_LINUX" == "0" && "$BUILD_OPNSENSE" == "0" ]]; then
printf '%s\n' "$REMOTE_SHA" | atomic_write "$DATA_DIR/state/last_commit"
log "no changes: Linux and OPNsense are complete for $REMOTE_SHA"
exit 0
fi
log "new commit detected: $REMOTE_SHA"
/opt/rust-package-builder/scripts/build-release.sh "$REMOTE_SHA" new-commit
if [[ "$BUILD_LINUX" == "1" && "$BUILD_OPNSENSE" == "1" ]]; then
REASON="new-or-missing-both"
elif [[ "$BUILD_LINUX" == "1" ]]; then
REASON="repair-linux"
else
REASON="repair-opnsense"
fi
log "build required: commit=$REMOTE_SHA linux=$BUILD_LINUX opnsense=$BUILD_OPNSENSE reason=$REASON"
/opt/rust-package-builder/scripts/build-release.sh "$REMOTE_SHA" "$REASON" "$BUILD_LINUX" "$BUILD_OPNSENSE"
+1 -1
View File
@@ -80,7 +80,7 @@ fi
log "builder started: branch=$SOURCE_BRANCH interval=${POLL_INTERVAL}s opnsense=$OPNSENSE_ABI freebsd=$FREEBSD_RELEASE"
while true; do
if ! /opt/rust-package-builder/scripts/check-and-build.sh; then
log "ERROR: build cycle failed; published repositories were not switched"
log "ERROR: one or more build branches failed; any successful branch was published independently"
fi
sleep "$POLL_INTERVAL"
done
+2 -1
View File
@@ -34,7 +34,7 @@ if ! aptly -config="$APTLY_CONF" repo list -raw | grep -Fx package-builder >/dev
package-builder >/dev/null
fi
for item in apt rpm opnsense repo-key.gpg index.html; do
for item in apt rpm opnsense repo-key.gpg index.html LINUX-BUILD-INFO.txt; do
if [[ -e "$REPOSITORY/$item" && ! -L "$REPOSITORY/$item" ]]; then
fatal "$REPOSITORY/$item exists and is not a symlink; refusing to overwrite"
fi
@@ -45,3 +45,4 @@ ln -sfn current/rpm "$REPOSITORY/rpm"
ln -sfn current/opnsense "$REPOSITORY/opnsense"
ln -sfn current/repo-key.gpg "$REPOSITORY/repo-key.gpg"
ln -sfn current/index.html "$REPOSITORY/index.html"
ln -sfn current/LINUX-BUILD-INFO.txt "$REPOSITORY/LINUX-BUILD-INFO.txt"