opn
This commit is contained in:
1 parent
cce1081f75
commit
2b8f11ef8c
36 files changed
+1868
-309
No files matched your search
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
source /opt/rust-package-builder/scripts/lib.sh
|
||||
|
||||
SOURCE_DIR=${1:?usage: apply-freebsd-patches.sh /path/to/source}
|
||||
PATCH_ROOT=/opt/rust-package-builder/opnsense/patches
|
||||
|
||||
[[ -f "$SOURCE_DIR/Cargo.toml" ]] || fatal "missing Cargo.toml in FreeBSD build source: $SOURCE_DIR"
|
||||
|
||||
VERSION=$(python3 - "$SOURCE_DIR/Cargo.toml" <<'PY'
|
||||
import sys, tomllib
|
||||
from pathlib import Path
|
||||
with Path(sys.argv[1]).open('rb') as f:
|
||||
print(tomllib.load(f)['package']['version'])
|
||||
PY
|
||||
)
|
||||
|
||||
PATCH_DIR="$PATCH_ROOT/$VERSION"
|
||||
if [[ ! -d "$PATCH_DIR" ]]; then
|
||||
log "no FreeBSD/OPNsense patch set for RustProxy $VERSION; using upstream source"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
shopt -s nullglob
|
||||
patches=("$PATCH_DIR"/*.patch)
|
||||
(( ${#patches[@]} > 0 )) || fatal "FreeBSD patch directory is empty: $PATCH_DIR"
|
||||
|
||||
log "applying FreeBSD/OPNsense patches for RustProxy $VERSION"
|
||||
for patch_file in "${patches[@]}"; do
|
||||
patch_name=$(basename "$patch_file")
|
||||
if git -C "$SOURCE_DIR" apply --check "$patch_file" 2>/dev/null; then
|
||||
git -C "$SOURCE_DIR" apply "$patch_file"
|
||||
log "applied patch: $patch_name"
|
||||
elif git -C "$SOURCE_DIR" apply --reverse --check "$patch_file" 2>/dev/null; then
|
||||
log "patch already present: $patch_name"
|
||||
else
|
||||
fatal "patch does not apply cleanly: $patch_name (RustProxy $VERSION)"
|
||||
fi
|
||||
done
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
source /opt/rust-package-builder/scripts/lib.sh
|
||||
|
||||
SOURCE_DIR=${1:?source path required}
|
||||
OUTPUT_BINARY=${2:?output binary required}
|
||||
BINARY_NAME=${3:?binary name required}
|
||||
WORK_DIR=${4:?work directory required}
|
||||
|
||||
BUILD_SOURCE="$WORK_DIR/freebsd-source"
|
||||
rm -rf "$BUILD_SOURCE"
|
||||
mkdir -p "$BUILD_SOURCE" "$(dirname "$OUTPUT_BINARY")"
|
||||
|
||||
rsync -a --delete \
|
||||
--exclude='.git/' \
|
||||
--exclude='target/' \
|
||||
"$SOURCE_DIR/" "$BUILD_SOURCE/"
|
||||
|
||||
if [[ "${APPLY_FREEBSD_PATCHES:-1}" == "1" ]]; then
|
||||
/opt/rust-package-builder/scripts/apply-freebsd-patches.sh "$BUILD_SOURCE"
|
||||
else
|
||||
log "FreeBSD/OPNsense source patches disabled"
|
||||
fi
|
||||
|
||||
FREEBSD_TARGET_VERSION=${FREEBSD_RELEASE%-RELEASE}
|
||||
export CFLAGS_x86_64_unknown_freebsd="--target=x86_64-unknown-freebsd${FREEBSD_TARGET_VERSION} --sysroot=/opt/freebsd-sysroot"
|
||||
export CXXFLAGS_x86_64_unknown_freebsd="--target=x86_64-unknown-freebsd${FREEBSD_TARGET_VERSION} --sysroot=/opt/freebsd-sysroot"
|
||||
export CARGO_TARGET_X86_64_UNKNOWN_FREEBSD_RUSTFLAGS="-C link-arg=--target=x86_64-unknown-freebsd${FREEBSD_TARGET_VERSION} -C link-arg=--sysroot=/opt/freebsd-sysroot -C link-arg=-fuse-ld=lld"
|
||||
|
||||
log "cross-compiling $BINARY_NAME for x86_64-unknown-freebsd${FREEBSD_TARGET_VERSION}"
|
||||
cd "$BUILD_SOURCE"
|
||||
if [[ -f Cargo.lock ]]; then
|
||||
cargo "+$FREEBSD_RUST_TOOLCHAIN" build --release --locked --target x86_64-unknown-freebsd
|
||||
else
|
||||
cargo "+$FREEBSD_RUST_TOOLCHAIN" build --release --target x86_64-unknown-freebsd
|
||||
fi
|
||||
|
||||
BUILT="$CARGO_TARGET_DIR/x86_64-unknown-freebsd/release/$BINARY_NAME"
|
||||
[[ -x "$BUILT" ]] || fatal "FreeBSD binary not found: $BUILT"
|
||||
install -m 0555 "$BUILT" "$OUTPUT_BINARY"
|
||||
|
||||
desc=$(file -b "$OUTPUT_BINARY")
|
||||
log "FreeBSD binary: $desc"
|
||||
[[ "$desc" == *FreeBSD* ]] || fatal "cross-built ELF is not identified as FreeBSD"
|
||||
|
||||
header=$(llvm-readelf -h "$OUTPUT_BINARY")
|
||||
printf '%s\n' "$header" | grep -qi FreeBSD || fatal "cross-built ELF header is not marked for FreeBSD"
|
||||
dynamic=$(llvm-readelf -d "$OUTPUT_BINARY" || true)
|
||||
if printf '%s\n' "$dynamic" | grep -q 'libc\.so\.6'; then
|
||||
fatal "Linux libc detected in FreeBSD target binary"
|
||||
fi
|
||||
@@ -11,8 +11,13 @@ PKG="$WORK/package"
|
||||
OUT="$WORK/out"
|
||||
REPOSITORY="$DATA_DIR/repository"
|
||||
APTLY_CONF="$DATA_DIR/state/aptly.conf"
|
||||
RELEASE_DIR=""
|
||||
PUBLISHED=0
|
||||
|
||||
cleanup() {
|
||||
if [[ "$PUBLISHED" != "1" && -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then
|
||||
rm -rf "$RELEASE_DIR"
|
||||
fi
|
||||
if [[ "${KEEP_WORKSPACE:-0}" != "1" ]]; then
|
||||
rm -rf "$WORK"
|
||||
else
|
||||
@@ -28,6 +33,7 @@ git clone --quiet --no-tags --single-branch --branch "$SOURCE_BRANCH" "$SOURCE_R
|
||||
git -C "$SRC" checkout --quiet --detach "$SHA"
|
||||
ACTUAL_SHA=$(git -C "$SRC" rev-parse HEAD)
|
||||
[[ "$ACTUAL_SHA" == "$SHA" ]] || fatal "checkout SHA mismatch"
|
||||
SOURCE_REVISION=$(git -C "$SRC" rev-list --count "$SHA")
|
||||
|
||||
cd "$SRC"
|
||||
META_JSON=$(cargo metadata --format-version 1 --no-deps)
|
||||
@@ -109,13 +115,24 @@ fi
|
||||
|
||||
export PACKAGE_NAME VERSION BINARY_NAME DESCRIPTION LICENSE HOMEPAGE MAINTAINER PACKAGE_RELEASE
|
||||
log "building version $VERSION ($PACKAGE_NAME / $BINARY_NAME)"
|
||||
log "cargo build started"
|
||||
|
||||
if [[ "${RUN_LINUX_TESTS:-1}" == "1" ]]; then
|
||||
log "cargo tests started"
|
||||
if [[ -f Cargo.lock ]]; then
|
||||
cargo test --release --locked
|
||||
else
|
||||
cargo test --release
|
||||
fi
|
||||
log "cargo tests completed"
|
||||
fi
|
||||
|
||||
log "Linux cargo build started"
|
||||
if [[ -f Cargo.lock ]]; then
|
||||
cargo build --release --locked
|
||||
else
|
||||
cargo build --release
|
||||
fi
|
||||
log "cargo build completed"
|
||||
log "Linux cargo build completed"
|
||||
|
||||
BINARY_PATH="$CARGO_TARGET_DIR/release/$BINARY_NAME"
|
||||
[[ -x "$BINARY_PATH" ]] || fatal "built binary not found: $BINARY_PATH"
|
||||
@@ -140,8 +157,24 @@ nfpm package --config "$PKG/nfpm-rpm.yaml" --packager rpm --target "$RPM_FILE"
|
||||
|
||||
ar t "$DEB_FILE" | grep '^_gpgorigin$' >/dev/null || fatal "DEB package does not contain nFPM GPG origin signature"
|
||||
rpm -qpi "$RPM_FILE" | awk -F: '/^Signature/ && $0 !~ /\(none\)/ {ok=1} END{exit !ok}' || fatal "RPM package is not signed"
|
||||
log "package signatures verified"
|
||||
log "Linux package signatures verified"
|
||||
|
||||
# OPNsense is built from an isolated source copy. FreeBSD-specific patches never
|
||||
# touch the Linux source checkout used above.
|
||||
FREEBSD_BINARY="$OUT/rust-proxy-freebsd"
|
||||
/opt/rust-package-builder/scripts/build-opnsense.sh "$SRC" "$FREEBSD_BINARY" "$BINARY_NAME" "$WORK"
|
||||
|
||||
if [[ "$SOURCE_REVISION" == "0" ]]; then
|
||||
OPNSENSE_PACKAGE_VERSION="$VERSION"
|
||||
else
|
||||
OPNSENSE_PACKAGE_VERSION="${VERSION}_${SOURCE_REVISION}"
|
||||
fi
|
||||
OPNSENSE_PACKAGE_FILE="$OUT/os-rustproxy-${OPNSENSE_PACKAGE_VERSION}.pkg"
|
||||
/opt/rust-package-builder/scripts/package-opnsense.sh \
|
||||
"$SRC" "$FREEBSD_BINARY" "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_VERSION"
|
||||
|
||||
# Only after all package formats have been built successfully do we update the
|
||||
# persistent Linux package stores and the internal APT repository.
|
||||
cp -f "$DEB_FILE" "$DATA_DIR/packages/deb/"
|
||||
cp -f "$RPM_FILE" "$DATA_DIR/packages/rpm/x86_64/"
|
||||
|
||||
@@ -178,7 +211,7 @@ gpg --batch --verify \
|
||||
"$APT_PUBLIC/dists/$REPO_DISTRIBUTION/Release" >/dev/null 2>&1 || fatal "APT Release signature verification failed"
|
||||
|
||||
RELEASE_DIR="$REPOSITORY/releases/$BUILD_ID"
|
||||
mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64"
|
||||
mkdir -p "$RELEASE_DIR/apt" "$RELEASE_DIR/rpm/x86_64" "$RELEASE_DIR/opnsense"
|
||||
|
||||
CURRENT_TARGET=""
|
||||
if [[ -L "$REPOSITORY/current" ]]; then
|
||||
@@ -214,24 +247,76 @@ gpg --batch --verify \
|
||||
"$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml.asc" \
|
||||
"$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" >/dev/null 2>&1 || fatal "RPM metadata signature verification failed"
|
||||
|
||||
log "staging OPNsense pkg repository"
|
||||
if [[ -n "$CURRENT_TARGET" && -d "$CURRENT_TARGET/opnsense" ]]; then
|
||||
rsync -a --delete --link-dest="$CURRENT_TARGET/opnsense"/ "$CURRENT_TARGET/opnsense"/ "$RELEASE_DIR/opnsense"/
|
||||
fi
|
||||
OPNSENSE_PACKAGE_DIR="$RELEASE_DIR/opnsense/$OPNSENSE_ABI/amd64"
|
||||
mkdir -p "$OPNSENSE_PACKAGE_DIR"
|
||||
# The previous OPNsense snapshot is hard-linked into this one. Unlink every
|
||||
# file we are about to replace so the currently published snapshot stays immutable.
|
||||
rm -f \
|
||||
"$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" \
|
||||
"$RELEASE_DIR/opnsense/RustProxy.conf" \
|
||||
"$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" \
|
||||
"$RELEASE_DIR/opnsense/SHA256" \
|
||||
"$RELEASE_DIR/opnsense/BUILD-INFO.txt" \
|
||||
"$RELEASE_DIR/opnsense/index.html"
|
||||
cp -f "$OPNSENSE_PACKAGE_FILE" "$OPNSENSE_PACKAGE_DIR/"
|
||||
/opt/rust-package-builder/scripts/generate-opnsense-repo.py \
|
||||
"$RELEASE_DIR/opnsense" --retention "$OPNSENSE_RETENTION"
|
||||
cat > "$RELEASE_DIR/opnsense/RustProxy.conf" <<REPOCONF
|
||||
RustProxy: {
|
||||
url: "${OPNSENSE_REPOSITORY_URL%/}",
|
||||
enabled: yes,
|
||||
mirror_type: "none",
|
||||
signature_type: "none"
|
||||
}
|
||||
REPOCONF
|
||||
cp -f "$OPNSENSE_PACKAGE_FILE" "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg"
|
||||
(cd "$RELEASE_DIR/opnsense" && sha256sum "$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")") > "$RELEASE_DIR/opnsense/SHA256"
|
||||
cat > "$RELEASE_DIR/opnsense/BUILD-INFO.txt" <<INFO
|
||||
app_version=$VERSION
|
||||
package_version=$OPNSENSE_PACKAGE_VERSION
|
||||
source_url=$SOURCE_REPO
|
||||
source_ref=$SOURCE_BRANCH
|
||||
source_commit=$SHA
|
||||
source_revision=$SOURCE_REVISION
|
||||
opnsense_series=$OPNSENSE_ABI
|
||||
freebsd_release=$FREEBSD_RELEASE
|
||||
freebsd_abi=$FREEBSD_ABI
|
||||
freebsd_arch=$FREEBSD_ARCH
|
||||
package=$OPNSENSE_ABI/amd64/$(basename "$OPNSENSE_PACKAGE_FILE")
|
||||
INFO
|
||||
/opt/rust-package-builder/scripts/generate-opnsense-index.sh "$RELEASE_DIR/opnsense/index.html"
|
||||
|
||||
gpg --batch --armor --export "$GPG_KEY_ID" > "$RELEASE_DIR/repo-key.gpg"
|
||||
[[ -s "$RELEASE_DIR/repo-key.gpg" ]] || fatal "public GPG key export failed"
|
||||
|
||||
/opt/rust-package-builder/scripts/generate-index.sh "$RELEASE_DIR/index.html" "$PACKAGE_NAME"
|
||||
|
||||
# Validate all externally visible endpoints before the atomic switch.
|
||||
# Validate both public repositories before one atomic switch exposes either one.
|
||||
[[ -s "$RELEASE_DIR/index.html" ]]
|
||||
[[ -f "$RELEASE_DIR/apt/dists/$REPO_DISTRIBUTION/InRelease" ]]
|
||||
[[ -f "$RELEASE_DIR/rpm/x86_64/repodata/repomd.xml" ]]
|
||||
[[ -s "$RELEASE_DIR/opnsense/index.html" ]]
|
||||
[[ -f "$RELEASE_DIR/opnsense/meta.conf" ]]
|
||||
[[ -f "$RELEASE_DIR/opnsense/packagesite.pkg" ]]
|
||||
[[ -f "$RELEASE_DIR/opnsense/data.pkg" ]]
|
||||
[[ -f "$RELEASE_DIR/opnsense/os-rustproxy-latest.pkg" ]]
|
||||
bsdtar -tf "$RELEASE_DIR/opnsense/packagesite.pkg" | grep -Fx 'packagesite.yaml' >/dev/null || fatal "invalid OPNsense packagesite.pkg"
|
||||
bsdtar -xOf "$OPNSENSE_PACKAGE_DIR/$(basename "$OPNSENSE_PACKAGE_FILE")" +COMPACT_MANIFEST >/dev/null || fatal "invalid OPNsense package manifest"
|
||||
|
||||
log "publishing repository atomically"
|
||||
log "publishing Linux and OPNsense repositories atomically"
|
||||
rm -f "$REPOSITORY/current.new"
|
||||
ln -s "releases/$BUILD_ID" "$REPOSITORY/current.new"
|
||||
mv -Tf "$REPOSITORY/current.new" "$REPOSITORY/current"
|
||||
PUBLISHED=1
|
||||
|
||||
printf '%s\n' "$SHA" | atomic_write "$DATA_DIR/state/last_commit"
|
||||
printf '%s\n' "$VERSION" | atomic_write "$DATA_DIR/state/last_version"
|
||||
printf '%s\n' "$PACKAGE_RELEASE" | atomic_write "$DATA_DIR/state/last_package_release"
|
||||
printf '%s\n' "$OPNSENSE_PACKAGE_VERSION" | atomic_write "$DATA_DIR/state/last_opnsense_package_version"
|
||||
printf '%s\n' "$BUILD_ID" | atomic_write "$DATA_DIR/state/last_release"
|
||||
|
||||
log "build completed: version=$VERSION package_release=$PACKAGE_RELEASE commit=$SHA release=$BUILD_ID"
|
||||
log "build completed: version=$VERSION linux_release=$PACKAGE_RELEASE opnsense=$OPNSENSE_PACKAGE_VERSION commit=$SHA release=$BUILD_ID"
|
||||
+24
-4
@@ -5,18 +5,33 @@ source /opt/rust-package-builder/scripts/lib.sh
|
||||
: "${SOURCE_REPO:?SOURCE_REPO is required}"
|
||||
SOURCE_BRANCH=${SOURCE_BRANCH:-master}
|
||||
: "${REPOSITORY_URL:?REPOSITORY_URL is required}"
|
||||
: "${OPNSENSE_REPOSITORY_URL:?OPNSENSE_REPOSITORY_URL is required}"
|
||||
POLL_INTERVAL=${POLL_INTERVAL:-300}
|
||||
PACKAGE_RELEASE=${PACKAGE_RELEASE:-auto}
|
||||
OPNSENSE_ABI=${OPNSENSE_ABI:-26.7}
|
||||
OPNSENSE_RETENTION=${OPNSENSE_RETENTION:-10}
|
||||
FREEBSD_RELEASE=${FREEBSD_RELEASE:-15.1-RELEASE}
|
||||
FREEBSD_ABI=${FREEBSD_ABI:-FreeBSD:15:amd64}
|
||||
FREEBSD_ARCH=${FREEBSD_ARCH:-freebsd:15:x86:64}
|
||||
FREEBSD_RUST_TOOLCHAIN=${FREEBSD_RUST_TOOLCHAIN:-1.88.0}
|
||||
APPLY_FREEBSD_PATCHES=${APPLY_FREEBSD_PATCHES:-1}
|
||||
RUN_LINUX_TESTS=${RUN_LINUX_TESTS:-1}
|
||||
|
||||
[[ "$POLL_INTERVAL" =~ ^[0-9]+$ ]] || fatal "POLL_INTERVAL must be an integer"
|
||||
(( POLL_INTERVAL >= 1 )) || fatal "POLL_INTERVAL must be >= 1"
|
||||
[[ "$PACKAGE_RELEASE" == "auto" || "$PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]] || fatal "PACKAGE_RELEASE must be auto or a positive integer"
|
||||
[[ "$OPNSENSE_RETENTION" =~ ^[0-9]+$ ]] || fatal "OPNSENSE_RETENTION must be >= 0"
|
||||
[[ "$APPLY_FREEBSD_PATCHES" =~ ^[01]$ ]] || fatal "APPLY_FREEBSD_PATCHES must be 0 or 1"
|
||||
[[ "$RUN_LINUX_TESTS" =~ ^[01]$ ]] || fatal "RUN_LINUX_TESTS must be 0 or 1"
|
||||
|
||||
export DATA_DIR=${DATA_DIR:-/var/lib/package-builder}
|
||||
export SOURCE_REPO SOURCE_BRANCH REPOSITORY_URL POLL_INTERVAL
|
||||
export SOURCE_REPO SOURCE_BRANCH REPOSITORY_URL OPNSENSE_REPOSITORY_URL POLL_INTERVAL
|
||||
export REPO_DISTRIBUTION=${REPO_DISTRIBUTION:-stable}
|
||||
export REPO_COMPONENT=${REPO_COMPONENT:-main}
|
||||
export PACKAGE_RELEASE
|
||||
export OPNSENSE_ABI OPNSENSE_RETENTION
|
||||
export FREEBSD_RELEASE FREEBSD_ABI FREEBSD_ARCH FREEBSD_RUST_TOOLCHAIN
|
||||
export APPLY_FREEBSD_PATCHES RUN_LINUX_TESTS
|
||||
export MAX_GLIBC_VERSION=${MAX_GLIBC_VERSION:-2.34}
|
||||
export GPG_PRIVATE_KEY_FILE=${GPG_PRIVATE_KEY_FILE:-/run/secrets/gpg/private-key.asc}
|
||||
export GPG_PASSPHRASE_FILE=${GPG_PASSPHRASE_FILE:-/run/secrets/gpg/passphrase}
|
||||
@@ -24,10 +39,15 @@ export GNUPGHOME=${GNUPGHOME:-/run/package-builder/gnupg}
|
||||
export CARGO_HOME="$DATA_DIR/cargo"
|
||||
export CARGO_TARGET_DIR="$DATA_DIR/cargo/target"
|
||||
|
||||
for cmd in git cargo rustc nfpm aptly createrepo_c gpg rsync python3 python3.11 objdump ar rpm; do
|
||||
for cmd in git cargo rustc rustup nfpm aptly createrepo_c gpg rsync python3 python3.11 \
|
||||
objdump ar rpm clang llvm-readelf llvm-ar llvm-ranlib file tar zstd bsdtar sha256sum; do
|
||||
require_cmd "$cmd"
|
||||
done
|
||||
|
||||
rustc "+$FREEBSD_RUST_TOOLCHAIN" --version >/dev/null 2>&1 || fatal "FreeBSD Rust toolchain is unavailable: $FREEBSD_RUST_TOOLCHAIN"
|
||||
rustup target list --toolchain "$FREEBSD_RUST_TOOLCHAIN" --installed | grep -Fx x86_64-unknown-freebsd >/dev/null \
|
||||
|| fatal "Rust target x86_64-unknown-freebsd is not installed for $FREEBSD_RUST_TOOLCHAIN"
|
||||
|
||||
mkdir -p \
|
||||
"$DATA_DIR/repository/releases" \
|
||||
"$DATA_DIR/state" \
|
||||
@@ -57,10 +77,10 @@ fi
|
||||
|
||||
/opt/rust-package-builder/scripts/init-storage.sh
|
||||
|
||||
log "builder started: branch=$SOURCE_BRANCH interval=${POLL_INTERVAL}s"
|
||||
log "builder started: branch=$SOURCE_BRANCH interval=${POLL_INTERVAL}s opnsense=$OPNSENSE_ABI freebsd=$FREEBSD_RELEASE"
|
||||
while true; do
|
||||
if ! /opt/rust-package-builder/scripts/check-and-build.sh; then
|
||||
log "ERROR: build cycle failed; published repository was not switched"
|
||||
log "ERROR: build cycle failed; published repositories were not switched"
|
||||
fi
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
+17
-10
@@ -6,7 +6,7 @@ OUT=${1:?output file required}
|
||||
PACKAGE=${2:?package name required}
|
||||
URL=$(trim_trailing_slash "$REPOSITORY_URL")
|
||||
|
||||
export OUT PACKAGE URL REPO_DISTRIBUTION REPO_COMPONENT
|
||||
export OUT PACKAGE URL OPNSENSE_REPOSITORY_URL REPO_DISTRIBUTION REPO_COMPONENT
|
||||
python3 <<'PY_END'
|
||||
import html, os
|
||||
out=os.environ['OUT']
|
||||
@@ -14,14 +14,15 @@ package=os.environ['PACKAGE']
|
||||
url=os.environ['URL']
|
||||
dist=os.environ['REPO_DISTRIBUTION']
|
||||
component=os.environ['REPO_COMPONENT']
|
||||
opn_url=os.environ['OPNSENSE_REPOSITORY_URL'].rstrip('/')
|
||||
h=html.escape
|
||||
content=f'''<!doctype html>
|
||||
<html lang="pl">
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<title>{h(package)} — repozytorium pakietów</title>
|
||||
<title>{h(package)} — package repository</title>
|
||||
<style>
|
||||
:root{{color-scheme:light dark;font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}}
|
||||
body{{max-width:920px;margin:0 auto;padding:42px 20px 70px;line-height:1.55}}
|
||||
@@ -34,9 +35,9 @@ a{{color:inherit}} .meta{{display:flex;gap:16px;flex-wrap:wrap;opacity:.82;font-
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>{h(package)} — repozytorium pakietów</h1>
|
||||
<p class="lead">Podpisane pakiety DEB i RPM dla x86_64/amd64.</p>
|
||||
<div class="meta"><span class="ok">Repozytorium aktywne</span><span>{h(url)}</span><span>APT + RPM/DNF</span></div>
|
||||
<h1>{h(package)} — package repository</h1>
|
||||
<p class="lead">Signed DEB and RPM packages for x86_64/amd64.</p>
|
||||
<div class="meta"><span class="ok">Repository active</span><span>{h(url)}</span><span>APT + RPM/DNF</span></div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Debian / Ubuntu</h2>
|
||||
@@ -68,17 +69,23 @@ sudo dnf install {h(package)}</code></pre>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Pliki repozytorium</h2>
|
||||
<h2>OPNsense / FreeBSD pkg</h2>
|
||||
<p>The OPNsense plugin is published separately because it uses the FreeBSD pkg format.</p>
|
||||
<p><a href="{h(opn_url)}/">{h(opn_url)}/</a></p>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<h2>Repository files</h2>
|
||||
<ul>
|
||||
<li><a href="{h(url)}/repo-key.gpg">Publiczny klucz GPG</a></li>
|
||||
<li><a href="{h(url)}/repo-key.gpg">Public GPG key</a></li>
|
||||
<li><a href="{h(url)}/apt/dists/{h(dist)}/InRelease">APT InRelease</a></li>
|
||||
<li><a href="{h(url)}/rpm/x86_64/repodata/repomd.xml">RPM repomd.xml</a></li>
|
||||
</ul>
|
||||
<p>Obsługiwane cele: Debian 12/13, Ubuntu 24.04/26.04, RHEL 9/10; architektura amd64/x86_64.</p>
|
||||
<p>Supported targets: Debian 12/13, Ubuntu 24.04/26.04, RHEL 9/10; architecture: amd64/x86_64.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open(out, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
PY_END
|
||||
PY_END
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
source /opt/rust-package-builder/scripts/lib.sh
|
||||
|
||||
OUT=${1:?output file required}
|
||||
URL=$(trim_trailing_slash "$OPNSENSE_REPOSITORY_URL")
|
||||
PACKAGE=${PACKAGE_NAME:-rustproxy}
|
||||
|
||||
export OUT URL PACKAGE OPNSENSE_ABI FREEBSD_ABI
|
||||
python3 <<'PY'
|
||||
import html
|
||||
import os
|
||||
|
||||
out = os.environ["OUT"]
|
||||
url = os.environ["URL"]
|
||||
package = os.environ["PACKAGE"]
|
||||
series = os.environ["OPNSENSE_ABI"]
|
||||
abi = os.environ["FREEBSD_ABI"]
|
||||
h = html.escape
|
||||
content = f'''<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>RustProxy OPNsense repository</title>
|
||||
<style>
|
||||
:root{{color-scheme:light dark;font-family:system-ui,-apple-system,"Segoe UI",sans-serif}}
|
||||
body{{max-width:900px;margin:0 auto;padding:42px 20px 70px;line-height:1.55}}
|
||||
.card{{border:1px solid #8885;border-radius:12px;padding:18px;margin:18px 0}}
|
||||
pre{{overflow:auto;padding:16px;border-radius:9px;background:#111;color:#eee}}
|
||||
code{{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>RustProxy OPNsense repository</h1>
|
||||
<p>FreeBSD pkg repository for the <code>os-rustproxy</code> OPNsense plugin.</p>
|
||||
<div class="card">
|
||||
<p><strong>Repository:</strong> {h(url)}</p>
|
||||
<p><strong>OPNsense series:</strong> {h(series)} <strong>ABI:</strong> {h(abi)}</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>pkg repository configuration</h2>
|
||||
<pre><code>mkdir -p /usr/local/etc/pkg/repos
|
||||
cat > /usr/local/etc/pkg/repos/RustProxy.conf <<'REPO'
|
||||
RustProxy: {{
|
||||
url: "{h(url)}",
|
||||
enabled: yes,
|
||||
mirror_type: "none",
|
||||
signature_type: "none"
|
||||
}}
|
||||
REPO
|
||||
|
||||
pkg update -f
|
||||
pkg install os-rustproxy</code></pre>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h2>Repository metadata</h2>
|
||||
<ul>
|
||||
<li><a href="{h(url)}/RustProxy.conf">RustProxy.conf</a> - ready pkg repo configuration</li>
|
||||
<li><a href="{h(url)}/meta.conf">meta.conf</a></li>
|
||||
<li><a href="{h(url)}/packagesite.pkg">packagesite.pkg</a></li>
|
||||
<li><a href="{h(url)}/data.pkg">data.pkg</a></li>
|
||||
<li><a href="{h(url)}/os-rustproxy-latest.pkg">latest package</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
with open(out, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
PY
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
CATALOG_FILES = {"data.pkg", "packagesite.pkg"}
|
||||
LATEST_NAMES = {"latest.pkg", "os-rustproxy-latest.pkg"}
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_manifest(pkg: Path) -> dict:
|
||||
for member in ("+COMPACT_MANIFEST", "+MANIFEST"):
|
||||
p = subprocess.run(
|
||||
["bsdtar", "-xOf", str(pkg), member],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
if p.returncode == 0 and p.stdout:
|
||||
return json.loads(p.stdout.decode("utf-8"))
|
||||
raise RuntimeError(f"{pkg.name}: missing +COMPACT_MANIFEST / +MANIFEST")
|
||||
|
||||
|
||||
def is_payload_pkg(path: Path) -> bool:
|
||||
return (
|
||||
path.is_file()
|
||||
and path.suffix == ".pkg"
|
||||
and path.name not in CATALOG_FILES
|
||||
and path.name not in LATEST_NAMES
|
||||
and not path.name.endswith(".tmp.pkg")
|
||||
)
|
||||
|
||||
|
||||
def collect_packages(repo: Path):
|
||||
packages = []
|
||||
for path in sorted(repo.rglob("*.pkg")):
|
||||
if not is_payload_pkg(path):
|
||||
continue
|
||||
manifest = read_manifest(path)
|
||||
manifest["path"] = path.relative_to(repo).as_posix()
|
||||
manifest["pkgsize"] = path.stat().st_size
|
||||
manifest["sum"] = sha256(path)
|
||||
packages.append((path, manifest))
|
||||
return packages
|
||||
|
||||
|
||||
def enforce_retention(packages, retention: int) -> bool:
|
||||
if retention <= 0:
|
||||
return False
|
||||
groups = defaultdict(list)
|
||||
for path, manifest in packages:
|
||||
# Keep the requested number per repository subdirectory/ABI and package name.
|
||||
key = (path.parent, manifest.get("name", path.stem))
|
||||
groups[key].append((path, manifest))
|
||||
|
||||
removed = False
|
||||
for (_parent, _name), entries in groups.items():
|
||||
entries.sort(key=lambda item: item[0].stat().st_mtime_ns, reverse=True)
|
||||
for path, _manifest in entries[retention:]:
|
||||
try:
|
||||
path.unlink()
|
||||
removed = True
|
||||
print(f"[opnsense-repo] retention: removed {path}", flush=True)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return removed
|
||||
|
||||
|
||||
def make_tar(dst: Path, member_name: str, payload: bytes):
|
||||
tmp = dst.with_name(dst.name + ".tmp")
|
||||
with tarfile.open(tmp, "w", format=tarfile.PAX_FORMAT) as tf:
|
||||
info = tarfile.TarInfo(member_name)
|
||||
info.size = len(payload)
|
||||
info.mode = 0o644
|
||||
info.mtime = int(time.time())
|
||||
tf.addfile(info, io.BytesIO(payload))
|
||||
os.replace(tmp, dst)
|
||||
|
||||
|
||||
def rebuild(repo: Path, retention: int):
|
||||
packages = collect_packages(repo)
|
||||
if enforce_retention(packages, retention):
|
||||
packages = collect_packages(repo)
|
||||
|
||||
manifests = [manifest for _, manifest in packages]
|
||||
packagesite = b"".join(
|
||||
json.dumps(m, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + b"\n"
|
||||
for m in manifests
|
||||
)
|
||||
data = json.dumps(
|
||||
{"groups": [], "expired_packages": [], "packages": manifests},
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
|
||||
make_tar(repo / "packagesite.pkg", "packagesite.yaml", packagesite)
|
||||
make_tar(repo / "data.pkg", "data", data)
|
||||
|
||||
meta = (
|
||||
'version: 1;\n'
|
||||
'packing_format: "tar";\n'
|
||||
'manifests: "packagesite.yaml";\n'
|
||||
'manifests_archive: "packagesite";\n'
|
||||
'data: "data";\n'
|
||||
'data_archive: "data";\n'
|
||||
'filesite: "files";\n'
|
||||
'filesite_archive: "files";\n'
|
||||
)
|
||||
tmp = repo / "meta.conf.tmp"
|
||||
tmp.write_text(meta, encoding="utf-8")
|
||||
os.replace(tmp, repo / "meta.conf")
|
||||
print(f"[opnsense-repo] rebuilt: {len(manifests)} package(s)", flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Build a static FreeBSD pkg repository catalogue")
|
||||
parser.add_argument("repo", type=Path, help="repository root")
|
||||
parser.add_argument("--retention", type=int, default=10, help="versions kept per package/ABI; 0 disables pruning")
|
||||
args = parser.parse_args()
|
||||
|
||||
repo = args.repo.resolve()
|
||||
repo.mkdir(parents=True, exist_ok=True)
|
||||
rebuild(repo, args.retention)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -34,7 +34,7 @@ if ! aptly -config="$APTLY_CONF" repo list -raw | grep -Fx package-builder >/dev
|
||||
package-builder >/dev/null
|
||||
fi
|
||||
|
||||
for item in apt rpm repo-key.gpg index.html; do
|
||||
for item in apt rpm opnsense repo-key.gpg index.html; do
|
||||
if [[ -e "$REPOSITORY/$item" && ! -L "$REPOSITORY/$item" ]]; then
|
||||
fatal "$REPOSITORY/$item exists and is not a symlink; refusing to overwrite"
|
||||
fi
|
||||
@@ -42,5 +42,6 @@ done
|
||||
|
||||
ln -sfn current/apt "$REPOSITORY/apt"
|
||||
ln -sfn current/rpm "$REPOSITORY/rpm"
|
||||
ln -sfn current/opnsense "$REPOSITORY/opnsense"
|
||||
ln -sfn current/repo-key.gpg "$REPOSITORY/repo-key.gpg"
|
||||
ln -sfn current/index.html "$REPOSITORY/index.html"
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
source /opt/rust-package-builder/scripts/lib.sh
|
||||
|
||||
SOURCE_DIR=${1:?source path required}
|
||||
BINARY=${2:?FreeBSD binary required}
|
||||
PACKAGE_FILE=${3:?output package file required}
|
||||
PACKAGE_VERSION=${4:?OPNsense package version required}
|
||||
|
||||
PLUGIN_ROOT=/opt/rust-package-builder/opnsense/plugin
|
||||
FREEBSD_ABI=${FREEBSD_ABI:-FreeBSD:15:amd64}
|
||||
FREEBSD_ARCH=${FREEBSD_ARCH:-freebsd:15:x86:64}
|
||||
|
||||
for tool in python3 sha256sum tar zstd file; do
|
||||
require_cmd "$tool"
|
||||
done
|
||||
|
||||
[[ -f "$SOURCE_DIR/Cargo.toml" ]] || fatal "missing Cargo.toml in $SOURCE_DIR"
|
||||
[[ -f "$BINARY" ]] || fatal "missing cross-built binary: $BINARY"
|
||||
[[ -d "$PLUGIN_ROOT/src" ]] || fatal "missing embedded OPNsense plugin source: $PLUGIN_ROOT/src"
|
||||
|
||||
binary_desc=$(file -b "$BINARY")
|
||||
[[ "$binary_desc" == *FreeBSD* ]] || fatal "refusing to package a non-FreeBSD binary: $binary_desc"
|
||||
|
||||
WORK=$(mktemp -d "${PACKAGE_FILE}.work.XXXXXX")
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
STAGE="$WORK/stage"
|
||||
META="$WORK/meta"
|
||||
ARCHIVE="$WORK/os-rustproxy.tar"
|
||||
PAYLOAD_LIST="$WORK/payload.list"
|
||||
mkdir -p "$STAGE/usr/local" "$META" "$(dirname "$PACKAGE_FILE")"
|
||||
|
||||
cp -a "$PLUGIN_ROOT/src/." "$STAGE/usr/local/"
|
||||
mkdir -p "$STAGE/usr/local/sbin"
|
||||
install -m 0555 "$BINARY" "$STAGE/usr/local/sbin/rust-proxy"
|
||||
|
||||
chmod 0555 "$STAGE/usr/local/etc/rc.d/rustproxy"
|
||||
chmod 0555 "$STAGE/usr/local/etc/rc.syshook.d/10-rustproxy.start"
|
||||
chmod 0555 "$STAGE/usr/local/opnsense/scripts/rustproxy/service.sh"
|
||||
|
||||
POST_INSTALL_FILE="$PLUGIN_ROOT/+POST_INSTALL.pre"
|
||||
PRE_DEINSTALL_FILE="$PLUGIN_ROOT/+PRE_DEINSTALL.pre"
|
||||
|
||||
python3 - "$STAGE" "$META" "$PACKAGE_VERSION" "$FREEBSD_ABI" "$FREEBSD_ARCH" \
|
||||
"$POST_INSTALL_FILE" "$PRE_DEINSTALL_FILE" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
stage = Path(sys.argv[1])
|
||||
meta_dir = Path(sys.argv[2])
|
||||
version = sys.argv[3]
|
||||
abi = sys.argv[4]
|
||||
arch = sys.argv[5]
|
||||
post_install = Path(sys.argv[6]).read_text(encoding="utf-8")
|
||||
pre_deinstall = Path(sys.argv[7]).read_text(encoding="utf-8")
|
||||
|
||||
files = {}
|
||||
flatsize = 0
|
||||
for p in sorted(stage.rglob("*")):
|
||||
if p.is_symlink():
|
||||
files["/" + p.relative_to(stage).as_posix()] = "-"
|
||||
elif p.is_file():
|
||||
files["/" + p.relative_to(stage).as_posix()] = hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
flatsize += p.stat().st_size
|
||||
|
||||
common = {
|
||||
"name": "os-rustproxy",
|
||||
"version": version,
|
||||
"origin": "opnsense/os-rustproxy",
|
||||
"comment": "RustProxy reverse proxy service",
|
||||
"desc": "RustProxy packaged as an OPNsense plugin with configurable HTTP, HTTPS/TLS and administration listeners.",
|
||||
"maintainer": "local@localhost",
|
||||
"www": "https://git.linuxiarz.pl/gru/RustProxy",
|
||||
"prefix": "/usr/local",
|
||||
"abi": abi,
|
||||
"arch": arch,
|
||||
"flatsize": flatsize,
|
||||
"licenselogic": "single",
|
||||
"licenses": ["UNKNOWN"],
|
||||
"categories": ["www"],
|
||||
"deps": {},
|
||||
"options": {},
|
||||
}
|
||||
manifest = dict(common)
|
||||
manifest["files"] = files
|
||||
manifest["scripts"] = {
|
||||
"post-install": post_install,
|
||||
"pre-deinstall": pre_deinstall,
|
||||
}
|
||||
|
||||
(meta_dir / "+COMPACT_MANIFEST").write_text(
|
||||
json.dumps(common, separators=(",", ":"), ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(meta_dir / "+MANIFEST").write_text(
|
||||
json.dumps(manifest, separators=(",", ":"), ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
|
||||
(
|
||||
cd "$STAGE"
|
||||
find . \( -type f -o -type l \) -print | LC_ALL=C sort | sed 's#^\./##' > "$PAYLOAD_LIST"
|
||||
)
|
||||
[[ -s "$PAYLOAD_LIST" ]] || fatal "OPNsense package payload is empty"
|
||||
|
||||
tar --format=ustar --owner=0 --group=0 --numeric-owner \
|
||||
-cf "$ARCHIVE" -C "$META" +COMPACT_MANIFEST +MANIFEST
|
||||
tar -P --format=ustar --owner=0 --group=0 --numeric-owner --no-recursion \
|
||||
--transform='s,^,/,S' \
|
||||
-rf "$ARCHIVE" -C "$STAGE" -T "$PAYLOAD_LIST"
|
||||
|
||||
python3 - "$ARCHIVE" "$META/+MANIFEST" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tarfile
|
||||
|
||||
archive = Path(sys.argv[1])
|
||||
manifest = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8"))
|
||||
expected = set(manifest.get("files", {}))
|
||||
with tarfile.open(archive, "r:") as tf:
|
||||
members = tf.getmembers()
|
||||
if [m.name for m in members[:2]] != ["+COMPACT_MANIFEST", "+MANIFEST"]:
|
||||
raise SystemExit("package metadata must be the first two tar members")
|
||||
payload = members[2:]
|
||||
if any(m.isdir() for m in payload):
|
||||
raise SystemExit("directory entries are not allowed in OPNsense package payload")
|
||||
actual = {m.name for m in payload if m.isfile() or m.issym() or m.islnk()}
|
||||
relative = sorted(name for name in actual if not name.startswith("/"))
|
||||
if relative:
|
||||
raise SystemExit(f"payload contains non-absolute paths: {relative[:5]}")
|
||||
missing = sorted(expected - actual)
|
||||
extra = sorted(actual - expected)
|
||||
if missing or extra:
|
||||
raise SystemExit(f"package payload does not match +MANIFEST; missing={missing[:5]} extra={extra[:5]}")
|
||||
PY
|
||||
|
||||
zstd -q -f -19 -T0 "$ARCHIVE" -o "$PACKAGE_FILE"
|
||||
zstd -dc "$PACKAGE_FILE" | tar -tf - >/dev/null
|
||||
log "created OPNsense package: $PACKAGE_FILE"
|
||||
Reference in new issue
Block a user