From 86aebcfc05ebf8eba95b38639ceecb6d6493a241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Thu, 8 Oct 2026 22:22:11 +0200 Subject: [PATCH] update --- Dockerfile | 1 + README.md | 25 ++++++ .../src/etc/newsyslog.conf.d/rustproxy.conf | 10 +++ opnsense/plugin/src/etc/rc.d/rustproxy | 3 +- packaging/rustproxy.logrotate | 11 +++ scripts/build-linux-release.sh | 7 ++ scripts/build-opnsense-release.sh | 22 ++++- scripts/check-and-build.sh | 6 +- scripts/lib.sh | 3 + scripts/package-opnsense.sh | 5 +- scripts/prepare-package.sh | 12 +-- tests/log-rotation.sh | 84 +++++++++++++++++++ 12 files changed, 177 insertions(+), 12 deletions(-) create mode 100644 opnsense/plugin/src/etc/newsyslog.conf.d/rustproxy.conf create mode 100644 packaging/rustproxy.logrotate create mode 100755 tests/log-rotation.sh diff --git a/Dockerfile b/Dockerfile index 964aca0..b866935 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,6 +83,7 @@ ENV FREEBSD_RUST_TOOLCHAIN=${FREEBSD_RUST_TOOLCHAIN} \ ENV CARGO_HOME=/var/lib/package-builder/cargo COPY scripts/ /opt/rust-package-builder/scripts/ +COPY packaging/ /opt/rust-package-builder/packaging/ COPY opnsense/ /opt/rust-package-builder/opnsense/ RUN chmod 0755 /opt/rust-package-builder/scripts/*.sh /opt/rust-package-builder/scripts/*.py diff --git a/README.md b/README.md index cef34f8..cc08bde 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,31 @@ Kod pluginu OPNsense znajduje się w: opnsense/plugin/ ``` +## Rotacja logów RustProxy + +- **DEB/RPM:** builder zawsze umieszcza `packaging/rustproxy.logrotate` w `/etc/logrotate.d/rustproxy` (nFPM: `config|noreplace`). Logrotate jest zależnością obu pakietów. Reguła obejmuje `/var/log/rustproxy/*.jsonl`, `*.log` i `vhosts/*.jsonl`, z rotacją codzienną lub po przekroczeniu 100 MiB, 14 archiwami, gzip z opóźnioną kompresją, `copytruncate` oraz `su rustproxy rustproxy`. +- **OPNsense/FreeBSD PKG:** pakiet dostarcza `/usr/local/etc/newsyslog.conf.d/rustproxy.conf` dla tych samych ścieżek; `newsyslog` wykonuje rotację codziennie lub przy rozmiarze 100 MiB, przechowuje 14 archiwów, pomija puste pliki i używa gzip z niekompresowanym najnowszym archiwum. Plik `*.log` jest ponownie otwierany przez `daemon(8)` po `SIGHUP` (opcja `-H` w skrypcie rc.d). OPNsense uruchamia usługę jako `root`, dlatego `newsyslog` używa `root:wheel`. + +**Uwaga FreeBSD:** `newsyslog` działa przez zmianę nazwy pliku, nie ma `copytruncate`. Pliki JSONL i inne logi otwarte bezpośrednio przez aplikację muszą być przez nią ponownie otwierane po rotacji. W przeciwnym razie proces może pisać do poprzedniego (przeniesionego) pliku; zachowanie nie jest identyczne z Linux. Reguły są instalowane, ale na działającym OPNsense należy zweryfikować sposób otwierania plików przez RustProxy. + +Zmiana konfiguracji pakietów ustawia `PACKAGING_REVISION=2`. `check-and-build.sh` porównuje rewizję z metadanymi aktualnych repozytoriów; po podmianie buildera `./start.sh` przebudowuje obie gałęzie **nawet przy tym samym SHA źródła**. Automatyczny release DEB/RPM oraz rewizja PKG są zwiększane, aby menedżer pakietów mógł zainstalować nową wersję. Konfiguracja w `/etc/logrotate.d/rustproxy` jest oznaczona jako `noreplace`, więc lokalna wersja zmodyfikowana przez administratora może zostać zachowana (zależnie od menedżera pakietów). + +Kontrola po zainstalowaniu: + +```sh +# DEB / RPM (nie wykonuje rotacji) +logrotate -d /etc/logrotate.d/rustproxy + +# OPNsense (nie wykonuje rotacji) +newsyslog -n -v -f /usr/local/etc/newsyslog.conf.d/rustproxy.conf +``` + +Pakietowe testy integracyjne (lokalnie, bez budowy aplikacji): + +```sh +./tests/log-rotation.sh +``` + ## Dane ```text diff --git a/opnsense/plugin/src/etc/newsyslog.conf.d/rustproxy.conf b/opnsense/plugin/src/etc/newsyslog.conf.d/rustproxy.conf new file mode 100644 index 0000000..dc2d52b --- /dev/null +++ b/opnsense/plugin/src/etc/newsyslog.conf.d/rustproxy.conf @@ -0,0 +1,10 @@ +# FreeBSD/OPNsense newsyslog equivalent for RustProxy log retention. +# Fields: path owner:group mode count size when flags [pidfile signal] +# 14 archives; rotate daily or above 100 MiB; gzip, keep newest uncompressed. +# G=glob, B=no injected text (important for JSONL), E=skip empty, P=delay gzip. +# The supervised *.log output is reopened on SIGHUP via daemon(8) -H. +# JSONL files and other application-owned logs cannot use Linux copytruncate: +# their writers must close/reopen the files to avoid writing to old inodes. +/var/log/rustproxy/*.jsonl root:wheel 0640 14 100M $D0 GBEZPN +/var/log/rustproxy/*.log root:wheel 0640 14 100M $D0 GBEZP /var/run/rustproxy/daemon.pid SIGHUP +/var/log/rustproxy/vhosts/*.jsonl root:wheel 0640 14 100M $D0 GBEZPN diff --git a/opnsense/plugin/src/etc/rc.d/rustproxy b/opnsense/plugin/src/etc/rc.d/rustproxy index 3338e84..78b4694 100755 --- a/opnsense/plugin/src/etc/rc.d/rustproxy +++ b/opnsense/plugin/src/etc/rc.d/rustproxy @@ -163,7 +163,8 @@ rustproxy_daemon_user_arg="" if [ "$rustproxy_user" != "root" ]; then rustproxy_daemon_user_arg="-u ${rustproxy_user}" fi -command_args="-r -R ${rustproxy_restart_delay} -P ${pidfile} -p ${child_pidfile} ${rustproxy_daemon_user_arg} -o ${rustproxy_log_file} ${rustproxy_command} ${rustproxy_runtime_args}" +# -H allows daemon(8) to reopen the stdout/stderr log when newsyslog sends SIGHUP. +command_args="-r -R ${rustproxy_restart_delay} -H -P ${pidfile} -p ${child_pidfile} ${rustproxy_daemon_user_arg} -o ${rustproxy_log_file} ${rustproxy_command} ${rustproxy_runtime_args}" required_files="${rustproxy_command}" start_precmd="rustproxy_prestart" start_postcmd="rustproxy_poststart" diff --git a/packaging/rustproxy.logrotate b/packaging/rustproxy.logrotate new file mode 100644 index 0000000..99770dd --- /dev/null +++ b/packaging/rustproxy.logrotate @@ -0,0 +1,11 @@ +/var/log/rustproxy/*.jsonl /var/log/rustproxy/*.log /var/log/rustproxy/vhosts/*.jsonl { + daily + maxsize 512M + rotate 14 + missingok + notifempty + compress + delaycompress + copytruncate + su rustproxy rustproxy +} diff --git a/scripts/build-linux-release.sh b/scripts/build-linux-release.sh index 096f422..15f5ee1 100755 --- a/scripts/build-linux-release.sh +++ b/scripts/build-linux-release.sh @@ -32,8 +32,14 @@ if [[ "$REQUESTED_PACKAGE_RELEASE" == "auto" ]]; then [[ -r "$DATA_DIR/state/last_version" ]] && LAST_VERSION=$(cat "$DATA_DIR/state/last_version") [[ -r "$DATA_DIR/state/last_package_release" ]] && LAST_PACKAGE_RELEASE=$(cat "$DATA_DIR/state/last_package_release") + CURRENT_LINUX_PACKAGING_REVISION="" + if [[ -r "$REPOSITORY/current/LINUX-BUILD-INFO.txt" ]]; then + CURRENT_LINUX_PACKAGING_REVISION=$(sed -n 's/^packaging_revision=//p' "$REPOSITORY/current/LINUX-BUILD-INFO.txt" | head -n1) + fi + if [[ "$LAST_LINUX_COMMIT" == "$SHA" \ && "$LAST_VERSION" == "$VERSION" \ + && "$CURRENT_LINUX_PACKAGING_REVISION" == "$PACKAGING_REVISION" \ && "$LAST_PACKAGE_RELEASE" =~ ^[1-9][0-9]*$ ]]; then PACKAGE_RELEASE=$LAST_PACKAGE_RELEASE log "Linux repair build: reusing package release $PACKAGE_RELEASE" @@ -169,6 +175,7 @@ gpg --batch --armor --export "$GPG_KEY_ID" > "$RELEASE_DIR/repo-key.gpg" cat > "$RELEASE_DIR/LINUX-BUILD-INFO.txt" <= OPNSENSE_PACKAGE_REVISION )); then + OPNSENSE_PACKAGE_REVISION=$((LAST_OPNSENSE_REVISION + 1)) + fi +elif [[ "$LAST_OPNSENSE_PACKAGE_VERSION" == "$VERSION" && "$OPNSENSE_PACKAGE_REVISION" == "0" ]]; then + OPNSENSE_PACKAGE_REVISION=1 +fi +if [[ "$OPNSENSE_PACKAGE_REVISION" == "0" ]]; then OPNSENSE_PACKAGE_VERSION="$VERSION" else - OPNSENSE_PACKAGE_VERSION="${VERSION}_${SOURCE_REVISION}" + OPNSENSE_PACKAGE_VERSION="${VERSION}_${OPNSENSE_PACKAGE_REVISION}" fi OPNSENSE_PACKAGE_FILE="$OUT/os-rustproxy-${OPNSENSE_PACKAGE_VERSION}.pkg" /opt/rust-package-builder/scripts/package-opnsense.sh \ @@ -83,6 +100,7 @@ source_ref=$SOURCE_BRANCH source_commit=$SHA build_reason=$BUILD_REASON source_revision=$SOURCE_REVISION +packaging_revision=$PACKAGING_REVISION opnsense_series=$OPNSENSE_ABI freebsd_release=$FREEBSD_RELEASE freebsd_abi=$FREEBSD_ABI diff --git a/scripts/check-and-build.sh b/scripts/check-and-build.sh index 7695e86..3505e4f 100755 --- a/scripts/check-and-build.sh +++ b/scripts/check-and-build.sh @@ -10,8 +10,11 @@ linux_release_complete_for_commit() { [[ -d "$CURRENT" ]] || return 1 [[ -f "$CURRENT/apt/dists/$REPO_DISTRIBUTION/InRelease" ]] || return 1 [[ -f "$CURRENT/rpm/x86_64/repodata/repomd.xml" ]] || return 1 + # A new builder revision must repackage even when RustProxy's SHA is unchanged. + [[ -f "$CURRENT/LINUX-BUILD-INFO.txt" ]] || return 1 + grep -Fqx "packaging_revision=$PACKAGING_REVISION" "$CURRENT/LINUX-BUILD-INFO.txt" || return 1 - if [[ -f "$CURRENT/LINUX-BUILD-INFO.txt" ]] && grep -Fqx "source_commit=$sha" "$CURRENT/LINUX-BUILD-INFO.txt"; then + if grep -Fqx "source_commit=$sha" "$CURRENT/LINUX-BUILD-INFO.txt"; then return 0 fi @@ -41,6 +44,7 @@ opnsense_release_complete_for_commit() { [[ -f "$opnsense/os-rustproxy-latest.pkg" ]] || return 1 grep -Fqx "source_commit=$sha" "$opnsense/BUILD-INFO.txt" || return 1 grep -Fqx "opnsense_series=$OPNSENSE_ABI" "$opnsense/BUILD-INFO.txt" || return 1 + grep -Fqx "packaging_revision=$PACKAGING_REVISION" "$opnsense/BUILD-INFO.txt" || return 1 [[ -d "$package_dir" ]] || return 1 find "$package_dir" -maxdepth 1 -type f -name 'os-rustproxy-*.pkg' -print -quit | grep -q . || return 1 return 0 diff --git a/scripts/lib.sh b/scripts/lib.sh index 098621a..5cd279b 100755 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -Eeuo pipefail +# Increment when package payload/installation semantics change without a source commit. +PACKAGING_REVISION=2 + log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" } diff --git a/scripts/package-opnsense.sh b/scripts/package-opnsense.sh index 4e18c0e..8eaf2cd 100755 --- a/scripts/package-opnsense.sh +++ b/scripts/package-opnsense.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash set -Eeuo pipefail -source /opt/rust-package-builder/scripts/lib.sh +BUILDER_ROOT=${BUILDER_ROOT:-/opt/rust-package-builder} +source "$BUILDER_ROOT/scripts/lib.sh" SOURCE_DIR=${1:?source path required} BINARY=${2:?FreeBSD binary required} PACKAGE_FILE=${3:?output package file required} PACKAGE_VERSION=${4:?OPNsense package version required} -PLUGIN_ROOT=/opt/rust-package-builder/opnsense/plugin +PLUGIN_ROOT="$BUILDER_ROOT/opnsense/plugin" FREEBSD_ABI=${FREEBSD_ABI:-FreeBSD:15:amd64} FREEBSD_ARCH=${FREEBSD_ARCH:-freebsd:15:x86:64} diff --git a/scripts/prepare-package.sh b/scripts/prepare-package.sh index e0e0c0d..55a52aa 100755 --- a/scripts/prepare-package.sh +++ b/scripts/prepare-package.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -source /opt/rust-package-builder/scripts/lib.sh +BUILDER_ROOT=${BUILDER_ROOT:-/opt/rust-package-builder} +source "$BUILDER_ROOT/scripts/lib.sh" SRC=${1:?source path required} PKG=${2:?package workspace required} @@ -12,7 +13,7 @@ install -D -m 0755 "$BINARY_PATH" "$PKG/payload/usr/bin/$BINARY_NAME" SERVICE_SRC="$SRC/debian/${PACKAGE_NAME}.service" ENV_SRC="$SRC/debian/${PACKAGE_NAME}.environment" -LOGROTATE_SRC="$SRC/debian/${PACKAGE_NAME}.logrotate" +LOGROTATE_SRC="$BUILDER_ROOT/packaging/rustproxy.logrotate" TMPFILES_SRC="$SRC/debian/${PACKAGE_NAME}.tmpfiles" POSTINST_SRC="$SRC/debian/${PACKAGE_NAME}.postinst" @@ -71,9 +72,8 @@ fi if [[ -r "$ENV_SRC" ]]; then install -D -m 0640 "$ENV_SRC" "$PKG/payload/etc/${PACKAGE_NAME}/environment" fi -if [[ -r "$LOGROTATE_SRC" ]]; then - install -D -m 0644 "$LOGROTATE_SRC" "$PKG/payload/etc/logrotate.d/${PACKAGE_NAME}" -fi +[[ -r "$LOGROTATE_SRC" ]] || fatal "missing bundled logrotate configuration: $LOGROTATE_SRC" +install -D -m 0644 "$LOGROTATE_SRC" "$PKG/payload/etc/logrotate.d/${PACKAGE_NAME}" if [[ -r "$TMPFILES_SRC" ]]; then install -D -m 0644 "$TMPFILES_SRC" "$PKG/payload/usr/lib/tmpfiles.d/${PACKAGE_NAME}.conf" fi @@ -182,7 +182,7 @@ add_content() { add_content "$PKG/payload/usr/bin/$BINARY_NAME" "/usr/bin/$BINARY_NAME" 0755 add_content "$PKG/payload/usr/lib/systemd/system/${PACKAGE_NAME}.service" "/usr/lib/systemd/system/${PACKAGE_NAME}.service" 0644 [[ -f "$PKG/payload/etc/${PACKAGE_NAME}/environment" ]] && add_content "$PKG/payload/etc/${PACKAGE_NAME}/environment" "/etc/${PACKAGE_NAME}/environment" 0640 'config|noreplace' -[[ -f "$PKG/payload/etc/logrotate.d/${PACKAGE_NAME}" ]] && add_content "$PKG/payload/etc/logrotate.d/${PACKAGE_NAME}" "/etc/logrotate.d/${PACKAGE_NAME}" 0644 'config|noreplace' +add_content "$PKG/payload/etc/logrotate.d/${PACKAGE_NAME}" "/etc/logrotate.d/${PACKAGE_NAME}" 0644 'config|noreplace' [[ -f "$PKG/payload/usr/lib/tmpfiles.d/${PACKAGE_NAME}.conf" ]] && add_content "$PKG/payload/usr/lib/tmpfiles.d/${PACKAGE_NAME}.conf" "/usr/lib/tmpfiles.d/${PACKAGE_NAME}.conf" 0644 [[ -f "$PKG/payload/usr/share/${PACKAGE_NAME}/grafana/rustproxy-overview.json" ]] && add_content "$PKG/payload/usr/share/${PACKAGE_NAME}/grafana/rustproxy-overview.json" "/usr/share/${PACKAGE_NAME}/grafana/rustproxy-overview.json" 0644 [[ -f "$PKG/payload/usr/share/doc/${PACKAGE_NAME}/grafana/README.md" ]] && add_content "$PKG/payload/usr/share/doc/${PACKAGE_NAME}/grafana/README.md" "/usr/share/doc/${PACKAGE_NAME}/grafana/README.md" 0644 diff --git a/tests/log-rotation.sh b/tests/log-rotation.sh new file mode 100755 index 0000000..27ed6ef --- /dev/null +++ b/tests/log-rotation.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) + +# The bundled Linux rules must match the expected user-provided settings. +diff -u "$ROOT/packaging/rustproxy.logrotate" - <<'CONFIG' +/var/log/rustproxy/*.jsonl /var/log/rustproxy/*.log /var/log/rustproxy/vhosts/*.jsonl { + daily + maxsize 512M + rotate 14 + missingok + notifempty + compress + delaycompress + copytruncate + su rustproxy rustproxy +} +CONFIG + +# All three log groups must also be tracked by native FreeBSD newsyslog. +OPNSENSE_CONF="$ROOT/opnsense/plugin/src/etc/newsyslog.conf.d/rustproxy.conf" +for pattern in '/var/log/rustproxy/*.jsonl' '/var/log/rustproxy/*.log' '/var/log/rustproxy/vhosts/*.jsonl'; do + grep -Fq "$pattern " "$OPNSENSE_CONF" +done +[[ $(grep -c '^/var/log/rustproxy/' "$OPNSENSE_CONF") == 3 ]] +grep -Fq '14 100M $D0 GBEZPN' "$OPNSENSE_CONF" +grep -Fq '14 100M $D0 GBEZP /var/run/rustproxy/daemon.pid SIGHUP' "$OPNSENSE_CONF" +grep -Fq ' -H -P ${pidfile}' "$ROOT/opnsense/plugin/src/etc/rc.d/rustproxy" +grep -Fq 'COPY packaging/ /opt/rust-package-builder/packaging/' "$ROOT/Dockerfile" +grep -Fq 'packaging_revision=$PACKAGING_REVISION' "$ROOT/scripts/build-linux-release.sh" +grep -Fq 'packaging_revision=$PACKAGING_REVISION' "$ROOT/scripts/build-opnsense-release.sh" + +# Test staging and native PKG manifest without compiling Rust or signing RPM/DEB. +for tool in file zstd tar sha256sum python3; do + command -v "$tool" >/dev/null 2>&1 || { echo "SKIP packaging integration: missing $tool"; exit 0; } +done +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/source" "$WORK/linux" + +if command -v logrotate >/dev/null 2>&1; then + # Test parser compatibility even if the local test machine has no rustproxy user. + sed 's/su rustproxy rustproxy/su root root/' "$ROOT/packaging/rustproxy.logrotate" > "$WORK/logrotate-test.conf" + logrotate -d -s "$WORK/logrotate.state" "$WORK/logrotate-test.conf" > "$WORK/logrotate-output" 2>&1 + grep -Fq 'reading config file' "$WORK/logrotate-output" +fi +printf '[package]\nname = "rust-proxy"\nversion = "1.0.0"\n' > "$WORK/source/Cargo.toml" +python3 - "$WORK/bin" <<'PY' +from pathlib import Path +import struct, sys +p = Path(sys.argv[1]) +# ELF header with FreeBSD OSABI is enough to check the OPNsense packaging path. +e_ident = b'\x7fELF\x02\x01\x01\x09' + bytes(8) +header = struct.pack(' "$WORK/manifest.json" +python3 - "$WORK/manifest.json" <<'PY' +import json, sys +with open(sys.argv[1]) as f: + manifest=json.load(f) +path='/usr/local/etc/newsyslog.conf.d/rustproxy.conf' +assert path in manifest['files'], f'{path} missing from PKG manifest' +assert path in manifest['files'] and len(manifest['files'][path]) == 64 +assert '/usr/local/etc/rc.d/rustproxy' in manifest['files'] +PY +zstd -dc "$WORK/os-rustproxy.pkg" | tar -xOf - /usr/local/etc/newsyslog.conf.d/rustproxy.conf > "$WORK/packaged-newsyslog.conf" +cmp "$OPNSENSE_CONF" "$WORK/packaged-newsyslog.conf" +echo 'log rotation package integration: OK'