diff --git a/AGENTS.md b/AGENTS.md index a6a19a6..e74dc31 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Read README.md, ROADMAP.md and docs/TEST_REPORT.md before making changes. The project name is RustProxy; the Cargo package/binary remains `rust-proxy` for -compatibility. Current source version: 0.24.8. +compatibility. Current source version: 0.24.9. ## Delivery diff --git a/CHANGELOG.md b/CHANGELOG.md index b0a7e70..522c992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.24.9 + +- Harden the built-in release checker for FreeBSD/OPNsense and other Unix platforms: supplement bundled Mozilla roots with common system CA bundles/`SSL_CERT_FILE`/`CURL_CA_BUNDLE`, follow bounded non-downgrading HTTP(S) redirects, and report failed checks as normal application `ERROR` events without changing dashboard behavior. +- Add a central AES-256-GCM encrypted Secret Store with private local master-key storage and migration of existing ACME DNS-provider credentials. +- Add encrypted full backup/restore with Argon2id, authenticated encryption, explicit manifests/checksums, staging validation and rollback-safe activation. +- Add an optional ERROR-only JSON/RAW file sink with bounded asynchronous writes, central secret redaction and system `logrotate` integration. +- Extend source/release regressions and update release identity to 0.24.9. + ## 0.24.8 - Expand the overview Cache RAM diagnostics with HOT usage against limit, transient RAM history for total/fill/COLD queue/COLD I/O, and COLD storage occupancy against its disk budget. diff --git a/Cargo.lock b/Cargo.lock index 5114ba3..e57c247 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1688,7 +1688,7 @@ dependencies = [ [[package]] name = "rust-proxy" -version = "0.24.8" +version = "0.24.9" dependencies = [ "anyhow", "arc-swap", @@ -1716,6 +1716,7 @@ dependencies = [ "rand 0.8.8", "rcgen", "regex", + "ring", "rpassword", "rustls", "rustls-pemfile", diff --git a/Cargo.toml b/Cargo.toml index 75f3a6f..442e216 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rust-proxy" -version = "0.24.8" +version = "0.24.9" edition = "2021" rust-version = "1.88" description = "RustProxy - independent reverse proxy with local administration" @@ -41,6 +41,7 @@ url = "2.5" base64 = "0.22" sha1 = "0.10" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +ring = "0.17" tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } hyper-rustls = { version = "0.27", default-features = false, features = ["ring", "http1", "http2", "tls12"] } rustls-pemfile = "2.2" diff --git a/README.md b/README.md index 162d8f1..1fe1a67 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# RustProxy 0.24.8 +# RustProxy 0.24.9 Independent Rust reverse proxy with a dashboard, SQLite configuration, streaming HTTP/TLS/WebSocket forwarding, raw TCP/UDP stream hosts, IP-bound vhosts and backend pools. -**0.24.8 release source.** +**0.24.9 release source.** ## Layer-4 streams @@ -11,6 +11,13 @@ Raw TCP/UDP forwarding has its own **Streams** page in the primary navigation, a Named HTTP/HTTPS listener bind addresses use the same chip-style editor as vhost hostnames, including multi-address listeners. +## What changed in 0.24.9 + +- Add a central encrypted Secret Store backed by SQLite ciphertext and a private local master key, with migration of Cloudflare, OVH and RFC2136 ACME credentials away from persistent plaintext files. +- Add password-encrypted full backup/restore with Argon2id-derived backup keys, authenticated AES-256-GCM encryption, explicit manifests/checksums, private staging and rollback-safe data-directory activation. +- Add an optional ERROR-only file sink integrated with the existing tracing pipeline, supporting JSON/RAW output, bounded asynchronous writes, secret redaction and external `logrotate` rotation. +- Update package/source/UI/OpenAPI/release identity to 0.24.9. + ## What changed in 0.24.8 - Expand the overview **Cache RAM** block with live HOT RAM occupancy against its configured limit, transient cache RAM history split into fill / COLD queue / COLD I/O, and COLD storage occupancy against its disk budget. @@ -631,8 +638,14 @@ default used when bootstrapping listeners. The release checker reads its compile-time defaults from repository-root `UPDATE_CHECK.toml`, which is embedded in the final binary. It checks the configured `Cargo.toml` URL immediately on startup and then every 3600 seconds by default. A failed request is ignored until the next -interval; it never blocks startup or proxy traffic and it never installs updates. Operators may -override the embedded values with global CLI options or equivalent environment variables: +interval; it never blocks startup or proxy traffic and it never installs updates. On Unix-like +systems the checker keeps its bundled public roots and also imports certificates from +`SSL_CERT_FILE`, `CURL_CA_BUNDLE` and common Linux/BSD CA bundles, including the +FreeBSD/OPNsense `ca_root_nss` path. It follows at most five credential-free HTTP(S) +redirects and never downgrades an HTTPS check to HTTP. Failed checks stay invisible in the dashboard, preserving the existing footer +behavior, but are emitted as application `ERROR` events so the optional error-file sink can +capture them. Operators may override the embedded values with global CLI options or equivalent +environment variables: ```text --update-repository-url RUST_PROXY_UPDATE_REPOSITORY_URL @@ -766,7 +779,8 @@ old server and extract this archive into a separate, non-nested directory. From the new source directory: ```bash -python3 scripts/backup.py --data-dir /actual/RustProxy/data /private/backup-before-0.13.2 +rust-proxy --data-dir /actual/RustProxy/data backup create \ + --output /private/rustproxy-before-upgrade.rpb bash upgrade.sh /actual/RustProxy --stopped # Also pass --data-dir /actual/private/data when it is external. cd /actual/RustProxy @@ -774,6 +788,11 @@ bash dev.sh test bash dev.sh ``` +The supported operator backup is the password-encrypted `rust-proxy backup create` +command. `scripts/backup.py` is retained only as an offline release-test helper and +must not be used as a Secret Store backup. See +[Secret Store and encrypted backup/restore](docs/SECRET_STORE_BACKUP.md). + `.env`, data, unmanaged files and previous managed sources are preserved. Extra language JSON files survive the update; shipped files replace their old names, whose previous bytes remain in the backup. Unknown custom source edits are not diff --git a/ROADMAP.md b/ROADMAP.md index 82bc44a..46c3fef 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,15 @@ # RustProxy roadmap +## 0.24.9 + +Current direction: 0.24.8 -> 0.24.9 + +- [x] Add the encrypted Secret Store and migrate ACME Cloudflare/OVH/RFC2136 credentials away from persistent plaintext files. +- [x] Add password-encrypted full backup/restore with manifest validation, integrity checks, staging and rollback-safe activation. +- [x] Add the optional bounded asynchronous ERROR file sink with JSON/RAW formats, shared secret redaction and external logrotate integration. +- [x] Extend offline source/release contracts and update release identity to 0.24.9. +- [ ] Run compiler-enabled `./dev.sh fmt`, `./dev.sh check`, `./dev.sh test` and end-to-end backup/restore/error-log acceptance on the release host. + ## 0.24.8 Current direction: 0.24.7 -> 0.24.8 diff --git a/SHA256SUMS b/SHA256SUMS index 94c1c8b..798c419 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,19 +1,19 @@ 61bf40a0572889dc8b0dda9eae4e93e44b0de9364b3c798106c85700ffb37c23 .dockerignore 4140a2956264b44f259e6a37fc00ba02097bbe68b09296e04d3d243c1287129b .env.example 9e6356475bf587091bab72246ec9e325b498f7e48a6220e9a18baa50d0a385a6 .gitignore -726a8227e45ed845a25e4cc4294eecd6037038130ecf58be711e8782515ff7b1 AGENTS.md -83d25221547ac491f073aae7a453f87ea072c059f7d87b61e348518ddedb5f84 CHANGELOG.md -14cee4fb07ab7667942ca7672e004a644a9659eb6091c28eb6aaba0c5ea39704 Cargo.lock -3c4543cf1006eb775ef4eb24aefef10654bcd51992c42bf8718041b575344870 Cargo.toml +60d7b6e6599269c73bb901383fc979be2403d41dbea221ff3100def26161baa3 AGENTS.md +6b70a0382728f3cbdaf69f17d2681fa233252cf9be93e744d992531c4cdbbbb5 CHANGELOG.md +31147611b139a69c531fbdafcbe53e3331c7afbcfcab1167ebbca8b24b5bc4cb Cargo.lock +5928784bf7f69a9b21f2420d63a82986ae03a08998ceca6bae1900fda16804a2 Cargo.toml d6d22ba63f4bc3d16a13012ecaabff8a156be57c824d2f20f66976502fb27d1f Dockerfile -ef6f82076a16922120c2d41d934087f167667e8530d162752d87e23d11fa0332 README.md -f36cc5e3438efdf7776365d1341d977c9cabf0de3b705d3fb155992d3199db08 ROADMAP.md +48fc6b2362af7db0cb5a5b60a0883d4ec05ba26c1936685c1884651f5b8fca11 README.md +219fe5a950e382f785054f697fe68d9a9754ab8c5490567c52c555fc5a45732c ROADMAP.md da8311e3db945290666324a2f24bf4521b365272d7ac3f862ded1be993eea646 UPDATE_CHECK.toml a55fafd4c9fc17f8af591407e7f86d222ce06658c64dcffda92acbf420f969f1 build.rs -5e3988e42e74fc513a734ef43fd7d98c7a496586742f8b328a489987fb467e63 debian/README.md -582f2a48e14174073a1c3375fa96242c149c101092fd4217282f417a6a5dd1dc debian/build-package.sh +cd246a44d67a769039183b59da238784d946cdfb5df843ee3464bf1c7dedf0db debian/README.md +299ed18788309061dcc69b1ba56a7ca7a43bf9e16d31f3ab4c50414d1e612c0a debian/build-package.sh 0763a4bd8b429f7922b23a7276ff694f3a751b83fb127f6ad7ec654e5e507ecd debian/build-repository.sh -d2a02c72bf26674ea5257e339dc031df8bedbadd314a6c340cf33c12ac1a4dd4 debian/changelog +8f0f9ab5368f2dc6577ca5bf8de42375cc97630994f89ac0552bec8a5cbb695f debian/changelog c90457b19b1b2f7998d3db59a656d215eb0fab23769197405960457b66d95cf7 debian/control dff2ecb9c14623416981f444bc6df1c4e2bf55d330260f36a312d8d42245400f debian/rules c537d54fd92a9b97c5f6e6ec63bc1276cda324513e2946b341b2193f63b62e6a debian/rustproxy.environment @@ -24,44 +24,46 @@ d1c483406fb53674ed8db5f30de599b5a91fa1c6b28242eb70a64d6a736430bb debian/rustpro e8e18df40bcd69d6aa404282679186d9b8256ac63a5b4e80d7f081e8a0095a2e debian/source/format 57d737d73b828bf763b210c8ddea39d7a63224f8d509a518d0b876ba2e6a805a deploy/README.md 50329bd9d6980fb1c90a5cb29966b19a257905791d1d07d185d6bd4e3b0bcb1a deploy/rustproxy.service -4888e8e7ca28a799e41e0798e1ff20c5491d284e70209036579ed3077fcb1e15 dev.sh +3096d5956723ccdeb09ba0c032057a1d40045f1ea98e92669a7f96ca3f0be48d dev.sh 5f51efe621f45512a466fa7f2656867c6502a135ca040f4bfeec06a6919b39b6 docker-agent/README.md 5e4f565c3c9de4dba897af2e801ea646a83395e72eae4717a6d502dff2fcec10 docker-agent/docker-agent.env.example 9e9f3f17f38f9b0730f598f65988ce8ef562d2182ef6778fc7292b990e9cf22f docker-agent/go.mod 9d77c90767fa929ee8338734d72ce83284d3d965673270f62cb2b1613d587a33 docker-agent/install.sh -6b571206aa171bee1ef72e7ac461a3c627b8c91f15bfe02efa940935466167f8 docker-agent/main.go +4902616a3763de27568529da910be7cc9d544c92cb36929bae098d5991035143 docker-agent/main.go 3d4051a1f805d145df3877476e4b87773e74be8e123045065955a5133afdc68c docker-agent/main_test.go 634790f35ff4627df2a78cec2ed371f6e9ec9df00afb386075595fc7f3ed72eb docker-agent/rustproxy-docker-agent.service 0dbacca69143bb1e38d74270e24562a6749e387b26ef869643e876df6e0b1edb docs/ACME_PROVIDERS.md -888bbf7812e7283202f303680145c99b3f1a7a52fb3923fc3de1a6cbcfb72d00 docs/ACME_RFC2136.md +b794dc1ba8519c0fa77f30cb52a2cf85b1b28bfe7ae51dee27ba2ddeda07764a docs/ACME_RFC2136.md 3bfc98b55e7e905250c4ded38d18ca064c6006ae9232acfa5d48517d0da14f5b docs/API.md -a1c6fe8761caa655970d8546843237905416f54309009fb809d3c3fcacfaf0a6 docs/API_ACCESS.md +2253520f5dc5a365f4a69c38308ea4c8c58f65b16cc06e83e192754791de49a7 docs/API_ACCESS.md dde6a6a0b99ac34e65fe80e5eb8aebc2bc852bb97eb5c13a3c1ab538ef37f5a0 docs/ARCHITECTURE.md 2a1a6b8e5783ef4a25fc7933b4faedfde44557a827782a6a900d448163d81df7 docs/CACHE.md 0450813e3c875371d75cc41835f6362abb5300cd4bdf545f3011d9df8101170c docs/CACHE_DIAGNOSTICS.md db5534427ff0f004940fd6960906d3eaa75347a110e6322827c51c5d63f20adb docs/CONTAINER.md c822cd1d6a055d5ddec06d4f6077f0b13837dfb25ede1ec1f4e9a52e182db946 docs/CUSTOM_LOCATIONS.md 7db65a96f696aff1d7f74e2041cfc6ecdf437b1412ece90cee26da1ceff4cf7e docs/DOCKER_DISCOVERY.md +9a882f07945b207d032936feb843f29467e4654f8255fa0104d988a54987f982 docs/ERROR_LOGGING.md 061e1ccee4f3e70126a27ba8bdbac10e895fdb4e6e4607c5c6a6ab1d164baf02 docs/HTTP3.md -4a466a1431db9972f07b3f19e34584484741a61d5350e68b7293f73fe0206863 docs/INSTALL.md +4deafcc78fdad53cb345e195d4a514fe7a6cf3d0c5c65d3be5e61f0b86f38cec docs/INSTALL.md 0a124bd3eafaa9c6b3bc2b811fdc3e6295a6a2978766c37cc0735c27f4cf6d59 docs/INTEGRATIONS_040.md 2baaa1d34979c33caead9b271a7d235c673acb4e188c94917d975c25c179c242 docs/LIVE_LOGS.md 4e1504d1a343b94c2252545cbd5248af79f158c4566754dfe11f3c15a19ec0da docs/MANAGEMENT.md -8b7d8106f19543f5e8f801e9f7a35513eb0d02769528b1175943e914297843aa docs/MIGRATION.md +c62f78dad867a484ca1d1a7386d4f06800662c29ab6ed9863ac4d94a8133cd3e docs/MIGRATION.md 5f090a190000c6badab2a74e5d4968e70c34de43b59c1452f5b05503b5c5c18a docs/MOCK_AUTH.md 1f85fac0e06e432ef6f744f0b5f9c1b7a27a1e9e1ad30dc1a218eab96b1caffe docs/MOCK_SERVER.md e7516646dfb107d6240370f4e7460b1189f2eb21f398740381d5d102847bbfe6 docs/NOT_FOUND_HOSTS.md 3da047768f32064c5ef0ad7a4c73c4b14930fe7317c54f5650f426af9f484593 docs/NPM_MIGRATION.md 8538816f6b64780ca4f652d189c5d2c00e40b0c8fcd1ad77ae2e11d39058dce6 docs/PROMETHEUS.md -31395c2c76bd0dcf78e49aa8e19f3d732b8c9ecee71b3c8af6b697f18729e03f docs/README.md +eb9bc3e69786df4abacd51d4bfd01fc0759d048b8a1ab1395f71b1aafe112b6b docs/README.md 6e46a137b9c88dac8cb9e3fdca877430092392de9913d746278dbff8652460c2 docs/REDIRECTS.md 0dfd0d431c5aedc67abac1ce5de040ffa790ec9578fd7d6f91a987c333d73358 docs/REFERENCES.md ce5afb4f531bb5fb4c453deb49f643f2a563f32f7908c995e5bc8e0d7de995b7 docs/RENEWAL.md +22c5ac843705fa34b2ed331b42b20b2e5fc657fe2601e42b1663d3d4114f7090 docs/SECRET_STORE_BACKUP.md c6fa646500c0acac18bff6349cbb1d821680a849c8648b4eaa64f2a4305f9b9d docs/SECURITY.md 620b69e566d27af04f0604f4ca3f88cf95cdfea7fbc6ae9dd92446d2b2b5d9be docs/SECURITY_AUDIT.md fce52cc908fb86cab16072c0297b6cdaf4857e84185783bc96240d9a6bc321fd docs/SETTINGS.md 3611defd206cde107e75cac006159856d12cd8f15df7ae2eb9ec3592ed2b4abd docs/STREAMS.md -3f30e5f47f6fc1da02ed803cd5504d715513017568357514b3990caefa765b46 docs/TEST_REPORT.md +2aab477150cea7e66de021676016a0872d244f7c7dca6e211fe3a25ac98ff539 docs/TEST_REPORT.md 8ee6187a5460ca45643b7fd789e27bffd264449e3a92ff70e9359edee011bbbb docs/TRANSFORM.md b02957b708174c9def830ced3c70c943a47af7a33467ef858255d04ab92c4af0 docs/TRANSPORT.md 96a6816915d526f00e261846131ee3f942ef332af996cf0f73c79ec1ba9f9c7a docs/V0.21.0_AUDIT.md @@ -95,18 +97,19 @@ b26833cb4251f9390a44192ecc827ff7f052fd19016a5ccf7498c94503ab4ce3 examples/mock- 17e7971fa9d9b141019fba3135d87cb1aa0cbc55d56b0718f879f880b615da26 examples/vhosts-balanced.json e06905b6363103b1bcee1a09b9e02292033f114d88f3edbea22ddab53e9e5071 examples/vhosts-private.json 529b47c6d4ea44d2a25d1cf9561cb5ee1e05f50a568c6dba5889741bcf936434 grafana/README.md -9fc541805900696d97251b8a6e38ffd4a6553d456862715f3a975eb1ce9fd9f5 grafana/rustproxy-overview.json +17f659f505bd6f06c98ac3778ddf5ad23f7326b99c633289205fd7620c14a0c8 grafana/rustproxy-overview.json f45e5cc3d0c253534978d4b752b0667e2a7906ccf5a22106400e0f6b70b00d23 install-debian13.sh 858b39fa6886b71d20690bf369e76580922364b3cc7f0cf6ba9b040df9cd76e3 install.sh 29cada0936762f26495064d1524a668262ed729b21f20aebddc8b6e5cfea363b langs/README.md -f735129fe2cf9af251467b06c8d003c717c34d80ce454a53b084a68b9df38056 langs/en_US.json -ee433b1e6f54e0b31bbd6e5502a38f36c6fc2f3b826fc8d4bdfe114e4b64770c langs/pl_pl.json -1f916b213663e8d5c7fb0ba759d8d803aa6bd58eaa1c63c367ee9ea9a154e2d6 make_zip.py +edcbe1fb0385038f94ad58e55923dd32e8c71073f526f7af182abc59dddfe8c0 langs/en_US.json +e99a9f1744a45e946c825814e9ba4c75288309a83f5cc56100c3f2c1ba6f4134 langs/pl_pl.json +bd1aedf3a88329cee660faaebac82a2d74e32fccaf92d4529799ab6ec7abe321 make_zip.py e830d37dd1f81a4b0aa7d535f5bba69e34e433b0fd99cc8ee0b296786c32379a migrations/0002_admin.sql 20112cc82192098b8d602f1c4f7ca08712e9e1366c845fabb92753187c370d35 migrations/0003_certificate_renewals.sql 5c62da30fa9db594a4d656c77ae676d3c388072b6a85f91170ec7d911d7cde43 migrations/0004_admin_preferences.sql 57dc28982f416874d5d00d1d438a18846d1ce35cb940c4bcf67fd9a8bc57339c migrations/0005_certificate_api.sql faa7e50922bf180c363f7627fbcfe8079831e36282220d17481cf69bc64b57f1 migrations/0006_certificate_api_credentials.sql +f093aef9639a959a2522946995a02d2e67c464e794c3ef665261a9a5b348c968 migrations/0007_secret_store.sql 478220552ffeea3de00e2866d427d9f87f4cfc4ae5232e5c3a774e62425b1e0c nagios/README.md 91233910aa4dab07ae917fef93d018a7a0beaab697f9a0bb889ac613e595cca4 nagios/check_rustproxy.py a42e01b341725c9da0928edf40c8c3b9c814ef90592f208c52b1f2c56a2ad383 nagios/check_rustproxy_api @@ -114,12 +117,12 @@ ee50f38c037765fab742e888d4b9fc4d874c31c785c6064d4eca3d2c4d24d944 nagios/check_r 1465396cf1358067e9fe48b3d61f28fd4ac65c577b67c502950ce6b42ecbe3b3 nagios/check_rustproxy_latency 6e036f736a2b40feb78eec17bf34871771524190f283591d1c89e48e6d2271e6 nagios/check_rustproxy_queues 7458b2d74495e6fb1c1edeea4d5a793705476431dadb60627187111f91441e13 rust-toolchain.toml -4c0a0a2131c5144ed97228db89ad0b78f08b93dc24b373bd0dc49ca137ad8324 scripts/acme_provider.py +edb6ee1445b01cd6f3a74a3a607f39fb107d411b49df028d2e0f4679e093de31 scripts/acme_provider.py e69813fb1d3a4cdcfb7243964cc323fc21a603087bf2e84188fa42b9d3cee4b5 scripts/acme_rfc2136.py ea2c670aa1fc2c145fb3666711642d526c2adb01537fc3b6398664e33136f0e4 scripts/admin_audit.py -761aadaffbfb08c32afca28af7e5379f933712563b00dddab7261a9505cb7d5b scripts/backup.py -0e7da27440c873af3007ceff83410f68ec16880217c9218fc1f13a9232e92978 scripts/check_source.py -47e01bdc0ebf659b956dda053073767045b12823588b83cb2dd9d49167559f64 scripts/deploy.py +20e83f8b6347b819d1d1ea3de5c691c36f0816954a4210b0d3f39557fefe1931 scripts/backup.py +b99f7bb40c05530b74948bd02fe9552b0c3c34767ac1c061fa4fb28c032a19d7 scripts/check_source.py +e19eba9a8b8a0a8eb26b1e70a78fdfea2661be3a32b3f769fe8129dc81b7aaf5 scripts/deploy.py 6facb2d65bfab234f09855097d12da8f72669a38ebceb4b1cbd03ab42b8d2469 scripts/langs_check.py 9f89ccdcd31da0f203113051a109ada8e51b44f91382ee2c5c75228d1e456540 scripts/memory_contracts.py 60504dd426195fca88d77e075d607c04540dbe8aae2cd806077b3818e0e0cf08 scripts/migrate_npm.py @@ -127,26 +130,26 @@ d1418f8e574a7390fd45a2aa879bdf5720b773162d65e295ab0350506aa5220a scripts/mock_a 2d3ff92300b09ce9cecbbaaa9309b4d33160fa4b2291c9952b784e8cfb599ab5 scripts/mock_server.py 2d4c859aae5ba357a6d30024c8487ae8f59f01b192711f13f9c2f8fee0f48b50 scripts/os-acme-client/README.md c73b814a42bb4ff703441dbd177ee625055eeb115f9b5bb0c2150aede573a159 scripts/os-acme-client/opnsense-rustproxy-cert-deploy.sh -5f8d0a41ea0eb97a063274d0c3ea72af218dc27b94e522e759ab1eaa33130756 scripts/preflight.py -38bddd454b6869822180c0134b410f81e0ba509b57992a7fcb43d2fb4105d9de scripts/release_smoke.py +a2b9cf7f003111f6b71adf2fe79853bb48d91a3eef53fb744e6c5ba8e6171eed scripts/preflight.py +ed4a944f5c25a9cf6e2be014e455fc8fd2167cd445696c869ef8acebf66983ef scripts/release_smoke.py c37847ae9663197d306ea55eadbe4a1892d61d031ced923a8eeffc1ae876bb4b scripts/routeros-oci-build/.env 45be151cd85fc2ef2f1fdc3003ace63381af262e73af8a685ab48cb697cc0de1 scripts/routeros-oci-build/.env.routeros.example 3b48cff8506e5a6ae7d3876f47721f5de7d63e2e02264a78cf5b6ddd52d9fbc6 scripts/routeros-oci-build/Dockerfile.routeros 7c7a3e51f43b43e1c962a4b0bb591f99c1f06977bf77f23719f7f867dfe7eb8e scripts/routeros-oci-build/Dockerfile.routeros.dockerignore -086148713830fe7c2cf24453d26b07e0ecf28d7bdef16ed1a9379a61ec58c09e scripts/routeros-oci-build/README.md +4578227b1734ad31f40837232c3d3c4c5923a896e05c426e887c37f323686488 scripts/routeros-oci-build/README.md d53b2ae7315ac143bd17d6327611a39e7c23666408bb95e39da2ca8517322ccb scripts/routeros-oci-build/build-routeros-oci.sh 3f267cafa16006e5456ade78ee2d5fae02cc7ce747839523068ffd8d7c98798a scripts/routeros-oci-build/docker-entrypoint.sh -9b8146e4fd56a8b68e6eae076382aa3027510296083c843361d922b88f8926f5 scripts/swagger_assets.py +c9be0bb829f45220653cf320b7466a567dd0745f511880eb6c27a3552c52e08b scripts/swagger_assets.py 102b23de19a88a8e60de392fb83aefd3dfb33e90cc97ed1690d9f3d4a33d8866 scripts/test_compression.py -38f88059efee4e7c495bddf05240e4ebf60978eb8b7b720fdb3130406a3a7eb0 scripts/upgrade.py +94b96941648424f44c86d3e7c6225c9afc1d91a13e2755b154591a2b9e950013 scripts/upgrade.py 081aad397b2b9485e20b684dc11682d3a16464c5b4f110cd461f8518eb74ad99 src/acl.rs -2d1ffc1c64e76c647abd3134719fb561b1f27d584c92d867011d8de7dd7f6662 src/acme.rs +c7e6874ba7bfc64d5e5071386fa8ee58c338740ea7bd1c024becd25ba2c77571 src/acme.rs fdba60fa997aa18cb0a7ef50e286a8b6fd877d14fe829d4b0bcb5c7c269a5ae6 src/admin/documentation.rs -3299668f6792805b648536357f90ddf059ebbb7569440ee766d8d130aaad7552 src/admin/mod.rs -0105c0cd9e682928aaabf03cca5c928f5fe3baa759783a3043d412f1800e41d8 src/admin/openapi.json +9457c2e3557df929da1118daab792373fffeb10ac5b71acf294bae5a2ed0f6e2 src/admin/mod.rs +358b21548909037c016356e9e2a7104fb175bdf7cd1ccf45cb0d8f5dc2c3b4c6 src/admin/openapi.json b4fa393b780b2676b5d72e0f90cd7290292f424b8062d4b5983e722d6ca72c32 src/admin/public.rs 23c55f23deeec844a9abc2289440c9e0583aaeb72416fc5f3632a43a864c0efd src/admin/web/acme.js -ccf01bfe2ca2c8fcc9c67629abf86881c81d1169284be99dfad47d774b3fac2a src/admin/web/app.js +70ce2072b5216cfe8e556343273714516bf503e7dcd53fae9b32fbff37008a2c src/admin/web/app.js d0be71555de13f732fd6ee44438a0110eb08c91e78bbc1aa36a1a84182780210 src/admin/web/cache-panel.js 50d1ebcbaf4d0b2968da23bb943fc3f67131c5d32df3f61d23ff37aad9da5b81 src/admin/web/demo.html 3b84a7a08f9f68e2c78fed4523b594c31a251addc5480a69c5f6949908230539 src/admin/web/docker.js @@ -161,7 +164,7 @@ f7d0955ab18c15e04369204e789f341d2e832f927c1ff20a3b2b9e35e7898f00 src/admin/web/ e6c4d91d1542d0a5b47bb8736f9539d43a6713df5a473b635519d5b7eac2eb9c src/admin/web/error_504.html 0a5fff35555153f59d4346933030a67b6e1a8768cee7a47cfa2fcf1a2c564f63 src/admin/web/favicon.svg dee3b113907f77138cdf919efdfb097e45b889cff3893bfc6f02ed998643bbae src/admin/web/host404.js -ea084ad0d95ff137d4cc277a810f958b47b7d5d7e426c2066086448d36aad700 src/admin/web/index.html +e8490ac3eaf5cd59f063f7a194e38bbc4cfcf89f7ee7c900fbbac4931f642558 src/admin/web/index.html 24db88842707f764064657712d3660c6ac129fbfb47d188c0903925cf8244ce5 src/admin/web/integrations.js cdab0c11a28deebdf401c0453fbc76b675e9e643722106fb20e812bb9648ccde src/admin/web/login-keys.json a253d202dc9a147f2add34c64e0f35c38fdc7f667f9908933ce5f959b1a92a2e src/admin/web/login.html @@ -172,10 +175,11 @@ dafc8b023f334091c7b5a04920038bce46ea3ef4b068e1f88cffe49d44eaeeee src/admin/web/ 1a30affd6a717095a1763998a5850a79ae0f86b7b694806f2aba2b880735dd1a src/admin/web/streams.js aad4a2f55f750c26ab98f125a8f7712b365e6c993cf4ee9a3636d05a6a0b41b6 src/admin/web/style.css 049be02e0f68c3f6702f3bf29be760c4e5bc1199393ed198107c0a264711300d src/admin/web/transform-panel.js -2b116e3b45a39078f52a0fd4cae3dd06bc61a111c312981716644f24e1314325 src/admin/web/vhosts.js +64f14f2be913d9c01efec7a3c00dfd081daf5f5a1d76a48ac9c40d80d0409215 src/admin/web/vhosts.js 1e5c6c6157f8b66d90751a0480cb90c6fdfa229330fc4c3f3504bc0c689c3d60 src/auth/files.rs 606f5491f94da007881a679051677a4609881d3151ac753de02556e0e76e1ce8 src/auth/mod.rs 1ceb57d97cbff655f185e126d98b8dc7884c7aac865e1f70e3c6d199685c13a7 src/auth/store.rs +b1950a91bf4e2ff0474000486c212a6390b7696f3f559b93aaedf96614408c9a src/backup.rs 168f55f75ad6ebd80f98c3376f5885cdc8d313f8b03f4e93066a28cf7b8c8b81 src/balance.rs e2bf9d4f8a8c8c7f9a8df96872a657b9706d05f3f3a83721fe912ca0af37b29a src/basic_auth.rs 9a2880276819a62fef40c4c926be688e38b7c0eb551995d5653c119ffced0db3 src/cache/cold.rs @@ -192,8 +196,8 @@ cae9d35be0e31b5f942e3d3b20e7496581ec6491d7974509ece4cf540271b1f5 src/cache/poli b20e2643084ac7f72f80c7f8fb8cf7ac4e16ee3d83d2f73767fcbb3e0ffe3cf8 src/cache/tests.rs bc7f52d55c707fbb409306eadd9aa31115e823590f81b24f68a45264a6f6c86e src/cache/tickets.rs 5d1788e6da0669546729e7bea56c64f3b989d399793e67d34fdbf427a6e0c65b src/compression.rs -09279a1f5e6e97ca6f856081bc1e710ae0b57b708bc6a62157741a5ace2fdfbb src/config.rs -a05a72ce53bebf5a060c55d0259bbef21dcd1b96808b4934852edc79486ad2ad src/db.rs +fed36acb2a8b44892061c83a23bab8028b7c26f84181be0382f6e192a5cbfb7b src/config.rs +ed08b39fad6e7f6a536f8fa7d09221209f99a92a6735d6d4ee9d3ddfa6503091 src/db.rs 19ab6c3ae37009c544291cb0b65c1bb9c721e168f95c1ed3fa88155aca7463ad src/demo.rs f93f0b1fdb894b29a271fc6f88ccc0afbd5c847ba3297cad3709858e17d05292 src/diagnostics.rs d7c1093b62dd37479dbefd1bc9f18d2fbc0ee5d4ebe155f7a7bbaccd058b1fd3 src/docker.rs @@ -201,9 +205,9 @@ d7c1093b62dd37479dbefd1bc9f18d2fbc0ee5d4ebe155f7a7bbaccd058b1fd3 src/docker.rs eff4484fd9630009471f824c57c5c74f3d3c5378f9e46a3bdda03ede37daec82 src/geoip.rs 185a7874d2ba45e6705fb69703fb99c96eef2a8134f43343e2b20f105f1a1be3 src/headers.rs 2f6dd8c74fe1d22a8fa4ec5a21d0d334a667ede1b11629541f4802252765b21b src/http3.rs -a6630962453c7e68a409aac45da51093148b9b3c8d044ab13897a8c9e476ae4d src/lib.rs -df14373ce3f9c5740d7501fdd4cb42d8b7b07b23ad86f2b69b5e534f0e0f01eb src/logging.rs -942a1bf335c2e0b3e000a4443b1d0ff88851d75d1362a4846420d8bf758a2377 src/main.rs +95bd6977db93be728b2597e942d9200e47474d6e0c7c78584df8c8bae6a1f9c7 src/lib.rs +681732a2ae47e99f3e86b2c627f22eaf3d54c3223e7e8477f426f085c4caf3ca src/logging.rs +26b704f3de0020488c09c8ece38aeec62a7d4c561ae8c63f0eb2347570cf6d6b src/main.rs 7022ec2e991e9906e30b06f5bca58f96b9d7e09cc92a27202b5196d54e341b75 src/metrics.rs 64494e5746912cfcb10612ea4e0b336004b1f738bd7999be7e58fc96abadb6e1 src/not_found.rs e4b1f61c12437cc840c36eb7ab7b69b03f310bca3f89288ade9992d9dd1b7e2c src/pages.rs @@ -214,7 +218,8 @@ f355d84c16cef89c43785fdc02181d6ad234fbc9c834367843822ff970088d00 src/prometheus 64665d778a90a22e93af7a17be037255d59fff3220d8924737da9be965acae69 src/proxy/websocket.rs 6b24aa692ed3cc7d20b561c90239e26e203f8e44f64d4bb45b8b63ba05906d4f src/rate_limit.rs acdca1bf861f95b85f08dbbcf69e8a2090335cab4d542f2509383bcc67a0e699 src/redirect.rs -2f160ccabf87d345b00dbabc7863bf96e9839e346b2def70501a33cc2ccb1673 src/runtime.rs +4e73633414a5d439447ec2cd1775dd0d8a81249c3183f0d351b830e7027c2f4a src/runtime.rs +a98a6a2a6274cc4d5cc4626f34eba7885dbf1c764ce5e5cf03dc3e084b3eb902 src/secret_store.rs c403b63a4192e9549b8a82f29e6966fddd2130be1a9a86a895e7692168de98ea src/security.rs d35fbad33476b465d456dde8984daa4e52e34b9d546ab9e48dffcc40c74a4669 src/server.rs a80e8fc2990e62b8fd28426c8e1d213c9761577462179e96637d646ad3d87518 src/settings.rs @@ -229,7 +234,7 @@ a026c13cc4757cb01bfd4595ac6b57a4d837482e40cd399e8d086a54700b2b97 src/tinyauth.r eca00dadae42aecfdcd918ec62b8603deb27caa20e1778befe211b57afa35001 src/transform/json.rs bef367ca40eb3c25e39b6f51f82745a50e8112496ae7303939722e30698908db src/transform/mod.rs c0c150a8799942c53500e48b6c9d3068210d6e6ef5693514ee9602127f6e5b0c src/transform/svg.rs -f255ed094a1cedf86f767bc062b1c589c5940a4f024274ca8238d2e83d0ede6c src/update_check.rs +7c32623d984adcf7e88ed29dd7d31ffea7f40aeac716eadd2d9da1d45456c119 src/update_check.rs 77cabbcd9730e01d19e1147ede94272f6100d97281f7892d2b1bc0279c780f63 src/upstream.rs ea3ec51c0ffac30670cca8580374cbd1f67a792dfab2e6ece46aaf22c70636b1 src/vhost.rs f00743af2cf4a5952caef7a1dcf68e7817b5599a252a9973e1c22045517c0b0e tests/fixtures/argon2id.json diff --git a/debian/README.md b/debian/README.md index 83a1249..09e73c3 100644 --- a/debian/README.md +++ b/debian/README.md @@ -1,6 +1,6 @@ # Debian packaging -This directory builds the `rustproxy` binary package for RustProxy 0.24.8. +This directory builds the `rustproxy` binary package for RustProxy 0.24.9. The package installs: - `/usr/bin/rust-proxy` diff --git a/debian/build-package.sh b/debian/build-package.sh index 89e7ef8..62c7712 100755 --- a/debian/build-package.sh +++ b/debian/build-package.sh @@ -21,6 +21,6 @@ cargo fetch --locked dpkg-buildpackage -us -uc -b mkdir -p dist -find .. -maxdepth 1 -type f \( -name 'rustproxy_0.24.8_*.deb' -o -name 'rustproxy_0.24.8_*.changes' -o -name 'rustproxy_0.24.8_*.buildinfo' \) \ +find .. -maxdepth 1 -type f \( -name 'rustproxy_0.24.9_*.deb' -o -name 'rustproxy_0.24.9_*.changes' -o -name 'rustproxy_0.24.9_*.buildinfo' \) \ -exec cp -f {} dist/ \; echo "Packages copied to $ROOT/dist" diff --git a/debian/changelog b/debian/changelog index fac249b..a3b8b96 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +rustproxy (0.24.9) stable; urgency=medium + + * Add encrypted Secret Store and migrate ACME provider credentials. + * Add password-encrypted full backup/restore with staged validation. + * Add optional bounded asynchronous ERROR file logging with secret redaction. + + -- RustProxy Release Sat, 03 Oct 2026 16:41:00 +0200 + rustproxy (0.24.8) stable; urgency=medium * Expand Cache RAM overview occupancy histories for HOT, transient and COLD storage. diff --git a/dev.sh b/dev.sh index 08a5377..a38a398 100755 --- a/dev.sh +++ b/dev.sh @@ -18,7 +18,7 @@ if [[ $# -gt 0 ]]; then shift; fi case "$mode" in help|-h|--help) cat <<'HELP' -RustProxy 0.24.8 +RustProxy 0.24.9 ./dev.sh Build and start proxy + local panel ./dev.sh run [serve flags] Same, with extra server options ./dev.sh admin setup-token Generate/print a one-time setup capability before first login @@ -123,7 +123,7 @@ case "$mode" in check) exec cargo check --locked --all-targets "$@" ;; release) exec cargo build --locked --release "$@" ;; esac -printf 'Building RustProxy 0.24.8...\n' >&2 +printf 'Building RustProxy 0.24.9...\n' >&2 cargo build --locked BIN="$CARGO_TARGET_DIR/debug/rust-proxy" case "$mode" in diff --git a/docker-agent/main.go b/docker-agent/main.go index 3036f5d..b2a03dd 100644 --- a/docker-agent/main.go +++ b/docker-agent/main.go @@ -32,7 +32,7 @@ import ( "time" ) -const version = "0.24.8" +const version = "0.24.9" type config struct { Listen string diff --git a/docs/ACME_RFC2136.md b/docs/ACME_RFC2136.md index 20821a1..898ae7a 100644 --- a/docs/ACME_RFC2136.md +++ b/docs/ACME_RFC2136.md @@ -96,16 +96,17 @@ can leave challenge TXT records: inspect DNS and clean only the failed challenge values, not unrelated records. Only the latest job status lives in memory; restart loses that status, not the private Certbot files. -TSIG is supplied through stdin to the helper, never argv, SQLite, exports or job -responses. The browser clears the secret field after submission and does not put -it in local/session storage. Private credentials/account files are deliberately -retained for renewal; protect them and private logs as secrets. Keep unused staged -pairs under operator control; active files are never silently deleted. +TSIG/API credentials are supplied through stdin to the helper, never argv, exports +or job responses. RustProxy encrypts DNS-provider credentials in the Secret Store; +Certbot-compatible `.ini` files are recreated with private permissions only for the +external operation and removed afterwards. The browser clears submitted secret +fields and does not put them in local/session storage. Protect the Certbot account +state and private logs as secrets. -The existing `dev.sh backup` includes SQLite and configured PEM files, **not the -ACME account/profile directory**. Back up `certs/acme` separately while no issuance -is active, preserving ownership/modes and encryption appropriate to private keys. -Source upgrade leaves the data directory alone. Do not ship profiles in source ZIPs. +Use `rust-proxy backup create` for current installations. The encrypted full backup +includes the durable ACME profile/account state, certificates and Secret Store key +material needed for renewal. The historical `dev.sh backup` helper is not sufficient +after Secret Store migration. See SECRET_STORE_BACKUP.md. ## API and local helper diff --git a/docs/API_ACCESS.md b/docs/API_ACCESS.md index f7b47c5..3fc2636 100644 --- a/docs/API_ACCESS.md +++ b/docs/API_ACCESS.md @@ -1,4 +1,4 @@ -# Administration API access - 0.24.8 +# Administration API access - 0.24.9 ## Sign-in boundary diff --git a/docs/ERROR_LOGGING.md b/docs/ERROR_LOGGING.md new file mode 100644 index 0000000..25193e8 --- /dev/null +++ b/docs/ERROR_LOGGING.md @@ -0,0 +1,39 @@ +# Optional error-file logging + +RustProxy can write an additional local file containing only `ERROR` events. This is an optional sink layered on the existing tracing/logging pipeline; journald/systemd and other configured sinks remain independent. + +## Configuration + +The settings document contains: + +```json +{ + "error_log": { + "enabled": true, + "path": "/var/log/rustproxy/error.log", + "format": "JSON" + } +} +``` + +`format` is `JSON` or `RAW`. The feature is disabled by default. The administration Settings UI exposes the same enable/path/format controls and applies changes at runtime. + +The sink accepts only `ERROR` events. JSON is one object per line; RAW is one human-readable line per event. + +## Runtime behavior + +Events are handed to a bounded queue and written by one asynchronous worker. The request path does not perform the file write. When the queue is full, new error-file events are dropped and counted instead of allowing unbounded memory growth. + +Filesystem errors do not stop RustProxy. The worker reports failures through the existing logger, rate-limits repeated diagnostics and retries opening the file. Changing the path or disabling the sink closes the current file; a format change takes effect without an application restart. + +Created log files use mode `0600` on POSIX. Symlink/non-regular targets are refused. + +## Secret redaction + +The common logging pipeline redacts sensitive keys and values before they reach the error-file sink. This includes authorization credentials, provider tokens/passwords, backup passwords, private-key PEM material and Secret Store/master-key related values. Structured secret fields are replaced with `[REDACTED]`. + +Error messages from dependencies should still be treated as untrusted diagnostic input; do not deliberately embed secrets in error text. + +## Rotation + +RustProxy does not implement file rollover, retention or compression. Use the system `logrotate`. The Debian package configuration in `debian/rustproxy.logrotate` already covers `/var/log/rustproxy/*.log` and uses `copytruncate`, so the default error-log path is included without a RustProxy restart. diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 510f90d..2953958 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,4 +1,4 @@ -# Installing and updating RustProxy 0.24.8 +# Installing and updating RustProxy 0.24.9 The scripts prepare a Linux/systemd installation. This source archive is **not a verified production build**: Rust/Cargo, real CA operations, live systemd startup diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 9aa4637..b7aa550 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -199,10 +199,10 @@ configuration publication. See CACHE_DIAGNOSTICS.md for explaining BYPASS. ACME/RFC2136 is optional and adds external Python/Certbot/plugin requirements. No DNS update or CA request happens simply by upgrading. Issuance is explicit, -staging-first and does not activate a certificate without import. Existing -`dev.sh backup` does not include the private `certs/acme` account/TSIG profiles; -back those up separately when issuance is idle. No automatic renewal is included. -See ACME_RFC2136.md for limitations and restoration responsibilities. +staging-first and does not activate a certificate without import. Current encrypted full backup includes the durable ACME profile/account state and +Secret Store material required for DNS-provider renewal; the historical `dev.sh backup` +helper does not. Use `rust-proxy backup create` for current installations. Automatic +renewal behavior is documented in RENEWAL.md. ## Reported auth module compilation failure @@ -230,16 +230,20 @@ source-tree update is preferable to extracting individual files over a checkout. ## Recommended source update -Extract the new archive beside, not inside, the installed project. First back up -the existing database using the new backup script. The destination must not exist: +Extract the new archive beside, not inside, the installed project. On installations +that provide the current encrypted backup command, stop RustProxy and create the +backup outside the data directory before replacing source files: ```bash +rust-proxy --data-dir /home/mg/RustProxy/data backup create \ + --output /home/mg/RustProxy-backup-before-upgrade.rpb unzip RustProxy-0.4.0.zip -d /home/mg/rustproxy-040-staging cd /home/mg/rustproxy-040-staging/RustProxy -python3 scripts/backup.py --data-dir /home/mg/RustProxy/data \ - /home/mg/RustProxy-backup-before-0.4.0 ``` +`scripts/backup.py` is a legacy release-test helper and is not a complete Secret +Store backup. For current backup/restore behavior see `SECRET_STORE_BACKUP.md`. + A missing database means there is no database to back up. Do not initialize an empty replacement accidentally. For data stored elsewhere, use the actual directory. @@ -283,25 +287,15 @@ never overwrite an existing configuration, including an empty existing one. ## Backup contents and restore -`./dev.sh backup NEW_DIRECTORY` uses SQLite's online backup API, so committed WAL -content is included. The result contains a private `config.sqlite3`, `backup.json` -with hashes/revision metadata and any configured/default certificate files under -`certs/`. Unreferenced staged certificate files are not included. Do not rotate or -manually replace certificate files while taking a backup. This is a consistent -SQLite snapshot plus file copies, not one cross-filesystem transaction. +Current RustProxy releases use the password-encrypted full-backup format documented +in `SECRET_STORE_BACKUP.md`. The supported interface is `rust-proxy backup create` +and CLI-only `rust-proxy backup restore`; it includes the encrypted Secret Store, +key material required to recover it, certificates and other durable application +state while excluding cache, logs and runtime files. -Backups contain credential hashes, session records and private keys. Keep them -private, encrypted at rest as appropriate, and outside public web directories. -They do not contain `.env`, source edits or a generated dependency lockfile; preserve -those separately. Existing backup destinations and certificate symlinks are refused. - -To restore, stop the server, verify backup hashes and keep the current data directory -as a separate recovery copy. Restore into a **new private directory**, including -`config.sqlite3` and `certs/` together, then point `RUST_PROXY_DATA_DIR` to it. -Use directory mode 700 and file mode 600. Do not copy stale `-wal` or `-shm` files -from another database beside the restored snapshot. Start the matching version and -run `./dev.sh admin logout-all` or reset the password to revoke restored sessions. -Verify routes, TLS identities and administration before resuming traffic. +The historical `./dev.sh backup` / `scripts/backup.py` snapshot helper described by +older release notes is not sufficient after Secret Store migration and is retained +only for offline compatibility/release tests. ## Downgrade diff --git a/docs/README.md b/docs/README.md index 6548312..c38f9b2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,8 @@ This index separates current operational documentation from historical/version-l - [CACHE.md](CACHE.md) and [CACHE_DIAGNOSTICS.md](CACHE_DIAGNOSTICS.md) - cache semantics and diagnostics. - [PROMETHEUS.md](PROMETHEUS.md), [HTTP3.md](HTTP3.md), [STREAMS.md](STREAMS.md) - monitoring and transport features. - [SECURITY.md](SECURITY.md) - current security model and limitations. +- [SECRET_STORE_BACKUP.md](SECRET_STORE_BACKUP.md) - encrypted credentials and password-protected full backup/restore. +- [ERROR_LOGGING.md](ERROR_LOGGING.md) - optional asynchronous ERROR-only file sink and external log rotation. - [TEST_REPORT.md](TEST_REPORT.md) - release verification status and explicit environment limitations. - [0.24.8 dashboard cache-memory audit](V0.24.8_AUDIT.md) - overview HOT/transient/COLD occupancy histories and compact RSS accounting help. - [0.24.7 transform resource-control audit](V0.24.7_AUDIT.md) - configurable transformation memory/concurrency budgets and resource presets. diff --git a/docs/SECRET_STORE_BACKUP.md b/docs/SECRET_STORE_BACKUP.md new file mode 100644 index 0000000..45114a5 --- /dev/null +++ b/docs/SECRET_STORE_BACKUP.md @@ -0,0 +1,58 @@ +# Secret Store and encrypted full backup/restore + +RustProxy keeps application credentials in a central encrypted Secret Store. Provider and ACME secrets are not stored as plaintext configuration values in SQLite. + +## Secret Store + +Secret records live in the `secret_store` SQLite table and contain an identifier, provider/type, encryption version, nonce, ciphertext, metadata and timestamps. Values are encrypted with AES-256-GCM using a random local data key stored at: + +```text +/secrets/master.key +``` + +The key file is created with mode `0600`; its directory is private. The application loads it during normal startup, so normal operation does not require an interactive password. + +Existing ACME DNS credential files are migrated on startup for Cloudflare, OVH and RFC2136. After successful import, the legacy plaintext credential file is removed. Certbot-compatible credential files are recreated only as short-lived private files around the external operation and removed afterwards. + +Do not copy only `config.sqlite3` as a Secret Store backup: the local key material is also required. Use the encrypted full-backup command below. + +## Create a full backup + +Stop RustProxy first so the data directory lock is available. The output file must be outside the RustProxy data directory and must not already exist. + +```bash +rust-proxy --data-dir /var/lib/rustproxy backup create \ + --output /private/rustproxy-backup.rpb +``` + +When run from a TTY, RustProxy asks for the backup password without putting it in process arguments or shell history. Press Enter at the password prompt to generate a strong random password. A generated password is printed exactly once and is required for restore. + +For non-interactive `backup create`, RustProxy generates the password automatically and prints it once to standard output. Protect that output as a secret. + +The encrypted backup contains the durable application state needed for recovery, including: + +- a consistent SQLite snapshot, +- configuration and Secret Store records, +- backup-wrapped Secret Store key material, +- certificates and private keys, +- ACME state and durable provider data, +- other durable RustProxy data accepted by the backup manifest. + +It excludes rebuildable/runtime data such as cache, logs, temporary files, sockets, PID/lock files and SQLite WAL/SHM sidecars. + +The outer container derives a backup key with Argon2id and encrypts authenticated chunks with AES-256-GCM. The backup password is independent from the local Secret Store key. + +## Restore + +Restore is CLI-only and requires an interactive TTY for the password: + +```bash +rust-proxy --data-dir /var/lib/rustproxy backup restore \ + /private/rustproxy-backup.rpb +``` + +Before replacing the destination, RustProxy verifies the encrypted container, password/authentication tags, manifest, checksums, backup format, RustProxy/schema compatibility, restored SQLite database and decryptability of all Secret Store records. + +Files are first restored into a private staging directory. Only after validation succeeds is the staged data directory atomically activated; activation failure attempts to roll back to the previous directory. Sensitive restored files use restrictive permissions. + +A lost backup password cannot be recovered by RustProxy. diff --git a/docs/TEST_REPORT.md b/docs/TEST_REPORT.md index 2fda60c..75e093c 100644 --- a/docs/TEST_REPORT.md +++ b/docs/TEST_REPORT.md @@ -1,11 +1,20 @@ -# Verification report - RustProxy 0.24.8 +# Verification report - RustProxy 0.24.9 ## Scope -RustProxy 0.24.8 expands the overview cache diagnostics with HOT RAM occupancy against its limit, transient cache RAM histories for total/fill/COLD queue/COLD I/O, and COLD storage occupancy against its disk budget. The process/cache RSS accounting explanation is now behind an accessible information control. It retains the 0.24.7 transform resource controls and the 0.24.6 cache/background-worker memory hardening. +RustProxy 0.24.9 implements the Secret Store + password-encrypted full backup/restore and optional ERROR-only file logging specifications. Secret Store adds schema version 7, encrypts ACME DNS-provider credentials with a local master key, migrates legacy provider `.ini` files, and recreates plaintext credential files only around the external Certbot operation. Full backup/restore uses Argon2id plus authenticated AES-256-GCM encryption, an explicit manifest, Secret Store key wrapping, staged validation and atomic data-directory activation. Error-file logging is a separate, disabled-by-default asynchronous bounded sink with JSON/RAW formats, central secret redaction and external `logrotate` rotation. + +The release also retains the 0.24.8 overview cache diagnostics, 0.24.7 transform resource controls and 0.24.6 cache/background-worker memory hardening described by the earlier audit material below. ## Source contracts reviewed +- SQLite schema version 7 includes the generic encrypted `secret_store` table; stored secret values are AES-256-GCM ciphertext with per-record nonces and the application master key lives outside SQLite in a private file. +- Cloudflare, OVH and RFC2136 ACME credentials are validated, imported into Secret Store, decrypted only when required, and legacy provider `.ini` files are removed after successful migration. Certbot credential files are temporary and removed in `finally` paths. +- `rust-proxy backup create` writes a new `.rpb` file outside the data directory, derives an independent backup key with Argon2id, encrypts authenticated chunks with AES-256-GCM, excludes cache/log/runtime data and wraps the Secret Store master key inside the encrypted backup. +- `rust-proxy backup restore` requires an interactive password, decrypts into private staging, validates manifest/checksums/schema/configuration/Secret Store decryptability before activation, then switches the data directory by rename with rollback on activation failure. +- `error_log` is independent from normal access logging, defaults disabled, accepts only `ERROR`, supports JSON/RAW, uses a bounded queue and a single asynchronous writer, handles filesystem failures without terminating RustProxy, and relies on system `logrotate` rather than application rollover. +- The stderr/journald path, access-log sinks and the optional error-file sink share secret-redaction helpers for authorization/provider/password/private-key material. +- The release checker preserves the existing dashboard contract while using bundled roots plus common system CA bundles (`SSL_CERT_FILE`, `CURL_CA_BUNDLE`, FreeBSD/OPNsense and Linux paths), follows at most five credential-free HTTP(S) redirects without HTTPS-to-HTTP downgrade, and reports network/TLS/HTTP/parser failures at `ERROR` level so the optional error-file sink receives them. - The overview Cache RAM block keeps separate histories for HOT allocated bytes, transient RAM, fill bytes, COLD queue bytes, COLD I/O bytes and COLD disk bytes; the disk series is never included in process RAM accounting. - HOT and transient histories are rendered against their runtime limits, while COLD storage is rendered against the configured COLD byte budget. - The long `memoryAccountingHint` remains translated and available on demand through the Process information control instead of consuming permanent dashboard height. @@ -32,10 +41,13 @@ RustProxy 0.24.8 expands the overview cache diagnostics with HOT RAM occupancy a | Browser JavaScript `node --check` | PASS for all administration JS bundles | | OpenAPI/language JSON parse | PASS | | Shell `bash -n` | PASS | -| Python syntax / bytecode compile | PASS | +| Python syntax / bytecode compile, including ACME provider worker | PASS | +| Secret Store / encrypted backup / error-log / portable update-check source-contract regression checks | PASS | +| SQLite schema 7 migration smoke (`secret_store`) | PASS | | Docker agent `go test ./...` / `go vet ./...` / `go build ./...` | PASS | | `python3 scripts/check_source.py` | PASS | | `python3 scripts/release_smoke.py` | PASS | +| FreeBSD/OPNsense live release-check execution | NOT RUN in this workspace; source contract and platform CA paths are covered, target runtime confirmation is still required | | `./dev.sh fmt` / `cargo fmt --all` | BLOCKED: Rust/Cargo toolchain unavailable in this workspace | | `./dev.sh check` / `cargo check --locked --all-targets` | BLOCKED: Rust/Cargo toolchain unavailable in this workspace | | `./dev.sh test` Rust phase | BLOCKED: Rust/Cargo toolchain unavailable in this workspace | diff --git a/grafana/rustproxy-overview.json b/grafana/rustproxy-overview.json index a8b3a71..6cedee6 100644 --- a/grafana/rustproxy-overview.json +++ b/grafana/rustproxy-overview.json @@ -3672,7 +3672,7 @@ }, "timepicker": {}, "timezone": "browser", - "title": "RustProxy 0.24.8 - Operations", + "title": "RustProxy 0.24.9 - Operations", "uid": "rustproxy-0170", "version": 2, "weekStart": "" diff --git a/langs/en_US.json b/langs/en_US.json index a8b0db8..d2f1dcf 100644 --- a/langs/en_US.json +++ b/langs/en_US.json @@ -906,6 +906,10 @@ "loggingDropped": "dropped", "loggingSent": "sent", "loggingCapacity": "capacity", + "errorLogTitle": "Error file", + "errorLogPath": "Error log path", + "errorLogFormat": "Format", + "errorLogHelp": "Only ERROR events are written. Rotation is handled by system logrotate.", "tls12": "TLS 1.2", "tls13": "TLS 1.3", "tlsPolicy": "TLS policy", diff --git a/langs/pl_pl.json b/langs/pl_pl.json index 3674cb9..819d2d2 100644 --- a/langs/pl_pl.json +++ b/langs/pl_pl.json @@ -906,6 +906,10 @@ "loggingDropped": "odrzuconych", "loggingSent": "wysłanych", "loggingCapacity": "pojemność", + "errorLogTitle": "Plik błędów", + "errorLogPath": "Ścieżka logu błędów", + "errorLogFormat": "Format", + "errorLogHelp": "Zapisywane są tylko zdarzenia ERROR. Rotację obsługuje systemowy logrotate.", "tls12": "TLS 1.2", "tls13": "TLS 1.3", "tlsPolicy": "Polityka TLS", diff --git a/make_zip.py b/make_zip.py index 90ebf1e..53f7e20 100755 --- a/make_zip.py +++ b/make_zip.py @@ -83,7 +83,7 @@ def add_file(archive: zipfile.ZipFile, rel: Path, data: bytes, timestamp: tuple[ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('output', nargs='?', type=Path, default=ROOT.parent / 'RustProxy-0.24.8.zip') + parser.add_argument('output', nargs='?', type=Path, default=ROOT.parent / 'RustProxy-0.24.9.zip') output = parser.parse_args().output.resolve() paths = files() checksum = ROOT / 'SHA256SUMS' diff --git a/migrations/0007_secret_store.sql b/migrations/0007_secret_store.sql new file mode 100644 index 0000000..2212936 --- /dev/null +++ b/migrations/0007_secret_store.sql @@ -0,0 +1,13 @@ +-- Encrypted application secrets. Ciphertext is protected by the local master +-- key stored outside SQLite. Plaintext credentials must never be persisted here. +CREATE TABLE IF NOT EXISTS secret_store ( + secret_id TEXT PRIMARY KEY CHECK (length(secret_id) BETWEEN 1 AND 256), + provider TEXT NOT NULL CHECK (length(provider) BETWEEN 1 AND 64), + encryption_version INTEGER NOT NULL CHECK (encryption_version > 0), + nonce BLOB NOT NULL CHECK (length(nonce) = 12), + ciphertext BLOB NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' CHECK (length(metadata) <= 4096), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); +CREATE INDEX IF NOT EXISTS secret_store_provider ON secret_store(provider); diff --git a/scripts/acme_provider.py b/scripts/acme_provider.py index e1b1878..0ca5a5f 100755 --- a/scripts/acme_provider.py +++ b/scripts/acme_provider.py @@ -28,7 +28,7 @@ ALGORITHMS = {"HMAC-SHA256", "HMAC-SHA384", "HMAC-SHA512"} MAX_INPUT = 16384 MAX_PEM = 131072 PROVIDERS = {"http01", "cloudflare", "ovh", "rfc2136"} -SECRET_FIELDS = {"key_secret", "api_token", "application_key", "application_secret", "consumer_key"} +SECRET_FIELDS = {"key_secret", "api_token", "application_key", "application_secret", "consumer_key", "credential_ini_b64"} COMMON_FIELDS = {"provider", "certificate_id", "email", "domains", "environment", "agree_tos", "confirm_production", "propagation_seconds"} PROVIDER_FIELDS = { "http01": set(), @@ -226,29 +226,33 @@ def issue(raw: dict, certificate_directory: Path, executable: str | None = None) raise AcmeError("acme.profile_busy") from None for folder in ["config", "work", "logs"]: private_directory(work / folder) - # Keep credentials for this lineage's renewal. Replace atomically; never - # follow an existing credential-file symlink or put secrets in SQLite. + # DNS credentials are supplied by RustProxy's encrypted Secret Store. + # Certbot needs a file path, but the plaintext file exists only while the + # provider command runs and is deleted before this worker returns. credential_file = work / (value["provider"] + ".ini") if value["provider"] == "http01": webroot = certificate_directory / "acme-http01" for path in (webroot, webroot / ".well-known", webroot / ".well-known" / "acme-challenge"): private_directory(path) else: - temporary = work / ("credentials-" + uuid.uuid4().hex) - write_new(temporary, credentials(value)) - os.replace(temporary, credential_file) - profile = {k: v for k, v in value.items() if k not in SECRET_FIELDS} - temporary = work / ("profile-" + uuid.uuid4().hex) - write_new(temporary, json.dumps(profile).encode()) - os.replace(temporary, work / "profile.json") - args = certbot_command(value, work, credential_file, executable) + credential_file.unlink(missing_ok=True) + write_new(credential_file, credentials(value)) try: - result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, timeout=1200 + value["propagation_seconds"], check=False) - except subprocess.TimeoutExpired: - raise AcmeError("acme.provider_timeout") from None - if result.returncode: - raise AcmeError("acme.certbot_failed_check_private_log") + profile = {k: v for k, v in value.items() if k not in SECRET_FIELDS} + temporary = work / ("profile-" + uuid.uuid4().hex) + write_new(temporary, json.dumps(profile).encode()) + os.replace(temporary, work / "profile.json") + args = certbot_command(value, work, credential_file, executable) + try: + result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=1200 + value["propagation_seconds"], check=False) + except subprocess.TimeoutExpired: + raise AcmeError("acme.provider_timeout") from None + if result.returncode: + raise AcmeError("acme.certbot_failed_check_private_log") + finally: + if value["provider"] != "http01": + credential_file.unlink(missing_ok=True) cert = read_generated(work, "fullchain.pem") key = read_generated(work, "privkey.pem") # Immutable copies allow validation/publication without changing the @@ -313,7 +317,7 @@ def certificate_dates(pem: bytes, executable: str | None = None) -> dict: def renewal_request(raw: object) -> dict: - if not isinstance(raw, dict) or set(raw) - {"action", "certificate_id", "provider", "environment", "cert_file", "key_file", "dry_run"}: + if not isinstance(raw, dict) or set(raw) - {"action", "certificate_id", "provider", "environment", "cert_file", "key_file", "dry_run", "credential_ini_b64"}: raise AcmeError("acme.invalid_renewal_request") if raw.get("action") != "renew" or raw.get("provider") not in PROVIDERS or raw.get("environment") not in DIRECTORIES: raise AcmeError("acme.invalid_renewal_request") @@ -325,7 +329,20 @@ def renewal_request(raw: object) -> dict: raise AcmeError("acme.invalid_renewal_request") if type(raw.get("dry_run", False)) is not bool: raise AcmeError("acme.invalid_renewal_request") - return dict(raw, dry_run=raw.get("dry_run", False)) + if raw["provider"] != "http01": + encoded = raw.get("credential_ini_b64") + try: + credential_ini = base64.b64decode(encoded, validate=True) if isinstance(encoded, str) else b"" + except (ValueError, TypeError): + credential_ini = b"" + if not credential_ini or len(credential_ini) > 65536 or b"\x00" in credential_ini: + raise AcmeError("acme.invalid_api_credential") + else: + credential_ini = b"" + value = dict(raw, dry_run=raw.get("dry_run", False)) + value["credential_ini"] = credential_ini + value.pop("credential_ini_b64", None) + return value def renew(raw: dict, certificate_directory: Path, executable: str | None = None) -> dict: @@ -367,30 +384,37 @@ def renew(raw: dict, certificate_directory: Path, executable: str | None = None) if not isinstance(names, list) or not 1 <= len(names) <= 16: raise AcmeError("acme.invalid_domains") names = [domain(n, wildcard=value["provider"] != "http01") for n in names] - # This exact renewal file must have been created by the original issuance. + # The renewal config persists, but provider credentials do not. Recreate + # the exact path referenced by Certbot only for this invocation. read_private(work / "config" / "renewal" / "rustproxy.conf", 65536) + credential_file = work / (value["provider"] + ".ini") if value["provider"] != "http01": - read_private(work / (value["provider"] + ".ini")) + credential_file.unlink(missing_ok=True) + write_new(credential_file, value["credential_ini"]) else: for path in [certificate_directory / "acme-http01", certificate_directory / "acme-http01" / ".well-known", certificate_directory / "acme-http01" / ".well-known" / "acme-challenge"]: private_directory(path) - installed_cert = read_private(certificate_directory / value["cert_file"], MAX_PEM) - installed_key = read_private(certificate_directory / value["key_file"], MAX_PEM) - # Check dependencies/installed PEM before invoking a CA operation. - old_dates = certificate_dates(installed_cert) - args = [executable, "renew", "--cert-name", "rustproxy", "--non-interactive", - "--no-random-sleep-on-renew", "--no-directory-hooks", - "--config-dir", str(work / "config"), "--work-dir", str(work / "work"), "--logs-dir", str(work / "logs")] - if value["dry_run"]: - args.append("--dry-run") try: - result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, timeout=2100, check=False) - except subprocess.TimeoutExpired: - raise AcmeError("acme.provider_timeout") from None - if result.returncode: - raise AcmeError("acme.certbot_failed_check_private_log") + installed_cert = read_private(certificate_directory / value["cert_file"], MAX_PEM) + installed_key = read_private(certificate_directory / value["key_file"], MAX_PEM) + # Check dependencies/installed PEM before invoking a CA operation. + old_dates = certificate_dates(installed_cert) + args = [executable, "renew", "--cert-name", "rustproxy", "--non-interactive", + "--no-random-sleep-on-renew", "--no-directory-hooks", + "--config-dir", str(work / "config"), "--work-dir", str(work / "work"), "--logs-dir", str(work / "logs")] + if value["dry_run"]: + args.append("--dry-run") + try: + result = subprocess.run(args, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=2100, check=False) + except subprocess.TimeoutExpired: + raise AcmeError("acme.provider_timeout") from None + if result.returncode: + raise AcmeError("acme.certbot_failed_check_private_log") + finally: + if value["provider"] != "http01": + credential_file.unlink(missing_ok=True) output = {k: value[k] for k in ("provider", "certificate_id", "environment")} output.update(domains=names, changed=False, staged=False, dry_run=value["dry_run"], **old_dates) if value["dry_run"]: diff --git a/scripts/backup.py b/scripts/backup.py index edde281..1737b6a 100755 --- a/scripts/backup.py +++ b/scripts/backup.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Create a private, consistent SQLite backup, including a live database in WAL mode. -This does not copy source code or accept arbitrary SQL. Restore instructions are in docs/MIGRATION.md. +"""Legacy SQLite snapshot helper retained for offline release tests. + +Operator backups must use `rust-proxy backup create`, which includes the encrypted +Secret Store key material and all durable data. This script is not a supported +backup/restore interface after the Secret Store migration. """ from __future__ import annotations -import argparse import hashlib import json import os @@ -85,14 +87,10 @@ def backup(data_dir: Path, destination: Path) -> Path: def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--data-dir", type=Path, default=Path("data")) - parser.add_argument("destination", type=Path, help="new directory; existing destinations are never overwritten") - args = parser.parse_args() - try: - print(f"Private SQLite and certificate backup created: {backup(args.data_dir, args.destination)}") - except (OSError, ValueError, sqlite3.Error, RuntimeError) as error: - parser.exit(1, f"Backup failed: {error}\nA partially-created destination must be inspected before reuse.\n") + raise SystemExit( + "scripts/backup.py is retired for operator backups. " + "Use: rust-proxy --data-dir backup create --output " + ) if __name__ == "__main__": diff --git a/scripts/check_source.py b/scripts/check_source.py index 7768ddd..2aff46d 100755 --- a/scripts/check_source.py +++ b/scripts/check_source.py @@ -16,7 +16,7 @@ def read_cache_source() -> str: preflight(root) manifest = tomllib.loads((root / "Cargo.toml").read_text()) -assert manifest["package"]["version"] == "0.24.8" +assert manifest["package"]["version"] == "0.24.9" assert manifest["dependencies"]["quinn"]["version"] == "0.11.12" h3dep = manifest["dependencies"]["h3"] assert h3dep["package"] == "hpx-h3" and h3dep["version"] == "=2.5.19" and "quinn" in h3dep["features"] @@ -24,14 +24,14 @@ assert "h3-quinn" not in manifest["dependencies"] lock_path = root / "Cargo.lock" if lock_path.exists(): lock = lock_path.read_text() - assert 'name = "rust-proxy"\nversion = "0.24.8"' in lock + assert 'name = "rust-proxy"\nversion = "0.24.9"' in lock assert 'name = "systemstat"' in lock and 'name = "metrics-process"' in lock and 'name = "sysinfo"' in lock, "Cargo.lock must be resolver-generated after adding cross-platform statistics libraries" for release_doc, marker in { - "README.md": "# RustProxy 0.24.8", - "CHANGELOG.md": "## 0.24.8", - "ROADMAP.md": "## 0.24.8", - "AGENTS.md": "Current source version: 0.24.8.", - "docs/TEST_REPORT.md": "# Verification report - RustProxy 0.24.8", + "README.md": "# RustProxy 0.24.9", + "CHANGELOG.md": "## 0.24.9", + "ROADMAP.md": "## 0.24.9", + "AGENTS.md": "Current source version: 0.24.9.", + "docs/TEST_REPORT.md": "# Verification report - RustProxy 0.24.9", }.items(): assert marker in (root / release_doc).read_text(), f"stale release identity in {release_doc}" assert tomllib.loads((root / "rust-toolchain.toml").read_text())["toolchain"]["channel"] == "stable" @@ -53,6 +53,18 @@ for marker in [ assert marker in update_check_source, marker assert 'update version document is too large or incomplete: {error}' in update_check_source assert '.context("update version document is too large or incomplete")' not in update_check_source +for marker in [ + 'tracing::error!(', + 'MAX_REDIRECTS: usize = 5', + 'SSL_CERT_FILE', + 'CURL_CA_BUNDLE', + '/usr/local/share/certs/ca-root-nss.crt', + 'response.status().is_redirection()', + 'redirect_target(¤t, location)', + 'update version redirect must not downgrade HTTPS to HTTP', +]: + assert marker in update_check_source, marker +assert 'pub last_error:' not in update_check_source, "update-check errors must not change dashboard/API state" for name in ["README.md", "ROADMAP.md", "AGENTS.md", "CHANGELOG.md", "UPDATE_CHECK.toml", "Dockerfile", ".dockerignore", "docs/TEST_REPORT.md", "docs/V0.24.8_AUDIT.md", "docs/V0.24.7_AUDIT.md", "docs/V0.24.6_AUDIT.md", "docs/V0.24.5_AUDIT.md", "docs/V0.24.1_AUDIT.md", "docs/V0.24.0_AUDIT.md", "docs/V0.23.2_AUDIT.md", "docs/V0.23.1_AUDIT.md", "docs/V0.22.3_AUDIT.md", "docs/LIVE_LOGS.md", "docs/README.md", "docs/API.md", "docs/ARCHITECTURE.md", "docs/VHOSTS.md", "docs/STREAMS.md", "docs/REDIRECTS.md", "docs/CONTAINER.md", "docs/NPM_MIGRATION.md", "docs/DOCKER_DISCOVERY.md", "docker-agent/main.go", "docker-agent/main_test.go", "docker-agent/rustproxy-docker-agent.service", "docker-agent/docker-agent.env.example", "docker-agent/install.sh", "src/docker.rs", "src/admin/web/docker.js", "scripts/migrate_npm.py", "docs/CUSTOM_LOCATIONS.md", "nagios/README.md", "nagios/check_rustproxy.py", "nagios/check_rustproxy_api", "nagios/check_rustproxy_backends", "nagios/check_rustproxy_latency", "nagios/check_rustproxy_queues", "src/admin/web/error_401.html", "src/admin/web/error_404.html", "src/admin/web/streams.js", "src/admin/web/redirects.js", "src/admin/web/host404.js", "docs/NOT_FOUND_HOSTS.md", "debian/control", "debian/rules", "debian/rustproxy.service", "debian/rustproxy.logrotate", "debian/build-package.sh", "debian/build-repository.sh", "grafana/rustproxy-overview.json", "grafana/README.md"]: assert (root / name).is_file(), name for path in (root / "examples").glob("*.json"): @@ -200,7 +212,7 @@ assert 'Opt-in health decisions based on ordinary user traffic. Defaults to disa assert '"force_ssl"' in openapi and '"forward_client_ip"' in openapi assert '"backend_header"' in openapi and '"backend_timing_header"' in openapi assert '"StreamMetrics"' in openapi and '"streams"' in openapi -for marker in ['"version": "0.24.8"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']: +for marker in ['"version": "0.24.9"', '"/api/config/{section}"', '"ConfigSectionUpdate"', '"cold_on_idle"', '"cold_on_age"', '"cold_max_hot_seconds"', '"cold_on_lru"', '"cold_on_shutdown"', '"cold_gc_interval_seconds"', '"cold_gc_max_delete"', '"collapse_enabled"', '"collapse_wait_timeout_ms"', '"collapse_timeout_fallback"', '"stale_if_error_timeout"', '"stale_if_error_connection"', '"stale_if_error_dns"', '"stale_if_error_tls"', '"stale_if_error_statuses"', '"proactive_refresh_enabled"', '"proactive_refresh_before_expiry_seconds"', '"proactive_refresh_percent"', '"proactive_refresh_max_inflight"', '"TransformSettings"', '"TransformOverrides"', '"/api/cache/cold/cleanup"', '"/api/cache/invalidate"', '"/api/cache/bans"', '"/api/cache/bans/{id}"', '"CacheInvalidationRequest"', '"CacheInvalidationResponse"', '"ColdCleanupJob"', '"match_type"', '"regex_insensitive"', '"rewrite"', '"response"', '"covered_hosts"', '"default": true']: assert marker in openapi, marker for marker in ['"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: assert marker in openapi, marker @@ -486,6 +498,28 @@ db_source = (root / "src/db.rs").read_text() acme_source = (root / "src/acme.rs").read_text() main_source = (root / "src/main.rs").read_text() logging_source = (root / "src/logging.rs").read_text() +secret_store_source = (root / "src/secret_store.rs").read_text() +backup_source = (root / "src/backup.rs").read_text() +config_source = (root / "src/config.rs").read_text() +acme_worker_source = (root / "scripts/acme_provider.py").read_text() +for feature_marker in [ + "AES_256_GCM", "master.key", "O_NOFOLLOW", "migrate_acme_credentials", + "secret_store", "SecretMetadata", +]: + assert feature_marker in secret_store_source or feature_marker in db_source, feature_marker +for feature_marker in [ + "Argon2", "RPROXYB1", "RPROXYA1", "backup create", "secret-store-key.enc", + "validate_backup_version", "stage", +]: + assert feature_marker in backup_source or feature_marker in main_source, feature_marker +for feature_marker in [ + "ErrorLogSettings", "ERROR_LOG_QUEUE", "RedactingStderr", "open_error_file", + "ErrorLogLayer", "ERROR", +]: + assert feature_marker in logging_source, feature_marker +assert "pub error_log: crate::logging::ErrorLogSettings" in config_source +assert "credential_ini_b64" in acme_source and "credential_ini_b64" in acme_worker_source +assert "finally:" in acme_worker_source and "credential_file.unlink" in acme_worker_source for marker in ['certificate_api_credentials', 'upsert_certificate_api_credential', 'revoke_certificate_api_credential', 'verify_certificate_api_credential', '0005_certificate_api.sql', '0006_certificate_api_credentials.sql']: assert marker in db_source, marker for marker in ['/api/certificate-api', 'certificate_api_credentials', 'certificate_api_operation', '/api/live-logs', 'CertificateApiBearer']: @@ -509,7 +543,7 @@ assert 'Ok(line) if command_tx.send(line).is_ok()' not in main_source, 'do not m assert 'Ok(line) => {' in main_source and 'command_tx.send(line).is_err()' in main_source for marker in ['LIVE_STREAM_CAPACITY', 'broadcast::Sender', 'pub fn subscribe']: assert marker in logging_source, marker -assert 'SCHEMA_VERSION: i64 = 6' in db_source and 'certificate_renewals' in db_source and '0004_admin_preferences.sql' in db_source and '0005_certificate_api.sql' in db_source and '0006_certificate_api_credentials.sql' in db_source +assert 'SCHEMA_VERSION: i64 = 7' in db_source and 'certificate_renewals' in db_source and '0004_admin_preferences.sql' in db_source and '0005_certificate_api.sql' in db_source and '0006_certificate_api_credentials.sql' in db_source and '0007_secret_store.sql' in db_source assert 'load_renewal_observations' not in acme_source and 'renewals.json' not in acme_source for marker in ['pub kind: &\'static str', 'pub log_directory: String', 'pub(crate) fn certbot_log_directory', 'kind: "issue"', '"renewal-dry-run"']: assert marker in acme_source, marker @@ -1002,12 +1036,13 @@ connection.executescript((root / "migrations/0003_certificate_renewals.sql").rea connection.executescript((root / "migrations/0004_admin_preferences.sql").read_text()) connection.executescript((root / "migrations/0005_certificate_api.sql").read_text()) connection.executescript((root / "migrations/0006_certificate_api_credentials.sql").read_text()) -connection.execute("PRAGMA user_version = 6") +connection.executescript((root / "migrations/0007_secret_store.sql").read_text()) +connection.execute("PRAGMA user_version = 7") assert connection.execute("PRAGMA quick_check").fetchone()[0] == "ok" assert connection.execute("SELECT COUNT(*) FROM auth_throttle WHERE id=1").fetchone()[0] == 1 assert connection.execute("SELECT COUNT(*) FROM certificate_renewals").fetchone()[0] == 0 assert connection.execute("SELECT COUNT(*) FROM admin_preferences").fetchone()[0] == 0 -assert connection.execute("PRAGMA user_version").fetchone()[0] == 6 +assert connection.execute("PRAGMA user_version").fetchone()[0] == 7 connection.close() for marker_0223 in [ '"/api/certificates/reload"', 'certificate_reload', @@ -1083,7 +1118,7 @@ assert 'id="process-platform"' in html and 'd.platform' in app_js assert 'id="panel-logging"' not in html assert 'id="settings-tab-logging"' in html and 'id="settings-page-logging"' in html and 'id="logging-form"' in html assert 'location.pathname === "/logging"' in settings_js_0241 -assert '"RuntimeInformation"' in openapi and '"version": "0.24.8"' in openapi +assert '"RuntimeInformation"' in openapi and '"version": "0.24.9"' in openapi # 0.24.1 compiler and hostname-cardinality regressions. tls_0241 = (root / "src/tls.rs").read_text() @@ -1128,5 +1163,5 @@ for schema_name in ["RedirectHost", "NotFoundHost"]: node = openapi_0241["components"]["schemas"][schema_name]["properties"]["frontend"]["properties"]["hosts"] assert "maxItems" not in node, schema_name -print("PASS: 0.24.8 manifest/package, shared system-statistics/runtime metadata, Settings logging, retained H2, Debian/Grafana/Prometheus/404-host/custom-location/path/cache/dashboard/Nagios UI/source contracts, stopped legacy-renewal removal, session/admin/transport contracts, deterministic package source and embedded SQLite schema-v6 migrations.") +print("PASS: 0.24.9 manifest/package, shared system-statistics/runtime metadata, Settings logging, retained H2, Debian/Grafana/Prometheus/404-host/custom-location/path/cache/dashboard/Nagios UI/source contracts, stopped legacy-renewal removal, session/admin/transport contracts, deterministic package source and embedded SQLite schema-v7 migrations.") print("NOT VERIFIED: Rust parsing, types, compilation, linking or runtime behavior. Run ./dev.sh test.") diff --git a/scripts/deploy.py b/scripts/deploy.py index c170f0b..d39976b 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -97,7 +97,7 @@ def digest(path: Path) -> str: def fetch_https(url: str, max_bytes: int) -> bytes: if urllib.parse.urlparse(url).scheme != 'https': raise ValueError('GeoIP downloads require HTTPS.') - request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.24.8'}) + request = urllib.request.Request(url, headers={'User-Agent': 'RustProxy-installer/0.24.9'}) with urllib.request.urlopen(request, timeout=30) as response: if urllib.parse.urlparse(response.geturl()).scheme != 'https': raise ValueError('GeoIP download redirected away from HTTPS.') diff --git a/scripts/preflight.py b/scripts/preflight.py index e951853..aedae93 100755 --- a/scripts/preflight.py +++ b/scripts/preflight.py @@ -33,10 +33,10 @@ def check(root: Path, repair_known: bool = False) -> list[str]: raise ValueError(f"Conflicting Rust modules: {flat.relative_to(root)} and {module.relative_to(root)}") for path in ["src/auth/mod.rs", "src/auth/store.rs", "src/upstream.rs", "src/tls.rs", "src/proxy/websocket.rs", "src/cache/mod.rs", "src/cache/model.rs", "src/cache/lifecycle.rs", "src/cache/cold.rs", "src/cache/dashboard.rs", "src/cache/tickets.rs", "src/cache/lookup.rs", "src/cache/policy.rs", "src/cache/fill.rs", "src/cache/tests.rs", "src/settings.rs", "src/compression.rs", "src/diagnostics.rs", "src/system_stats/mod.rs", "src/system_stats/linux.rs", "src/vhost.rs", "src/balance.rs", "src/security.rs", "src/admin/web/vhosts.js", "src/admin/web/streams.js", "src/admin/web/host404.js", "src/admin/web/management.js", "src/pages.rs", "scripts/mock_server.py", "scripts/mock_auth.py", "src/forwarding.rs", "src/tinyauth.rs", "src/http3.rs", "src/stream.rs", "src/admin/web/integrations.js", "src/prometheus.rs", "src/admin/web/settings.js", "src/admin/web/docker.js", "src/docker.rs", "docker-agent/main.go", "docker-agent/main_test.go", "docker-agent/rustproxy-docker-agent.service", "docker-agent/install.sh", "docs/DOCKER_DISCOVERY.md", "scripts/acme_provider.py", "src/basic_auth.rs", "scripts/deploy.py", "scripts/migrate_npm.py", "docs/NPM_MIGRATION.md", "docs/STREAMS.md", "install.sh", "install-debian13.sh", "update.sh", "build.rs", "scripts/swagger_assets.py", "src/admin/documentation.rs", "src/admin/openapi.json", "langs/en_US.json", "langs/pl_pl.json", "Dockerfile", ".dockerignore", "docs/CONTAINER.md", "docs/PROMETHEUS.md", "docs/NOT_FOUND_HOSTS.md", "debian/control", "debian/rules", "debian/rustproxy.service", "debian/rustproxy.logrotate", "grafana/rustproxy-overview.json"]: if not (root / path).is_file(): - raise ValueError(f"Incomplete RustProxy 0.24.8 source tree: missing {path}") + raise ValueError(f"Incomplete RustProxy 0.24.9 source tree: missing {path}") text = (root / "src/lib.rs").read_text(encoding="utf-8") if '#[path = "auth/mod.rs"]' not in text: - raise ValueError("src/lib.rs is not the 0.24.8 entry point; extract the complete package, not individual files.") + raise ValueError("src/lib.rs is not the 0.24.9 entry point; extract the complete package, not individual files.") return messages diff --git a/scripts/release_smoke.py b/scripts/release_smoke.py index 9135ca7..8bb4725 100755 --- a/scripts/release_smoke.py +++ b/scripts/release_smoke.py @@ -50,15 +50,16 @@ def sql_and_backup_smoke(tmp: Path) -> None: ) connection.execute("PRAGMA user_version = 3") connection.commit() - # Simulate additive startup migrations through the current schema-v6. + # Simulate additive startup migrations through the current schema-v7. connection.executescript((ROOT / "migrations/0004_admin_preferences.sql").read_text(encoding="utf-8")) connection.executescript((ROOT / "migrations/0005_certificate_api.sql").read_text(encoding="utf-8")) connection.executescript((ROOT / "migrations/0006_certificate_api_credentials.sql").read_text(encoding="utf-8")) + connection.executescript((ROOT / "migrations/0007_secret_store.sql").read_text(encoding="utf-8")) connection.execute( "INSERT INTO admin_preferences(administrator_id,document) VALUES(1,?)", (json.dumps({"live_log_columns": ["time", "status"]}),), ) - connection.execute("PRAGMA user_version = 6") + connection.execute("PRAGMA user_version = 7") connection.commit() assert connection.execute("PRAGMA quick_check").fetchone()[0] == "ok" tables = { @@ -67,7 +68,7 @@ def sql_and_backup_smoke(tmp: Path) -> None: "SELECT name FROM sqlite_master WHERE type='table'" ) } - for table in ("configuration", "administrator", "admin_sessions", "admin_audit", "auth_throttle", "certificate_renewals", "admin_preferences", "certificate_api_credential", "certificate_api_credentials"): + for table in ("configuration", "administrator", "admin_sessions", "admin_audit", "auth_throttle", "certificate_renewals", "admin_preferences", "certificate_api_credential", "certificate_api_credentials", "secret_store"): assert table in tables, table finally: connection.close() @@ -78,7 +79,7 @@ def sql_and_backup_smoke(tmp: Path) -> None: assert copied.is_file() metadata = json.loads((destination / "backup.json").read_text(encoding="utf-8")) assert metadata["project"] == "RustProxy" - assert metadata["schema"] == 6 + assert metadata["schema"] == 7 assert metadata["configuration_revision"] == 4 restored = sqlite3.connect(copied) try: @@ -113,13 +114,13 @@ def upgrade_smoke(tmp: Path) -> None: assert backup.is_dir() assert (backup / "src/obsolete.rs").is_file() assert not (target / "src/obsolete.rs").exists() - assert 'version = "0.24.8"' in (target / "Cargo.toml").read_text(encoding="utf-8") + assert 'version = "0.24.9"' in (target / "Cargo.toml").read_text(encoding="utf-8") assert (target / "data/keep.txt").read_text(encoding="utf-8") == "preserve\n" assert (target / ".env").read_text(encoding="utf-8") == "RUST_PROXY_DEMO=0\n" assert (target / "langs/site.json").is_file() assert (target / "docker-agent/main.go").is_file() manifest = json.loads((backup / "upgrade.json").read_text(encoding="utf-8")) - assert manifest["to_version"] == "0.24.8" + assert manifest["to_version"] == "0.24.9" assert manifest["removed_obsolete_runtime_state"] is True assert not legacy.exists() assert not (backup / "legacy-data").exists() @@ -428,6 +429,16 @@ def source_contract_smoke() -> None: main = (ROOT / "src/main.rs").read_text(encoding="utf-8") server = (ROOT / "src/server.rs").read_text(encoding="utf-8") logging = (ROOT / "src/logging.rs").read_text(encoding="utf-8") + secret_store = (ROOT / "src/secret_store.rs").read_text(encoding="utf-8") + backup = (ROOT / "src/backup.rs").read_text(encoding="utf-8") + acme_worker = (ROOT / "scripts/acme_provider.py").read_text(encoding="utf-8") + for feature_marker in ["AES_256_GCM", "master.key", "O_NOFOLLOW", "migrate_acme_credentials"]: + assert feature_marker in secret_store, feature_marker + for feature_marker in ["Argon2", "RPROXYB1", "secret-store-key.enc", "validate_backup_version"]: + assert feature_marker in backup, feature_marker + for feature_marker in ["ErrorLogSettings", "ERROR_LOG_QUEUE", "RedactingStderr", "open_error_file", "ErrorLogLayer"]: + assert feature_marker in logging, feature_marker + assert "credential_ini_b64" in acme_worker and "credential_file.unlink" in acme_worker cache = read_cache_source() settings = (ROOT / "src/settings.rs").read_text(encoding="utf-8") websocket = (ROOT / "src/proxy/websocket.rs").read_text(encoding="utf-8") @@ -495,7 +506,7 @@ def source_contract_smoke() -> None: assert marker in admin, marker anonymous_block = admin.split("fn anonymous_allowed", 1)[1].split("const CERTIFICATE_API_KEY_HEADER", 1)[0] assert '"/api/status"' not in anonymous_block - assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.24.8"' in openapi_0241 + assert '"RuntimeInformation"' in openapi_0241 and '"version": "0.24.9"' in openapi_0241 assert 'id="panel-logging"' not in html for marker in ['id="settings-tab-logging"', 'id="settings-page-logging"', 'id="logging-form"']: assert marker in html, marker @@ -550,6 +561,9 @@ def source_contract_smoke() -> None: assert marker in update_check, marker assert 'update version document is too large or incomplete: {error}' in update_check assert '.context("update version document is too large or incomplete")' not in update_check + for marker in ['tracing::error!(', 'MAX_REDIRECTS: usize = 5', 'SSL_CERT_FILE', 'CURL_CA_BUNDLE', '/usr/local/share/certs/ca-root-nss.crt', 'response.status().is_redirection()', 'redirect_target(¤t, location)']: + assert marker in update_check, marker + assert 'pub last_error:' not in update_check for marker in ['RUST_PROXY_UPDATE_REPOSITORY_URL', 'RUST_PROXY_UPDATE_VERSION_URL', 'RUST_PROXY_UPDATE_CHECK_INTERVAL_SECS', 'state.start_update_check(update_check);']: assert marker in main, marker assert 'pub software_update: Arc' in runtime @@ -867,7 +881,7 @@ def source_contract_smoke() -> None: assert '/Ubuntu%2014.04%20LTS/' in acl and 'repeated_slashes_are_normalized_for_routing' in acl and 'percent_encoded_spaces_are_valid_request_paths' in policies and 'repeated_slashes_route_normally_but_reach_upstream_unchanged' in policies and 'local_location_response_uses_response_rules_without_opening_backend' in policies and 'local_location_response_still_respects_route_acl' in policies db_source = (ROOT / "src/db.rs").read_text(encoding="utf-8") acme_source = (ROOT / "src/acme.rs").read_text(encoding="utf-8") - assert "SCHEMA_VERSION: i64 = 6" in db_source and "certificate_renewals" in db_source and "0004_admin_preferences.sql" in db_source and "0005_certificate_api.sql" in db_source and "0006_certificate_api_credentials.sql" in db_source + assert "SCHEMA_VERSION: i64 = 7" in db_source and "certificate_renewals" in db_source and "0004_admin_preferences.sql" in db_source and "0005_certificate_api.sql" in db_source and "0006_certificate_api_credentials.sql" in db_source and "0007_secret_store.sql" in db_source assert "load_renewal_observations" not in acme_source and "renewals.json" not in acme_source for marker in ["pub kind: &'static str", "pub log_directory: String", "pub(crate) fn certbot_log_directory", 'kind: "issue"', '"renewal-dry-run"']: assert marker in acme_source, marker @@ -936,7 +950,7 @@ def source_contract_smoke() -> None: assert "removed_obsolete_runtime_state" in upgrade_source and "legacy-data" not in upgrade_source and "certs_dir.is_symlink()" in upgrade_source assert '.managed-update' in main and "flush_hot_to_cold_for_update" in main openapi = (ROOT / "src/admin/openapi.json").read_text(encoding="utf-8") - for marker in ['"version": "0.24.8"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: + for marker in ['"version": "0.24.9"', '"memory_preset"', '"cold_preset"', '"memory_budget_bytes"', '"max_inflight"', '"preset"', '"session_days"', '"session_lifetime_seconds"', '"accept_local_listener_risk"', '"backend_header"', '"backend_timing_header"', '"cold_on_age"', '"match_type"', '"regex_insensitive"', '"NotFoundHost"', '"not_found_hosts"', '"PrometheusSettings"', '"allow_trusted_network_without_token"', '"include_docker_vhosts"', '"include_streams"', '"include_logging"']: assert marker in openapi, marker assert '"software_update"' in openapi and '"update_available"' in openapi for marker in ['"/api/certificates/deploy"', '"CertificateDeploy"', '"CertificateDeployResult"', '"/api/certificates/reload"']: @@ -1084,7 +1098,7 @@ def main() -> None: source_contract_smoke() container_contract_smoke() debian_grafana_contract_smoke() - print("PASS: 0.24.8 SQLite/upgrade, local-response 404 hosts, Prometheus/Grafana diagnostics, Debian packaging, cache/dashboard/logging/Nagios, retained admin/H3/stream/TLS contracts and reproducible source archives.") + print("PASS: 0.24.9 SQLite/upgrade, local-response 404 hosts, Prometheus/Grafana diagnostics, Debian packaging, cache/dashboard/logging/Nagios, retained admin/H3/stream/TLS contracts and reproducible source archives.") if __name__ == "__main__": diff --git a/scripts/routeros-oci-build/README.md b/scripts/routeros-oci-build/README.md index b0b1f39..f5ac5b9 100644 --- a/scripts/routeros-oci-build/README.md +++ b/scripts/routeros-oci-build/README.md @@ -76,10 +76,10 @@ ENV_FILE=scripts/routeros-oci-build/.env \ bash scripts/routeros-oci-build/build-routeros-oci.sh push ``` -With RustProxy version `0.24.8` and the defaults this publishes: +With RustProxy version `0.24.9` and the defaults this publishes: ```text -zot.example.net/network/rustproxy:0.24.8 +zot.example.net/network/rustproxy:0.24.9 zot.example.net/network/rustproxy:latest zot.example.net/network/rustproxy:routeros ``` diff --git a/scripts/swagger_assets.py b/scripts/swagger_assets.py index 85740b4..c26d73d 100755 --- a/scripts/swagger_assets.py +++ b/scripts/swagger_assets.py @@ -85,7 +85,7 @@ def download(destination: Path) -> None: """TLS verification uses Python's default trust store; never executes a shell.""" started = time.monotonic() opener = build_opener(HttpsRedirects()) - request = Request(DOWNLOAD_URL, headers={"User-Agent": "RustProxy-build/0.24.8"}) + request = Request(DOWNLOAD_URL, headers={"User-Agent": "RustProxy-build/0.24.9"}) with opener.open(request, timeout=30) as response, destination.open("wb") as output: length = response.headers.get("Content-Length") if length and int(length) > MAX_ARCHIVE: diff --git a/scripts/upgrade.py b/scripts/upgrade.py index 23bcffa..b40c827 100755 --- a/scripts/upgrade.py +++ b/scripts/upgrade.py @@ -46,7 +46,7 @@ def upgrade(source: Path, target: Path, *, stopped: bool, data_dir: Path | None for name in MANAGED: if (target / name).is_symlink(): raise ValueError(f"Managed target {name} is a symlink; refusing to replace it.") - backup = target / ".upgrade-backup" / ("before-0.24.8-" + uuid.uuid4().hex[:12]) + backup = target / ".upgrade-backup" / ("before-0.24.9-" + uuid.uuid4().hex[:12]) backup.mkdir(parents=True, mode=0o700) installed: list[str] = [] moved: list[str] = [] @@ -84,7 +84,7 @@ def upgrade(source: Path, target: Path, *, stopped: bool, data_dir: Path | None remove_legacy = legacy.exists() if remove_legacy and (legacy.is_symlink() or not legacy.is_file()): raise ValueError("Obsolete renewal observation sidecar must be a regular file.") - (backup / "upgrade.json").write_text(json.dumps({"from_directory":str(target), "to_version":"0.24.8", "managed_paths":MANAGED, "removed_obsolete_runtime_state": remove_legacy}, indent=2)+"\n", encoding="utf-8") + (backup / "upgrade.json").write_text(json.dumps({"from_directory":str(target), "to_version":"0.24.9", "managed_paths":MANAGED, "removed_obsolete_runtime_state": remove_legacy}, indent=2)+"\n", encoding="utf-8") if remove_legacy: legacy.unlink() except BaseException: @@ -115,7 +115,7 @@ def main() -> None: backup = upgrade(Path(__file__).resolve().parents[1], args.target, stopped=args.stopped, data_dir=data_dir) except (OSError, ValueError) as error: parser.exit(2, f"Upgrade refused or rolled back: {error}\n") - print(f"Installed RustProxy 0.24.8. Previous managed source files: {backup}") + print(f"Installed RustProxy 0.24.9. Previous managed source files: {backup}") print("Preserved data, .env, .git, target and other unmanaged files. No database migration was run.") print(f"Next: cd {args.target}; ./dev.sh test; ./dev.sh") diff --git a/src/acme.rs b/src/acme.rs index e474ea9..d1087f3 100644 --- a/src/acme.rs +++ b/src/acme.rs @@ -1,9 +1,11 @@ //! Bounded control-plane adapter for Certbot HTTP-01/DNS-01 providers. Never on traffic paths. -//! Provider credentials enter via stdin, not argv, logs, SQLite or API responses. +//! Provider credentials enter via stdin, are encrypted in Secret Store, and never +//! travel through argv, logs or API responses. use crate::{ runtime::{ApiError, App}, tls::Certificate, }; +use base64::Engine as _; use http::StatusCode; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -39,6 +41,100 @@ pub(crate) fn certbot_log_directory(directory: &std::path::Path, provider: &str, .to_string_lossy() .into_owned() } +fn provider_credential_bytes(request: &Value, provider: &str) -> Result>, ApiError> { + fn credential<'a>(request: &'a Value, name: &str) -> Result<&'a str, ApiError> { + let invalid = || ApiError(StatusCode::BAD_REQUEST, "acme.invalid_api_credential".into()); + let value = request.get(name).and_then(Value::as_str).ok_or_else(invalid)?; + let valid = (8..=4096).contains(&value.len()) + && value.bytes().all(|byte| { + byte.is_ascii_alphanumeric() + || matches!(byte, b'_' | b'.' | b'/' | b'+' | b'-' | b'=') + }); + if valid { Ok(value) } else { Err(invalid()) } + } + let bytes = match provider { + "cloudflare" => format!("dns_cloudflare_api_token = {}\n", credential(request, "api_token")?).into_bytes(), + "ovh" => { + if request.get("endpoint").is_some_and(|value| !value.is_string()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_ovh_endpoint".into())); + } + let endpoint = request.get("endpoint").and_then(Value::as_str).unwrap_or("ovh-eu"); + if !matches!(endpoint, "ovh-eu" | "ovh-ca") { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_ovh_endpoint".into())); + } + format!( + "dns_ovh_endpoint = {}\ndns_ovh_application_key = {}\ndns_ovh_application_secret = {}\ndns_ovh_consumer_key = {}\n", + endpoint, + credential(request, "application_key")?, + credential(request, "application_secret")?, + credential(request, "consumer_key")? + ) + .into_bytes() + } + "rfc2136" => { + let dns_server = request + .get("dns_server") + .and_then(Value::as_str) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_dns_server".into()))?; + if request.get("dns_port").is_some_and(|value| value.as_u64().is_none()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_limits".into())); + } + let dns_port = request.get("dns_port").and_then(Value::as_u64).unwrap_or(53); + if !(1..=65535).contains(&dns_port) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_limits".into())); + } + let raw_name = request.get("key_name").and_then(Value::as_str).unwrap_or(""); + let key_name = raw_name.to_ascii_lowercase(); + let key_name = key_name.trim_end_matches('.'); + let valid_name = !key_name.is_empty() + && key_name.len() <= 253 + && key_name.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label.bytes().enumerate().all(|(index, byte)| { + (byte.is_ascii_lowercase() || byte.is_ascii_digit()) + || (byte == b'-' && index > 0 && index + 1 < label.len()) + }) + }); + if !valid_name { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_domain".into())); + } + if request.get("key_algorithm").is_some_and(|value| !value.is_string()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_algorithm".into())); + } + let key_algorithm = request + .get("key_algorithm") + .and_then(Value::as_str) + .unwrap_or("HMAC-SHA256"); + if !matches!(key_algorithm, "HMAC-SHA256" | "HMAC-SHA384" | "HMAC-SHA512") { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_algorithm".into())); + } + let key_secret = request.get("key_secret").and_then(Value::as_str).ok_or_else(|| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_api_credential".into()))?; + let decoded = Zeroizing::new( + base64::engine::general_purpose::STANDARD + .decode(key_secret) + .map_err(|_| ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_secret".into()))?, + ); + if key_secret.len() > 1024 || !(16..=512).contains(&decoded.len()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_key_secret".into())); + } + if request.get("sign_query").is_some_and(|value| !value.is_boolean()) { + return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_sign_query".into())); + } + let sign_query = request.get("sign_query").and_then(Value::as_bool).unwrap_or(false); + format!( + "dns_rfc2136_server = {dns_server}\ndns_rfc2136_port = {dns_port}\ndns_rfc2136_name = {key_name}.\ndns_rfc2136_secret = {key_secret}\ndns_rfc2136_algorithm = {key_algorithm}\ndns_rfc2136_sign_query = {sign_query}\n" + ) + .into_bytes() + } + "http01" => Vec::new(), + _ => return Err(ApiError(StatusCode::BAD_REQUEST, "acme.invalid_provider".into())), + }; + Ok(Zeroizing::new(bytes)) +} + + pub struct Manager { slot: Arc, job: Mutex>, @@ -137,6 +233,22 @@ impl Manager { .clone() .try_acquire_owned() .map_err(|_| ApiError(StatusCode::TOO_MANY_REQUESTS, "acme.job_busy".into()))?; + if provider != "http01" { + let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| { + ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into()) + })?; + let credential = provider_credential_bytes(&request, &provider)?; + crate::secret_store::put( + &state.store, + data_dir, + &crate::secret_store::acme_secret_id(&id, &provider), + &provider, + credential.as_ref(), + serde_json::json!({"certificate_id": id.clone(), "purpose": "acme_dns"}), + ) + .await + .map_err(crate::auth::internal)?; + } let raw = Zeroizing::new(serde_json::to_string(&request).map_err(crate::auth::internal)?); for key in [ "key_secret", @@ -789,9 +901,28 @@ impl Manager { .await .map_err(crate::auth::internal)?; *state.acme.job.lock().unwrap_or_else(|e| e.into_inner()) = Some(job.clone()); - let request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider, + let mut request = serde_json::json!({"action":"renew","certificate_id":cert.id,"provider":issuer.provider, "environment":issuer.environment,"cert_file":cert.cert_file,"key_file":cert.key_file,"dry_run":dry_run}); + if issuer.provider != "http01" { + let data_dir = state.policy.data_dir.as_ref().ok_or_else(|| { + ApiError(StatusCode::CONFLICT, "acme.secret_store_unavailable".into()) + })?; + let credential = crate::secret_store::get( + &state.store, + data_dir, + &crate::secret_store::acme_secret_id(&cert.id, &issuer.provider), + Some(&issuer.provider), + ) + .await + .map_err(|_| ApiError(StatusCode::CONFLICT, "acme.secret_missing".into()))?; + request["credential_ini_b64"] = Value::String( + base64::engine::general_purpose::STANDARD.encode(credential.as_ref()) + ); + } let raw = Zeroizing::new(request.to_string()); + if let Some(Value::String(secret)) = request.get_mut("credential_ini_b64") { + secret.zeroize(); + } let task_job = job.clone(); let tasks = state.tunnels.clone(); tasks.spawn(async move { diff --git a/src/admin/mod.rs b/src/admin/mod.rs index eeee43c..395a250 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1324,8 +1324,11 @@ fn status_payload(state: &App) -> Value { payload["transform_metrics"] = json!(state.transform_metrics.total()); payload["logs_dropped"] = json!(state.logs.dropped.load(Ordering::Relaxed)); payload["software_update"] = json!(state.software_update.snapshot()); - payload["logging"] = - json!({"settings": &snapshot.envelope.config.logging, "sinks": state.logs.stats()}); + payload["logging"] = json!({ + "settings": &snapshot.envelope.config.logging, + "sinks": state.logs.stats(), + "error_file": crate::logging::error_log_stats(), + }); payload["tls"] = json!({ "policy": &snapshot.envelope.config.tls.policy, "cipher_suites": crate::tls::cipher_suite_names(), diff --git a/src/admin/openapi.json b/src/admin/openapi.json index 62ecf70..a8568d8 100644 --- a/src/admin/openapi.json +++ b/src/admin/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.3", "info": { "title": "RustProxy Administration API", - "version": "0.24.8", + "version": "0.24.9", "description": "Administration API for the active panel origin. Sign in through the panel; the same-origin HttpOnly cookie authenticates all private operations. Mutations also require the exact Origin, X-RustProxy-Request: 1 and a session-bound X-CSRF-Token. Loopback HTTP is the default; Settings can enable HTTPS before using a non-loopback administration bind. This document contains no live configuration or credentials. Object schemas marked extensible are summaries: full configuration validation is authoritative in Rust." }, "servers": [ @@ -693,6 +693,30 @@ }, "transform": { "$ref": "#/components/schemas/TransformSettings" + }, + "error_log": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "path": { + "type": "string", + "default": "/var/log/rustproxy/error.log", + "maxLength": 1024 + }, + "format": { + "type": "string", + "enum": [ + "JSON", + "RAW" + ], + "default": "JSON" + } + }, + "description": "Optional asynchronous ERROR-only file sink. Rotation and retention are handled externally by logrotate." } }, "additionalProperties": true @@ -916,29 +940,80 @@ "transform_resources": { "type": "object", "properties": { - "memory_reserved_bytes": { "type": "integer", "format": "int64" }, - "memory_budget_bytes": { "type": "integer", "format": "int64" }, - "inflight": { "type": "integer", "format": "int64" }, - "max_inflight": { "type": "integer", "format": "int64" } + "memory_reserved_bytes": { + "type": "integer", + "format": "int64" + }, + "memory_budget_bytes": { + "type": "integer", + "format": "int64" + }, + "inflight": { + "type": "integer", + "format": "int64" + }, + "max_inflight": { + "type": "integer", + "format": "int64" + } }, "additionalProperties": false }, "memory_accounting": { "type": "object", "properties": { - "process_rss_bytes": { "type": "integer", "format": "int64", "nullable": true }, - "cache_hot_bytes": { "type": "integer", "format": "int64" }, - "cache_hot_limit_bytes": { "type": "integer", "format": "int64" }, - "cache_transient_bytes": { "type": "integer", "format": "int64" }, - "cache_transient_limit_bytes": { "type": "integer", "format": "int64" }, - "cache_total_bytes": { "type": "integer", "format": "int64" }, - "transform_reserved_bytes": { "type": "integer", "format": "int64" }, - "transform_budget_bytes": { "type": "integer", "format": "int64" }, - "transform_inflight": { "type": "integer", "format": "int64" }, - "transform_max_inflight": { "type": "integer", "format": "int64" }, - "non_cache_rss_bytes": { "type": "integer", "format": "int64", "nullable": true }, - "non_cache_rss_includes_transform": { "type": "boolean" }, - "method": { "type": "string" } + "process_rss_bytes": { + "type": "integer", + "format": "int64", + "nullable": true + }, + "cache_hot_bytes": { + "type": "integer", + "format": "int64" + }, + "cache_hot_limit_bytes": { + "type": "integer", + "format": "int64" + }, + "cache_transient_bytes": { + "type": "integer", + "format": "int64" + }, + "cache_transient_limit_bytes": { + "type": "integer", + "format": "int64" + }, + "cache_total_bytes": { + "type": "integer", + "format": "int64" + }, + "transform_reserved_bytes": { + "type": "integer", + "format": "int64" + }, + "transform_budget_bytes": { + "type": "integer", + "format": "int64" + }, + "transform_inflight": { + "type": "integer", + "format": "int64" + }, + "transform_max_inflight": { + "type": "integer", + "format": "int64" + }, + "non_cache_rss_bytes": { + "type": "integer", + "format": "int64", + "nullable": true + }, + "non_cache_rss_includes_transform": { + "type": "boolean" + }, + "method": { + "type": "string" + } }, "additionalProperties": false }, @@ -3239,10 +3314,22 @@ "type": "object", "additionalProperties": false, "properties": { - "memory_reserved_bytes": { "type": "integer", "format": "int64" }, - "memory_budget_bytes": { "type": "integer", "format": "int64" }, - "inflight": { "type": "integer", "format": "int64" }, - "max_inflight": { "type": "integer", "format": "int64" } + "memory_reserved_bytes": { + "type": "integer", + "format": "int64" + }, + "memory_budget_bytes": { + "type": "integer", + "format": "int64" + }, + "inflight": { + "type": "integer", + "format": "int64" + }, + "max_inflight": { + "type": "integer", + "format": "int64" + } }, "required": [ "memory_reserved_bytes", @@ -4332,7 +4419,7 @@ } } }, - "description": "Sets an HttpOnly SameSite=Strict session cookie. Browser sessions use the configured 7\u201330 day absolute lifetime. Use the panel sign-in page. No bearer token or URL token is accepted.", + "description": "Sets an HttpOnly SameSite=Strict session cookie. Browser sessions use the configured 7–30 day absolute lifetime. Use the panel sign-in page. No bearer token or URL token is accepted.", "security": [], "parameters": [ { diff --git a/src/admin/web/app.js b/src/admin/web/app.js index 60ee51d..dbd0b6c 100644 --- a/src/admin/web/app.js +++ b/src/admin/web/app.js @@ -778,7 +778,7 @@ function renderSettings() { if (!settingsDirty.has("response-form")) { const server = config.server_header || {}, compression = config.compression || {}, diagnostics = config.diagnostic_headers || {}; $("expose-request-id").checked = !!diagnostics.request_id; - $("server-mode").value = server.mode || "preserve"; $("server-value").value = server.value || "RustProxy/v0.24.8"; + $("server-mode").value = server.mode || "preserve"; $("server-value").value = server.value || "RustProxy/v0.24.9"; for (const name of ["gzip", "br", "zstd", "deflate"]) $(`compress-${name}`).checked = !!compression[name]; $("compress-min").value = compression.min_bytes ?? 1024; syncServerMode(); } @@ -821,14 +821,23 @@ function renderSettings() { updateTransformResourceSizing(); } if (!settingsDirty.has("logging-form")) { - const l = config.logging || {}, file = l.file || {}, logstash = l.logstash || {}, loki = l.loki || {}, format = l.format || {}; + const l = config.logging || {}, file = l.file || {}, logstash = l.logstash || {}, loki = l.loki || {}, format = l.format || {}, errorLog = config.error_log || {}; $("log-file-enabled").checked = !!file.enabled; $("log-file-per-vhost").checked = !!file.per_vhost; $("log-file-path").value = file.path || "/var/log/rustproxy/access.jsonl"; $("log-file-directory").value = file.directory || "/var/log/rustproxy/vhosts"; syncLoggingFileMode(); $("log-logstash-enabled").checked = !!logstash.enabled; $("log-logstash-address").value = logstash.address || "127.0.0.1:5044"; $("log-loki-enabled").checked = !!loki.enabled; $("log-loki-url").value = loki.url || "http://127.0.0.1:3100/loki/api/v1/push"; $("log-loki-token").value = loki.bearer_token_env || ""; $("log-loki-labels").value = JSON.stringify(loki.labels || { job: "rustproxy" }, null, 2); $("log-format-fields").value = JSON.stringify(format.fields || {}, null, 2); $("log-format-static").value = JSON.stringify(format.static_fields || {}, null, 2); $("log-format-preset").value = detectLoggingPreset(format.fields || {}); + $("error-log-enabled").checked = !!errorLog.enabled; + $("error-log-path").value = errorLog.path || "/var/log/rustproxy/error.log"; + $("error-log-format").value = ["JSON", "RAW"].includes(errorLog.format) ? errorLog.format : "JSON"; + syncErrorLogMode(); + } + const sink = latestStatus?.logging?.sinks, errorSink = latestStatus?.logging?.error_file; + if (sink) { + let status = `${t("loggingQueue")}: ${fmt(sink.queue_dropped)} ${t("loggingDropped")} / ${fmt(sink.sent)} ${t("loggingSent")} / ${fmt(sink.errors)} ${t("errors")} · ${t("loggingCapacity")}: ${fmt(sink.queue_capacity)}`; + if (errorSink) status += ` · ${t("errorLogTitle")}: ${fmt(errorSink.queue_dropped)} ${t("loggingDropped")} / ${fmt(errorSink.written)} ${t("loggingSent")} / ${fmt(errorSink.errors)} ${t("errors")}`; + $("logging-status").textContent = status; } - const sink = latestStatus?.logging?.sinks; if (sink) $("logging-status").textContent = `${t("loggingQueue")}: ${fmt(sink.queue_dropped)} ${t("loggingDropped")} / ${fmt(sink.sent)} ${t("loggingSent")} / ${fmt(sink.errors)} ${t("errors")} · ${t("loggingCapacity")}: ${fmt(sink.queue_capacity)}`; if (!settingsDirty.has("forwarding-form")) { $("trusted-proxies").value = (config.forwarding?.trusted_proxies || []).join("\n"); $("trusted-proto").checked = !!config.forwarding?.trust_forwarded_proto; } if (!settingsDirty.has("headers-form")) { for (const direction of ["request", "response"]) renderHeaderRules(direction, config.headers?.[direction] || []); } } @@ -991,6 +1000,12 @@ function syncLoggingFileMode() { $("log-file-path").disabled = !enabled || perVhost; $("log-file-directory").disabled = !enabled || !perVhost; } +function syncErrorLogMode() { + const disabled = !$("error-log-enabled").checked; + $("error-log-path").disabled = disabled; + $("error-log-format").disabled = disabled; +} + function syncColdCachePolicy() { const enabled = $("cache-cold-enabled").checked; for (const id of ["cache-cold-path", "cache-cold-bytes", "cache-cold-entries", "cache-cold-queue-bytes", "cache-cold-gc-interval", "cache-cold-gc-max-delete", "cache-cold-on-idle", "cache-cold-on-age", "cache-cold-on-lru", "cache-cold-on-shutdown"]) $(id).disabled = !enabled; @@ -1113,10 +1128,16 @@ bindSettingsForm("logging-form", next => { loki: { enabled: $("log-loki-enabled").checked, url: $("log-loki-url").value.trim(), bearer_token_env: $("log-loki-token").value.trim() || null, labels: parseObject("log-loki-labels") }, format: { fields: parseObject("log-format-fields"), static_fields: parseObject("log-format-static") } }; + next.error_log = { + enabled: $("error-log-enabled").checked, + path: $("error-log-path").value.trim(), + format: $("error-log-format").value + }; }, "logging"); bindSettingsForm("forwarding-form", next => { next.forwarding = { trusted_proxies: $("trusted-proxies").value.split(/[\n,]/).map(s => s.trim()).filter(Boolean), trust_forwarded_proto: $("trusted-proto").checked }; }, "forwarding"); bindSettingsForm("headers-form", next => { next.headers = { request: collectHeaderRules("request"), response: collectHeaderRules("response") }; }, "headers"); $("log-format-preset").addEventListener("change", event => applyLoggingPreset(event.target.value)); +$("error-log-enabled").addEventListener("change", syncErrorLogMode); $("log-format-open").addEventListener("click", openLoggingBuilder); $("logging-format-close").addEventListener("click", () => $("logging-format-dialog").close()); $("logging-format-cancel").addEventListener("click", () => $("logging-format-dialog").close()); diff --git a/src/admin/web/index.html b/src/admin/web/index.html index e7974e1..85050f6 100644 --- a/src/admin/web/index.html +++ b/src/admin/web/index.html @@ -1391,7 +1391,7 @@

Diagnostic headers

@@ -1798,6 +1798,20 @@ +
+

Error file

+ + + +

Only ERROR events are written. Rotation is handled by system logrotate.

+

Configurable JSON format

Map output names to source fields. Leave @@ -2098,7 +2112,7 @@ -