v0.34.0_p1

scripts/tests/cache_age_policy_034_model.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Deterministic oracle for HTTP Age / local freshness-cap algebra.
+No binary execution, no claim of measured throughput. Run with Python 3 only.
+"""
+import json
+import random
+from pathlib import Path
+
+
+def old_freshness(max_age, header_age, apparent_age, response_delay, local_limit):
+    age = max(header_age, apparent_age) + response_delay
+    return max(0, min(max_age, local_limit) - age)
+
+
+def new_freshness(max_age, header_age, apparent_age, response_delay, local_limit):
+    # RFC 9111 corrected_initial_age = max(apparent_age, age_value + response_delay)
+    age = max(apparent_age, header_age + response_delay)
+    lifetime = min(max_age, age + local_limit)
+    return max(0, lifetime - age)
+
+
+def run():
+    cases = [
+        ('user_cdn_example', 2_592_000, 1_553_483, 0, 0, 86_400),
+        ('user_cdn_example_7d', 2_592_000, 1_553_483, 0, 0, 604_800),
+        ('origin_nearly_stale', 2_592_000, 2_591_990, 0, 0, 86_400),
+        ('origin_stale', 2_592_000, 2_592_010, 0, 0, 86_400),
+        ('normal_small', 3_600, 0, 0, 0, 86_400),
+        ('date_is_older', 3_600, 10, 600, 2, 86_400),
+        ('rfc_delay_case', 3_600, 3, 10, 2, 86_400),
+    ]
+    report = {'reference': 'RFC 9111 4.2.3 (Age) and 4.2.1 (freshness lifetime)',
+              'kind': 'algebra simulation; not RustProxy RPS benchmark',
+              'scenarios': [], 'randomized_checks': 0, 'false_miss_cases_old': 0}
+    for name, max_age, age, apparent, delay, local in cases:
+        old = old_freshness(max_age, age, apparent, delay, local)
+        new = new_freshness(max_age, age, apparent, delay, local)
+        report['scenarios'].append({'scenario': name, 'origin_max_age_seconds': max_age,
+                                    'received_age_seconds': age, 'apparent_age_seconds': apparent,
+                                    'response_delay_seconds': delay, 'local_max_ttl_seconds': local,
+                                    'old_freshness_seconds': old, 'new_freshness_seconds': new})
+    rng = random.Random(0x3400)
+    for _ in range(100_000):
+        origin = rng.randrange(0, 1 << 24)
+        header_age = rng.randrange(0, 1 << 24)
+        apparent = rng.randrange(0, 1 << 24)
+        delay = rng.randrange(0, 300)
+        local = rng.randrange(0, 604_801)
+        corrected_age = max(apparent, header_age + delay)
+        new = new_freshness(origin, header_age, apparent, delay, local)
+        expected = min(max(0, origin - corrected_age), local)
+        assert new == expected, (origin, header_age, apparent, delay, local, new, expected)
+        assert new <= local and new <= max(0, origin - corrected_age)
+        old = old_freshness(origin, header_age, apparent, delay, local)
+        report['false_miss_cases_old'] += old == 0 < new
+        report['randomized_checks'] += 1
+    return report
+
+
+if __name__ == '__main__':
+    report = run()
+    print(json.dumps(report, ensure_ascii=False, indent=2))

scripts/tests/cache_age_transform_034_live.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""Loopback integration regression: transformed Age, HOT/COLD, diagnostics, isolation.
+
+Run after compiling:
+  python3 scripts/tests/cache_age_transform_034_live.py --binary target/release/rust-proxy
+To reproduce the original regression with the user-supplied 0.34.0 binary:
+  python3 scripts/tests/cache_age_transform_034_live.py --binary /path/to/old/rust-proxy --expect-old-age-bug
+No internet and no user data; test config/data/passwords are in a disposable temporary dir.
+"""
+import argparse
+import collections
+import copy
+import json
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'performance'))
+from common import Fixture, http, sha256, write_json
+
+BODY = b'/* test cache */\nbody { color: red; padding: 0px; }\n'
+# Fixed sample from Brotli encoder, embedded to avoid Python dependency on brotli.
+BROTLI_BODY = b"\x1b3\x00\xe8\x1d\x07\xee\xbe\r\xa2\xbc3'\x8cP\xe9\xad\xa0\xf2\xb2\xa7\xe4\xfa\x12\x84\xac\x0c\x1cr\xc0\xfe\xc0>\x0fR*a\x9c\x9e2\xe8\x08\xcb\x1cP\x881\xac\xdd\x88s5(\x9d\x01"
+COUNTS = collections.Counter()
+
+class Origin(BaseHTTPRequestHandler):
+    def log_message(self, *_):
+        pass
+
+    def do_GET(self):
+        path = self.path.split('?', 1)[0]
+        if path not in {'/aged.css', '/normal.css', '/private.css', '/no-store.css', '/other.css', '/br.css', '/aged-br.css'}:
+            self.send_error(404)
+            return
+        COUNTS[path] += 1
+        self.send_response(200)
+        if path in ('/aged.css', '/aged-br.css'):
+            policy = 'public, max-age=2592000'
+        elif path == '/private.css':
+            policy = 'private, max-age=2592000'
+        elif path == '/no-store.css':
+            policy = 'no-store'
+        else:
+            policy = 'public, max-age=3600'
+        self.send_header('Cache-Control', policy)
+        self.send_header('Content-Type', 'text/css')
+        payload = BROTLI_BODY if path in ('/br.css', '/aged-br.css') else BODY
+        self.send_header('Content-Length', str(len(payload)))
+        if path in ('/br.css', '/aged-br.css'):
+            self.send_header('Content-Encoding', 'br')
+        self.send_header('ETag', '"origin-css"')
+        self.send_header('Last-Modified', 'Thu, 07 Jan 2016 20:04:59 GMT')
+        self.send_header('Vary', 'Accept-Encoding')
+        if path in ('/aged.css', '/aged-br.css'):
+            self.send_header('Age', '1553483')
+        self.end_headers()
+        self.wfile.write(payload)
+
+
+def request(port, path, encoding='identity'):
+    status, headers, body = http(port, path, headers={'Accept-Encoding': encoding})
+    return {'status': status, 'x_cache': headers.get('x-cache'),
+            'x_cache_hits': headers.get('x-cache-hits'), 'x_cache_resident_age': headers.get('x-cache-resident-age'),
+            'cache_status': headers.get('cache-status'), 'age': headers.get('age'),
+            'body': body.decode('utf-8', 'replace'), 'content_length': headers.get('content-length'),
+            'etag': headers.get('etag'), 'encoding': headers.get('content-encoding'), 'origin_count': COUNTS[path]}
+
+
+def dash(rust):
+    status, _, body = rust.admin('/api/cache?limit=200')
+    assert status == 200, ('cache dashboard', status, body[:200])
+    data = json.loads(body)['dashboard']
+    return {'objects': [{k: row.get(k) for k in ('vhost','path','tier','transformed',
+                     'hits','age_seconds','resident_age_seconds','remaining_ttl_seconds')} for row in data['objects']],
+            'domains': data['domains']}
+
+
+def find(d, path):
+    matches = [item for item in d['objects'] if item['path'] == path and item['vhost'] == 'reverse']
+    assert len(matches) == 1, (path, matches)
+    return matches[0]
+
+
+def run(binary: Path, output: Path, expect_old_age_bug: bool):
+    outcome = {'binary_sha256': sha256(binary), 'expect_old_age_bug': expect_old_age_bug,
+               'tests': [], 'passed': False}
+    output.mkdir(parents=True, exist_ok=True)
+    with Fixture(output, workers=2) as fixture:
+        origin = ThreadingHTTPServer(('127.0.0.1', fixture.origin_port), Origin)
+        thread = threading.Thread(target=origin.serve_forever, daemon=True)
+        thread.start()
+        try:
+            rust = fixture.rust(binary, 'cache-age-034')
+            rust.login()
+            cfg = copy.deepcopy(rust.base)
+            cfg['cache'] = {'enabled': True, 'max_ttl_seconds': 86_400,
+                            'expose_status': True, 'expose_hits': True,
+                            'expose_cache_status': True, 'expose_resident_age': True}
+            cfg['transform'] = {'enabled': True, 'css': True}
+            for vhost in cfg['vhosts']:
+                vhost['transform'] = {'enabled': True, 'css': True}
+            rust.apply(cfg, 'fresh-cache-transform')
+            for path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css', '/private.css', '/no-store.css'):
+                before = COUNTS[path]
+                encoding = 'br' if path in ('/br.css', '/aged-br.css') else 'identity'
+                first, second = request(rust.reverse_port, path, encoding), request(rust.reverse_port, path, encoding)
+                row = {'path': path, 'first': first, 'second': second,
+                       'origin_delta': COUNTS[path] - before}
+                outcome['tests'].append(row)
+                if path in ('/aged.css', '/aged-br.css') and expect_old_age_bug:
+                    assert first['x_cache'] == second['x_cache'] == 'MISS', row
+                    assert row['origin_delta'] == 2, row
+                elif path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css'):
+                    assert first['x_cache'] == 'MISS' and second['x_cache'] == 'HIT', row
+                    assert row['origin_delta'] == 1, row
+                    assert second['cache_status'] and 'transformed' in second['cache_status'], row
+                    assert first['etag'] == second['etag'] and first['body'] == second['body'], row
+                else:
+                    assert first['x_cache'] == second['x_cache'] == 'MISS', row
+                print(path, first['x_cache'], second['x_cache'], 'origin+', row['origin_delta'], flush=True)
+            snapshot = dash(rust)
+            outcome['tests'].append({'dashboard': snapshot})
+            assert find(snapshot, '/normal.css')['transformed'], snapshot
+            aged = find(snapshot, '/aged.css')
+            if expect_old_age_bug:
+                assert aged['remaining_ttl_seconds'] == 0, aged
+            else:
+                assert aged['transformed'] and aged['remaining_ttl_seconds'] >= 86_000, aged
+                assert aged['age_seconds'] >= 1_553_483, aged
+                assert aged['resident_age_seconds'] < 10, aged
+                aged_br = find(snapshot, '/aged-br.css')
+                assert aged_br['transformed'] and aged_br['remaining_ttl_seconds'] >= 86_000, aged_br
+
+            # Operational LB tuning must not invalidate a cached representation.
+            before = COUNTS['/normal.css']
+            cfg['vhosts'][0]['load_balancing']['keepalive'] = {'max_idle_per_host': 16}
+            rust.apply(cfg, 'only-backend-keepalive-changed')
+            normal = request(rust.reverse_port, '/normal.css')
+            assert normal['x_cache'] == 'HIT' and COUNTS['/normal.css'] == before
+            outcome['tests'].append({'keepalive_update_without_purge': normal})
+
+            # Diagnostics can be intentionally hidden on MISS and remain visible on HIT.
+            cfg['cache']['expose_only_hits'] = True
+            rust.apply(cfg, 'only-show-hit-headers')
+            before = COUNTS['/other.css']
+            first, second = request(rust.reverse_port, '/other.css'), request(rust.reverse_port, '/other.css')
+            assert first['x_cache'] is None and second['x_cache'] == 'HIT'
+            assert COUNTS['/other.css'] - before == 1
+            outcome['tests'].append({'only_hits': [first, second]})
+
+            if not expect_old_age_bug:
+                # Cold path on a 1-entry HOT tier: demotion and on-disk reuse.
+                cfg['cache'].update({'max_entries': 1, 'cold_enabled': True, 'cold_on_lru': True,
+                                     'cold_path': 'cache/age-regression-cold'})
+                cfg['cache'].pop('expose_only_hits', None)
+                rust.apply(cfg, 'cold-enabled-one-hot-entry')
+                assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'MISS'
+                assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'HIT'
+                assert request(rust.reverse_port, '/other.css')['x_cache'] == 'MISS'
+                deadline = time.monotonic() + 4
+                while True:
+                    snapshot = dash(rust)
+                    matches = [x for x in snapshot['objects'] if x['path'] == '/aged.css' and x['tier'] == 'cold']
+                    if matches:
+                        assert matches[0]['remaining_ttl_seconds'] >= 86_000, matches
+                        break
+                    if time.monotonic() > deadline:
+                        raise AssertionError('aged transformed CSS never demoted to COLD: ' + str(snapshot))
+                    time.sleep(.05)
+                cold_hit = request(rust.reverse_port, '/aged.css')
+                assert cold_hit['x_cache'] == 'HIT' and 'transformed' in cold_hit['cache_status'], cold_hit
+                outcome['tests'].append({'cold': {'row': matches[0], 'hit': cold_hit}})
+            outcome['passed'] = True
+        finally:
+            origin.shutdown()
+            origin.server_close()
+    write_json(output / 'cache-age-transform-result.json', outcome)
+    return outcome
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--binary', type=Path, required=True)
+    parser.add_argument('--output', type=Path, default=Path('/tmp/rustproxy-cache-age-034-live'))
+    parser.add_argument('--expect-old-age-bug', action='store_true')
+    args = parser.parse_args()
+    try:
+        report = run(args.binary.resolve(), args.output.resolve(), args.expect_old_age_bug)
+    except Exception as error:
+        print('FAIL', type(error).__name__, str(error)[:1000], file=sys.stderr)
+        raise
+    print('PASS', len(report['tests']), 'scenarios; see', args.output / 'cache-age-transform-result.json')