#!/usr/bin/env python3 """Loopback integration regression: transformed Age, HOT/COLD, diagnostics, isolation. Run after compiling: python3 scripts/tests/cache_age_transform_034_live.py --binary target/release/rust-proxy To reproduce the original regression with the user-supplied 0.34.0 binary: python3 scripts/tests/cache_age_transform_034_live.py --binary /path/to/old/rust-proxy --expect-old-age-bug No internet and no user data; test config/data/passwords are in a disposable temporary dir. """ import argparse import collections import copy import json import sys import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'performance')) from common import Fixture, http, sha256, write_json BODY = b'/* test cache */\nbody { color: red; padding: 0px; }\n' # Fixed sample from Brotli encoder, embedded to avoid Python dependency on brotli. BROTLI_BODY = b"\x1b3\x00\xe8\x1d\x07\xee\xbe\r\xa2\xbc3'\x8cP\xe9\xad\xa0\xf2\xb2\xa7\xe4\xfa\x12\x84\xac\x0c\x1cr\xc0\xfe\xc0>\x0fR*a\x9c\x9e2\xe8\x08\xcb\x1cP\x881\xac\xdd\x88s5(\x9d\x01" COUNTS = collections.Counter() class Origin(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_GET(self): path = self.path.split('?', 1)[0] if path not in {'/aged.css', '/normal.css', '/private.css', '/no-store.css', '/other.css', '/br.css', '/aged-br.css'}: self.send_error(404) return COUNTS[path] += 1 self.send_response(200) if path in ('/aged.css', '/aged-br.css'): policy = 'public, max-age=2592000' elif path == '/private.css': policy = 'private, max-age=2592000' elif path == '/no-store.css': policy = 'no-store' else: policy = 'public, max-age=3600' self.send_header('Cache-Control', policy) self.send_header('Content-Type', 'text/css') payload = BROTLI_BODY if path in ('/br.css', '/aged-br.css') else BODY self.send_header('Content-Length', str(len(payload))) if path in ('/br.css', '/aged-br.css'): self.send_header('Content-Encoding', 'br') self.send_header('ETag', '"origin-css"') self.send_header('Last-Modified', 'Thu, 07 Jan 2016 20:04:59 GMT') self.send_header('Vary', 'Accept-Encoding') if path in ('/aged.css', '/aged-br.css'): self.send_header('Age', '1553483') self.end_headers() self.wfile.write(payload) def request(port, path, encoding='identity'): status, headers, body = http(port, path, headers={'Accept-Encoding': encoding}) return {'status': status, 'x_cache': headers.get('x-cache'), 'x_cache_hits': headers.get('x-cache-hits'), 'x_cache_resident_age': headers.get('x-cache-resident-age'), 'cache_status': headers.get('cache-status'), 'age': headers.get('age'), 'body': body.decode('utf-8', 'replace'), 'content_length': headers.get('content-length'), 'etag': headers.get('etag'), 'encoding': headers.get('content-encoding'), 'origin_count': COUNTS[path.split('?', 1)[0]]} def dash(rust): status, _, body = rust.admin('/api/cache?limit=200') assert status == 200, ('cache dashboard', status, body[:200]) data = json.loads(body)['dashboard'] return {'objects': [{k: row.get(k) for k in ('vhost','path','tier','transformed', 'hits','age_seconds','resident_age_seconds','remaining_ttl_seconds')} for row in data['objects']], 'domains': data['domains']} def find(d, path): matches = [item for item in d['objects'] if item['path'] == path and item['vhost'] == 'reverse'] assert len(matches) == 1, (path, matches) return matches[0] def run(binary: Path, output: Path, expect_old_age_bug: bool): outcome = {'binary_sha256': sha256(binary), 'expect_old_age_bug': expect_old_age_bug, 'tests': [], 'passed': False} output.mkdir(parents=True, exist_ok=True) with Fixture(output, workers=2) as fixture: origin = ThreadingHTTPServer(('127.0.0.1', fixture.origin_port), Origin) thread = threading.Thread(target=origin.serve_forever, daemon=True) thread.start() try: rust = fixture.rust(binary, 'cache-age-034') rust.login() cfg = copy.deepcopy(rust.base) cfg['cache'] = {'enabled': True, 'max_ttl_seconds': 86_400, 'expose_status': True, 'expose_hits': True, 'expose_cache_status': True, 'expose_resident_age': True} cfg['transform'] = {'enabled': True, 'css': True} for vhost in cfg['vhosts']: vhost['transform'] = {'enabled': True, 'css': True} rust.apply(cfg, 'fresh-cache-transform') for path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css', '/private.css', '/no-store.css'): before = COUNTS[path] encoding = 'br' if path in ('/br.css', '/aged-br.css') else 'identity' first, second = request(rust.reverse_port, path, encoding), request(rust.reverse_port, path, encoding) row = {'path': path, 'first': first, 'second': second, 'origin_delta': COUNTS[path] - before} outcome['tests'].append(row) if path in ('/aged.css', '/aged-br.css') and expect_old_age_bug: assert first['x_cache'] == second['x_cache'] == 'MISS', row assert row['origin_delta'] == 2, row elif path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css'): assert first['x_cache'] == 'MISS' and second['x_cache'] == 'HIT', row assert row['origin_delta'] == 1, row assert second['cache_status'] and 'transformed' in second['cache_status'], row assert first['etag'] == second['etag'] and first['body'] == second['body'], row else: assert first['x_cache'] == second['x_cache'] == 'MISS', row print(path, first['x_cache'], second['x_cache'], 'origin+', row['origin_delta'], flush=True) snapshot = dash(rust) outcome['tests'].append({'dashboard': snapshot}) assert find(snapshot, '/normal.css')['transformed'], snapshot aged = find(snapshot, '/aged.css') if expect_old_age_bug: assert aged['remaining_ttl_seconds'] == 0, aged else: assert aged['transformed'] and aged['remaining_ttl_seconds'] >= 86_000, aged assert aged['age_seconds'] >= 1_553_483, aged assert aged['resident_age_seconds'] < 10, aged aged_br = find(snapshot, '/aged-br.css') assert aged_br['transformed'] and aged_br['remaining_ttl_seconds'] >= 86_000, aged_br # Query-string bypass is an explicit administrator rule. Even a # transformed CSS with public max-age stays BYPASS while it is ON, # then the *same* URL must become MISS -> HIT after disabling it. # This is not a transform regression: it is a request policy. cfg['cache']['bypass_query'] = True rust.apply(cfg, 'query-bypass-enabled') query_path = '/normal.css?v=transformed-public' before = COUNTS['/normal.css'] q_bypass = [request(rust.reverse_port, query_path) for _ in range(2)] assert [row['x_cache'] for row in q_bypass] == ['BYPASS', 'BYPASS'], q_bypass assert COUNTS['/normal.css'] - before == 2, q_bypass cfg['cache']['bypass_query'] = False rust.apply(cfg, 'query-bypass-disabled') q_cached = [request(rust.reverse_port, query_path) for _ in range(2)] assert [row['x_cache'] for row in q_cached] == ['MISS', 'HIT'], q_cached assert q_cached[1]['cache_status'] and 'transformed' in q_cached[1]['cache_status'] assert COUNTS['/normal.css'] - before == 3, q_cached outcome['tests'].append({'query_bypass_policy': {'bypass': q_bypass, 'cached': q_cached}}) # Operational LB tuning must not invalidate a cached representation. before = COUNTS['/normal.css'] cfg['vhosts'][0]['load_balancing']['keepalive'] = {'max_idle_per_host': 16} rust.apply(cfg, 'only-backend-keepalive-changed') normal = request(rust.reverse_port, '/normal.css') assert normal['x_cache'] == 'HIT' and COUNTS['/normal.css'] == before outcome['tests'].append({'keepalive_update_without_purge': normal}) # Diagnostics can be intentionally hidden on MISS and remain visible on HIT. cfg['cache']['expose_only_hits'] = True rust.apply(cfg, 'only-show-hit-headers') before = COUNTS['/other.css'] first, second = request(rust.reverse_port, '/other.css'), request(rust.reverse_port, '/other.css') assert first['x_cache'] is None and second['x_cache'] == 'HIT' assert COUNTS['/other.css'] - before == 1 outcome['tests'].append({'only_hits': [first, second]}) if not expect_old_age_bug: # Cold path on a 1-entry HOT tier: demotion and on-disk reuse. cfg['cache'].update({'max_entries': 1, 'cold_enabled': True, 'cold_on_lru': True, 'cold_path': 'cache/age-regression-cold'}) cfg['cache'].pop('expose_only_hits', None) rust.apply(cfg, 'cold-enabled-one-hot-entry') assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'MISS' assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'HIT' assert request(rust.reverse_port, '/other.css')['x_cache'] == 'MISS' deadline = time.monotonic() + 4 while True: snapshot = dash(rust) matches = [x for x in snapshot['objects'] if x['path'] == '/aged.css' and x['tier'] == 'cold'] if matches: assert matches[0]['remaining_ttl_seconds'] >= 86_000, matches break if time.monotonic() > deadline: raise AssertionError('aged transformed CSS never demoted to COLD: ' + str(snapshot)) time.sleep(.05) cold_hit = request(rust.reverse_port, '/aged.css') assert cold_hit['x_cache'] == 'HIT' and 'transformed' in cold_hit['cache_status'], cold_hit outcome['tests'].append({'cold': {'row': matches[0], 'hit': cold_hit}}) outcome['passed'] = True finally: origin.shutdown() origin.server_close() write_json(output / 'cache-age-transform-result.json', outcome) return outcome if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--binary', type=Path, required=True) parser.add_argument('--output', type=Path, default=Path('/tmp/rustproxy-cache-age-034-live')) parser.add_argument('--expect-old-age-bug', action='store_true') args = parser.parse_args() try: report = run(args.binary.resolve(), args.output.resolve(), args.expect_old_age_bug) except Exception as error: print('FAIL', type(error).__name__, str(error)[:1000], file=sys.stderr) raise print('PASS', len(report['tests']), 'scenarios; see', args.output / 'cache-age-transform-result.json')