v0.34.0_p1
This commit is contained in:
Mateusz Gruszczyński committed
1 parent
00f29f92c8
commit
0a8bc6f040
21 files changed
+672
-144
No files matched your search
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Loopback integration regression: transformed Age, HOT/COLD, diagnostics, isolation.
|
||||
|
||||
Run after compiling:
|
||||
python3 scripts/tests/cache_age_transform_034_live.py --binary target/release/rust-proxy
|
||||
To reproduce the original regression with the user-supplied 0.34.0 binary:
|
||||
python3 scripts/tests/cache_age_transform_034_live.py --binary /path/to/old/rust-proxy --expect-old-age-bug
|
||||
No internet and no user data; test config/data/passwords are in a disposable temporary dir.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'performance'))
|
||||
from common import Fixture, http, sha256, write_json
|
||||
|
||||
BODY = b'/* test cache */\nbody { color: red; padding: 0px; }\n'
|
||||
# Fixed sample from Brotli encoder, embedded to avoid Python dependency on brotli.
|
||||
BROTLI_BODY = b"\x1b3\x00\xe8\x1d\x07\xee\xbe\r\xa2\xbc3'\x8cP\xe9\xad\xa0\xf2\xb2\xa7\xe4\xfa\x12\x84\xac\x0c\x1cr\xc0\xfe\xc0>\x0fR*a\x9c\x9e2\xe8\x08\xcb\x1cP\x881\xac\xdd\x88s5(\x9d\x01"
|
||||
COUNTS = collections.Counter()
|
||||
|
||||
class Origin(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
path = self.path.split('?', 1)[0]
|
||||
if path not in {'/aged.css', '/normal.css', '/private.css', '/no-store.css', '/other.css', '/br.css', '/aged-br.css'}:
|
||||
self.send_error(404)
|
||||
return
|
||||
COUNTS[path] += 1
|
||||
self.send_response(200)
|
||||
if path in ('/aged.css', '/aged-br.css'):
|
||||
policy = 'public, max-age=2592000'
|
||||
elif path == '/private.css':
|
||||
policy = 'private, max-age=2592000'
|
||||
elif path == '/no-store.css':
|
||||
policy = 'no-store'
|
||||
else:
|
||||
policy = 'public, max-age=3600'
|
||||
self.send_header('Cache-Control', policy)
|
||||
self.send_header('Content-Type', 'text/css')
|
||||
payload = BROTLI_BODY if path in ('/br.css', '/aged-br.css') else BODY
|
||||
self.send_header('Content-Length', str(len(payload)))
|
||||
if path in ('/br.css', '/aged-br.css'):
|
||||
self.send_header('Content-Encoding', 'br')
|
||||
self.send_header('ETag', '"origin-css"')
|
||||
self.send_header('Last-Modified', 'Thu, 07 Jan 2016 20:04:59 GMT')
|
||||
self.send_header('Vary', 'Accept-Encoding')
|
||||
if path in ('/aged.css', '/aged-br.css'):
|
||||
self.send_header('Age', '1553483')
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
|
||||
def request(port, path, encoding='identity'):
|
||||
status, headers, body = http(port, path, headers={'Accept-Encoding': encoding})
|
||||
return {'status': status, 'x_cache': headers.get('x-cache'),
|
||||
'x_cache_hits': headers.get('x-cache-hits'), 'x_cache_resident_age': headers.get('x-cache-resident-age'),
|
||||
'cache_status': headers.get('cache-status'), 'age': headers.get('age'),
|
||||
'body': body.decode('utf-8', 'replace'), 'content_length': headers.get('content-length'),
|
||||
'etag': headers.get('etag'), 'encoding': headers.get('content-encoding'), 'origin_count': COUNTS[path]}
|
||||
|
||||
|
||||
def dash(rust):
|
||||
status, _, body = rust.admin('/api/cache?limit=200')
|
||||
assert status == 200, ('cache dashboard', status, body[:200])
|
||||
data = json.loads(body)['dashboard']
|
||||
return {'objects': [{k: row.get(k) for k in ('vhost','path','tier','transformed',
|
||||
'hits','age_seconds','resident_age_seconds','remaining_ttl_seconds')} for row in data['objects']],
|
||||
'domains': data['domains']}
|
||||
|
||||
|
||||
def find(d, path):
|
||||
matches = [item for item in d['objects'] if item['path'] == path and item['vhost'] == 'reverse']
|
||||
assert len(matches) == 1, (path, matches)
|
||||
return matches[0]
|
||||
|
||||
|
||||
def run(binary: Path, output: Path, expect_old_age_bug: bool):
|
||||
outcome = {'binary_sha256': sha256(binary), 'expect_old_age_bug': expect_old_age_bug,
|
||||
'tests': [], 'passed': False}
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
with Fixture(output, workers=2) as fixture:
|
||||
origin = ThreadingHTTPServer(('127.0.0.1', fixture.origin_port), Origin)
|
||||
thread = threading.Thread(target=origin.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
rust = fixture.rust(binary, 'cache-age-034')
|
||||
rust.login()
|
||||
cfg = copy.deepcopy(rust.base)
|
||||
cfg['cache'] = {'enabled': True, 'max_ttl_seconds': 86_400,
|
||||
'expose_status': True, 'expose_hits': True,
|
||||
'expose_cache_status': True, 'expose_resident_age': True}
|
||||
cfg['transform'] = {'enabled': True, 'css': True}
|
||||
for vhost in cfg['vhosts']:
|
||||
vhost['transform'] = {'enabled': True, 'css': True}
|
||||
rust.apply(cfg, 'fresh-cache-transform')
|
||||
for path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css', '/private.css', '/no-store.css'):
|
||||
before = COUNTS[path]
|
||||
encoding = 'br' if path in ('/br.css', '/aged-br.css') else 'identity'
|
||||
first, second = request(rust.reverse_port, path, encoding), request(rust.reverse_port, path, encoding)
|
||||
row = {'path': path, 'first': first, 'second': second,
|
||||
'origin_delta': COUNTS[path] - before}
|
||||
outcome['tests'].append(row)
|
||||
if path in ('/aged.css', '/aged-br.css') and expect_old_age_bug:
|
||||
assert first['x_cache'] == second['x_cache'] == 'MISS', row
|
||||
assert row['origin_delta'] == 2, row
|
||||
elif path in ('/normal.css', '/aged.css', '/br.css', '/aged-br.css'):
|
||||
assert first['x_cache'] == 'MISS' and second['x_cache'] == 'HIT', row
|
||||
assert row['origin_delta'] == 1, row
|
||||
assert second['cache_status'] and 'transformed' in second['cache_status'], row
|
||||
assert first['etag'] == second['etag'] and first['body'] == second['body'], row
|
||||
else:
|
||||
assert first['x_cache'] == second['x_cache'] == 'MISS', row
|
||||
print(path, first['x_cache'], second['x_cache'], 'origin+', row['origin_delta'], flush=True)
|
||||
snapshot = dash(rust)
|
||||
outcome['tests'].append({'dashboard': snapshot})
|
||||
assert find(snapshot, '/normal.css')['transformed'], snapshot
|
||||
aged = find(snapshot, '/aged.css')
|
||||
if expect_old_age_bug:
|
||||
assert aged['remaining_ttl_seconds'] == 0, aged
|
||||
else:
|
||||
assert aged['transformed'] and aged['remaining_ttl_seconds'] >= 86_000, aged
|
||||
assert aged['age_seconds'] >= 1_553_483, aged
|
||||
assert aged['resident_age_seconds'] < 10, aged
|
||||
aged_br = find(snapshot, '/aged-br.css')
|
||||
assert aged_br['transformed'] and aged_br['remaining_ttl_seconds'] >= 86_000, aged_br
|
||||
|
||||
# Operational LB tuning must not invalidate a cached representation.
|
||||
before = COUNTS['/normal.css']
|
||||
cfg['vhosts'][0]['load_balancing']['keepalive'] = {'max_idle_per_host': 16}
|
||||
rust.apply(cfg, 'only-backend-keepalive-changed')
|
||||
normal = request(rust.reverse_port, '/normal.css')
|
||||
assert normal['x_cache'] == 'HIT' and COUNTS['/normal.css'] == before
|
||||
outcome['tests'].append({'keepalive_update_without_purge': normal})
|
||||
|
||||
# Diagnostics can be intentionally hidden on MISS and remain visible on HIT.
|
||||
cfg['cache']['expose_only_hits'] = True
|
||||
rust.apply(cfg, 'only-show-hit-headers')
|
||||
before = COUNTS['/other.css']
|
||||
first, second = request(rust.reverse_port, '/other.css'), request(rust.reverse_port, '/other.css')
|
||||
assert first['x_cache'] is None and second['x_cache'] == 'HIT'
|
||||
assert COUNTS['/other.css'] - before == 1
|
||||
outcome['tests'].append({'only_hits': [first, second]})
|
||||
|
||||
if not expect_old_age_bug:
|
||||
# Cold path on a 1-entry HOT tier: demotion and on-disk reuse.
|
||||
cfg['cache'].update({'max_entries': 1, 'cold_enabled': True, 'cold_on_lru': True,
|
||||
'cold_path': 'cache/age-regression-cold'})
|
||||
cfg['cache'].pop('expose_only_hits', None)
|
||||
rust.apply(cfg, 'cold-enabled-one-hot-entry')
|
||||
assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'MISS'
|
||||
assert request(rust.reverse_port, '/aged.css')['x_cache'] == 'HIT'
|
||||
assert request(rust.reverse_port, '/other.css')['x_cache'] == 'MISS'
|
||||
deadline = time.monotonic() + 4
|
||||
while True:
|
||||
snapshot = dash(rust)
|
||||
matches = [x for x in snapshot['objects'] if x['path'] == '/aged.css' and x['tier'] == 'cold']
|
||||
if matches:
|
||||
assert matches[0]['remaining_ttl_seconds'] >= 86_000, matches
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError('aged transformed CSS never demoted to COLD: ' + str(snapshot))
|
||||
time.sleep(.05)
|
||||
cold_hit = request(rust.reverse_port, '/aged.css')
|
||||
assert cold_hit['x_cache'] == 'HIT' and 'transformed' in cold_hit['cache_status'], cold_hit
|
||||
outcome['tests'].append({'cold': {'row': matches[0], 'hit': cold_hit}})
|
||||
outcome['passed'] = True
|
||||
finally:
|
||||
origin.shutdown()
|
||||
origin.server_close()
|
||||
write_json(output / 'cache-age-transform-result.json', outcome)
|
||||
return outcome
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--binary', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, default=Path('/tmp/rustproxy-cache-age-034-live'))
|
||||
parser.add_argument('--expect-old-age-bug', action='store_true')
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
report = run(args.binary.resolve(), args.output.resolve(), args.expect_old_age_bug)
|
||||
except Exception as error:
|
||||
print('FAIL', type(error).__name__, str(error)[:1000], file=sys.stderr)
|
||||
raise
|
||||
print('PASS', len(report['tests']), 'scenarios; see', args.output / 'cache-age-transform-result.json')
|
||||
Reference in new issue
Block a user