fix in aggregate

This commit is contained in:
Mateusz Gruszczyński
2026-02-17 13:24:20 +01:00
parent 4485aa37ba
commit d82926a4d1

63
api.py
View File

@@ -49,11 +49,10 @@ def get_country_networks_cached(country_code: str, use_cache: bool = True):
try: try:
cached_data = redis_cache.redis_client.get(redis_key) cached_data = redis_cache.redis_client.get(redis_key)
if cached_data: if cached_data:
# FIX: Handle both bytes and str
if isinstance(cached_data, bytes): if isinstance(cached_data, bytes):
networks = json.loads(cached_data.decode('utf-8')) networks = json.loads(cached_data.decode('utf-8'))
else: else:
networks = json.loads(cached_data) # Already string networks = json.loads(cached_data)
return networks, 'redis' return networks, 'redis'
except Exception as e: except Exception as e:
@@ -157,7 +156,6 @@ def cache_status():
total_size_bytes = 0 total_size_bytes = 0
try: try:
# Count country keys
pattern_country = "geoban:country:*" pattern_country = "geoban:country:*"
cursor = 0 cursor = 0
while True: while True:
@@ -173,7 +171,6 @@ def cache_status():
if cursor == 0: if cursor == 0:
break break
# Count config keys (old format: geoip:config:*)
pattern_config = "geoip:config:*" pattern_config = "geoip:config:*"
cursor = 0 cursor = 0
while True: while True:
@@ -189,7 +186,6 @@ def cache_status():
if cursor == 0: if cursor == 0:
break break
# Also check for new format: geoban:config:*
pattern_config_new = "geoban:config:*" pattern_config_new = "geoban:config:*"
cursor = 0 cursor = 0
while True: while True:
@@ -410,17 +406,29 @@ def generate_raw_cidr():
countries = data.get('countries', []) countries = data.get('countries', [])
aggregate = data.get('aggregate', True) aggregate = data.get('aggregate', True)
format_type = data.get('app_variant', 'txt') app_type = data.get('app_type', 'raw-cidr_txt')
use_cache = data.get('use_cache', True) use_cache = data.get('use_cache', True)
if app_type == 'raw-cidr':
app_type = 'raw-cidr_txt'
if not countries: if not countries:
return jsonify({'success': False, 'error': 'No countries selected'}), 400 return jsonify({'success': False, 'error': 'No countries selected'}), 400
if use_cache and redis_cache: if use_cache and redis_cache:
cached = redis_cache.get_cached_config(countries, f"raw-cidr_{format_type}", aggregate) cached = redis_cache.get_cached_config(countries, app_type, aggregate)
if cached: if cached:
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.{format_type}" if 'json' in app_type:
mimetype = 'text/csv' if format_type == 'csv' else 'text/plain' extension = 'json'
mimetype = 'application/json'
elif 'csv' in app_type:
extension = 'csv'
mimetype = 'text/csv'
else:
extension = 'txt'
mimetype = 'text/plain'
filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
return Response( return Response(
cached['config'], cached['config'],
@@ -467,15 +475,39 @@ def generate_raw_cidr():
update_progress('Generating file...', 85, 100) update_progress('Generating file...', 85, 100)
if format_type == 'txt': if 'txt' in app_type or 'cidr' in app_type or 'newline' in app_type:
config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None)
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.txt" filename = f"blocklist_{'_'.join(sorted(countries))}.txt"
mimetype = 'text/plain' mimetype = 'text/plain'
else:
elif 'json' in app_type:
all_networks = []
for nets in country_networks.values():
all_networks.extend(nets)
if aggregate:
all_networks = ConfigGenerator.aggregate_networks(all_networks)
else:
all_networks = sorted(list(set(all_networks)))
config_text = json.dumps({
'countries': countries,
'networks': all_networks,
'count': len(all_networks),
'aggregated': aggregate
}, indent=2)
filename = f"blocklist_{'_'.join(sorted(countries))}.json"
mimetype = 'application/json'
elif 'csv' in app_type:
config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None)
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.csv" filename = f"blocklist_{'_'.join(sorted(countries))}.csv"
mimetype = 'text/csv' mimetype = 'text/csv'
else:
clear_progress()
return jsonify({'success': False, 'error': f'Unknown format: {app_type}'}), 400
total_networks = sum(len(nets) for nets in country_networks.values()) total_networks = sum(len(nets) for nets in country_networks.values())
stats = { stats = {
'countries': len(country_networks), 'countries': len(country_networks),
@@ -485,7 +517,7 @@ def generate_raw_cidr():
if redis_cache: if redis_cache:
update_progress('Saving to Redis cache...', 95, 100) update_progress('Saving to Redis cache...', 95, 100)
redis_cache.save_config(countries, f"raw-cidr_{format_type}", aggregate, config_text, stats) redis_cache.save_config(countries, app_type, aggregate, config_text, stats)
update_progress('Complete!', 100, 100) update_progress('Complete!', 100, 100)
clear_progress() clear_progress()
@@ -511,6 +543,8 @@ def generate_raw_cidr():
except Exception as e: except Exception as e:
clear_progress() clear_progress()
import traceback
traceback.print_exc()
return jsonify({'success': False, 'error': str(e)}), 500 return jsonify({'success': False, 'error': str(e)}), 500
@api_blueprint.route('/api/generate', methods=['POST']) @api_blueprint.route('/api/generate', methods=['POST'])
@@ -653,7 +687,6 @@ def sqlite_status():
try: try:
file_size = db_path.stat().st_size file_size = db_path.stat().st_size
modified_time = datetime.fromtimestamp(db_path.stat().st_mtime) modified_time = datetime.fromtimestamp(db_path.stat().st_mtime)