fix in aggregate
This commit is contained in:
63
api.py
63
api.py
@@ -49,11 +49,10 @@ def get_country_networks_cached(country_code: str, use_cache: bool = True):
|
|||||||
try:
|
try:
|
||||||
cached_data = redis_cache.redis_client.get(redis_key)
|
cached_data = redis_cache.redis_client.get(redis_key)
|
||||||
if cached_data:
|
if cached_data:
|
||||||
# FIX: Handle both bytes and str
|
|
||||||
if isinstance(cached_data, bytes):
|
if isinstance(cached_data, bytes):
|
||||||
networks = json.loads(cached_data.decode('utf-8'))
|
networks = json.loads(cached_data.decode('utf-8'))
|
||||||
else:
|
else:
|
||||||
networks = json.loads(cached_data) # Already string
|
networks = json.loads(cached_data)
|
||||||
|
|
||||||
return networks, 'redis'
|
return networks, 'redis'
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -157,7 +156,6 @@ def cache_status():
|
|||||||
total_size_bytes = 0
|
total_size_bytes = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Count country keys
|
|
||||||
pattern_country = "geoban:country:*"
|
pattern_country = "geoban:country:*"
|
||||||
cursor = 0
|
cursor = 0
|
||||||
while True:
|
while True:
|
||||||
@@ -173,7 +171,6 @@ def cache_status():
|
|||||||
if cursor == 0:
|
if cursor == 0:
|
||||||
break
|
break
|
||||||
|
|
||||||
# Count config keys (old format: geoip:config:*)
|
|
||||||
pattern_config = "geoip:config:*"
|
pattern_config = "geoip:config:*"
|
||||||
cursor = 0
|
cursor = 0
|
||||||
while True:
|
while True:
|
||||||
@@ -189,7 +186,6 @@ def cache_status():
|
|||||||
if cursor == 0:
|
if cursor == 0:
|
||||||
break
|
break
|
||||||
|
|
||||||
# Also check for new format: geoban:config:*
|
|
||||||
pattern_config_new = "geoban:config:*"
|
pattern_config_new = "geoban:config:*"
|
||||||
cursor = 0
|
cursor = 0
|
||||||
while True:
|
while True:
|
||||||
@@ -410,17 +406,29 @@ def generate_raw_cidr():
|
|||||||
|
|
||||||
countries = data.get('countries', [])
|
countries = data.get('countries', [])
|
||||||
aggregate = data.get('aggregate', True)
|
aggregate = data.get('aggregate', True)
|
||||||
format_type = data.get('app_variant', 'txt')
|
app_type = data.get('app_type', 'raw-cidr_txt')
|
||||||
use_cache = data.get('use_cache', True)
|
use_cache = data.get('use_cache', True)
|
||||||
|
|
||||||
|
if app_type == 'raw-cidr':
|
||||||
|
app_type = 'raw-cidr_txt'
|
||||||
|
|
||||||
if not countries:
|
if not countries:
|
||||||
return jsonify({'success': False, 'error': 'No countries selected'}), 400
|
return jsonify({'success': False, 'error': 'No countries selected'}), 400
|
||||||
|
|
||||||
if use_cache and redis_cache:
|
if use_cache and redis_cache:
|
||||||
cached = redis_cache.get_cached_config(countries, f"raw-cidr_{format_type}", aggregate)
|
cached = redis_cache.get_cached_config(countries, app_type, aggregate)
|
||||||
if cached:
|
if cached:
|
||||||
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.{format_type}"
|
if 'json' in app_type:
|
||||||
mimetype = 'text/csv' if format_type == 'csv' else 'text/plain'
|
extension = 'json'
|
||||||
|
mimetype = 'application/json'
|
||||||
|
elif 'csv' in app_type:
|
||||||
|
extension = 'csv'
|
||||||
|
mimetype = 'text/csv'
|
||||||
|
else:
|
||||||
|
extension = 'txt'
|
||||||
|
mimetype = 'text/plain'
|
||||||
|
|
||||||
|
filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
|
||||||
|
|
||||||
return Response(
|
return Response(
|
||||||
cached['config'],
|
cached['config'],
|
||||||
@@ -467,14 +475,38 @@ def generate_raw_cidr():
|
|||||||
|
|
||||||
update_progress('Generating file...', 85, 100)
|
update_progress('Generating file...', 85, 100)
|
||||||
|
|
||||||
if format_type == 'txt':
|
if 'txt' in app_type or 'cidr' in app_type or 'newline' in app_type:
|
||||||
config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None)
|
config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None)
|
||||||
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.txt"
|
filename = f"blocklist_{'_'.join(sorted(countries))}.txt"
|
||||||
mimetype = 'text/plain'
|
mimetype = 'text/plain'
|
||||||
else:
|
|
||||||
|
elif 'json' in app_type:
|
||||||
|
all_networks = []
|
||||||
|
for nets in country_networks.values():
|
||||||
|
all_networks.extend(nets)
|
||||||
|
|
||||||
|
if aggregate:
|
||||||
|
all_networks = ConfigGenerator.aggregate_networks(all_networks)
|
||||||
|
else:
|
||||||
|
all_networks = sorted(list(set(all_networks)))
|
||||||
|
|
||||||
|
config_text = json.dumps({
|
||||||
|
'countries': countries,
|
||||||
|
'networks': all_networks,
|
||||||
|
'count': len(all_networks),
|
||||||
|
'aggregated': aggregate
|
||||||
|
}, indent=2)
|
||||||
|
filename = f"blocklist_{'_'.join(sorted(countries))}.json"
|
||||||
|
mimetype = 'application/json'
|
||||||
|
|
||||||
|
elif 'csv' in app_type:
|
||||||
config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None)
|
config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None)
|
||||||
filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.csv"
|
filename = f"blocklist_{'_'.join(sorted(countries))}.csv"
|
||||||
mimetype = 'text/csv'
|
mimetype = 'text/csv'
|
||||||
|
|
||||||
|
else:
|
||||||
|
clear_progress()
|
||||||
|
return jsonify({'success': False, 'error': f'Unknown format: {app_type}'}), 400
|
||||||
|
|
||||||
total_networks = sum(len(nets) for nets in country_networks.values())
|
total_networks = sum(len(nets) for nets in country_networks.values())
|
||||||
stats = {
|
stats = {
|
||||||
@@ -485,7 +517,7 @@ def generate_raw_cidr():
|
|||||||
|
|
||||||
if redis_cache:
|
if redis_cache:
|
||||||
update_progress('Saving to Redis cache...', 95, 100)
|
update_progress('Saving to Redis cache...', 95, 100)
|
||||||
redis_cache.save_config(countries, f"raw-cidr_{format_type}", aggregate, config_text, stats)
|
redis_cache.save_config(countries, app_type, aggregate, config_text, stats)
|
||||||
|
|
||||||
update_progress('Complete!', 100, 100)
|
update_progress('Complete!', 100, 100)
|
||||||
clear_progress()
|
clear_progress()
|
||||||
@@ -511,6 +543,8 @@ def generate_raw_cidr():
|
|||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
clear_progress()
|
clear_progress()
|
||||||
|
import traceback
|
||||||
|
traceback.print_exc()
|
||||||
return jsonify({'success': False, 'error': str(e)}), 500
|
return jsonify({'success': False, 'error': str(e)}), 500
|
||||||
|
|
||||||
@api_blueprint.route('/api/generate', methods=['POST'])
|
@api_blueprint.route('/api/generate', methods=['POST'])
|
||||||
@@ -653,7 +687,6 @@ def sqlite_status():
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
|
|
||||||
file_size = db_path.stat().st_size
|
file_size = db_path.stat().st_size
|
||||||
modified_time = datetime.fromtimestamp(db_path.stat().st_mtime)
|
modified_time = datetime.fromtimestamp(db_path.stat().st_mtime)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user