""" API endpoints for GeoIP Ban Generator """ from flask import Blueprint, request, jsonify, Response from geoip_handler import GeoIPHandler, ConfigGenerator, generate_metadata from datetime import datetime import json import config import sqlite3 api_blueprint = Blueprint('api', __name__) handler = GeoIPHandler() redis_cache = None if config.REDIS_ENABLED: try: from redis_cache import RedisCache redis_cache = RedisCache() print("[REDIS] Cache enabled", flush=True) except Exception as e: print(f"[REDIS] Failed to initialize: {e}", flush=True) redis_cache = None progress_state = { 'active': False, 'message': '', 'progress': 0, 'total': 0 } def update_progress(message, progress=0, total=0): progress_state['active'] = True progress_state['message'] = message progress_state['progress'] = progress progress_state['total'] = total def clear_progress(): progress_state['active'] = False progress_state['message'] = '' progress_state['progress'] = 0 progress_state['total'] = 0 def get_country_networks_cached(country_code: str, use_cache: bool = True): country_code = country_code.upper() if use_cache and redis_cache: redis_key = f"geoban:country:{country_code}" try: cached_data = redis_cache.redis_client.get(redis_key) if cached_data: if isinstance(cached_data, bytes): networks = json.loads(cached_data.decode('utf-8')) else: networks = json.loads(cached_data) return networks, 'redis' except Exception as e: print(f"[{country_code}] Redis read error: {e}", flush=True) networks = handler._get_cached_networks(country_code) if networks is not None: if redis_cache: try: redis_key = f"geoban:country:{country_code}" redis_cache.redis_client.setex( redis_key, 86400, json.dumps(networks) ) except Exception as e: print(f"[{country_code}] Redis save error: {e}", flush=True) return networks, 'sqlite' networks = handler.fetch_country_networks(country_code) if networks and redis_cache: try: redis_key = f"geoban:country:{country_code}" redis_cache.redis_client.setex( redis_key, 86400, json.dumps(networks) ) except Exception as e: print(f"[{country_code}] Redis save error: {e}", flush=True) return networks, 'maxmind' class ProgressTracker: def __init__(self, country, country_idx, total_countries, base_progress, next_progress): self.country = country self.country_idx = country_idx self.total_countries = total_countries self.base_progress = base_progress self.next_progress = next_progress self.current_progress = base_progress def callback(self, message): progress_range = self.next_progress - self.base_progress increment = max(1, progress_range // 10) self.current_progress = min(self.current_progress + increment, self.next_progress - 1) update_progress( f'[{self.country_idx}/{self.total_countries}] {self.country}: {message}', self.current_progress, 100 ) @api_blueprint.route('/api/progress', methods=['GET']) def get_progress(): return jsonify(progress_state) @api_blueprint.route('/api/database/status', methods=['GET']) def database_status(): cfg = handler.load_config() exists = handler.mmdb_file.exists() needs_update = handler.needs_update() return jsonify({ 'success': True, 'exists': exists, 'needs_update': needs_update, 'last_update': cfg.get('last_update'), 'file_size': cfg.get('file_size', 0) if exists else 0, 'auto_update': config.MAXMIND_AUTO_UPDATE }) @api_blueprint.route('/api/database/update', methods=['POST']) def update_database(): result = handler.download_database() return jsonify(result) @api_blueprint.route('/api/countries', methods=['GET']) def get_countries(): return jsonify({ 'success': True, 'countries': config.COMMON_COUNTRIES }) @api_blueprint.route('/api/cache/redis/status', methods=['GET']) def cache_status(): if not redis_cache: return jsonify({ 'success': False, 'enabled': False, 'message': 'Redis cache is not enabled' }) health = None try: health = redis_cache.health_check() except Exception as health_error: print(f"[REDIS] Health check failed: {health_error}", flush=True) health = {'connected': False, 'status': 'disconnected', 'error': str(health_error)} country_keys_count = 0 config_keys_count = 0 total_size_bytes = 0 patterns = [ ("geoban:country:*", "country"), ("geoip:config:*", "config"), ("geoban:config:*", "config") ] for pattern, key_type in patterns: try: cursor = 0 while True: cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern, count=1000) key_count = len(keys) if key_type == "country": country_keys_count += key_count else: config_keys_count += key_count for key in keys: try: size = redis_cache.redis_client.memory_usage(key) if size: total_size_bytes += size except: pass if cursor == 0: break except Exception as pattern_error: print(f"[REDIS] Pattern '{pattern}' failed: {pattern_error}", flush=True) continue return jsonify({ 'success': True, 'enabled': True, 'health': health, 'stats': { 'country_keys': country_keys_count, 'config_keys': config_keys_count, 'total_keys': country_keys_count + config_keys_count, 'total_size_mb': round(total_size_bytes / 1024 / 1024, 2), 'memory_used_mb': health.get('memory_used_mb', 0) if isinstance(health, dict) else 0, 'total_keys_in_db': health.get('keys', 0) if isinstance(health, dict) else 0 } }) @api_blueprint.route('/api/cache/flush', methods=['POST']) def cache_flush(): if not redis_cache: return jsonify({'success': False, 'error': 'Redis not enabled'}), 503 try: success = redis_cache.flush_all() return jsonify({ 'success': success, 'message': 'Cache flushed successfully' if success else 'Failed to flush cache' }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @api_blueprint.route('/api/cache/invalidate/', methods=['POST']) def cache_invalidate(country_code): if not redis_cache: return jsonify({'success': False, 'error': 'Redis not enabled'}), 503 try: country_code = country_code.upper() country_key = f"geoban:country:{country_code}" deleted = redis_cache.redis_client.delete(country_key) pattern = f"geoban:config:*{country_code}*" cursor = 0 config_deleted = 0 while True: cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern, count=100) if keys: config_deleted += redis_cache.redis_client.delete(*keys) if cursor == 0: break total_deleted = deleted + config_deleted return jsonify({ 'success': True, 'deleted': total_deleted, 'country': country_code, 'details': { 'country_cache': deleted, 'config_caches': config_deleted } }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @api_blueprint.route('/api/generate/preview', methods=['POST']) def generate_preview(): try: clear_progress() data = request.get_json() countries = data.get('countries', []) app_type = data.get('app_type', 'nginx') app_variant = data.get('app_variant', 'geo') aggregate = data.get('aggregate', True) use_cache = data.get('use_cache', True) if not countries: return jsonify({'success': False, 'error': 'No countries selected'}), 400 if use_cache and redis_cache: cached = redis_cache.get_cached_config(countries, f"{app_type}_{app_variant}", aggregate) if cached: return jsonify({ 'success': True, 'config': cached['config'], 'stats': cached['stats'], 'from_cache': True, 'cache_type': 'redis-full', 'generated_at': cached['generated_at'] }) if handler.needs_update(): handler.check_and_update() update_progress(f'Loading data for {len(countries)} countries...', 0, 100) country_networks = {} cache_sources = {} total_countries = len(countries) for idx, country in enumerate(countries, 1): base_progress = int((idx - 1) / total_countries * 80) update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) networks, source = get_country_networks_cached(country, use_cache=use_cache) if networks: country_networks[country] = networks cache_sources[country] = source update_progress( f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', base_progress + 10, 100 ) else: update_progress(f'[{idx}/{total_countries}] {country}: No networks found', base_progress + 10, 100) if not country_networks: clear_progress() return jsonify({'success': False, 'error': 'No networks found'}), 404 update_progress('Generating configuration...', 90, 100) if app_type == 'raw-cidr': if app_variant == 'csv': config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) else: config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) else: generators = { 'nginx_geo': ConfigGenerator.generate_nginx_geo, 'nginx_map': ConfigGenerator.generate_nginx_map, 'nginx_deny': ConfigGenerator.generate_nginx_deny, 'apache_22': ConfigGenerator.generate_apache_22, 'apache_24': ConfigGenerator.generate_apache_24, 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, "haproxy_map": ConfigGenerator.generate_haproxy_map, } generator_key = f"{app_type}_{app_variant}" generator = generators.get(generator_key) if not generator: clear_progress() return jsonify({'success': False, 'error': f'Invalid configuration type: {generator_key}'}), 400 config_text = generator(country_networks, aggregate=aggregate, redis_ips=None) total_networks = sum(len(nets) for nets in country_networks.values()) stats = { 'countries': len(country_networks), 'total_networks': total_networks, 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} } if redis_cache: update_progress('Saving to Redis cache for future use...', 95, 100) redis_cache.save_config(countries, f"{app_type}_{app_variant}", aggregate, config_text, stats) update_progress('Complete!', 100, 100) clear_progress() source_summary = { 'redis': sum(1 for s in cache_sources.values() if s == 'redis'), 'sqlite': sum(1 for s in cache_sources.values() if s == 'sqlite'), 'maxmind': sum(1 for s in cache_sources.values() if s == 'maxmind') } return jsonify({ 'success': True, 'config': config_text, 'stats': stats, 'from_cache': False, 'cache_type': 'hybrid', 'cache_sources': cache_sources, 'source_summary': source_summary, 'generated_at': datetime.now().isoformat() }) except Exception as e: clear_progress() import traceback traceback.print_exc() return jsonify({'success': False, 'error': str(e)}), 500 @api_blueprint.route('/api/generate/raw', methods=['POST']) def generate_raw_cidr(): try: clear_progress() data = request.get_json() countries = data.get('countries', []) aggregate = data.get('aggregate', True) app_type = data.get('app_type', 'raw-cidr_txt') use_cache = data.get('use_cache', True) as_js = bool(data.get('as_js', False)) js_var = data.get('js_var', 'geoipBlocklist') if app_type == 'raw-cidr': app_type = 'raw-cidr_txt' if not countries: return jsonify({'success': False, 'error': 'No countries selected'}), 400 if use_cache and redis_cache: cached = redis_cache.get_cached_config(countries, app_type, aggregate) if cached: if 'json' in app_type: if as_js: extension = 'js' mimetype = 'application/javascript' filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" body = f"const {js_var} = {cached['config']};\n" else: extension = 'json' mimetype = 'application/json' filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" body = cached['config'] elif 'csv' in app_type: extension = 'csv' mimetype = 'text/csv' filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" body = cached['config'] else: extension = 'txt' mimetype = 'text/plain' filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" body = cached['config'] return Response( body, mimetype=mimetype, headers={ 'Content-Disposition': f'attachment; filename="{filename}"', 'X-From-Cache': 'true', 'X-Cache-Type': 'redis-full', 'X-Generated-At': cached['generated_at'], 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': '0' } ) if handler.needs_update(): handler.check_and_update() update_progress(f'Loading data for {len(countries)} countries...', 0, 100) country_networks = {} cache_sources = {} total_countries = len(countries) for idx, country in enumerate(countries, 1): base_progress = int((idx - 1) / total_countries * 80) update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) networks, source = get_country_networks_cached(country, use_cache=use_cache) if networks: country_networks[country] = networks cache_sources[country] = source next_progress = int(idx / total_countries * 80) update_progress( f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', next_progress, 100 ) if not country_networks: clear_progress() return jsonify({'success': False, 'error': 'No networks found'}), 404 update_progress('Generating file...', 85, 100) if 'txt' in app_type or 'cidr' in app_type or 'newline' in app_type: config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) filename = f"blocklist_{'_'.join(sorted(countries))}.txt" mimetype = 'text/plain' elif 'json' in app_type: all_networks = [] for nets in country_networks.values(): all_networks.extend(nets) if aggregate: all_networks = ConfigGenerator._aggregate_networks(all_networks) else: all_networks = sorted(list(set(all_networks))) json_text = json.dumps({ 'countries': countries, 'networks': all_networks, 'count': len(all_networks), 'aggregated': aggregate }, indent=2) if as_js: config_text = f"const {js_var} = {json_text};\n" filename = f"blocklist_{'_'.join(sorted(countries))}.js" mimetype = 'application/javascript' else: config_text = json_text filename = f"blocklist_{'_'.join(sorted(countries))}.json" mimetype = 'application/json' elif 'csv' in app_type: config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) filename = f"blocklist_{'_'.join(sorted(countries))}.csv" mimetype = 'text/csv' else: clear_progress() return jsonify({'success': False, 'error': f'Unknown format: {app_type}'}), 400 total_networks = sum(len(nets) for nets in country_networks.values()) stats = { 'countries': len(country_networks), 'total_networks': total_networks, 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} } if redis_cache: update_progress('Saving to Redis cache...', 95, 100) redis_cache.save_config(countries, app_type, aggregate, config_text, stats) update_progress('Complete!', 100, 100) clear_progress() cache_type = 'hybrid' if cache_sources: most_common = max(set(cache_sources.values()), key=list(cache_sources.values()).count) cache_type = most_common return Response( config_text, mimetype=mimetype, headers={ 'Content-Disposition': f'attachment; filename="{filename}"', 'X-From-Cache': 'false', 'X-Cache-Type': cache_type, 'X-Generated-At': datetime.now().isoformat(), 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': '0' } ) except Exception as e: clear_progress() import traceback traceback.print_exc() return jsonify({'success': False, 'error': str(e)}), 500 @api_blueprint.route('/api/generate', methods=['POST']) def generate_config(): try: clear_progress() data = request.get_json() countries = data.get('countries', []) app_type = data.get('app_type', 'nginx') app_variant = data.get('app_variant', 'geo') aggregate = data.get('aggregate', True) use_cache = data.get('use_cache', True) if not countries: return jsonify({'success': False, 'error': 'No countries selected'}), 400 if use_cache and redis_cache: cached = redis_cache.get_cached_config(countries, f"{app_type}_{app_variant}", aggregate) if cached: filename = f"geoblock_{app_type}_{app_variant}.conf" if app_variant == 'lua': filename = f"geoblock_{app_type}.lua" return Response( cached['config'], mimetype='text/plain', headers={ 'Content-Disposition': f'attachment; filename="{filename}"', 'X-From-Cache': 'true', 'X-Cache-Type': 'redis-full', 'X-Generated-At': cached['generated_at'], 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': '0' } ) if handler.needs_update(): handler.check_and_update() update_progress(f'Loading data for {len(countries)} countries...', 0, 100) country_networks = {} cache_sources = {} total_countries = len(countries) for idx, country in enumerate(countries, 1): base_progress = int((idx - 1) / total_countries * 80) update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) networks, source = get_country_networks_cached(country, use_cache=use_cache) if networks: country_networks[country] = networks cache_sources[country] = source next_progress = int(idx / total_countries * 80) update_progress( f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', next_progress, 100 ) if not country_networks: clear_progress() return jsonify({'success': False, 'error': 'No networks found'}), 404 update_progress('Generating configuration...', 85, 100) generators = { 'nginx_geo': ConfigGenerator.generate_nginx_geo, 'nginx_map': ConfigGenerator.generate_nginx_map, 'nginx_deny': ConfigGenerator.generate_nginx_deny, 'apache_22': ConfigGenerator.generate_apache_22, 'apache_24': ConfigGenerator.generate_apache_24, 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, 'haproxy_map': ConfigGenerator.generate_haproxy_map, } generator_key = f"{app_type}_{app_variant}" generator = generators.get(generator_key) if not generator: clear_progress() return jsonify({'success': False, 'error': 'Invalid configuration type'}), 400 config_text = generator(country_networks, aggregate=aggregate, redis_ips=None) stats = { 'countries': len(country_networks), 'total_networks': sum(len(nets) for nets in country_networks.values()), 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} } if redis_cache: update_progress('Saving to Redis cache...', 95, 100) redis_cache.save_config(countries, f"{app_type}_{app_variant}", aggregate, config_text, stats) filename = f"geoblock_{app_type}_{app_variant}.conf" if app_variant == 'lua': filename = f"geoblock_{app_type}.lua" update_progress('Complete!', 100, 100) clear_progress() cache_type = 'hybrid' if cache_sources: most_common = max(set(cache_sources.values()), key=list(cache_sources.values()).count) cache_type = most_common return Response( config_text, mimetype='text/plain', headers={ 'Content-Disposition': f'attachment; filename="{filename}"', 'X-From-Cache': 'false', 'X-Cache-Type': cache_type, 'X-Generated-At': datetime.now().isoformat(), 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', 'Expires': '0' } ) except Exception as e: clear_progress() return jsonify({'success': False, 'error': str(e)}), 500 @api_blueprint.route('/api/cache/sqlite/status', methods=['GET']) def sqlite_status(): """Get SQLite cache database statistics""" db_path = config.GEOIP_DB_DIR / 'networks_cache.db' if not db_path.exists(): return jsonify({ 'success': False, 'exists': False, 'message': 'SQLite cache database not found' }) try: file_size = db_path.stat().st_size modified_time = datetime.fromtimestamp(db_path.stat().st_mtime) conn = sqlite3.connect(str(db_path), timeout=10.0) cursor = conn.cursor() cursor.execute("SELECT COUNT(*) FROM cache_metadata") total_countries = cursor.fetchone()[0] cursor.execute("SELECT SUM(network_count) FROM cache_metadata") total_networks = cursor.fetchone()[0] or 0 cursor.execute("SELECT MIN(last_scan), MAX(last_scan) FROM cache_metadata") oldest, newest = cursor.fetchone() cursor.execute(""" SELECT country_code, network_count FROM cache_metadata ORDER BY network_count DESC LIMIT 5 """) top_countries = [{'code': row[0], 'networks': row[1]} for row in cursor.fetchall()] conn.close() return jsonify({ 'success': True, 'exists': True, 'file_size': file_size, 'file_size_mb': round(file_size / 1024 / 1024, 2), 'modified': modified_time.isoformat(), 'total_countries': total_countries, 'total_networks': total_networks, 'oldest_scan': oldest, 'newest_scan': newest, 'top_countries': top_countries }) except Exception as e: import traceback traceback.print_exc() return jsonify({ 'success': False, 'exists': True, 'error': str(e) }), 500 api = api_blueprint