commit c0afc1554d5cc0586602bdf72396996a604f1d87 Author: Mateusz Gruszczyński Date: Tue Feb 17 09:04:09 2026 +0100 first commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d829c45 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,84 @@ +# Git +.git +.gitignore +.gitattributes + +# Python cache +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +venv/ +env/ +ENV/ +.venv/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ +tests/ +test_*.py +*_test.py + +# Documentation +README.md +docs/ +*.md +LICENSE + +# CI/CD +.github/ +.gitlab-ci.yml +.travis.yml +Jenkinsfile + +# Docker +Dockerfile* +docker-compose*.yml +.dockerignore + +# Environment files (will be injected at runtime) +.env +.env.* + +# Development files +*.log +logs/ + +# OS files +.DS_Store +Thumbs.db +desktop.ini + +# Temporary files +*.tmp +*.bak +*~ + +# GeoIP database (downloaded at runtime or mounted) +geoip_db/*.mmdb +geoip_db/config.json + +# Build artifacts +build/ +dist/ +*.egg-info/ + +# Node modules (if any frontend tools) +node_modules/ +npm-debug.log +yarn-error.log + +geoip_db \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4b558fc --- /dev/null +++ b/.env.example @@ -0,0 +1,51 @@ +# Flask Configuration +FLASK_HOST=0.0.0.0 +FLASK_PORT=5000 +FLASK_DEBUG=False +SECRET_KEY=change-me-in-production-use-random-string + +# Application Settings +APP_NAME=GeoIP Ban Generator +LOGO_URL= +LOGO_LINK=/ + +# Footer +FOOTER_TEXT=© 2026 GeoIP Ban Generator +FOOTER_LINK= +FOOTER_LINK_TEXT=Documentation + +# MaxMind Database +MAXMIND_PRIMARY_URL=https://github.com/P3TERX/GeoLite.mmdb/releases/download/2026.02.07/GeoLite2-Country.mmdb +MAXMIND_FALLBACK_URL=https://git.io/GeoLite2-Country.mmdb +MAXMIND_UPDATE_INTERVAL_DAYS=7 +MAXMIND_AUTO_UPDATE=True + +# Cache Settings +CACHE_ENABLED=True +CACHE_TTL_SECONDS=3600 + +# MaxMind Database +MAXMIND_UPDATE_INTERVAL_DAYS=7 +MAXMIND_AUTO_UPDATE=True + +# Background Scheduler Settings +SCHEDULER_ENABLED=true +SCAN_INTERVAL=7d +SCAN_TIME=02:00 +SCAN_ON_STARTUP=true +CACHE_MAX_AGE_HOURS=168 + +# Parallel scanning +PARALLEL_WORKERS=8 # 0=auto + +# Redis +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_DB=0 +REDIS_PASSWORD= +REDIS_ENABLED=true +REDIS_CACHE_TTL=86400 + +# Precache Daemon Settings +PRECACHE_INTERVAL_HOURS=168 +PRECACHE_CHECK_INTERVAL=3600 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e6b834e --- /dev/null +++ b/.gitignore @@ -0,0 +1,121 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual Environment +venv/ +env/ +ENV/ +env.bak/ +venv.bak/ +.venv/ + +# Flask +instance/ +.webassets-cache +*.log + +# Environment variables +.env +.env.local +.env.*.local + +# GeoIP Database +geoip_db/*.mmdb +geoip_db/config.json + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store + +# Testing +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.nox/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# Celery +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Temporary files +*.tmp +*.bak +*.swp +*~ + +# Logs +logs/ +*.log + +# OS files +.DS_Store +Thumbs.db +desktop.ini + +# Cache +.cache/ +*.cache + +# Local config overrides +config.local.py +local_settings.py + +geoip_db/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b54b6ec --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.14-alpine + +WORKDIR /app + +RUN apk add --no-cache \ + gcc \ + musl-dev \ + linux-headers \ + curl + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p /app/geoip_db /app/static /app/templates + +ENV FLASK_HOST=0.0.0.0 +ENV FLASK_PORT=5000 +ENV FLASK_DEBUG=False +ENV PYTHONUNBUFFERED=1 + +EXPOSE ${FLASK_PORT:-5000} + +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:${FLASK_PORT:-5000}/ || exit 1 + +CMD gunicorn --bind 0.0.0.0:${FLASK_PORT:-5000} \ + --workers 1 \ + --threads 8 \ + --timeout 900 \ + --graceful-timeout 900 \ + --keep-alive 300 \ + --access-logfile - \ + --error-logfile - \ + --log-level info \ + app:app \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..313c606 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# Pierwsze uruchomienie - pobierze bazę automatycznie +python3 generate_ban.py --country CN --app nginx --output china.conf + +# Wiele krajów po przecinku +python3 generate_ban.py --country CN,RU,KP,IR --app haproxy --output multi_country.conf + +# Ręczna aktualizacja bazy +python3 generate_ban.py --update-db + +# Zmiana URL bazy danych +python3 generate_ban.py --set-config database_url=https://twoj-serwer.pl/GeoIP.zip + +# Zmiana interwału aktualizacji (dni) +python3 generate_ban.py --set-config update_interval_days=14 + +# Wyłączenie auto-update +python3 generate_ban.py --set-config auto_update=false + +# Podgląd konfiguracji +python3 generate_ban.py --show-config + +# Użycie niestandardowej lokalizacji config +python3 generate_ban.py --config /etc/geoip/config.json --country US --app nginx + +# Bez agregacji sieci (wszystkie oryginalne zakresy) +python3 generate_ban.py --country CN --app nginx --no-aggregate + +# Apache z wieloma krajami +python3 generate_ban.py --country CN,RU,BY,KP --app apache --output apache_geoblock.conf + + diff --git a/api.py b/api.py new file mode 100644 index 0000000..17ea2d8 --- /dev/null +++ b/api.py @@ -0,0 +1,705 @@ +""" +API endpoints for GeoIP Ban Generator +""" + +from flask import Blueprint, request, jsonify, Response +from geoip_handler import GeoIPHandler, ConfigGenerator, generate_metadata +from datetime import datetime +import json +import config +import sqlite3 + +api_blueprint = Blueprint('api', __name__) +handler = GeoIPHandler() + +redis_cache = None +if config.REDIS_ENABLED: + try: + from redis_cache import RedisCache + redis_cache = RedisCache() + print("[REDIS] Cache enabled", flush=True) + except Exception as e: + print(f"[REDIS] Failed to initialize: {e}", flush=True) + redis_cache = None + +progress_state = { + 'active': False, + 'message': '', + 'progress': 0, + 'total': 0 +} + +def update_progress(message, progress=0, total=0): + progress_state['active'] = True + progress_state['message'] = message + progress_state['progress'] = progress + progress_state['total'] = total + +def clear_progress(): + progress_state['active'] = False + progress_state['message'] = '' + progress_state['progress'] = 0 + progress_state['total'] = 0 + +def get_country_networks_cached(country_code: str, use_cache: bool = True): + country_code = country_code.upper() + + if use_cache and redis_cache: + redis_key = f"geoban:country:{country_code}" + try: + cached_data = redis_cache.redis_client.get(redis_key) + if cached_data: + # FIX: Handle both bytes and str + if isinstance(cached_data, bytes): + networks = json.loads(cached_data.decode('utf-8')) + else: + networks = json.loads(cached_data) # Already string + + return networks, 'redis' + except Exception as e: + print(f"[{country_code}] Redis read error: {e}", flush=True) + + networks = handler._get_cached_networks(country_code) + + if networks is not None: + if redis_cache: + try: + redis_key = f"geoban:country:{country_code}" + redis_cache.redis_client.setex( + redis_key, + 86400, + json.dumps(networks) + ) + except Exception as e: + print(f"[{country_code}] Redis save error: {e}", flush=True) + + return networks, 'sqlite' + + networks = handler.fetch_country_networks(country_code) + + if networks and redis_cache: + try: + redis_key = f"geoban:country:{country_code}" + redis_cache.redis_client.setex( + redis_key, + 86400, + json.dumps(networks) + ) + except Exception as e: + print(f"[{country_code}] Redis save error: {e}", flush=True) + + return networks, 'maxmind' + +class ProgressTracker: + def __init__(self, country, country_idx, total_countries, base_progress, next_progress): + self.country = country + self.country_idx = country_idx + self.total_countries = total_countries + self.base_progress = base_progress + self.next_progress = next_progress + self.current_progress = base_progress + + def callback(self, message): + progress_range = self.next_progress - self.base_progress + increment = max(1, progress_range // 10) + self.current_progress = min(self.current_progress + increment, self.next_progress - 1) + update_progress( + f'[{self.country_idx}/{self.total_countries}] {self.country}: {message}', + self.current_progress, + 100 + ) + +@api_blueprint.route('/api/progress', methods=['GET']) +def get_progress(): + return jsonify(progress_state) + +@api_blueprint.route('/api/database/status', methods=['GET']) +def database_status(): + cfg = handler.load_config() + exists = handler.mmdb_file.exists() + needs_update = handler.needs_update() + + return jsonify({ + 'success': True, + 'exists': exists, + 'needs_update': needs_update, + 'last_update': cfg.get('last_update'), + 'file_size': cfg.get('file_size', 0) if exists else 0, + 'auto_update': config.MAXMIND_AUTO_UPDATE + }) + +@api_blueprint.route('/api/database/update', methods=['POST']) +def update_database(): + result = handler.download_database() + return jsonify(result) + +@api_blueprint.route('/api/countries', methods=['GET']) +def get_countries(): + return jsonify({ + 'success': True, + 'countries': config.COMMON_COUNTRIES + }) + +@api_blueprint.route('/api/cache/status', methods=['GET']) +def cache_status(): + if not redis_cache: + return jsonify({ + 'success': False, + 'enabled': False, + 'message': 'Redis cache is not enabled' + }) + + try: + health = redis_cache.health_check() + + country_keys_count = 0 + config_keys_count = 0 + total_size_bytes = 0 + + try: + # Count country keys + pattern_country = "geoban:country:*" + cursor = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern_country, count=1000) + country_keys_count += len(keys) + for key in keys: + try: + size = redis_cache.redis_client.memory_usage(key) + if size: + total_size_bytes += size + except: + pass + if cursor == 0: + break + + # Count config keys (old format: geoip:config:*) + pattern_config = "geoip:config:*" + cursor = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern_config, count=1000) + config_keys_count += len(keys) + for key in keys: + try: + size = redis_cache.redis_client.memory_usage(key) + if size: + total_size_bytes += size + except: + pass + if cursor == 0: + break + + # Also check for new format: geoban:config:* + pattern_config_new = "geoban:config:*" + cursor = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern_config_new, count=1000) + config_keys_count += len(keys) + for key in keys: + try: + size = redis_cache.redis_client.memory_usage(key) + if size: + total_size_bytes += size + except: + pass + if cursor == 0: + break + + except Exception as e: + print(f"[REDIS] Error counting keys: {e}", flush=True) + import traceback + traceback.print_exc() + + return jsonify({ + 'success': True, + 'enabled': True, + 'health': health, + 'stats': { + 'country_keys': country_keys_count, + 'config_keys': config_keys_count, + 'total_keys': country_keys_count + config_keys_count, + 'total_size_mb': round(total_size_bytes / 1024 / 1024, 2), + 'memory_used_mb': health.get('memory_used_mb', 0), + 'total_keys_in_db': health.get('keys', 0) + } + }) + except Exception as e: + import traceback + traceback.print_exc() + return jsonify({ + 'success': False, + 'enabled': True, + 'error': str(e) + }), 500 + +@api_blueprint.route('/api/cache/flush', methods=['POST']) +def cache_flush(): + if not redis_cache: + return jsonify({'success': False, 'error': 'Redis not enabled'}), 503 + + try: + success = redis_cache.flush_all() + return jsonify({ + 'success': success, + 'message': 'Cache flushed successfully' if success else 'Failed to flush cache' + }) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}), 500 + +@api_blueprint.route('/api/cache/invalidate/', methods=['POST']) +def cache_invalidate(country_code): + if not redis_cache: + return jsonify({'success': False, 'error': 'Redis not enabled'}), 503 + + try: + country_code = country_code.upper() + + country_key = f"geoban:country:{country_code}" + deleted = redis_cache.redis_client.delete(country_key) + + pattern = f"geoban:config:*{country_code}*" + cursor = 0 + config_deleted = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match=pattern, count=100) + if keys: + config_deleted += redis_cache.redis_client.delete(*keys) + if cursor == 0: + break + + total_deleted = deleted + config_deleted + + return jsonify({ + 'success': True, + 'deleted': total_deleted, + 'country': country_code, + 'details': { + 'country_cache': deleted, + 'config_caches': config_deleted + } + }) + except Exception as e: + return jsonify({'success': False, 'error': str(e)}), 500 + +@api_blueprint.route('/api/generate/preview', methods=['POST']) +def generate_preview(): + try: + clear_progress() + data = request.get_json() + + countries = data.get('countries', []) + app_type = data.get('app_type', 'nginx') + app_variant = data.get('app_variant', 'geo') + aggregate = data.get('aggregate', True) + use_cache = data.get('use_cache', True) + + if not countries: + return jsonify({'success': False, 'error': 'No countries selected'}), 400 + + if use_cache and redis_cache: + cached = redis_cache.get_cached_config(countries, f"{app_type}_{app_variant}", aggregate) + if cached: + return jsonify({ + 'success': True, + 'config': cached['config'], + 'stats': cached['stats'], + 'from_cache': True, + 'cache_type': 'redis-full', + 'generated_at': cached['generated_at'] + }) + + if handler.needs_update(): + handler.check_and_update() + + update_progress(f'Loading data for {len(countries)} countries...', 0, 100) + + country_networks = {} + cache_sources = {} + total_countries = len(countries) + + for idx, country in enumerate(countries, 1): + base_progress = int((idx - 1) / total_countries * 80) + update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) + + networks, source = get_country_networks_cached(country, use_cache=use_cache) + + if networks: + country_networks[country] = networks + cache_sources[country] = source + update_progress( + f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', + base_progress + 10, + 100 + ) + else: + update_progress(f'[{idx}/{total_countries}] {country}: No networks found', base_progress + 10, 100) + + if not country_networks: + clear_progress() + return jsonify({'success': False, 'error': 'No networks found'}), 404 + + update_progress('Generating configuration...', 90, 100) + + if app_type == 'raw-cidr': + if app_variant == 'csv': + config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) + else: + config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) + else: + generators = { + 'nginx_geo': ConfigGenerator.generate_nginx_geo, + 'nginx_map': ConfigGenerator.generate_nginx_map, + 'nginx_deny': ConfigGenerator.generate_nginx_deny, + 'apache_22': ConfigGenerator.generate_apache_22, + 'apache_24': ConfigGenerator.generate_apache_24, + 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, + 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + } + + generator_key = f"{app_type}_{app_variant}" + generator = generators.get(generator_key) + + if not generator: + clear_progress() + return jsonify({'success': False, 'error': f'Invalid configuration type: {generator_key}'}), 400 + + config_text = generator(country_networks, aggregate=aggregate, redis_ips=None) + + total_networks = sum(len(nets) for nets in country_networks.values()) + stats = { + 'countries': len(country_networks), + 'total_networks': total_networks, + 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} + } + + if redis_cache: + update_progress('Saving to Redis cache for future use...', 95, 100) + redis_cache.save_config(countries, f"{app_type}_{app_variant}", aggregate, config_text, stats) + + update_progress('Complete!', 100, 100) + clear_progress() + + source_summary = { + 'redis': sum(1 for s in cache_sources.values() if s == 'redis'), + 'sqlite': sum(1 for s in cache_sources.values() if s == 'sqlite'), + 'maxmind': sum(1 for s in cache_sources.values() if s == 'maxmind') + } + + return jsonify({ + 'success': True, + 'config': config_text, + 'stats': stats, + 'from_cache': False, + 'cache_type': 'hybrid', + 'cache_sources': cache_sources, + 'source_summary': source_summary, + 'generated_at': datetime.now().isoformat() + }) + + except Exception as e: + clear_progress() + import traceback + traceback.print_exc() + return jsonify({'success': False, 'error': str(e)}), 500 + +@api_blueprint.route('/api/generate/raw', methods=['POST']) +def generate_raw_cidr(): + try: + clear_progress() + data = request.get_json() + + countries = data.get('countries', []) + aggregate = data.get('aggregate', True) + format_type = data.get('app_variant', 'txt') + use_cache = data.get('use_cache', True) + + if not countries: + return jsonify({'success': False, 'error': 'No countries selected'}), 400 + + if use_cache and redis_cache: + cached = redis_cache.get_cached_config(countries, f"raw-cidr_{format_type}", aggregate) + if cached: + filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.{format_type}" + mimetype = 'text/csv' if format_type == 'csv' else 'text/plain' + + return Response( + cached['config'], + mimetype=mimetype, + headers={ + 'Content-Disposition': f'attachment; filename="{filename}"', + 'X-From-Cache': 'true', + 'X-Cache-Type': 'redis-full', + 'X-Generated-At': cached['generated_at'], + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + ) + + if handler.needs_update(): + handler.check_and_update() + + update_progress(f'Loading data for {len(countries)} countries...', 0, 100) + + country_networks = {} + cache_sources = {} + total_countries = len(countries) + + for idx, country in enumerate(countries, 1): + base_progress = int((idx - 1) / total_countries * 80) + update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) + + networks, source = get_country_networks_cached(country, use_cache=use_cache) + + if networks: + country_networks[country] = networks + cache_sources[country] = source + next_progress = int(idx / total_countries * 80) + update_progress( + f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', + next_progress, + 100 + ) + + if not country_networks: + clear_progress() + return jsonify({'success': False, 'error': 'No networks found'}), 404 + + update_progress('Generating file...', 85, 100) + + if format_type == 'txt': + config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) + filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.txt" + mimetype = 'text/plain' + else: + config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) + filename = f"cidr_blocklist_{'_'.join(sorted(countries))}.csv" + mimetype = 'text/csv' + + total_networks = sum(len(nets) for nets in country_networks.values()) + stats = { + 'countries': len(country_networks), + 'total_networks': total_networks, + 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} + } + + if redis_cache: + update_progress('Saving to Redis cache...', 95, 100) + redis_cache.save_config(countries, f"raw-cidr_{format_type}", aggregate, config_text, stats) + + update_progress('Complete!', 100, 100) + clear_progress() + + cache_type = 'hybrid' + if cache_sources: + most_common = max(set(cache_sources.values()), key=list(cache_sources.values()).count) + cache_type = most_common + + return Response( + config_text, + mimetype=mimetype, + headers={ + 'Content-Disposition': f'attachment; filename="{filename}"', + 'X-From-Cache': 'false', + 'X-Cache-Type': cache_type, + 'X-Generated-At': datetime.now().isoformat(), + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + ) + + except Exception as e: + clear_progress() + return jsonify({'success': False, 'error': str(e)}), 500 + +@api_blueprint.route('/api/generate', methods=['POST']) +def generate_config(): + try: + clear_progress() + data = request.get_json() + + countries = data.get('countries', []) + app_type = data.get('app_type', 'nginx') + app_variant = data.get('app_variant', 'geo') + aggregate = data.get('aggregate', True) + use_cache = data.get('use_cache', True) + + if not countries: + return jsonify({'success': False, 'error': 'No countries selected'}), 400 + + if use_cache and redis_cache: + cached = redis_cache.get_cached_config(countries, f"{app_type}_{app_variant}", aggregate) + if cached: + filename = f"geoblock_{app_type}_{app_variant}.conf" + if app_variant == 'lua': + filename = f"geoblock_{app_type}.lua" + + return Response( + cached['config'], + mimetype='text/plain', + headers={ + 'Content-Disposition': f'attachment; filename="{filename}"', + 'X-From-Cache': 'true', + 'X-Cache-Type': 'redis-full', + 'X-Generated-At': cached['generated_at'], + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + ) + + if handler.needs_update(): + handler.check_and_update() + + update_progress(f'Loading data for {len(countries)} countries...', 0, 100) + + country_networks = {} + cache_sources = {} + total_countries = len(countries) + + for idx, country in enumerate(countries, 1): + base_progress = int((idx - 1) / total_countries * 80) + update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) + + networks, source = get_country_networks_cached(country, use_cache=use_cache) + + if networks: + country_networks[country] = networks + cache_sources[country] = source + next_progress = int(idx / total_countries * 80) + update_progress( + f'[{idx}/{total_countries}] {country}: {len(networks):,} networks ({source})', + next_progress, + 100 + ) + + if not country_networks: + clear_progress() + return jsonify({'success': False, 'error': 'No networks found'}), 404 + + update_progress('Generating configuration...', 85, 100) + + generators = { + 'nginx_geo': ConfigGenerator.generate_nginx_geo, + 'nginx_map': ConfigGenerator.generate_nginx_map, + 'nginx_deny': ConfigGenerator.generate_nginx_deny, + 'apache_22': ConfigGenerator.generate_apache_22, + 'apache_24': ConfigGenerator.generate_apache_24, + 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, + 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + } + + generator_key = f"{app_type}_{app_variant}" + generator = generators.get(generator_key) + + if not generator: + clear_progress() + return jsonify({'success': False, 'error': 'Invalid configuration type'}), 400 + + config_text = generator(country_networks, aggregate=aggregate, redis_ips=None) + + stats = { + 'countries': len(country_networks), + 'total_networks': sum(len(nets) for nets in country_networks.values()), + 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} + } + + if redis_cache: + update_progress('Saving to Redis cache...', 95, 100) + redis_cache.save_config(countries, f"{app_type}_{app_variant}", aggregate, config_text, stats) + + filename = f"geoblock_{app_type}_{app_variant}.conf" + if app_variant == 'lua': + filename = f"geoblock_{app_type}.lua" + + update_progress('Complete!', 100, 100) + clear_progress() + + cache_type = 'hybrid' + if cache_sources: + most_common = max(set(cache_sources.values()), key=list(cache_sources.values()).count) + cache_type = most_common + + return Response( + config_text, + mimetype='text/plain', + headers={ + 'Content-Disposition': f'attachment; filename="{filename}"', + 'X-From-Cache': 'false', + 'X-Cache-Type': cache_type, + 'X-Generated-At': datetime.now().isoformat(), + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + 'Expires': '0' + } + ) + + except Exception as e: + clear_progress() + return jsonify({'success': False, 'error': str(e)}), 500 + +@api_blueprint.route('/api/database/sqlite/status', methods=['GET']) +def sqlite_status(): + """Get SQLite cache database statistics""" + db_path = config.GEOIP_DB_DIR / 'networks_cache.db' + + if not db_path.exists(): + return jsonify({ + 'success': False, + 'exists': False, + 'message': 'SQLite cache database not found' + }) + + try: + + + file_size = db_path.stat().st_size + modified_time = datetime.fromtimestamp(db_path.stat().st_mtime) + + conn = sqlite3.connect(str(db_path), timeout=10.0) + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM cache_metadata") + total_countries = cursor.fetchone()[0] + + cursor.execute("SELECT SUM(network_count) FROM cache_metadata") + total_networks = cursor.fetchone()[0] or 0 + + cursor.execute("SELECT MIN(last_scan), MAX(last_scan) FROM cache_metadata") + oldest, newest = cursor.fetchone() + + cursor.execute(""" + SELECT country_code, network_count + FROM cache_metadata + ORDER BY network_count DESC + LIMIT 5 + """) + top_countries = [{'code': row[0], 'networks': row[1]} for row in cursor.fetchall()] + + conn.close() + + return jsonify({ + 'success': True, + 'exists': True, + 'file_size': file_size, + 'file_size_mb': round(file_size / 1024 / 1024, 2), + 'modified': modified_time.isoformat(), + 'total_countries': total_countries, + 'total_networks': total_networks, + 'oldest_scan': oldest, + 'newest_scan': newest, + 'top_countries': top_countries + }) + + except Exception as e: + import traceback + traceback.print_exc() + return jsonify({ + 'success': False, + 'exists': True, + 'error': str(e) + }), 500 + + +api = api_blueprint diff --git a/app.py b/app.py new file mode 100644 index 0000000..2c8b7bb --- /dev/null +++ b/app.py @@ -0,0 +1,242 @@ +""" +GeoIP Ban Generator - Web Application +""" + +from flask import Flask, render_template, request, Response, jsonify +import hashlib +import os +import sqlite3 +from pathlib import Path +from functools import wraps +from datetime import datetime + +import config +from api import api +from geoip_handler import GeoIPHandler + +app = Flask(__name__, + static_folder=str(config.STATIC_DIR), + template_folder=str(config.TEMPLATE_DIR)) + +app.config['SECRET_KEY'] = config.SECRET_KEY +app.register_blueprint(api) + +handler = GeoIPHandler() + +redis_cache = None +if config.REDIS_ENABLED: + try: + from redis_cache import RedisCache + redis_cache = RedisCache() + redis_health = redis_cache.health_check() + if redis_health['connected']: + print(f"[REDIS] Connected successfully - {redis_health['memory_used_mb']}MB used", flush=True) + else: + print(f"[REDIS] Connection failed: {redis_health.get('error')}", flush=True) + except Exception as e: + print(f"[REDIS] Failed to initialize: {e}", flush=True) + redis_cache = None + +def get_file_hash(filepath: Path) -> str: + """Generate MD5 hash for cache busting""" + try: + if filepath.exists(): + with open(filepath, 'rb') as f: + return hashlib.md5(f.read()).hexdigest()[:8] + except: + pass + return 'v1' + +def get_sqlite_status(): + """Get SQLite cache database status""" + db_path = config.GEOIP_DB_DIR / 'networks_cache.db' + + if not db_path.exists(): + return { + 'exists': False, + 'status': 'missing' + } + + try: + file_size = db_path.stat().st_size + modified_time = datetime.fromtimestamp(db_path.stat().st_mtime) + + conn = sqlite3.connect(str(db_path), timeout=5.0) + cursor = conn.cursor() + + cursor.execute("SELECT COUNT(*) FROM cache_metadata") + total_countries = cursor.fetchone()[0] + + cursor.execute("SELECT SUM(network_count) FROM cache_metadata") + total_networks = cursor.fetchone()[0] or 0 + + conn.close() + + return { + 'exists': True, + 'status': 'ok', + 'file_size_mb': round(file_size / 1024 / 1024, 2), + 'total_countries': total_countries, + 'total_networks': total_networks, + 'modified': modified_time.isoformat() + } + except Exception as e: + return { + 'exists': True, + 'status': 'error', + 'error': str(e) + } + +@app.context_processor +def inject_globals(): + """Inject global variables into templates""" + return { + 'app_name': config.APP_NAME, + 'app_version': config.APP_VERSION, + 'logo_url': config.LOGO_URL, + 'logo_link': config.LOGO_LINK, + 'footer_text': config.FOOTER_TEXT, + 'footer_link': config.FOOTER_LINK, + 'footer_link_text': config.FOOTER_LINK_TEXT, + 'css_hash': get_file_hash(config.STATIC_DIR / 'css' / 'style.css'), + 'js_hash': get_file_hash(config.STATIC_DIR / 'js' / 'app.js'), + 'get_country_flag': config.get_country_flag, + 'redis_enabled': config.REDIS_ENABLED, + 'redis_connected': redis_cache.health_check()['connected'] if redis_cache else False, + } + +@app.route('/') +def index(): + """Main page""" + if config.MAXMIND_AUTO_UPDATE: + handler.check_and_update() + + return render_template('index.html', countries=config.COMMON_COUNTRIES) + +@app.route('/api-docs') +def api_docs(): + """API documentation page""" + return render_template('api.html') + +@app.route('/favicon.ico') +def favicon(): + return '', 204 + +@app.route('/health') +def health(): + """Health check endpoint for HAProxy/monitoring""" + health_status = { + 'status': 'healthy', + 'timestamp': datetime.now().isoformat(), + 'components': {} + } + + health_status['components']['flask'] = { + 'status': 'ok', + 'version': config.APP_VERSION + } + + maxmind_exists = handler.mmdb_file.exists() + health_status['components']['maxmind_db'] = { + 'status': 'ok' if maxmind_exists else 'missing', + 'exists': maxmind_exists + } + + if not maxmind_exists: + health_status['status'] = 'degraded' + + sqlite_status = get_sqlite_status() + health_status['components']['sqlite_cache'] = sqlite_status + + if not sqlite_status['exists'] or sqlite_status.get('status') != 'ok': + health_status['status'] = 'degraded' + + if config.REDIS_ENABLED and redis_cache: + redis_health = redis_cache.health_check() + if redis_health.get('connected'): + health_status['components']['redis'] = { + 'status': 'ok', + 'memory_mb': redis_health.get('memory_used_mb', 0), + 'keys': redis_health.get('keys', 0) + } + else: + health_status['components']['redis'] = { + 'status': 'error', + 'error': redis_health.get('error', 'Connection failed') + } + health_status['status'] = 'degraded' + else: + health_status['components']['redis'] = { + 'status': 'disabled', + 'enabled': False + } + + status_code = 200 if health_status['status'] in ['healthy', 'degraded'] else 503 + return jsonify(health_status), status_code + +@app.route('/api/stats/summary') +def stats_summary(): + """Combined stats endpoint for dashboard""" + stats = { + 'maxmind': { + 'exists': handler.mmdb_file.exists(), + 'needs_update': handler.needs_update() + }, + 'sqlite': get_sqlite_status(), + 'redis': {'enabled': False} + } + + if config.REDIS_ENABLED and redis_cache: + redis_health = redis_cache.health_check() + stats['redis'] = { + 'enabled': True, + 'connected': redis_health.get('connected', False), + 'memory_mb': redis_health.get('memory_used_mb', 0), + 'keys': redis_health.get('keys', 0) + } + + return jsonify(stats) + +def cache_control(max_age: int = None): + """Decorator for static file cache control""" + def decorator(f): + @wraps(f) + def decorated_function(*args, **kwargs): + response = f(*args, **kwargs) + if isinstance(response, Response): + if max_age: + response.headers['Cache-Control'] = f'public, max-age={max_age}' + else: + response.headers['Cache-Control'] = 'no-cache, no-store' + response.headers['Pragma'] = 'no-cache' + response.headers['Expires'] = '0' + return response + return decorated_function + return decorator + +@app.after_request +def add_headers(response): + """Add cache control headers based on request path""" + + if request.path == '/' or request.path.startswith('/api/'): + response.headers['Cache-Control'] = 'no-cache, no-store' + + elif request.path.startswith('/static/'): + if 'Content-Disposition' in response.headers: + del response.headers['Content-Disposition'] + if config.ENABLE_CACHE_BUSTING: + response.headers['Cache-Control'] = f'public, max-age={config.CACHE_TTL_SECONDS}, immutable' + else: + response.headers['Cache-Control'] = f'public, max-age={config.CACHE_TTL_SECONDS}' + + elif request.path == '/api-docs': + response.headers['Cache-Control'] = 'public, max-age=300' + + return response + +if __name__ == '__main__': + app.run( + host=config.FLASK_HOST, + port=config.FLASK_PORT, + debug=config.FLASK_DEBUG + ) diff --git a/blocklist.txt b/blocklist.txt new file mode 100644 index 0000000..e69de29 diff --git a/config.py b/config.py new file mode 100644 index 0000000..a07a6b8 --- /dev/null +++ b/config.py @@ -0,0 +1,254 @@ +""" +Configuration file for GeoIP Ban Generator +""" + +import os +from pathlib import Path + +# Base paths +BASE_DIR = Path(__file__).parent +STATIC_DIR = BASE_DIR / 'static' +TEMPLATE_DIR = BASE_DIR / 'templates' +GEOIP_DB_DIR = BASE_DIR / 'geoip_db' + +# Flask settings +FLASK_HOST = os.getenv('FLASK_HOST', '0.0.0.0') +FLASK_PORT = int(os.getenv('FLASK_PORT', 5000)) +FLASK_DEBUG = os.getenv('FLASK_DEBUG', 'False').lower() == 'true' +SECRET_KEY = os.getenv('SECRET_KEY', 'change-me-in-production') + +# Application settings +APP_NAME = os.getenv('APP_NAME', 'GeoIP Ban Generator') +APP_VERSION = '1.0.0' + +# Logo settings +LOGO_URL = os.getenv('LOGO_URL', '') +LOGO_LINK = os.getenv('LOGO_LINK', '/') + +# Footer settings +FOOTER_TEXT = os.getenv('FOOTER_TEXT', '© 2026 GeoIP Ban Generator') +FOOTER_LINK = os.getenv('FOOTER_LINK', '') +FOOTER_LINK_TEXT = os.getenv('FOOTER_LINK_TEXT', 'Documentation') + +# MaxMind database settings +MAXMIND_PRIMARY_URL = os.getenv( + 'MAXMIND_PRIMARY_URL', + 'https://github.com/P3TERX/GeoLite.mmdb/releases/download/2026.02.07/GeoLite2-Country.mmdb' +) +MAXMIND_FALLBACK_URL = os.getenv( + 'MAXMIND_FALLBACK_URL', + 'https://git.io/GeoLite2-Country.mmdb' +) +MAXMIND_UPDATE_INTERVAL_DAYS = int(os.getenv('MAXMIND_UPDATE_INTERVAL_DAYS', 7)) +MAXMIND_AUTO_UPDATE = os.getenv('MAXMIND_AUTO_UPDATE', 'True').lower() == 'true' + +# IP range sources +IP_RANGE_SOURCES = { + 'github': 'https://raw.githubusercontent.com/herrbischoff/country-ip-blocks/master/ipv4/{country_lower}.cidr', + 'ipdeny': 'https://www.ipdeny.com/ipblocks/data/aggregated/{country_lower}-aggregated.zone' +} + +# Cache settings +CACHE_ENABLED = os.getenv('CACHE_ENABLED', 'True').lower() == 'true' +CACHE_TTL_SECONDS = int(os.getenv('CACHE_TTL_SECONDS', 3600)) + +# Static file cache busting +ENABLE_CACHE_BUSTING = True + + +def get_country_flag(country_code: str) -> str: + """ + Convert ISO 3166-1 alpha-2 country code to flag emoji. + Uses Unicode Regional Indicator Symbols. + + Example: 'PL' -> '🇵🇱' + """ + if not country_code or len(country_code) != 2: + return '' + + # Convert to uppercase and get Unicode regional indicators + # Regional Indicator Symbol Letter A starts at 0x1F1E6 + code = country_code.upper() + return chr(0x1F1E6 + ord(code[0]) - ord('A')) + chr(0x1F1E6 + ord(code[1]) - ord('A')) + + +# Available countries (ISO 3166-1 alpha-2) +# Focus on high-risk scammer countries and commonly blocked regions +COMMON_COUNTRIES = [ + # High-risk Asian countries + {'code': 'CN', 'name': 'China'}, + {'code': 'IN', 'name': 'India'}, + {'code': 'PK', 'name': 'Pakistan'}, + {'code': 'BD', 'name': 'Bangladesh'}, + {'code': 'ID', 'name': 'Indonesia'}, + {'code': 'PH', 'name': 'Philippines'}, + {'code': 'VN', 'name': 'Vietnam'}, + {'code': 'TH', 'name': 'Thailand'}, + {'code': 'MY', 'name': 'Malaysia'}, + {'code': 'SG', 'name': 'Singapore'}, + {'code': 'KH', 'name': 'Cambodia'}, + {'code': 'MM', 'name': 'Myanmar'}, + {'code': 'LA', 'name': 'Laos'}, + {'code': 'NP', 'name': 'Nepal'}, + {'code': 'LK', 'name': 'Sri Lanka'}, + + # Middle East + {'code': 'IR', 'name': 'Iran'}, + {'code': 'IQ', 'name': 'Iraq'}, + {'code': 'SY', 'name': 'Syria'}, + {'code': 'YE', 'name': 'Yemen'}, + {'code': 'SA', 'name': 'Saudi Arabia'}, + {'code': 'AE', 'name': 'United Arab Emirates'}, + {'code': 'QA', 'name': 'Qatar'}, + {'code': 'KW', 'name': 'Kuwait'}, + {'code': 'BH', 'name': 'Bahrain'}, + {'code': 'OM', 'name': 'Oman'}, + {'code': 'JO', 'name': 'Jordan'}, + {'code': 'LB', 'name': 'Lebanon'}, + + # Africa - West + {'code': 'NG', 'name': 'Nigeria'}, + {'code': 'GH', 'name': 'Ghana'}, + {'code': 'CI', 'name': 'Ivory Coast'}, + {'code': 'SN', 'name': 'Senegal'}, + {'code': 'BJ', 'name': 'Benin'}, + {'code': 'TG', 'name': 'Togo'}, + {'code': 'ML', 'name': 'Mali'}, + {'code': 'BF', 'name': 'Burkina Faso'}, + {'code': 'NE', 'name': 'Niger'}, + {'code': 'LR', 'name': 'Liberia'}, + {'code': 'SL', 'name': 'Sierra Leone'}, + + # Africa - East + {'code': 'KE', 'name': 'Kenya'}, + {'code': 'ET', 'name': 'Ethiopia'}, + {'code': 'TZ', 'name': 'Tanzania'}, + {'code': 'UG', 'name': 'Uganda'}, + {'code': 'SO', 'name': 'Somalia'}, + {'code': 'SD', 'name': 'Sudan'}, + {'code': 'SS', 'name': 'South Sudan'}, + {'code': 'ER', 'name': 'Eritrea'}, + {'code': 'DJ', 'name': 'Djibouti'}, + + # Africa - South + {'code': 'ZA', 'name': 'South Africa'}, + {'code': 'ZW', 'name': 'Zimbabwe'}, + {'code': 'MZ', 'name': 'Mozambique'}, + {'code': 'AO', 'name': 'Angola'}, + {'code': 'ZM', 'name': 'Zambia'}, + {'code': 'MW', 'name': 'Malawi'}, + {'code': 'BW', 'name': 'Botswana'}, + + # Africa - Central + {'code': 'CM', 'name': 'Cameroon'}, + {'code': 'CD', 'name': 'DR Congo'}, + {'code': 'CG', 'name': 'Congo'}, + {'code': 'CF', 'name': 'Central African Republic'}, + {'code': 'TD', 'name': 'Chad'}, + {'code': 'GA', 'name': 'Gabon'}, + + # Africa - North + {'code': 'EG', 'name': 'Egypt'}, + {'code': 'DZ', 'name': 'Algeria'}, + {'code': 'MA', 'name': 'Morocco'}, + {'code': 'TN', 'name': 'Tunisia'}, + {'code': 'LY', 'name': 'Libya'}, + + # Eastern Europe + {'code': 'RU', 'name': 'Russia'}, + {'code': 'UA', 'name': 'Ukraine'}, + {'code': 'BY', 'name': 'Belarus'}, + {'code': 'MD', 'name': 'Moldova'}, + {'code': 'GE', 'name': 'Georgia'}, + {'code': 'AM', 'name': 'Armenia'}, + {'code': 'AZ', 'name': 'Azerbaijan'}, + {'code': 'KZ', 'name': 'Kazakhstan'}, + {'code': 'UZ', 'name': 'Uzbekistan'}, + {'code': 'TM', 'name': 'Turkmenistan'}, + {'code': 'KG', 'name': 'Kyrgyzstan'}, + {'code': 'TJ', 'name': 'Tajikistan'}, + + # Balkans + {'code': 'RO', 'name': 'Romania'}, + {'code': 'BG', 'name': 'Bulgaria'}, + {'code': 'AL', 'name': 'Albania'}, + {'code': 'RS', 'name': 'Serbia'}, + {'code': 'BA', 'name': 'Bosnia and Herzegovina'}, + {'code': 'MK', 'name': 'North Macedonia'}, + {'code': 'XK', 'name': 'Kosovo'}, + {'code': 'ME', 'name': 'Montenegro'}, + + # Latin America + {'code': 'BR', 'name': 'Brazil'}, + {'code': 'MX', 'name': 'Mexico'}, + {'code': 'CO', 'name': 'Colombia'}, + {'code': 'VE', 'name': 'Venezuela'}, + {'code': 'AR', 'name': 'Argentina'}, + {'code': 'PE', 'name': 'Peru'}, + {'code': 'CL', 'name': 'Chile'}, + {'code': 'EC', 'name': 'Ecuador'}, + {'code': 'BO', 'name': 'Bolivia'}, + {'code': 'PY', 'name': 'Paraguay'}, + + # Caribbean + {'code': 'CU', 'name': 'Cuba'}, + {'code': 'HT', 'name': 'Haiti'}, + {'code': 'DO', 'name': 'Dominican Republic'}, + {'code': 'JM', 'name': 'Jamaica'}, + {'code': 'TT', 'name': 'Trinidad and Tobago'}, + + # Other high-risk + {'code': 'KP', 'name': 'North Korea'}, + {'code': 'AF', 'name': 'Afghanistan'}, + {'code': 'TR', 'name': 'Turkey'}, + + # Western countries (for reference/testing) + {'code': 'US', 'name': 'United States'}, + {'code': 'GB', 'name': 'United Kingdom'}, + {'code': 'DE', 'name': 'Germany'}, + {'code': 'FR', 'name': 'France'}, + {'code': 'IT', 'name': 'Italy'}, + {'code': 'ES', 'name': 'Spain'}, + {'code': 'PL', 'name': 'Poland'}, + {'code': 'NL', 'name': 'Netherlands'}, + {'code': 'BE', 'name': 'Belgium'}, + {'code': 'SE', 'name': 'Sweden'}, + {'code': 'NO', 'name': 'Norway'}, + {'code': 'DK', 'name': 'Denmark'}, + {'code': 'FI', 'name': 'Finland'}, + {'code': 'CH', 'name': 'Switzerland'}, + {'code': 'AT', 'name': 'Austria'}, + {'code': 'CA', 'name': 'Canada'}, + {'code': 'AU', 'name': 'Australia'}, + {'code': 'NZ', 'name': 'New Zealand'}, + {'code': 'JP', 'name': 'Japan'}, + {'code': 'KR', 'name': 'South Korea'}, +] + +# Sort countries by name +COMMON_COUNTRIES = sorted(COMMON_COUNTRIES, key=lambda x: x['name']) + +# Add flags dynamically to countries +for country in COMMON_COUNTRIES: + country['flag'] = get_country_flag(country['code']) + +PRECACHE_APP_TYPES = [ + 'nginx_geo', + 'nginx_deny', + 'apache_24', + 'haproxy_acl', + 'raw-cidr_txt', +] + +PRECACHE_AGGREGATE_VARIANTS = [True] + + +# Redis Configuration +REDIS_HOST = os.getenv('REDIS_HOST', 'localhost') +REDIS_PORT = int(os.getenv('REDIS_PORT', '6379')) +REDIS_DB = int(os.getenv('REDIS_DB', '0')) +REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', None) +REDIS_CACHE_TTL = int(os.getenv('REDIS_CACHE_TTL', '86400')) # 24h default +REDIS_ENABLED = os.getenv('REDIS_ENABLED', 'true').lower() == 'true' + +CACHE_MAX_AGE_HOURS = 168 # 7 dni (7 * 24h) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..367daf0 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,127 @@ +version: '3.8' + +services: + redis: + image: redis:7-alpine + container_name: geoip-redis + restart: unless-stopped + command: redis-server --appendonly yes --maxmemory 512mb --maxmemory-policy allkeys-lru + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis_data:/data + networks: + - geoip-network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 3 + + geoip-ban: + build: . + container_name: geoip-ban-generator + restart: unless-stopped + ports: + - "${FLASK_PORT:-5000}:${FLASK_PORT:-5000}" + environment: + - FLASK_HOST=0.0.0.0 + - FLASK_PORT=${FLASK_PORT:-5000} + - FLASK_DEBUG=${FLASK_DEBUG:-False} + - SECRET_KEY=${SECRET_KEY:-change-me-in-production} + - APP_NAME=${APP_NAME:-GeoIP Ban Generator} + - LOGO_URL=${LOGO_URL:-} + - LOGO_LINK=${LOGO_LINK:-/} + - FOOTER_TEXT=${FOOTER_TEXT:-© 2026 GeoIP Ban Generator} + - FOOTER_LINK=${FOOTER_LINK:-} + - FOOTER_LINK_TEXT=${FOOTER_LINK_TEXT:-Documentation} + - MAXMIND_PRIMARY_URL=${MAXMIND_PRIMARY_URL:-https://github.com/P3TERX/GeoLite.mmdb/releases/download/2026.02.07/GeoLite2-Country.mmdb} + - MAXMIND_FALLBACK_URL=${MAXMIND_FALLBACK_URL:-https://git.io/GeoLite2-Country.mmdb} + - MAXMIND_UPDATE_INTERVAL_DAYS=${MAXMIND_UPDATE_INTERVAL_DAYS:-7} + - MAXMIND_AUTO_UPDATE=${MAXMIND_AUTO_UPDATE:-True} + - CACHE_ENABLED=${CACHE_ENABLED:-True} + - CACHE_TTL_SECONDS=${CACHE_TTL_SECONDS:-3600} + + # Redis configuration + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DB=${REDIS_DB:-0} + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + - REDIS_ENABLED=${REDIS_ENABLED:-true} + - REDIS_CACHE_TTL=${REDIS_CACHE_TTL:-86400} + + volumes: + - geoip_data:/app/geoip_db + networks: + - geoip-network + depends_on: + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${FLASK_PORT:-5000}/"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + geoip-scheduler: + build: . + container_name: geoip-scheduler + restart: unless-stopped + environment: + - MAXMIND_PRIMARY_URL=${MAXMIND_PRIMARY_URL:-https://github.com/P3TERX/GeoLite.mmdb/releases/download/2026.02.07/GeoLite2-Country.mmdb} + - MAXMIND_FALLBACK_URL=${MAXMIND_FALLBACK_URL:-https://git.io/GeoLite2-Country.mmdb} + - MAXMIND_UPDATE_INTERVAL_DAYS=${MAXMIND_UPDATE_INTERVAL_DAYS:-7} + - SCHEDULER_ENABLED=${SCHEDULER_ENABLED:-true} + - SCAN_INTERVAL=${SCAN_INTERVAL:-30d} + - SCAN_TIME=${SCAN_TIME:-02:00} + - SCAN_ON_STARTUP=${SCAN_ON_STARTUP:-true} + - CACHE_MAX_AGE_HOURS=${CACHE_MAX_AGE_HOURS:-720} + - PARALLEL_WORKERS=${PARALLEL_WORKERS:-8} + + # Redis configuration (for cache invalidation after scan) + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DB=${REDIS_DB:-0} + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + - REDIS_ENABLED=${REDIS_ENABLED:-true} + + volumes: + - geoip_data:/app/geoip_db + networks: + - geoip-network + command: python scheduler.py + depends_on: + - geoip-ban + - redis + + geoip-precache: + build: . + container_name: geoip-precache + restart: "no" + environment: + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_DB=${REDIS_DB:-0} + - REDIS_PASSWORD=${REDIS_PASSWORD:-} + - REDIS_ENABLED=true + volumes: + - geoip_data:/app/geoip_db + networks: + - geoip-network + command: python precache_configs.py + depends_on: + - redis + - geoip-ban + profiles: + - tools + +volumes: + geoip_data: + driver: local + redis_data: + driver: local + +networks: + geoip-network: + driver: bridge diff --git a/generate_ban.py b/generate_ban.py new file mode 100644 index 0000000..d2932e7 --- /dev/null +++ b/generate_ban.py @@ -0,0 +1,648 @@ +#!/usr/bin/env python3 + +import argparse +import sys +import os +import json +import ipaddress +import urllib.request +from pathlib import Path +from typing import List, Dict, Set +from datetime import datetime, timedelta +import geoip2.database +from geoip2.errors import AddressNotFoundError + + +class Config: + """Configuration manager""" + + DEFAULT_CONFIG = { + "database_url": "https://github.com/P3TERX/GeoLite.mmdb/releases/download/2026.02.07/GeoLite2-Country.mmdb", + "database_file": "GeoLite2-Country.mmdb", + "last_update": None, + "update_interval_days": 7, + "geoip_db_dir": "geoip_db", + "cache_enabled": True, + "auto_update": True, + "ip_range_sources": { + "github": "https://raw.githubusercontent.com/herrbischoff/country-ip-blocks/master/ipv4/{country_lower}.cidr", + "alternative": "https://www.ipdeny.com/ipblocks/data/aggregated/{country_lower}-aggregated.zone" + } + } + + def __init__(self, config_path: str = "geoip_db/config.json"): + self.config_path = Path(config_path) + self.config = self.load() + + def load(self) -> Dict: + """Load configuration from file""" + if self.config_path.exists(): + try: + with open(self.config_path, 'r') as f: + config = json.load(f) + return {**self.DEFAULT_CONFIG, **config} + except Exception as e: + print(f"Warning: Could not load config: {e}", file=sys.stderr) + return self.DEFAULT_CONFIG.copy() + + def save(self): + """Save configuration to file""" + self.config_path.parent.mkdir(parents=True, exist_ok=True) + with open(self.config_path, 'w') as f: + json.dump(self.config, f, indent=2, default=str) + + def get(self, key: str, default=None): + """Get configuration value""" + return self.config.get(key, default) + + def set(self, key: str, value): + """Set configuration value""" + self.config[key] = value + self.save() + + def needs_update(self) -> bool: + """Check if database needs update""" + if not self.config.get('auto_update', True): + return False + + last_update = self.config.get('last_update') + if not last_update: + return True + + try: + last_date = datetime.fromisoformat(last_update) + interval = timedelta(days=self.config.get('update_interval_days', 7)) + return datetime.now() - last_date > interval + except: + return True + + +class GeoIPDatabase: + """GeoIP database handler using MMDB format""" + + def __init__(self, config: Config): + self.config = config + self.db_dir = Path(config.get('geoip_db_dir', 'geoip_db')) + self.db_dir.mkdir(parents=True, exist_ok=True) + self.mmdb_file = self.db_dir / config.get('database_file', 'GeoLite2-Country.mmdb') + self.cache = {} + self.reader = None + + def download_database(self, url: str = None): + """Download MMDB database""" + url = url or self.config.get('database_url') + + print(f"Downloading database from: {url}", file=sys.stderr) + print(f"Saving to: {self.mmdb_file}", file=sys.stderr) + + try: + urllib.request.urlretrieve(url, self.mmdb_file) + + # Update config + self.config.set('last_update', datetime.now().isoformat()) + + print("Database downloaded successfully", file=sys.stderr) + print(f"File size: {self.mmdb_file.stat().st_size / 1024 / 1024:.2f} MB", file=sys.stderr) + return True + + except Exception as e: + print(f"Error downloading database: {e}", file=sys.stderr) + return False + + def check_and_update(self): + """Check if update is needed and download if necessary""" + if not self.mmdb_file.exists(): + print("Database not found, downloading...", file=sys.stderr) + return self.download_database() + + if self.config.needs_update(): + print("Database is outdated, updating...", file=sys.stderr) + return self.download_database() + + return True + + def open_reader(self): + """Open MMDB reader""" + if self.reader is None: + try: + self.reader = geoip2.database.Reader(str(self.mmdb_file)) + print(f"Opened database: {self.mmdb_file}", file=sys.stderr) + except Exception as e: + print(f"Error opening database: {e}", file=sys.stderr) + print("Install geoip2: pip install geoip2", file=sys.stderr) + sys.exit(1) + + def close_reader(self): + """Close MMDB reader""" + if self.reader: + self.reader.close() + self.reader = None + + def get_country_networks_from_source(self, country_code: str) -> List[ipaddress.IPv4Network]: + """Download IP ranges from external source (fallback method)""" + sources = self.config.get('ip_range_sources', {}) + networks = [] + + country_lower = country_code.lower() + + # Try multiple sources + for source_name, url_template in sources.items(): + try: + url = url_template.format(country_lower=country_lower, country_upper=country_code.upper()) + print(f"Fetching from {source_name}: {url}", file=sys.stderr) + + response = urllib.request.urlopen(url, timeout=30) + data = response.read().decode('utf-8') + + for line in data.strip().split('\n'): + line = line.strip() + if line and not line.startswith('#'): + try: + networks.append(ipaddress.IPv4Network(line)) + except ValueError: + continue + + if networks: + print(f"Loaded {len(networks)} networks from {source_name}", file=sys.stderr) + break + + except Exception as e: + print(f"Could not fetch from {source_name}: {e}", file=sys.stderr) + continue + + return networks + + def get_country_networks(self, country_codes: List[str]) -> Dict[str, List[ipaddress.IPv4Network]]: + """Get IP networks for specified countries""" + + # Check cache + cache_key = ','.join(sorted(country_codes)) + if self.config.get('cache_enabled') and cache_key in self.cache: + print(f"Using cached data for {cache_key}", file=sys.stderr) + return self.cache[cache_key] + + country_networks = {code: [] for code in country_codes} + + print(f"Loading networks for: {', '.join(country_codes)}", file=sys.stderr) + + # Use external IP range sources (more efficient than scanning MMDB) + for country_code in country_codes: + networks = self.get_country_networks_from_source(country_code) + country_networks[country_code] = networks + print(f" {country_code}: {len(networks)} networks", file=sys.stderr) + + # Cache results + if self.config.get('cache_enabled'): + self.cache[cache_key] = country_networks + + return country_networks + + +class ConfigGenerator: + + @staticmethod + def _aggregate_networks(networks: list) -> list: + """Aggregate IP networks to minimize list size""" + if not networks: + return [] + + try: + ip_objects = [] + for network in networks: + try: + ip_objects.append(ipaddress.IPv4Network(network, strict=False)) + except: + continue + + if ip_objects: + # Remove duplicates and aggregate + collapsed = list(ipaddress.collapse_addresses(ip_objects)) + return sorted([str(net) for net in collapsed]) + return sorted(list(set(networks))) # At least remove duplicates + except: + return sorted(list(set(networks))) + + @staticmethod + def generate_nginx_geo(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) # Remove duplicates anyway + + config = f"""# Nginx Geo Module Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {len(all_networks)} + +geo $blocked_country {{ + default 0; + +""" + + for network in all_networks: + config += f" {network} 1;\n" + + config += "}\n" + return config + + @staticmethod + def generate_nginx_map(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + # Process each country separately + processed_networks = {} + for country_code, networks in country_networks.items(): + if aggregate: + processed_networks[country_code] = ConfigGenerator._aggregate_networks(networks) + else: + processed_networks[country_code] = sorted(list(set(networks))) + + # Calculate total + total_networks = sum(len(nets) for nets in processed_networks.values()) + + config = f"""# Nginx Map Module Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {total_networks} + +map $remote_addr $blocked_country {{ + default 0; + +""" + + for country_code in sorted(processed_networks.keys()): + networks = processed_networks[country_code] + config += f" # {country_code} - {len(networks)} networks\n" + for network in networks: + config += f" {network} 1;\n" + config += "\n" + + config += "}\n" + return config + + @staticmethod + def generate_nginx_deny(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + config = f"""# Nginx Deny Directives Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {len(all_networks)} + +""" + + for network in all_networks: + config += f"deny {network};\n" + + config += "allow all;\n" + return config + + @staticmethod + def generate_apache_24(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + config = f"""# Apache 2.4 Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {len(all_networks)} + + + Require all granted +""" + + for network in all_networks: + config += f" Require not ip {network}\n" + + config += "\n" + return config + + @staticmethod + def generate_apache_22(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + config = f"""# Apache 2.2 Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {len(all_networks)} + +Order Allow,Deny +Allow from all +""" + + for network in all_networks: + config += f"Deny from {network}\n" + + return config + + @staticmethod + def generate_haproxy_acl(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + config = f"""# HAProxy ACL Configuration +# Generated: {timestamp} +# Countries: {countries_list} +# Total networks: {len(all_networks)} + +frontend http-in + bind *:80 + +""" + + for network in all_networks: + config += f" acl blocked_ip src {network}\n" + + config += """ + http-request deny if blocked_ip + default_backend servers +""" + return config + + @staticmethod + def generate_haproxy_lua(country_networks: dict, aggregate: bool = True) -> str: + timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + countries_list = ', '.join(sorted(country_networks.keys())) + + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + config = f"""-- HAProxy Lua Script +-- Generated: {timestamp} +-- Countries: {countries_list} +-- Total networks: {len(all_networks)} + +local blocked_networks = {{ +""" + + for network in all_networks: + config += f' "{network}",\n' + + config += """}} + +function check_blocked(txn) + local src_ip = txn.f:src() + for _, network in ipairs(blocked_networks) do + if string.match(src_ip, network) then + return true + end + end + return false +end + +core.register_fetches("is_blocked", check_blocked) +""" + return config + + +def main(): + parser = argparse.ArgumentParser( + description='Advanced GeoIP ban configuration generator using MaxMind MMDB', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Generate nginx config for China + %(prog)s --country CN --app nginx --output china.conf + + # Multiple countries (comma-separated) + %(prog)s --country CN,RU,KP --app haproxy --output blocked.conf + + # Update database manually + %(prog)s --update-db + + # Use custom database URL + %(prog)s --db-url https://example.com/GeoLite2-Country.mmdb --country US --app nginx + + # Disable aggregation for all original networks + %(prog)s --country CN --app nginx --no-aggregate + + # Set custom configuration options + %(prog)s --set-config update_interval_days=14 + %(prog)s --set-config auto_update=false + + # Output to console + %(prog)s --country RU,BY --app nginx + """ + ) + + parser.add_argument( + '--country', + help='Country code(s) - comma-separated (e.g., CN,RU,KP)' + ) + + parser.add_argument( + '--app', + choices=['nginx', 'haproxy', 'apache'], + help='Target application type' + ) + + parser.add_argument( + '--output', + help='Output file path (default: stdout)' + ) + + parser.add_argument( + '--config', + default='geoip_db/config.json', + help='Config file path (default: geoip_db/config.json)' + ) + + parser.add_argument( + '--db-url', + help='Custom database URL (MMDB format)' + ) + + parser.add_argument( + '--update-db', + action='store_true', + help='Force database update' + ) + + parser.add_argument( + '--no-aggregate', + action='store_true', + help='Disable network aggregation' + ) + + parser.add_argument( + '--no-auto-update', + action='store_true', + help='Disable automatic database updates' + ) + + parser.add_argument( + '--set-config', + metavar='KEY=VALUE', + help='Set configuration option (e.g., update_interval_days=14)' + ) + + parser.add_argument( + '--show-config', + action='store_true', + help='Show current configuration' + ) + + parser.add_argument( + '--list-countries', + action='store_true', + help='List available country codes' + ) + + args = parser.parse_args() + + # Load configuration + config = Config(args.config) + + # Handle list-countries + if args.list_countries: + common_countries = [ + "CN - China", "RU - Russia", "US - United States", "KP - North Korea", + "IR - Iran", "BY - Belarus", "SY - Syria", "VE - Venezuela", + "CU - Cuba", "SD - Sudan", "IQ - Iraq", "LY - Libya", + "IN - India", "BR - Brazil", "DE - Germany", "FR - France", + "GB - United Kingdom", "JP - Japan", "KR - South Korea" + ] + print("Common country codes:") + for country in common_countries: + print(f" {country}") + print("\nUse ISO 3166-1 alpha-2 codes (2 letters)") + return + + # Handle set-config + if args.set_config: + try: + key, value = args.set_config.split('=', 1) + try: + value = json.loads(value) + except: + pass + config.set(key, value) + print(f"Configuration updated: {key} = {value}", file=sys.stderr) + return + except ValueError: + print("Error: --set-config format should be KEY=VALUE", file=sys.stderr) + sys.exit(1) + + # Handle show-config + if args.show_config: + print(json.dumps(config.config, indent=2, default=str)) + return + + # Override config with command line args + if args.db_url: + config.set('database_url', args.db_url) + + if args.no_auto_update: + config.set('auto_update', False) + + # Initialize database + db = GeoIPDatabase(config) + + # Handle database update + if args.update_db: + db.download_database() + print("Database updated successfully", file=sys.stderr) + return + + # Check if we need to generate config + if not args.country or not args.app: + if not args.update_db and not args.set_config and not args.show_config and not args.list_countries: + parser.print_help() + sys.exit(1) + return + + # Auto-update database if needed + if not args.no_auto_update: + db.check_and_update() + + # Parse countries + countries = [c.strip().upper() for c in args.country.split(',')] + + print(f"Processing countries: {', '.join(countries)}", file=sys.stderr) + + # Get networks + country_networks = db.get_country_networks(countries) + + # Check if we got any data + if not any(country_networks.values()): + print("Error: No networks found for specified countries", file=sys.stderr) + sys.exit(1) + + # Generate configuration + generators = { + 'nginx': ConfigGenerator.generate_nginx, + 'haproxy': ConfigGenerator.generate_haproxy, + 'apache': ConfigGenerator.generate_apache + } + + aggregate = not args.no_aggregate + config_output = generators[args.app](country_networks, aggregate) + + # Output + if args.output: + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'w') as f: + f.write(config_output) + print(f"Configuration written to: {output_path}", file=sys.stderr) + else: + print(config_output) + + # Close database + db.close_reader() + + +if __name__ == '__main__': + main() diff --git a/geoip_handler.py b/geoip_handler.py new file mode 100644 index 0000000..1197d29 --- /dev/null +++ b/geoip_handler.py @@ -0,0 +1,1367 @@ +""" +GeoIP Handler - Database management and IP network fetching +""" + +import geoip2.database +import requests +import json +import ipaddress +import sqlite3 +from pathlib import Path +from datetime import datetime, timedelta +from concurrent.futures import ThreadPoolExecutor, as_completed +import threading +import config +import ipaddress + + +def generate_metadata(countries: list, country_data: dict, redis_stats: dict = None, handler: 'GeoIPHandler' = None) -> dict: + """ + Generate metadata about the configuration for headers + + Args: + countries: List of country codes + country_data: Dict mapping country codes to their networks + redis_stats: Optional dict with Redis statistics {'total': int, 'unique': int, 'deduped': int} + handler: Optional GeoIPHandler instance (will create new if None) + + Returns: + Dict with metadata fields + """ + if handler is None: + handler = GeoIPHandler() + + now = datetime.now() + timestamp = now.strftime('%Y-%m-%d %H:%M:%S %Z') + + total_networks = sum(len(networks) for networks in country_data.values()) + + # Build data sources info per country + sources_info = [] + + conn = sqlite3.connect(str(handler.cache_db)) + cursor = conn.cursor() + + for country in countries: + count = len(country_data.get(country, [])) + + # Get cache metadata + cursor.execute( + 'SELECT last_scan, source FROM cache_metadata WHERE country_code = ?', + (country.upper(),) + ) + row = cursor.fetchone() + + if row: + last_scan_str, source = row + try: + last_scan = datetime.fromisoformat(last_scan_str) + age_hours = (now - last_scan).total_seconds() / 3600 + age_days = age_hours / 24 + + sources_info.append({ + 'country': country, + 'count': count, + 'source_type': 'cache', + 'source_detail': source, + 'last_scan': last_scan_str[:19], + 'age_hours': age_hours, + 'age_days': age_days, + 'formatted': f"# [{country}] {count:,} networks - SQLite cache (source: {source}, scanned: {last_scan_str[:19]}, age: {age_days:.1f} days)" + }) + except Exception as e: + sources_info.append({ + 'country': country, + 'count': count, + 'source_type': 'cache', + 'source_detail': source, + 'last_scan': last_scan_str[:19] if last_scan_str else 'unknown', + 'age_hours': None, + 'age_days': None, + 'formatted': f"# [{country}] {count:,} networks - SQLite cache (source: {source}, scanned: {last_scan_str[:19]})" + }) + else: + sources_info.append({ + 'country': country, + 'count': count, + 'source_type': 'fresh', + 'source_detail': 'live_scan', + 'last_scan': None, + 'age_hours': 0, + 'age_days': 0, + 'formatted': f"# [{country}] {count:,} networks - Fresh scan (no cache)" + }) + + conn.close() + + # Redis statistics + redis_info = {} + if redis_stats: + redis_info = { + 'total': redis_stats.get('total', 0), + 'unique': redis_stats.get('unique', 0), + 'deduped': redis_stats.get('deduped', 0), + 'formatted': f"Redis bad IPs: {redis_stats.get('total', 0)} entries ({redis_stats.get('unique', 0)} unique after deduplication)" + } + + return { + 'timestamp': timestamp, + 'timestamp_iso': now.isoformat(), + 'countries': countries, + 'countries_string': ', '.join(countries), + 'country_count': len(countries), + 'total_networks': total_networks, + 'sources': sources_info, + 'sources_formatted': '\n'.join([s['formatted'] for s in sources_info]), + 'redis': redis_info, + 'cache_max_age_hours': getattr(config, 'CACHE_MAX_AGE_HOURS', 168), + 'cache_max_age_days': getattr(config, 'CACHE_MAX_AGE_HOURS', 168) / 24, + 'cache_db_path': str(handler.cache_db) + } + +def _generate_range_regex(start: int, end: int) -> str: + """Generate optimal regex for numeric range 0-255""" + + if start == 0 and end == 255: + return "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])" + + if end - start < 10: + return "(" + "|".join(str(i) for i in range(start, end + 1)) + ")" + + parts = [] + current = start + while current <= end: + first_digit = current // 10 + last_digit = current % 10 + max_in_decade = min(end, (first_digit + 1) * 10 - 1) + + if last_digit == 0 and max_in_decade == (first_digit + 1) * 10 - 1: + if first_digit == 0: + parts.append("[0-9]") + else: + parts.append(f"{first_digit}[0-9]") + current = max_in_decade + 1 + elif current == max_in_decade: + parts.append(str(current)) + current += 1 + else: + if first_digit == 0: + parts.append(f"[{last_digit}-{max_in_decade % 10}]") + else: + parts.append(f"{first_digit}[{last_digit}-{max_in_decade % 10}]") + current = max_in_decade + 1 + + return "(" + "|".join(parts) + ")" + + +def cidr_to_nginx_regex(cidr: str) -> str: + + try: + network = ipaddress.IPv4Network(cidr, strict=False) + prefix = network.prefixlen + octets = str(network.network_address).split('.') + + if prefix == 32: + return f"~^{octets[0]}\\.{octets[1]}\\.{octets[2]}\\.{octets[3]}$" + + if prefix >= 24: + return f"~^{octets[0]}\\.{octets[1]}\\.{octets[2]}\\." + + if prefix >= 16: + start_third = int(octets[2]) + num_subnets = 2 ** (24 - prefix) + end_third = start_third + num_subnets - 1 + + if start_third == end_third: + return f"~^{octets[0]}\\.{octets[1]}\\.{start_third}\\." + elif end_third - start_third == 1: + return f"~^{octets[0]}\\.{octets[1]}\\.({start_third}|{end_third})\\." + else: + range_regex = _generate_range_regex(start_third, end_third) + return f"~^{octets[0]}\\.{octets[1]}\\.{range_regex}\\." + + if prefix >= 8: + start_second = int(octets[1]) + num_subnets = 2 ** (16 - prefix) + end_second = start_second + num_subnets - 1 + + if start_second == end_second: + return f"~^{octets[0]}\\.{start_second}\\." + else: + range_regex = _generate_range_regex(start_second, end_second) + return f"~^{octets[0]}\\.{range_regex}\\." + + start_first = int(octets[0]) + num_subnets = 2 ** (8 - prefix) + end_first = start_first + num_subnets - 1 + range_regex = _generate_range_regex(start_first, end_first) + return f"~^{range_regex}\\." + + except Exception as e: + print(f"[ERROR] CIDR conversion failed for {cidr}: {e}", flush=True) + return None + +class GeoIPHandler: + def __init__(self): + self.mmdb_file = config.GEOIP_DB_DIR / 'GeoLite2-Country.mmdb' + self.config_file = config.GEOIP_DB_DIR / 'config.json' + self.cache_db = config.GEOIP_DB_DIR / 'networks_cache.db' + config.GEOIP_DB_DIR.mkdir(parents=True, exist_ok=True) + self._init_cache_db() + + def _init_cache_db(self): + conn = sqlite3.connect(str(self.cache_db), timeout=30.0) + cursor = conn.cursor() + + cursor.execute('PRAGMA journal_mode=WAL;') + cursor.execute('PRAGMA synchronous=NORMAL;') + cursor.execute('PRAGMA cache_size=10000;') + cursor.execute('PRAGMA temp_store=MEMORY;') + + cursor.execute(''' + CREATE TABLE IF NOT EXISTS networks_cache ( + country_code TEXT NOT NULL, + network TEXT NOT NULL, + source TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (country_code, network) + ) + ''') + cursor.execute(''' + CREATE TABLE IF NOT EXISTS cache_metadata ( + country_code TEXT PRIMARY KEY, + last_scan TEXT NOT NULL, + network_count INTEGER NOT NULL, + source TEXT DEFAULT 'unknown' + ) + ''') + cursor.execute(''' + CREATE INDEX IF NOT EXISTS idx_networks_country + ON networks_cache(country_code) + ''') + + conn.commit() + conn.close() + + def _get_cached_networks(self, country_code: str) -> list: + """Get networks from cache with chunked reading for large datasets""" + conn = sqlite3.connect(str(self.cache_db), timeout=600.0) + cursor = conn.cursor() + + cursor.execute( + 'SELECT last_scan, network_count FROM cache_metadata WHERE country_code = ?', + (country_code.upper(),) + ) + row = cursor.fetchone() + + if row: + last_scan_str, count = row + last_scan = datetime.fromisoformat(last_scan_str) + age_hours = (datetime.now() - last_scan).total_seconds() / 3600 + + if age_hours < config.CACHE_MAX_AGE_HOURS: + # Chunked reading for large datasets + chunk_size = 100000 + all_networks = [] + offset = 0 + + while offset < count: + cursor.execute( + 'SELECT network FROM networks_cache WHERE country_code = ? LIMIT ? OFFSET ?', + (country_code.upper(), chunk_size, offset) + ) + chunk = [row[0] for row in cursor.fetchall()] + + if not chunk: + break + + all_networks.extend(chunk) + offset += chunk_size + + conn.close() + return all_networks + + conn.close() + return None + + def _save_to_cache(self, country_code: str, networks: list, source: str): + if not networks: + print(f"[CACHE] Skipping {country_code} - no networks to save", flush=True) + return False + + max_retries = 3 + country_code = country_code.upper() + chunk_size = 50000 + + for attempt in range(max_retries): + conn = None + try: + conn = sqlite3.connect( + str(self.cache_db), + timeout=300.0, + isolation_level='DEFERRED' + ) + cursor = conn.cursor() + + cursor.execute('DELETE FROM networks_cache WHERE country_code = ?', (country_code,)) + cursor.execute('DELETE FROM cache_metadata WHERE country_code = ?', (country_code,)) + + timestamp = datetime.now().isoformat() + + total_inserted = 0 + for i in range(0, len(networks), chunk_size): + chunk = networks[i:i+chunk_size] + cursor.executemany( + 'INSERT INTO networks_cache (country_code, network, source, created_at) VALUES (?, ?, ?, ?)', + [(country_code, network, source, timestamp) for network in chunk] + ) + total_inserted += len(chunk) + + if len(networks) > chunk_size: + print(f"[CACHE] {country_code}: Inserted {total_inserted}/{len(networks)} networks...", flush=True) + + cursor.execute( + 'INSERT INTO cache_metadata (country_code, last_scan, network_count, source) VALUES (?, ?, ?, ?)', + (country_code, timestamp, len(networks), source) + ) + + conn.commit() + print(f"[CACHE] ✓ Saved {country_code}: {len(networks)} networks from {source}", flush=True) + return True + + except sqlite3.OperationalError as e: + if 'locked' in str(e).lower() or 'busy' in str(e).lower(): + print(f"[CACHE] Database locked for {country_code}, attempt {attempt+1}/{max_retries}", flush=True) + if attempt < max_retries - 1: + import time + time.sleep(10 * (attempt + 1)) + else: + print(f"[ERROR] Failed to save {country_code} after {max_retries} attempts", flush=True) + return False + else: + print(f"[ERROR] SQLite error for {country_code}: {e}", flush=True) + import traceback + traceback.print_exc() + return False + + except Exception as e: + print(f"[ERROR] Failed to save cache for {country_code}: {e}", flush=True) + import traceback + traceback.print_exc() + return False + + finally: + if conn: + try: + conn.close() + except: + pass + + return False + + def _update_cache_incremental(self, country_code: str, new_networks: list, source: str): + if not new_networks: + print(f"[CACHE] No networks to update for {country_code}", flush=True) + return False + + max_retries = 3 + country_code = country_code.upper() + chunk_size = 50000 + + for attempt in range(max_retries): + conn = None + try: + conn = sqlite3.connect( + str(self.cache_db), + timeout=300.0, + isolation_level='DEFERRED' + ) + cursor = conn.cursor() + + cursor.execute( + 'SELECT network FROM networks_cache WHERE country_code = ?', + (country_code,) + ) + old_networks = set(row[0] for row in cursor.fetchall()) + new_networks_set = set(new_networks) + + to_add = new_networks_set - old_networks + to_remove = old_networks - new_networks_set + + timestamp = datetime.now().isoformat() + + if to_remove: + to_remove_list = list(to_remove) + for i in range(0, len(to_remove_list), chunk_size): + chunk = to_remove_list[i:i+chunk_size] + cursor.executemany( + 'DELETE FROM networks_cache WHERE country_code = ? AND network = ?', + [(country_code, net) for net in chunk] + ) + print(f"[CACHE] Removed {len(to_remove)} old networks from {country_code}", flush=True) + + if to_add: + to_add_list = list(to_add) + total_added = 0 + for i in range(0, len(to_add_list), chunk_size): + chunk = to_add_list[i:i+chunk_size] + cursor.executemany( + 'INSERT INTO networks_cache (country_code, network, source, created_at) VALUES (?, ?, ?, ?)', + [(country_code, network, source, timestamp) for network in chunk] + ) + total_added += len(chunk) + + if len(to_add_list) > chunk_size: + print(f"[CACHE] {country_code}: Added {total_added}/{len(to_add_list)} new networks...", flush=True) + + print(f"[CACHE] Added {len(to_add)} new networks to {country_code}", flush=True) + + cursor.execute('DELETE FROM cache_metadata WHERE country_code = ?', (country_code,)) + cursor.execute( + 'INSERT INTO cache_metadata (country_code, last_scan, network_count, source) VALUES (?, ?, ?, ?)', + (country_code, timestamp, len(new_networks), source) + ) + + conn.commit() + + unchanged = len(old_networks & new_networks_set) + print(f"[CACHE] ✓ Updated {country_code}: +{len(to_add)} new, -{len(to_remove)} removed, ={unchanged} unchanged (total: {len(new_networks)})", flush=True) + return True + + except sqlite3.OperationalError as e: + if 'locked' in str(e).lower() or 'busy' in str(e).lower(): + print(f"[CACHE] Database locked for {country_code}, attempt {attempt+1}/{max_retries}", flush=True) + if attempt < max_retries - 1: + import time + time.sleep(10 * (attempt + 1)) + else: + print(f"[ERROR] Failed to update {country_code} after {max_retries} attempts", flush=True) + return False + else: + print(f"[ERROR] SQLite error for {country_code}: {e}", flush=True) + return False + + except Exception as e: + print(f"[ERROR] Failed to update cache for {country_code}: {e}", flush=True) + import traceback + traceback.print_exc() + return False + + finally: + if conn: + try: + conn.close() + except: + pass + + return False + + def get_countries_needing_scan(self, max_age_hours: int = 168) -> tuple: + import sys + sys.path.insert(0, '/opt/geoip_block_generator') + + all_countries = [c['code'] for c in config.COMMON_COUNTRIES] + + try: + conn = sqlite3.connect(str(self.cache_db), timeout=30.0) + cursor = conn.cursor() + + cursor.execute('SELECT country_code, last_scan FROM cache_metadata') + cached_data = {row[0]: row[1] for row in cursor.fetchall()} + conn.close() + + missing = [] + stale = [] + cutoff_time = datetime.now() - timedelta(hours=max_age_hours) + + for country in all_countries: + if country not in cached_data: + missing.append(country) + else: + try: + last_scan = datetime.fromisoformat(cached_data[country]) + if last_scan < cutoff_time: + stale.append(country) + except: + stale.append(country) + + return missing, stale + + except Exception as e: + print(f"[ERROR] Failed to check cache status: {e}", flush=True) + return all_countries, [] + + def load_config(self) -> dict: + if self.config_file.exists(): + try: + with open(self.config_file, 'r') as f: + return json.load(f) + except: + pass + return {} + + def save_config(self, data: dict): + with open(self.config_file, 'w') as f: + json.dump(data, f, indent=2) + + def needs_update(self) -> bool: + if not self.mmdb_file.exists(): + return True + + cfg = self.load_config() + last_update = cfg.get('last_update') + + if not last_update: + return True + + try: + last_update_date = datetime.fromisoformat(last_update) + days_old = (datetime.now() - last_update_date).days + return days_old >= config.MAXMIND_UPDATE_INTERVAL_DAYS + except: + return True + + def download_database(self) -> dict: + urls = [config.MAXMIND_PRIMARY_URL, config.MAXMIND_FALLBACK_URL] + + for url in urls: + try: + print(f"Downloading database from {url}") + response = requests.get(url, timeout=60, stream=True) + response.raise_for_status() + + with open(self.mmdb_file, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + file_size = self.mmdb_file.stat().st_size + + self.save_config({ + 'last_update': datetime.now().isoformat(), + 'url': url, + 'file_size': file_size + }) + + print(f"Database downloaded successfully ({file_size} bytes)") + return {'success': True, 'url': url, 'size': file_size} + + except Exception as e: + print(f"Failed to download from {url}: {e}") + continue + + return {'success': False, 'error': 'All download sources failed'} + + def check_and_update(self): + if self.needs_update(): + print("Database update needed, downloading...") + self.download_database() + + def _get_scan_ranges(self) -> list: + scan_ranges = [] + + for first_octet in range(1, 224): + if first_octet in [10, 127, 169, 172, 192]: + continue + + for second_octet in range(0, 256): + scan_ranges.append(f"{first_octet}.{second_octet}.0.0/16") + + return scan_ranges + + def _scan_maxmind_for_country(self, country_code: str, progress_callback=None) -> list: + if not self.mmdb_file.exists(): + return [] + + country_code = country_code.upper() + found_networks = set() + found_networks_lock = threading.Lock() + + try: + if progress_callback: + progress_callback(f"Starting parallel MaxMind scan with 32 workers...") + + scan_ranges = self._get_scan_ranges() + total_ranges = len(scan_ranges) + + if progress_callback: + progress_callback(f"Scanning {total_ranges} IP ranges...") + + completed = 0 + completed_lock = threading.Lock() + + def scan_range(network_str): + nonlocal completed + + reader = geoip2.database.Reader(str(self.mmdb_file)) + local_networks = set() + + try: + network = ipaddress.IPv4Network(network_str, strict=False) + + for subnet in network.subnets(new_prefix=24): + sample_ip = str(subnet.network_address + 1) + + try: + response = reader.country(sample_ip) + if response.country.iso_code == country_code: + local_networks.add(str(subnet)) + except: + pass + + except Exception as e: + pass + finally: + reader.close() + + with completed_lock: + completed += 1 + if completed % 2000 == 0 and progress_callback: + with found_networks_lock: + progress_pct = (completed / total_ranges) * 100 + progress_callback(f"Scanning: {completed}/{total_ranges} ranges ({progress_pct:.1f}%), found {len(found_networks)} networks") + + return local_networks + + with ThreadPoolExecutor(max_workers=32) as executor: + futures = {executor.submit(scan_range, r): r for r in scan_ranges} + + for future in as_completed(futures): + local_nets = future.result() + + with found_networks_lock: + found_networks.update(local_nets) + + result = list(found_networks) + + if progress_callback: + progress_callback(f"MaxMind scan complete: {len(result)} networks") + + return result + + except Exception as e: + print(f"[ERROR] MaxMind scan failed for {country_code}: {e}", flush=True) + import traceback + traceback.print_exc() + return [] + + def fetch_country_networks(self, country_code: str, progress_callback=None) -> list: + country_code = country_code.upper() + + cached = self._get_cached_networks(country_code) + if cached is not None: + if progress_callback: + progress_callback(f"Using cached data") + return cached + + if progress_callback: + progress_callback(f"No cache, starting parallel MaxMind scan") + + maxmind_networks = self._scan_maxmind_for_country(country_code, progress_callback) + + if maxmind_networks: + if progress_callback: + progress_callback(f"Checking GitHub for validation") + + github_networks = self._fetch_from_github(country_code) + if github_networks: + maxmind_set = set(maxmind_networks) + github_set = set(github_networks) + missing = github_set - maxmind_set + + if missing: + maxmind_networks.extend(missing) + + self._save_to_cache(country_code, maxmind_networks, 'maxmind+github') + return maxmind_networks + + github_networks = self._fetch_from_github(country_code) + if github_networks: + self._save_to_cache(country_code, github_networks, 'github') + return github_networks + + ipdeny_networks = self._fetch_from_ipdeny(country_code) + if ipdeny_networks: + self._save_to_cache(country_code, ipdeny_networks, 'ipdeny') + return ipdeny_networks + + return [] + + def _fetch_from_github(self, country_code: str) -> list: + url = config.IP_RANGE_SOURCES['github'].format(country_lower=country_code.lower()) + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + networks = [line.strip() for line in response.text.split('\n') if line.strip() and not line.startswith('#')] + return networks + except Exception as e: + return [] + + def _fetch_from_ipdeny(self, country_code: str) -> list: + url = config.IP_RANGE_SOURCES['ipdeny'].format(country_lower=country_code.lower()) + try: + response = requests.get(url, timeout=10) + response.raise_for_status() + networks = [line.strip() for line in response.text.split('\n') if line.strip() and not line.startswith('#')] + return networks + except Exception as e: + return [] + + +class ConfigGenerator: + + @staticmethod + def _aggregate_networks(networks: list) -> list: + try: + if not networks: + return [] + + unique_networks = list(set(networks)) + + ip_objects = [] + for network in unique_networks: + try: + ip_objects.append(ipaddress.IPv4Network(network, strict=False)) + except ValueError: + continue + + if ip_objects: + collapsed = list(ipaddress.collapse_addresses(ip_objects)) + return sorted([str(net) for net in collapsed]) + + return sorted(unique_networks) + except Exception as e: + print(f"[ERROR] Aggregation failed: {e}") + return sorted(list(set(networks))) + + @staticmethod + def generate_nginx_geo(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate Nginx Geo Module configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Nginx Geo Module Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate geo block + config += "geo $blocked_country {\n" + config += " default 0;\n" + config += " \n" + + for network in all_networks: + config += f" {network} 1;\n" + + config += "}\n" + return config + + @staticmethod + def generate_nginx_map(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate Nginx Map Module configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Process networks per country + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Nginx Map Module Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# Note: Using regex patterns for CIDR matching (map module doesn't support CIDR natively)\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate map block with regex conversion + config += "map $remote_addr $blocked_country {\n" + config += " default 0;\n" + config += " \n" + + converted_count = 0 + failed_count = 0 + + for network in all_networks: + regex = cidr_to_nginx_regex(network) + if regex: + config += f" {regex} 1;\n" + converted_count += 1 + else: + # Fallback - zapisz z ostrzeżeniem + config += f" # ERROR: Failed to convert: {network}\n" + failed_count += 1 + + config += "}\n" + + # Log conversion statistics + print(f"[INFO] Generated nginx map: {converted_count} regex patterns", flush=True) + + if failed_count > 0: + print(f"[WARNING] Failed to convert {failed_count} networks to regex - check config file", flush=True) + + return config + + + @staticmethod + def generate_nginx_deny(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate Nginx Deny Directives configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Nginx Deny Directives Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate deny directives + for network in all_networks: + config += f"deny {network};\n" + + config += "allow all;\n" + return config + + @staticmethod + def generate_apache_24(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate Apache 2.4 configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Apache 2.4 Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate Apache 2.4 rules + config += "\n" + config += " Require all granted\n" + + for network in all_networks: + config += f" Require not ip {network}\n" + + config += "\n" + return config + + + @staticmethod + def generate_apache_22(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate Apache 2.2 configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Apache 2.2 Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate Apache 2.2 rules + config += "Order Allow,Deny\n" + config += "Allow from all\n" + + for network in all_networks: + config += f"Deny from {network}\n" + + return config + + + @staticmethod + def generate_haproxy_acl(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate HAProxy ACL configuration with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# HAProxy ACL Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# Usage in HAProxy:\n" + config += "# acl banned_ips src -f /path/to/this_file.acl\n" + config += "# http-request deny if banned_ips\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate ACL rules + config += "frontend http-in\n" + config += " bind *:80\n" + config += " \n" + + for network in all_networks: + config += f" acl blocked_ip src {network}\n" + + config += """ + http-request deny if blocked_ip + default_backend servers + """ + return config + + @staticmethod + def generate_haproxy_lua(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate HAProxy Lua script with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "-- " + "="*76 + "\n" + config += "-- HAProxy Lua Script\n" + config += f"-- Generated: {metadata['timestamp']}\n" + config += "-- " + "="*76 + "\n" + config += "-- \n" + config += f"-- Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"-- Total networks: {len(all_networks):,}\n" + config += "-- \n" + config += "-- Data sources:\n" + + for line in metadata['sources_formatted'].split('\n'): + config += f"-- # {line}\n" + config += "-- \n" + + if metadata['redis']: + config += f"-- {metadata['redis']['formatted']}\n" + config += "-- \n" + + config += "-- Cache settings:\n" + config += f"-- Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"-- Database: {metadata['cache_db_path']}\n" + config += "-- \n" + config += "-- " + "="*76 + "\n" + config += "\n" + + # Generate Lua code + config += "local blocked_networks = {\n" + + for network in all_networks: + config += f' "{network}",\n' + + config += "}\n\n" + config += """ + function check_blocked(txn) + local src_ip = txn.f:src() + for _, network in ipairs(blocked_networks) do + if string.match(src_ip, network) then + return true + end + end + return false + end + + core.register_fetches("is_blocked", check_blocked) + """ + return config + + @staticmethod + def generate_raw_cidr(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate raw CIDR list with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# Raw CIDR List\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += f"# Aggregated: {aggregate}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # Generate CIDR list + for network in all_networks: + config += f"{network}\n" + + return config + + + @staticmethod + def generate_csv(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """Generate CSV format with detailed metadata header""" + + # Get metadata + countries = sorted(country_networks.keys()) + redis_stats = None + if redis_ips: + redis_stats = { + 'total': len(redis_ips), + 'unique': len(redis_ips), + 'deduped': 0 + } + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redis_stats, handler) + + # Calculate totals before aggregation + total_before = sum(len(nets) for nets in country_networks.values()) + if redis_ips: + total_before += len(redis_ips) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# CSV Export\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Aggregated: {aggregate}\n" + config += f"# Networks before aggregation: {total_before:,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + config += "country,network,source\n" + + # Generate CSV data + total_after = 0 + for country_code, networks in sorted(country_networks.items()): + if aggregate: + networks = ConfigGenerator._aggregate_networks(networks) + else: + networks = sorted(list(set(networks))) + + total_after += len(networks) + + for network in networks: + config += f"{country_code},{network},cache\n" + + # Add Redis IPs if present + if redis_ips: + redis_list = list(redis_ips) + if aggregate: + redis_list = ConfigGenerator._aggregate_networks(redis_list) + else: + redis_list = sorted(redis_list) + + total_after += len(redis_list) + + for network in redis_list: + config += f"REDIS,{network},redis\n" + + # Update header with final count + config = config.replace( + f"# Networks before aggregation: {total_before:,}", + f"# Networks before aggregation: {total_before:,}\n# Networks after aggregation: {total_after:,}" + ) + + return config diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg new file mode 100644 index 0000000..f071903 --- /dev/null +++ b/haproxy/haproxy.cfg @@ -0,0 +1,55 @@ +global + log /dev/log local0 + log /dev/log local1 notice + chroot /var/lib/haproxy + stats socket /var/lib/haproxy/admin.sock mode 660 level admin + + stats timeout 30s + user haproxy + group haproxy + daemon + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 900000 + timeout server 900000 + +listen stats + bind *:8404 + stats enable + stats uri /stats + stats refresh 10s + stats admin if TRUE + stats auth admin:geoip2024 + +frontend http_front + bind *:80 + option httplog + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" + default_backend webapp_backend + +backend webapp_backend + balance source + option httpchk GET /health + http-response del-header Server + http-check expect status 200 + retries 3 + option redispatch + option http-server-close + option forwardfor + http-request add-header X-Forwarded-Proto http + compression algo gzip + compression type text/html text/plain text/css application/javascript application/json + + server webapp1 127.0.0.1:5001 check inter 5s fall 3 rise 2 maxconn 50 + server webapp2 127.0.0.1:5002 check inter 5s fall 3 rise 2 maxconn 50 + server webapp3 127.0.0.1:5003 check inter 5s fall 3 rise 2 maxconn 50 + server webapp4 127.0.0.1:5004 check inter 5s fall 3 rise 2 maxconn 50 diff --git a/precache_daemon.py b/precache_daemon.py new file mode 100644 index 0000000..b62f933 --- /dev/null +++ b/precache_daemon.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +""" +Pre-cache individual countries in ALL config variants to Redis +""" + +import sys +import os +import sqlite3 +import json +from datetime import datetime + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPT_DIR) +os.chdir(SCRIPT_DIR) + +print(f"[PRE-CACHE] Working from: {SCRIPT_DIR}", flush=True) + +from redis_cache import RedisCache +from geoip_handler import ConfigGenerator +import config + +DB_PATH = config.GEOIP_DB_DIR / 'networks_cache.db' + +if not DB_PATH.exists(): + print(f"[ERROR] SQLite database not found: {DB_PATH}", flush=True) + sys.exit(1) + +redis_cache = RedisCache() +health = redis_cache.health_check() +print(f"[PRE-CACHE] Redis: {health['status']} ({health.get('memory_used_mb', 0):.1f} MB used)", flush=True) + +APP_TYPES = [ + 'nginx_geo', + 'nginx_map', + 'nginx_deny', + 'apache_24', + 'haproxy_acl', + 'raw-cidr_txt', + 'raw-newline_txt', + 'raw-json', + 'raw-csv', +] + +def get_available_countries(): + conn = sqlite3.connect(str(DB_PATH), timeout=30.0) + cursor = conn.cursor() + cursor.execute('SELECT country_code, network_count FROM cache_metadata ORDER BY country_code') + countries_info = {} + for row in cursor.fetchall(): + countries_info[row[0]] = row[1] + conn.close() + return countries_info + +def fetch_country_networks(country_code): + conn = sqlite3.connect(str(DB_PATH), timeout=600.0) + cursor = conn.cursor() + + cursor.execute('SELECT network_count FROM cache_metadata WHERE country_code = ?', (country_code.upper(),)) + row = cursor.fetchone() + if not row: + conn.close() + return [] + + total_count = row[0] + chunk_size = 100000 + all_networks = [] + offset = 0 + + while offset < total_count: + cursor.execute('SELECT network FROM networks_cache WHERE country_code = ? LIMIT ? OFFSET ?', + (country_code.upper(), chunk_size, offset)) + chunk = [row[0] for row in cursor.fetchall()] + if not chunk: + break + all_networks.extend(chunk) + offset += chunk_size + + conn.close() + return all_networks + +start_time = datetime.now() + +print(f"\n{'='*70}", flush=True) +print(f"[STRATEGY] Per-country cache (all config variants)", flush=True) +print(f" Each country: raw data + {len(APP_TYPES)} types × 2 aggregation = {len(APP_TYPES)*2} configs", flush=True) +print(f" Multi-country combos: generated on-demand", flush=True) +print(f"{'='*70}\n", flush=True) + +available_countries = get_available_countries() +print(f"Found {len(available_countries)} countries\n", flush=True) + +country_data_generated = 0 +country_data_cached = 0 +config_generated = 0 +config_cached = 0 +errors = 0 + +for idx, (country, count) in enumerate(available_countries.items(), 1): + print(f"[{idx}/{len(available_countries)}] {country}: {count:,} networks", flush=True) + + redis_key_data = f"geoban:country:{country}" + data_exists = redis_cache.redis_client.exists(redis_key_data) + + if data_exists: + country_data_cached += 1 + print(f" ✓ Raw data: cached", flush=True) + try: + data = redis_cache.redis_client.get(redis_key_data) + if isinstance(data, bytes): + networks = json.loads(data.decode('utf-8')) + else: + networks = json.loads(data) + country_networks = {country: networks} + except Exception as e: + print(f" ✗ Error loading: {e}", flush=True) + errors += 1 + continue + else: + networks = fetch_country_networks(country) + if not networks: + print(f" ✗ No data", flush=True) + errors += 1 + continue + + redis_cache.redis_client.setex(redis_key_data, 86400, json.dumps(networks)) + country_data_generated += 1 + print(f" ✓ Raw data: generated", flush=True) + country_networks = {country: networks} + + configs_generated_this_country = 0 + configs_cached_this_country = 0 + + for app_type in APP_TYPES: + for aggregate in [True, False]: + try: + cached_config = redis_cache.get_cached_config([country], app_type, aggregate) + if cached_config: + config_cached += 1 + configs_cached_this_country += 1 + continue + + if app_type.startswith('raw-'): + format_type = app_type.split('-')[1] + + if format_type == 'cidr_txt': + config_text = '\n'.join(networks) + elif format_type == 'newline_txt': + config_text = '\n'.join(networks) + elif format_type == 'json': + config_text = json.dumps({ + 'country': country, + 'networks': networks, + 'count': len(networks) + }, indent=2) + elif format_type == 'csv': + config_text = 'network\n' + '\n'.join(networks) + else: + print(f" ✗ Unknown raw format: {format_type}", flush=True) + continue + + else: + generators = { + 'nginx_geo': ConfigGenerator.generate_nginx_geo, + 'nginx_map': ConfigGenerator.generate_nginx_map, + 'nginx_deny': ConfigGenerator.generate_nginx_deny, + 'apache_22': ConfigGenerator.generate_apache_22, + 'apache_24': ConfigGenerator.generate_apache_24, + 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, + 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + } + + generator = generators.get(app_type) + if not generator: + continue + + config_text = generator(country_networks, aggregate=aggregate, redis_ips=None) + + stats = { + 'countries': 1, + 'total_networks': len(networks), + 'per_country': {country: len(networks)} + } + + success = redis_cache.save_config([country], app_type, aggregate, config_text, stats) + + if success: + config_generated += 1 + configs_generated_this_country += 1 + else: + errors += 1 + + except Exception as e: + print(f" ✗ {app_type} ({aggregate}): {e}", flush=True) + errors += 1 + + if configs_generated_this_country > 0: + print(f" → New configs: {configs_generated_this_country}", flush=True) + if configs_cached_this_country > 0: + print(f" → Cached configs: {configs_cached_this_country}", flush=True) + + progress_pct = (idx / len(available_countries)) * 100 + print(f" → Progress: {progress_pct:.1f}%\n", flush=True) + +duration = (datetime.now() - start_time).total_seconds() + +print(f"{'='*70}", flush=True) +print(f"[SUMMARY] Complete in {duration/60:.1f} minutes", flush=True) +print(f"\n[Raw Country Data]", flush=True) +print(f" Generated: {country_data_generated}", flush=True) +print(f" Cached: {country_data_cached}", flush=True) +print(f"\n[Config Files]", flush=True) +print(f" Generated: {config_generated}", flush=True) +print(f" Cached: {config_cached}", flush=True) +print(f" Errors: {errors}", flush=True) + +try: + total_keys = redis_cache.redis_client.dbsize() + + cursor = 0 + country_keys = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match="geoban:country:*", count=1000) + country_keys += len(keys) + if cursor == 0: + break + + cursor = 0 + config_keys = 0 + while True: + cursor, keys = redis_cache.redis_client.scan(cursor, match="geoip:config:*", count=1000) + config_keys += len(keys) + if cursor == 0: + break + + health = redis_cache.health_check() + + print(f"\n[REDIS]", flush=True) + print(f" Total keys: {total_keys}", flush=True) + print(f" Country keys: {country_keys}", flush=True) + print(f" Config keys: {config_keys}", flush=True) + print(f" Memory: {health.get('memory_used_mb', 0):.2f} MB", flush=True) + +except Exception as e: + print(f"\n[REDIS] Error: {e}", flush=True) + +print(f"{'='*70}\n", flush=True) diff --git a/redis_cache.py b/redis_cache.py new file mode 100644 index 0000000..b1bc39c --- /dev/null +++ b/redis_cache.py @@ -0,0 +1,204 @@ +""" +Redis Cache Handler for pre-generated GeoIP configs +""" + +import redis +import json +import hashlib +from datetime import datetime +from typing import Optional, Dict, List +import config + + +class RedisCache: + def __init__(self): + self.redis_client = redis.Redis( + host=config.REDIS_HOST, + port=config.REDIS_PORT, + db=config.REDIS_DB, + password=config.REDIS_PASSWORD if config.REDIS_PASSWORD else None, + decode_responses=True, + socket_connect_timeout=5, + socket_timeout=5, + #decode_responses=True + ) + self.default_ttl = config.REDIS_CACHE_TTL + + def _generate_key(self, countries: List[str], app_type: str, aggregate: bool) -> str: + """Generate cache key with normalization""" + + normalized_countries = sorted([c.upper().strip() for c in countries]) + normalized_app_type = app_type.lower().strip() + normalized_aggregate = bool(aggregate) + + key_data = { + 'countries': normalized_countries, + 'app_type': normalized_app_type, + 'aggregate': normalized_aggregate + } + + key_str = json.dumps(key_data, sort_keys=True) + key_hash = hashlib.md5(key_str.encode()).hexdigest()[:16] + + # DEBUG + #print(f"[CACHE KEY] {normalized_countries} + {normalized_app_type} + {normalized_aggregate} -> geoip:config:{key_hash}", flush=True) + + return f"geoip:config:{key_hash}" + + def get_cached_config(self, countries: List[str], app_type: str, aggregate: bool) -> Optional[Dict]: + """Get pre-generated config from Redis""" + try: + cache_key = self._generate_key(countries, app_type, aggregate) + data = self.redis_client.get(cache_key) + + if data: + cached = json.loads(data) + print(f"[REDIS] Cache HIT: {cache_key}", flush=True) + return cached + + print(f"[REDIS] Cache MISS: {cache_key}", flush=True) + return None + + except redis.RedisError as e: + print(f"[REDIS] Error getting cache: {e}", flush=True) + return None + + def save_config(self, countries: List[str], app_type: str, aggregate: bool, + config_text: str, stats: Dict, ttl: Optional[int] = None) -> bool: + """Save generated config to Redis""" + try: + cache_key = self._generate_key(countries, app_type, aggregate) + + cache_data = { + 'config': config_text, + 'stats': stats, + 'generated_at': datetime.now().isoformat(), + 'countries': sorted(countries), + 'app_type': app_type, + 'aggregate': aggregate + } + + ttl = ttl or self.default_ttl + self.redis_client.setex( + cache_key, + ttl, + json.dumps(cache_data, ensure_ascii=False) + ) + + print(f"[REDIS] Saved config: {cache_key} (TTL: {ttl}s)", flush=True) + return True + + except redis.RedisError as e: + print(f"[REDIS] Error saving cache: {e}", flush=True) + return False + + def invalidate_country(self, country_code: str) -> int: + """Invalidate all cached configs containing specific country""" + try: + pattern = f"geoip:config:*" + deleted = 0 + + for key in self.redis_client.scan_iter(match=pattern, count=100): + data = self.redis_client.get(key) + if data: + try: + cached = json.loads(data) + if country_code in cached.get('countries', []): + self.redis_client.delete(key) + deleted += 1 + except: + continue + + print(f"[REDIS] Invalidated {deleted} cache entries for {country_code}", flush=True) + return deleted + + except redis.RedisError as e: + print(f"[REDIS] Error invalidating cache: {e}", flush=True) + return 0 + + def get_cache_stats(self) -> Dict: + """Get Redis cache statistics""" + try: + pattern = f"geoip:config:*" + keys = list(self.redis_client.scan_iter(match=pattern, count=1000)) + + total_size = 0 + entries = [] + + for key in keys[:100]: + try: + data = self.redis_client.get(key) + ttl = self.redis_client.ttl(key) + + if data: + cached = json.loads(data) + size = len(data) + total_size += size + + entries.append({ + 'countries': cached.get('countries', []), + 'app_type': cached.get('app_type'), + 'aggregate': cached.get('aggregate'), + 'generated_at': cached.get('generated_at'), + 'size_bytes': size, + 'ttl_seconds': ttl + }) + except: + continue + + return { + 'total_entries': len(keys), + 'total_size_bytes': total_size, + 'total_size_mb': round(total_size / 1024 / 1024, 2), + 'entries_sample': entries[:20] + } + + except redis.RedisError as e: + print(f"[REDIS] Error getting stats: {e}", flush=True) + return {'error': str(e)} + + def flush_all(self): + """Flush all geoban-related keys from Redis""" + try: + patterns = [ + 'geoban:country:*', + 'geoban:config:*', + 'geoip:config:*' + ] + + deleted = 0 + for pattern in patterns: + cursor = 0 + while True: + cursor, keys = self.redis_client.scan(cursor, match=pattern, count=1000) + if keys: + deleted += self.redis_client.delete(*keys) + if cursor == 0: + break + + print(f"[REDIS] Flushed {deleted} keys", flush=True) + return True + except Exception as e: + print(f"[REDIS] Flush error: {e}", flush=True) + return False + + + def health_check(self) -> Dict: + """Check Redis connection health""" + try: + self.redis_client.ping() + info = self.redis_client.info('memory') + + return { + 'status': 'healthy', + 'connected': True, + 'memory_used_mb': round(info.get('used_memory', 0) / 1024 / 1024, 2), + 'memory_peak_mb': round(info.get('used_memory_peak', 0) / 1024 / 1024, 2) + } + + except redis.RedisError as e: + return { + 'status': 'unhealthy', + 'connected': False, + 'error': str(e) + } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b9960f3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask +Werkzeug +gunicorn +geoip2 +schedule +requests +redis diff --git a/rescan_github_merge.py b/rescan_github_merge.py new file mode 100644 index 0000000..5a1d79c --- /dev/null +++ b/rescan_github_merge.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Rescan script - merges GitHub networks with existing MaxMind cache +Updates source to 'maxmind+github' without rescanning MaxMind +""" + +import sys +sys.path.insert(0, '/opt/geoip_block_generator') + +import sqlite3 +from geoip_handler import GeoIPHandler +from pathlib import Path + +handler = GeoIPHandler() + +conn = sqlite3.connect(str(handler.cache_db), timeout=30) +cursor = conn.cursor() + +cursor.execute("SELECT country_code, network_count, source FROM cache_metadata ORDER BY country_code") +countries = cursor.fetchall() + +print(f"Found {len(countries)} countries in cache\n") + +for country_code, current_count, current_source in countries: + print(f"[{country_code}] Current: {current_count:,} networks, source: {current_source}") + + cursor.execute( + "SELECT network FROM networks_cache WHERE country_code = ?", + (country_code,) + ) + maxmind_networks = [row[0] for row in cursor.fetchall()] + + if not maxmind_networks: + print(f" ⚠ Empty cache, skipping...") + continue + + github_networks = handler._fetch_from_github(country_code) + + if not github_networks: + print(f" ℹ GitHub: no data") + + if current_source in ['unknown', None]: + cursor.execute( + "UPDATE cache_metadata SET source = ? WHERE country_code = ?", + ('maxmind', country_code) + ) + conn.commit() + print(f" ✓ Updated source: unknown → maxmind") + continue + + maxmind_set = set(maxmind_networks) + github_set = set(github_networks) + missing = github_set - maxmind_set + + if missing: + print(f" + GitHub: {len(github_networks):,} networks, {len(missing):,} NEW") + + import datetime + timestamp = datetime.datetime.now().isoformat() + + cursor.executemany( + "INSERT OR IGNORE INTO networks_cache (country_code, network, source, created_at) VALUES (?, ?, ?, ?)", + [(country_code, net, 'github', timestamp) for net in missing] + ) + + new_count = current_count + len(missing) + cursor.execute( + "UPDATE cache_metadata SET network_count = ?, source = ?, last_scan = ? WHERE country_code = ?", + (new_count, 'maxmind+github', timestamp, country_code) + ) + + conn.commit() + print(f" ✓ Updated: {current_count:,} → {new_count:,} networks, source: maxmind+github") + else: + print(f" ℹ GitHub: {len(github_networks):,} networks, 0 new (all covered by MaxMind)") + + cursor.execute( + "UPDATE cache_metadata SET source = ? WHERE country_code = ?", + ('maxmind+github', country_code) + ) + conn.commit() + print(f" ✓ Updated source: {current_source} → maxmind+github") + +conn.close() + +print("\n=== Summary ===") +conn = sqlite3.connect(str(handler.cache_db), timeout=30) +cursor = conn.cursor() + +cursor.execute("SELECT source, COUNT(*), SUM(network_count) FROM cache_metadata GROUP BY source") +for source, count, total in cursor.fetchall(): + print(f"{source}: {count} countries, {total:,} networks") + +conn.close() diff --git a/scheduler.py b/scheduler.py new file mode 100644 index 0000000..134736d --- /dev/null +++ b/scheduler.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +""" +GeoIP Country Scanner Daemon - Incremental Update Mode +""" +import schedule +import time +import sys +import signal +import os +import sqlite3 +import concurrent.futures +from datetime import datetime +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor, as_completed +from multiprocessing import cpu_count +import threading + + +sys.path.insert(0, str(Path(__file__).parent)) + + +from geoip_handler import GeoIPHandler +import config + + +running = True +log_lock = threading.Lock() +write_lock = threading.Lock() +active_scans = {} +active_scans_lock = threading.Lock() + + +def signal_handler(signum, frame): + global running + print(f"\n[{datetime.now()}] Received signal {signum}, shutting down...", flush=True) + sys.stdout.flush() + running = False + + +def log_safe(message): + with log_lock: + print(message, flush=True) + sys.stdout.flush() + + +def update_scan_progress(country_code, progress_msg): + with active_scans_lock: + if country_code in active_scans: + active_scans[country_code]['progress'] = progress_msg + active_scans[country_code]['last_update'] = datetime.now() + + +def progress_callback_factory(country_code): + def callback(msg): + update_scan_progress(country_code, msg) + return callback + + +def print_active_scans(): + with active_scans_lock: + if not active_scans: + return + + print("\n" + "=" * 70, flush=True) + print("ACTIVE SCANS STATUS:", flush=True) + print("=" * 70, flush=True) + + for country, info in sorted(active_scans.items()): + elapsed = (datetime.now() - info['start_time']).total_seconds() + progress = info.get('progress', 'Unknown') + is_update = info.get('is_update', False) + mode = "UPDATE" if is_update else "SCAN" + print(f" {country} [{mode}]: {progress} | {elapsed:.0f}s", flush=True) + + print("=" * 70 + "\n", flush=True) + sys.stdout.flush() + + +def scan_single_country(country_code, is_update=False): + try: + with active_scans_lock: + active_scans[country_code] = { + 'start_time': datetime.now(), + 'progress': 'Starting...', + 'last_update': datetime.now(), + 'is_update': is_update + } + + start_time = time.time() + mode = "INCREMENTAL UPDATE" if is_update else "FULL SCAN" + print(f"[START] {country_code} - {mode}...", flush=True) + sys.stdout.flush() + + progress_cb = progress_callback_factory(country_code) + handler = GeoIPHandler() + + print(f"[{country_code}] Scanning MaxMind + GitHub...", flush=True) + + maxmind_networks = handler._scan_maxmind_for_country(country_code, progress_callback=progress_cb) + + if maxmind_networks: + print(f"[{country_code}] MaxMind: {len(maxmind_networks):,} networks, checking GitHub...", flush=True) + + github_networks = handler._fetch_from_github(country_code) + if github_networks: + maxmind_set = set(maxmind_networks) + github_set = set(github_networks) + missing = github_set - maxmind_set + + if missing: + maxmind_networks.extend(missing) + print(f"[{country_code}] GitHub added {len(missing):,} new networks", flush=True) + else: + print(f"[{country_code}] GitHub: {len(github_networks):,} networks (no new)", flush=True) + + source = 'maxmind+github' + else: + print(f"[{country_code}] GitHub: no data", flush=True) + source = 'maxmind' + + networks = maxmind_networks + else: + print(f"[{country_code}] MaxMind found nothing, trying GitHub...", flush=True) + networks = handler._fetch_from_github(country_code) + source = 'github' if networks else None + + if networks: + with write_lock: + print(f"[{country_code}] Acquired write lock, saving to database...", flush=True) + + if is_update: + saved = handler._update_cache_incremental(country_code, networks, source) + else: + saved = handler._save_to_cache(country_code, networks, source) + + print(f"[{country_code}] Released write lock", flush=True) + + elapsed = time.time() - start_time + + with active_scans_lock: + active_scans.pop(country_code, None) + + if saved: + print(f"[DONE] {country_code}: {len(networks)} networks in {elapsed:.1f}s ({mode})", flush=True) + sys.stdout.flush() + return {'country': country_code, 'success': True, 'networks': len(networks), 'error': None, 'mode': mode} + else: + print(f"[ERROR] {country_code}: Failed to save to cache", flush=True) + sys.stdout.flush() + return {'country': country_code, 'success': False, 'networks': 0, 'error': 'Failed to save', 'mode': mode} + else: + with active_scans_lock: + active_scans.pop(country_code, None) + + print(f"[ERROR] {country_code}: No data found", flush=True) + sys.stdout.flush() + return {'country': country_code, 'success': False, 'networks': 0, 'error': 'No data found', 'mode': mode} + + except Exception as e: + with active_scans_lock: + active_scans.pop(country_code, None) + + print(f"[ERROR] {country_code}: {e}", flush=True) + sys.stdout.flush() + import traceback + traceback.print_exc() + return {'country': country_code, 'success': False, 'networks': 0, 'error': str(e), 'mode': 'UNKNOWN'} + + +def scan_all_countries_incremental(parallel_workers=None, max_age_hours=168): + log_safe(f"[{datetime.now()}] Starting INCREMENTAL country scan...") + + try: + handler = GeoIPHandler() + + if handler.needs_update(): + log_safe("Updating MaxMind database...") + result = handler.download_database() + if not result.get('success'): + log_safe(f"Warning: Database update failed - {result.get('error')}") + + log_safe("\nChecking cache status...") + missing, stale = handler.get_countries_needing_scan(max_age_hours) + + log_safe(f"Missing countries (never scanned): {len(missing)}") + log_safe(f"Stale countries (needs update): {len(stale)}") + + if missing: + log_safe(f"Missing: {', '.join(sorted(missing))}") + if stale: + log_safe(f"Stale: {', '.join(sorted(stale))}") + + total = len(missing) + len(stale) + + if total == 0: + log_safe("\n✓ All countries are up to date!") + return True + + if parallel_workers is None: + parallel_workers = min(cpu_count(), 16) + + log_safe(f"\nProcessing {total} countries using {parallel_workers} parallel workers...") + log_safe(f" - {len(missing)} new countries (full scan)") + log_safe(f" - {len(stale)} stale countries (incremental update)") + log_safe(f"Note: Database writes are serialized with write lock") + log_safe(f"Estimated time: {total / parallel_workers * 3:.1f} minutes\n") + + start_time = datetime.now() + completed = 0 + success_count = 0 + failed_countries = [] + results_list = [] + last_progress_time = time.time() + last_status_print = time.time() + + def print_progress(force=False): + nonlocal last_progress_time + current_time = time.time() + + if not force and (current_time - last_progress_time) < 30: + return + + last_progress_time = current_time + elapsed = (datetime.now() - start_time).total_seconds() + avg_time = elapsed / completed if completed > 0 else 0 + remaining = (total - completed) * avg_time if completed > 0 else 0 + progress_bar = "█" * int(completed / total * 40) + progress_bar += "░" * (40 - int(completed / total * 40)) + + msg = (f"[{progress_bar}] {completed}/{total} ({100*completed/total:.1f}%) | " + f"Elapsed: {elapsed:.0f}s | ETA: {remaining:.0f}s") + print(msg, flush=True) + sys.stdout.flush() + + log_safe("Starting parallel execution...") + sys.stdout.flush() + + tasks = [(country, False) for country in missing] + [(country, True) for country in stale] + + with ThreadPoolExecutor(max_workers=parallel_workers) as executor: + future_to_country = { + executor.submit(scan_single_country, country, is_update): country + for country, is_update in tasks + } + + log_safe(f"Submitted {len(future_to_country)} tasks\n") + sys.stdout.flush() + + pending = set(future_to_country.keys()) + + while pending: + done, pending = concurrent.futures.wait( + pending, + timeout=10, + return_when=concurrent.futures.FIRST_COMPLETED + ) + + for future in done: + result = future.result() + results_list.append(result) + completed += 1 + + if result['success']: + success_count += 1 + else: + failed_countries.append(result['country']) + + print_progress(force=bool(done)) + + current_time = time.time() + if current_time - last_status_print >= 30: + print_active_scans() + last_status_print = current_time + + print("\n", flush=True) + sys.stdout.flush() + + elapsed = (datetime.now() - start_time).total_seconds() + + log_safe("=" * 70) + log_safe("SCAN RESULTS (sorted by country):") + log_safe("=" * 70) + + for result in sorted(results_list, key=lambda x: x['country']): + mode_str = f"[{result.get('mode', 'UNKNOWN')}]" + if result['success']: + log_safe(f" {result['country']}: ✓ {result['networks']:,} networks {mode_str}") + else: + log_safe(f" {result['country']}: ✗ {result['error']} {mode_str}") + + log_safe("=" * 70) + log_safe(f"\n[{datetime.now()}] Incremental scan complete!") + log_safe(f"✓ Success: {success_count}/{total} countries") + log_safe(f" - New countries: {len([r for r in results_list if r.get('mode') == 'FULL SCAN' and r['success']])}") + log_safe(f" - Updated countries: {len([r for r in results_list if r.get('mode') == 'INCREMENTAL UPDATE' and r['success']])}") + log_safe(f" Time: {elapsed:.1f}s ({elapsed/60:.1f} minutes)") + log_safe(f" Average: {elapsed/total:.1f}s per country\n") + + if failed_countries: + log_safe(f"✗ Failed: {', '.join(failed_countries)}\n") + + return True + except Exception as e: + log_safe(f"[{datetime.now()}] ERROR: {e}") + import traceback + traceback.print_exc() + sys.stdout.flush() + return False + + +if __name__ == '__main__': + print("=" * 70, flush=True) + print("GeoIP Country Scanner Daemon", flush=True) + print("=" * 70, flush=True) + print(f"Started: {datetime.now()}", flush=True) + print(f"Data dir: {config.GEOIP_DB_DIR}", flush=True) + print(f"CPU cores: {cpu_count()}", flush=True) + sys.stdout.flush() + + scheduler_enabled = os.getenv('SCHEDULER_ENABLED', 'true').lower() == 'true' + + if not scheduler_enabled: + print("\n[DISABLED] SCHEDULER_ENABLED=false - exiting", flush=True) + print("=" * 70, flush=True) + sys.stdout.flush() + sys.exit(0) + + print("=" * 70, flush=True) + sys.stdout.flush() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + + scan_time = os.getenv('SCAN_TIME', '02:00') + scan_interval = os.getenv('SCAN_INTERVAL', '7d') + scan_on_startup = os.getenv('SCAN_ON_STARTUP', 'true').lower() == 'true' + cache_max_age_hours = int(os.getenv('CACHE_MAX_AGE_HOURS', '168')) + parallel_workers = int(os.getenv('PARALLEL_WORKERS', '16')) + + if parallel_workers == 0: + parallel_workers = min(cpu_count(), 16) + + print(f"\n[CONFIG] Scheduler: enabled", flush=True) + print(f"[CONFIG] Parallel: {parallel_workers} workers", flush=True) + print(f"[CONFIG] Interval: {scan_interval}", flush=True) + print(f"[CONFIG] Time: {scan_time}", flush=True) + print(f"[CONFIG] Startup scan: {scan_on_startup}", flush=True) + print(f"[CONFIG] Cache max age: {cache_max_age_hours}h ({cache_max_age_hours/24:.1f} days)", flush=True) + sys.stdout.flush() + + scan_function = lambda: scan_all_countries_incremental(parallel_workers, cache_max_age_hours) + + if scan_on_startup: + print("\n[STARTUP] Running incremental scan...\n", flush=True) + sys.stdout.flush() + scan_function() + else: + print("\n[STARTUP] Skipping (SCAN_ON_STARTUP=false)", flush=True) + sys.stdout.flush() + + if scan_interval == 'daily': + schedule.every().day.at(scan_time).do(scan_function) + print(f"\n[SCHEDULER] Daily at {scan_time}", flush=True) + elif scan_interval == 'weekly': + schedule.every().monday.at(scan_time).do(scan_function) + print(f"\n[SCHEDULER] Weekly (Monday at {scan_time})", flush=True) + elif scan_interval == 'monthly': + schedule.every(30).days.do(scan_function) + print(f"\n[SCHEDULER] Monthly (every 30 days)", flush=True) + elif scan_interval.endswith('h'): + hours = int(scan_interval[:-1]) + schedule.every(hours).hours.do(scan_function) + print(f"\n[SCHEDULER] Every {hours} hours", flush=True) + elif scan_interval.endswith('d'): + days = int(scan_interval[:-1]) + schedule.every(days).days.do(scan_function) + print(f"\n[SCHEDULER] Every {days} days", flush=True) + else: + print(f"\n[ERROR] Invalid SCAN_INTERVAL: {scan_interval}", flush=True) + sys.stdout.flush() + sys.exit(1) + + next_run = schedule.next_run() + if next_run: + print(f"[SCHEDULER] Next run: {next_run}", flush=True) + print("\nScheduler running. Press Ctrl+C to stop.\n", flush=True) + sys.stdout.flush() + + while running: + schedule.run_pending() + time.sleep(60) + + print("\n[SHUTDOWN] Stopped gracefully.", flush=True) + sys.stdout.flush() + sys.exit(0) diff --git a/start-instance.sh b/start-instance.sh new file mode 100755 index 0000000..86cb44b --- /dev/null +++ b/start-instance.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +PORT=$1 + +if [ -z "$PORT" ]; then + echo "ERROR: Port not specified" + echo "Usage: $0 " + exit 1 +fi + +cd /opt/geoip_block_generator + +# Safe .env parser +if [ -f /opt/geoip_block_generator/.env ]; then + echo "Loading environment from .env..." + + while IFS='=' read -r key value || [ -n "$key" ]; do + [[ "$key" =~ ^[[:space:]]*# ]] && continue + [[ -z "$key" ]] && continue + + key=$(echo "$key" | xargs) + value=$(echo "$value" | xargs) + + if [[ "$value" =~ ^\"(.*)\"$ ]]; then + value="${BASH_REMATCH[1]}" + elif [[ "$value" =~ ^\'(.*)\'$ ]]; then + value="${BASH_REMATCH[1]}" + fi + + export "$key=$value" + done < /opt/geoip_block_generator/.env +fi + +# Override port for this instance +export FLASK_PORT=$PORT +export FLASK_HOST=127.0.0.1 + +# Create log directory if not exists +mkdir -p /var/log/geoip-ban + +# Log startup +echo "========================================" +echo "GeoIP WebApp Instance Starting" +echo "Port: $PORT" +echo "User: $(whoami)" +echo "Time: $(date)" +echo "========================================" + +exec /opt/geoip_block_generator/venv/bin/gunicorn \ + --bind "127.0.0.1:${PORT}" \ + --workers 1 \ + --threads 8 \ + --worker-class sync \ + --timeout 900 \ + --access-logfile - \ + --error-logfile - \ + --log-level info \ + --access-logformat '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" [Instance:'"${PORT}"']' \ + app:app diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..5cd04d1 --- /dev/null +++ b/start.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e + +if [ -f /opt/geoip_block_generator/.env ]; then + echo "Loading environment from .env..." + + while IFS='=' read -r key value || [ -n "$key" ]; do + [[ "$key" =~ ^[[:space:]]*# ]] && continue + [[ -z "$key" ]] && continue + + key=$(echo "$key" | xargs) + value=$(echo "$value" | xargs) + + if [[ "$value" =~ ^\"(.*)\"$ ]]; then + value="${BASH_REMATCH[1]}" + elif [[ "$value" =~ ^\'(.*)\'$ ]]; then + value="${BASH_REMATCH[1]}" + fi + + export "$key=$value" + done < /opt/geoip_block_generator/.env +fi + +# Defaults +FLASK_HOST=${FLASK_HOST:-127.0.0.1} +FLASK_PORT=${FLASK_PORT:-5000} + +# Start gunicorn +exec /opt/geoip_block_generator/venv/bin/gunicorn \ + --bind "${FLASK_HOST}:${FLASK_PORT}" \ + --workers 1 \ + --threads 8 \ + --worker-class sync \ + --timeout 900 \ + --access-logfile /var/log/geoip-ban/access.log \ + --error-logfile /var/log/geoip-ban/error.log \ + app:app \ No newline at end of file diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000..72fc762 --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,496 @@ +body { + background-color: #f5f5f5; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; + padding-bottom: 2rem; +} + +html { + scroll-behavior: smooth; +} + +.card { + border: 1px solid #e0e0e0; + border-radius: 0.5rem; + margin-bottom: 1.5rem; + animation: fadeIn 0.4s ease-out; +} + +.card-header { + background-color: #ffffff; + border-bottom: 2px solid #e0e0e0; + padding: 1.25rem 1.5rem; +} + +.card-header h4 { + color: #212529; + font-weight: 600; +} + +.card-body { + background-color: #ffffff; +} + +.shadow-sm { + box-shadow: 0 0.125rem 0.25rem rgba(0, 0, 0, 0.075) !important; +} + +#countryList { + max-height: 600px; + overflow-y: auto; + padding: 0.75rem; + background-color: #fafafa; + border: 1px solid #e0e0e0; + border-radius: 0.375rem; + margin-bottom: 0.75rem; +} + +#countryList::-webkit-scrollbar { + width: 8px; +} + +#countryList::-webkit-scrollbar-track { + background: #f1f1f1; + border-radius: 4px; +} + +#countryList::-webkit-scrollbar-thumb { + background: #888; + border-radius: 4px; +} + +#countryList::-webkit-scrollbar-thumb:hover { + background: #555; +} + +.form-check-compact { + padding: 0.25rem 0.5rem; + margin-bottom: 0.15rem; + transition: background-color 0.1s; + border-radius: 0.25rem; + display: flex; + align-items: center; + gap: 0.4rem; +} + +.form-check-compact:hover { + background-color: #e9ecef; +} + +.form-check-compact .form-check-input { + width: 1rem; + height: 1rem; + margin: 0; + cursor: pointer; + flex-shrink: 0; + position: relative; +} + +.form-check-compact .form-check-label { + cursor: pointer; + user-select: none; + font-size: 0.75rem; + padding: 0; + margin: 0; + font-family: 'Courier New', monospace; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + line-height: 1.4; + flex: 1; + min-width: 0; +} + +#countryList .col-lg-2, +#countryList .col-md-3, +#countryList .col-sm-4, +#countryList .col-6 { + padding: 0.15rem; +} + +#countryList .form-check { + padding-left: 0; + min-height: auto; +} + +.form-label { + color: #495057; + font-size: 1rem; + margin-bottom: 0.75rem; +} + +.form-select, +.form-control { + border-radius: 0.375rem; + border: 1px solid #ced4da; + transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out; +} + +.form-select:focus, +.form-control:focus { + border-color: #80bdff; + box-shadow: 0 0 0 0.2rem rgba(0, 123, 255, 0.25); +} + +.form-select-lg { + padding: 0.75rem 1rem; + font-size: 1.05rem; +} + +.form-switch { + padding-left: 0; + min-height: auto; +} + +.form-switch .form-check-input { + width: 3rem; + height: 1.5rem; + margin-left: 0; + margin-right: 1rem; + float: left; + cursor: pointer; +} + +.form-switch .form-check-label { + display: inline-block; + padding-left: 0; + padding-top: 0.125rem; +} + +.aggregate-card { + background-color: #f8f9fa; + border: 1px solid #e0e0e0; + border-radius: 0.375rem; + padding: 1rem; +} + +.aggregate-card .form-check { + padding: 0; + margin-bottom: 0; +} + +.btn { + border-radius: 0.375rem; + font-weight: 500; + transition: all 0.2s ease; +} + +.btn-primary { + background-color: #0d6efd; + border-color: #0d6efd; +} + +.btn-primary:hover { + background-color: #0b5ed7; + border-color: #0a58ca; + transform: translateY(-1px); + box-shadow: 0 4px 8px rgba(13, 110, 253, 0.3); +} + +.btn-lg { + padding: 0.875rem 1.5rem; + font-size: 1.125rem; +} + +.btn-outline-primary { + color: #0d6efd; + border-color: #0d6efd; +} + +.btn-outline-primary:hover { + background-color: #0d6efd; + border-color: #0d6efd; + color: white; +} + +.btn-outline-secondary { + color: #6c757d; + border-color: #6c757d; +} + +.btn-outline-secondary:hover { + background-color: #6c757d; + border-color: #6c757d; + color: white; +} + +.btn:disabled { + cursor: not-allowed; + opacity: 0.65; +} + +.alert { + border-radius: 0.5rem; + border: none; + padding: 1rem 1.25rem; +} + +.alert i { + font-size: 1.1rem; + vertical-align: middle; +} + +.alert-info { + background-color: #d1ecf1; + color: #0c5460; +} + +.alert-success { + background-color: #d4edda; + color: #155724; +} + +.alert-warning { + background-color: #fff3cd; + color: #856404; +} + +.alert-danger { + background-color: #f8d7da; + color: #721c24; +} + +.progress { + border-radius: 0.5rem; + background-color: #e9ecef; + overflow: hidden; +} + +.progress-bar { + font-size: 0.95rem; + font-weight: 500; +} + +.navbar { + background-color: #ffffff; + border-bottom: 1px solid #e0e0e0; + padding: 1rem 0; +} + +.navbar-brand { + font-weight: 600; + font-size: 1.25rem; + color: #212529; +} + +.navbar-brand img { + max-height: 30px; +} + +.footer { + background-color: #ffffff; + border-top: 1px solid #e0e0e0; + padding: 1.5rem 0; + margin-top: 3rem; +} + +.footer a { + color: #0d6efd; + text-decoration: none; + transition: color 0.2s; +} + +.footer a:hover { + color: #0a58ca; + text-decoration: underline; +} + +.modal-xl { + max-width: 90%; +} + +#previewContent { + background-color: #282c34 !important; + color: #abb2bf !important; + padding: 1.5rem; + border-radius: 0.375rem; + font-family: 'Courier New', Consolas, Monaco, monospace; + font-size: 0.875rem; + line-height: 1.5; + max-height: 70vh; + overflow: auto; + white-space: pre; + word-wrap: normal; + display: block !important; +} + +.modal-body { + padding: 1.5rem; +} + +.modal-body pre { + margin-bottom: 0; + background-color: transparent; +} + +.modal-body pre code { + display: block; + background-color: #282c34; + color: #abb2bf; +} + +#previewContent::-webkit-scrollbar { + width: 10px; + height: 10px; +} + +#previewContent::-webkit-scrollbar-track { + background: #21252b; +} + +#previewContent::-webkit-scrollbar-thumb { + background: #4b5263; + border-radius: 5px; +} + +#previewContent::-webkit-scrollbar-thumb:hover { + background: #5c6370; +} + +.api-header-get { + background-color: #e7f3ff; + border-left: 4px solid #0dcaf0; + cursor: pointer; + transition: background-color 0.2s; +} + +.api-header-get:hover { + background-color: #d1ecf1; +} + +.api-header-post { + background-color: #d4edda; + border-left: 4px solid #198754; + cursor: pointer; + transition: background-color 0.2s; +} + +.api-header-post:hover { + background-color: #c3e6cb; +} + +.api-path { + font-size: 1rem; + font-weight: 600; + color: #212529; +} + +.api-endpoint pre { + background-color: #282c34; + color: #abb2bf; + padding: 1rem; + border-radius: 0.375rem; + overflow-x: auto; + margin-bottom: 0; +} + +.api-endpoint pre code { + background-color: transparent; + color: inherit; + padding: 0; + font-size: 0.9rem; + line-height: 1.6; +} + +.api-endpoint .text-success { + color: #98c379 !important; +} + +.api-endpoint .text-warning { + color: #e5c07b !important; +} + +.api-endpoint .text-info { + color: #61afef !important; +} + +.api-endpoint .text-danger { + color: #e06c75 !important; +} + +@keyframes fadeIn { + from { + opacity: 0; + transform: translateY(20px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +@media (min-width: 1400px) { + #countryList .col-lg-2 { + flex: 0 0 12.5%; + max-width: 12.5%; + } +} + +@media (max-width: 1199px) { + #countryList .col-lg-2 { + flex: 0 0 16.666%; + max-width: 16.666%; + } +} + +@media (max-width: 991px) { + #countryList .col-md-3 { + flex: 0 0 20%; + max-width: 20%; + } +} + +@media (max-width: 767px) { + .card-body { + padding: 1.5rem !important; + } + + #countryList { + max-height: 300px; + } + + #countryList .col-sm-4 { + flex: 0 0 25%; + max-width: 25%; + } + + .form-check-compact .form-check-label { + font-size: 0.75rem; + } + + .form-select-lg, + .btn-lg { + font-size: 1rem; + padding: 0.75rem 1.25rem; + } + + .navbar-brand { + font-size: 1rem; + } + + .modal-xl { + max-width: 95%; + } +} + +@media (max-width: 575px) { + #countryList .col-6 { + flex: 0 0 33.333%; + max-width: 33.333%; + } +} + +@media (max-width: 399px) { + #countryList .col-6 { + flex: 0 0 50%; + max-width: 50%; + } +} + +#variantDescription { + animation: fadeIn 0.3s ease-in; +} + +@keyframes fadeIn { + from { opacity: 0; transform: translateY(-5px); } + to { opacity: 1; transform: translateY(0); } +} \ No newline at end of file diff --git a/static/js/api.js b/static/js/api.js new file mode 100644 index 0000000..2fab749 --- /dev/null +++ b/static/js/api.js @@ -0,0 +1,75 @@ +const baseUrl = window.location.origin; + +document.addEventListener('DOMContentLoaded', function() { + document.getElementById('baseUrl').textContent = baseUrl; + document.querySelectorAll('[id^="curlUrl"]').forEach(element => { + element.textContent = baseUrl; + }); +}); + +function toggleEndpoint(id) { + const element = document.getElementById(id); + const bsCollapse = new bootstrap.Collapse(element, { + toggle: true + }); +} + +function tryEndpoint(endpoint, method = 'GET') { + const url = baseUrl + '/api/' + endpoint; + const responseId = 'response-' + endpoint.replace(/\//g, '-'); + const responseDiv = document.getElementById(responseId); + const responseBody = document.getElementById(responseId + '-body'); + + responseDiv.style.display = 'block'; + responseBody.textContent = 'Loading...'; + + const options = { + method: method, + headers: { + 'Content-Type': 'application/json' + } + }; + + fetch(url, options) + .then(response => response.json()) + .then(data => { + responseBody.textContent = JSON.stringify(data, null, 2); + }) + .catch(error => { + responseBody.textContent = 'Error: ' + error.message; + }); +} + +function tryInvalidateCountry() { + const countryInput = document.getElementById('invalidateCountry'); + const country = countryInput.value.trim().toUpperCase(); + + if (!country || country.length !== 2) { + alert('Please enter a valid 2-letter country code (e.g., CN, RU, US)'); + return; + } + + const url = baseUrl + '/api/cache/invalidate/' + country; + const responseDiv = document.getElementById('response-cache-invalidate'); + const responseBody = document.getElementById('response-cache-invalidate-body'); + + responseDiv.style.display = 'block'; + responseBody.textContent = 'Loading...'; + + fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + } + }) + .then(response => response.json()) + .then(data => { + responseBody.textContent = JSON.stringify(data, null, 2); + if (data.success) { + countryInput.value = ''; + } + }) + .catch(error => { + responseBody.textContent = 'Error: ' + error.message; + }); +} diff --git a/static/js/app.js b/static/js/app.js new file mode 100644 index 0000000..0827d0e --- /dev/null +++ b/static/js/app.js @@ -0,0 +1,375 @@ +const BASE_URL = window.location.origin; + +const APP_VARIANTS = { + 'raw-cidr': [ + { + value: 'txt', + text: 'Plain Text (.txt)', + description: 'Simple list of CIDR blocks, one per line' + }, + { + value: 'csv', + text: 'CSV Format (.csv)', + description: 'Structured CSV with country codes and networks' + } + ], + nginx: [ + { + value: 'deny', + text: 'Deny Directives', + description: 'Simple and fast. Works everywhere. Recommended for large lists.', + recommended: true + }, + { + value: 'geo', + text: 'Geo Module', + description: 'Fast with native CIDR support. Requires http_geo_module compiled in nginx.' + }, + { + value: 'map', + text: 'Map Module (regex)', + description: 'Slow with 10k+ rules. Uses regex patterns. Not recommended for production.', + warning: true + } + ], + apache: [ + { + value: '24', + text: 'Apache 2.4 (Require)', + description: 'Modern Apache 2.4+ syntax using Require directives' + }, + { + value: '22', + text: 'Apache 2.2 (Allow/Deny)', + description: 'Legacy Apache 2.2 syntax with Allow/Deny directives' + } + ], + haproxy: [ + { + value: 'acl', + text: 'ACL Rules', + description: 'Native HAProxy ACL rules for frontend/backend blocking' + }, + { + value: 'lua', + text: 'Lua Script', + description: 'Lua-based blocking script for advanced HAProxy setups' + } + ] +}; + + +document.addEventListener('DOMContentLoaded', function() { + updateVariants(); + checkDatabaseStatus(); +}); + + +function updateVariants() { + const appType = document.getElementById('appType').value; + const variantSelect = document.getElementById('appVariant'); + const variantSection = document.getElementById('variantSection'); + const variants = APP_VARIANTS[appType] || []; + + variantSelect.innerHTML = ''; + variantSection.style.display = 'block'; + + variants.forEach(variant => { + const option = document.createElement('option'); + option.value = variant.value; + option.textContent = variant.text; + option.dataset.description = variant.description || ''; + option.dataset.warning = variant.warning || false; + option.dataset.recommended = variant.recommended || false; + variantSelect.appendChild(option); + }); + + updateVariantDescription(); +} + + +function updateVariantDescription() { + const variantSelect = document.getElementById('appVariant'); + const descriptionDiv = document.getElementById('variantDescription'); + + if (!descriptionDiv) return; + + const selectedOption = variantSelect.options[variantSelect.selectedIndex]; + + if (selectedOption && selectedOption.dataset.description) { + const isWarning = selectedOption.dataset.warning === 'true'; + const isRecommended = selectedOption.dataset.recommended === 'true'; + + let alertClass = 'alert-info'; + let borderClass = 'border-info'; + let icon = 'fa-info-circle'; + + if (isRecommended) { + alertClass = 'alert-success'; + borderClass = 'border-success'; + icon = 'fa-check-circle'; + } else if (isWarning) { + alertClass = 'alert-warning'; + borderClass = 'border-warning'; + icon = 'fa-exclamation-triangle'; + } + + descriptionDiv.innerHTML = ` +
+ + + ${selectedOption.dataset.description} + +
+ `; + descriptionDiv.style.display = 'block'; + } else { + descriptionDiv.style.display = 'none'; + } +} + + +function checkDatabaseStatus() { + fetch(BASE_URL + '/api/database/status') + .then(response => response.json()) + .then(data => { + const statusDiv = document.getElementById('dbStatus'); + + if (data.success) { + if (data.exists && !data.needs_update) { + statusDiv.className = 'alert alert-success mb-0'; + statusDiv.innerHTML = 'Database ready (Last update: ' + formatDate(data.last_update) + ')'; + } else if (data.needs_update) { + statusDiv.className = 'alert alert-warning mb-0'; + statusDiv.innerHTML = 'Database needs update '; + } else { + statusDiv.className = 'alert alert-info mb-0'; + statusDiv.innerHTML = 'Downloading database...'; + } + } + }) + .catch(error => { + console.error('Error:', error); + }); +} + + +function updateDatabase() { + const statusDiv = document.getElementById('dbStatus'); + statusDiv.className = 'alert alert-info mb-0'; + statusDiv.innerHTML = 'Updating database...'; + + fetch(BASE_URL + '/api/database/update', { method: 'POST' }) + .then(response => response.json()) + .then(data => { + if (data.success) { + statusDiv.className = 'alert alert-success mb-0'; + statusDiv.innerHTML = 'Database updated successfully'; + setTimeout(checkDatabaseStatus, 2000); + } else { + statusDiv.className = 'alert alert-danger mb-0'; + statusDiv.innerHTML = 'Update failed: ' + data.error; + } + }); +} + + +function selectAll() { + const checkboxes = document.querySelectorAll('input[name="countries"]'); + checkboxes.forEach(cb => cb.checked = true); +} + + +function deselectAll() { + const checkboxes = document.querySelectorAll('input[name="countries"]'); + checkboxes.forEach(cb => cb.checked = false); +} + + +function formatDate(dateString) { + if (!dateString) return 'Never'; + try { + const date = new Date(dateString); + return date.toLocaleDateString() + ' ' + date.toLocaleTimeString(); + } catch (e) { + return dateString; + } +} + + +function copyToClipboard() { + const content = document.getElementById('previewContent').textContent; + + navigator.clipboard.writeText(content).then(() => { + showResult('Copied to clipboard!', 'success'); + }).catch(err => { + const textarea = document.createElement('textarea'); + textarea.value = content; + textarea.style.position = 'fixed'; + textarea.style.opacity = '0'; + document.body.appendChild(textarea); + textarea.select(); + + try { + document.execCommand('copy'); + showResult('Copied to clipboard!', 'success'); + } catch (e) { + showResult('Failed to copy to clipboard', 'danger'); + } + + document.body.removeChild(textarea); + }); +} + + +function getFormData() { + const countries = Array.from(document.querySelectorAll('input[name="countries"]:checked')) + .map(input => input.value); + + if (countries.length === 0) { + return null; + } + + const useCacheCheckbox = document.getElementById('useCache'); + + return { + countries: countries, + app_type: document.getElementById('appType').value, + app_variant: document.getElementById('appVariant').value, + aggregate: document.getElementById('aggregate').checked, + use_cache: useCacheCheckbox ? useCacheCheckbox.checked : true + }; +} + + +function showCacheBadge(fromCache, generatedAt) { + if (fromCache) { + const badge = document.createElement('div'); + badge.className = 'alert alert-success alert-dismissible fade show mt-3'; + badge.innerHTML = ` + + Lightning fast! Config loaded from Redis cache in <100ms + Generated: ${new Date(generatedAt).toLocaleString()} + + `; + + const container = document.querySelector('.container > .row > .col-lg-10'); + container.insertBefore(badge, container.firstChild); + + setTimeout(() => { + badge.classList.remove('show'); + setTimeout(() => badge.remove(), 150); + }, 5000); + } +} + + +async function previewConfiguration() { + const formData = getFormData(); + + if (!formData) { + showResult('Please select at least one country to continue', 'warning'); + return; + } + + showProgress(); + + try { + const endpoint = formData.app_type === 'raw-cidr' + ? '/api/generate/raw' + : '/api/generate/preview'; + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(formData) + }); + + hideProgress(); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); + showResult('Error: ' + (errorData.error || 'Request failed'), 'danger'); + return; + } + + if (formData.app_type === 'raw-cidr') { + const text = await response.text(); + const fromCache = response.headers.get('X-From-Cache') === 'true'; + const generatedAt = response.headers.get('X-Generated-At'); + + const contentDisposition = response.headers.get('Content-Disposition'); + let filename = 'blocklist.txt'; + if (contentDisposition) { + const matches = /filename="?(.+)"?/.exec(contentDisposition); + if (matches) filename = matches[1]; + } + + document.getElementById('previewContent').textContent = text; + + const cacheIndicator = document.getElementById('cacheIndicator'); + if (fromCache) { + cacheIndicator.innerHTML = ' From Cache'; + showCacheBadge(true, generatedAt); + } else { + cacheIndicator.innerHTML = ' Fresh'; + } + + window.lastGeneratedConfig = text; + window.lastGeneratedFilename = filename; + + const modal = new bootstrap.Modal(document.getElementById('previewModal')); + modal.show(); + } else { + const result = await response.json(); + + if (result.success) { + document.getElementById('previewContent').textContent = result.config; + + const cacheIndicator = document.getElementById('cacheIndicator'); + if (result.from_cache) { + cacheIndicator.innerHTML = ' From Cache'; + showCacheBadge(true, result.generated_at); + } else { + cacheIndicator.innerHTML = ' Fresh'; + } + + if (result.stats) { + const statsText = `${result.stats.countries} countries, ${result.stats.total_networks.toLocaleString()} networks`; + document.getElementById('previewStats').textContent = statsText; + } + + window.lastGeneratedConfig = result.config; + window.currentStats = result.stats; + + const modal = new bootstrap.Modal(document.getElementById('previewModal')); + modal.show(); + } else { + showResult(result.error || 'An error occurred while generating the preview', 'danger'); + } + } + } catch (error) { + hideProgress(); + showResult('Network error: ' + error.message, 'danger'); + console.error('Preview error:', error); + } +} + + +async function downloadFromPreview() { + const formData = getFormData(); + + if (!formData) { + showResult('Please select at least one country', 'warning'); + return; + } + + const modal = bootstrap.Modal.getInstance(document.getElementById('previewModal')); + if (modal) { + modal.hide(); + } + + await downloadConfiguration(formData); +} diff --git a/static/js/cache.js b/static/js/cache.js new file mode 100644 index 0000000..1602fce --- /dev/null +++ b/static/js/cache.js @@ -0,0 +1,276 @@ +async function loadCacheStats() { + const container = document.getElementById('cacheStatsContent'); + + if (!container) return; + + container.innerHTML = ` +
+ + Loading statistics... +
+ `; + + try { + const [cacheResponse, sqliteResponse] = await Promise.all([ + fetch('/api/cache/status'), + fetch('/api/database/sqlite/status') + ]); + + const cacheData = await cacheResponse.json(); + const sqliteData = await sqliteResponse.json(); + + if (!cacheData.success) { + container.innerHTML = `
Redis cache unavailable: ${cacheData.error || 'Unknown error'}
`; + return; + } + + const stats = cacheData.stats || {}; + const health = cacheData.health || {}; + + let html = ` +
Redis Cache
+
+
+
+

${stats.country_keys || 0}

+ Country Keys +
+
+
+
+

${stats.config_keys || 0}

+ Config Keys +
+
+
+
+

${stats.total_size_mb || 0} MB

+ Cache Size +
+
+
+
+

${health.memory_used_mb || 0} MB

+ Memory Used +
+
+
+ `; + + if (sqliteData.success && sqliteData.exists) { + const modifiedDate = new Date(sqliteData.modified).toLocaleString(); + + html += ` +
SQLite Cache Database
+
+
+
+

${sqliteData.total_countries || 0}

+ Countries +
+
+
+
+

${(sqliteData.total_networks || 0).toLocaleString()}

+ Total Networks +
+
+
+
+

${sqliteData.file_size_mb || 0} MB

+ Database Size +
+
+
+
+ Last Modified + ${modifiedDate} +
+
+
+ `; + + if (sqliteData.top_countries && sqliteData.top_countries.length > 0) { + html += ` +
+ Top countries: + ${sqliteData.top_countries.map(c => + `${c.code}: ${c.networks.toLocaleString()}` + ).join('')} +
+ `; + } + } else { + html += ` +
+ + SQLite cache database not available +
+ `; + } + + container.innerHTML = html; + + } catch (error) { + console.error('Error loading cache stats:', error); + container.innerHTML = ` +
+ + Failed to load statistics: ${error.message} +
+ `; + } +} + +async function flushCache() { + const confirmed = await showConfirmModal( + 'Flush Redis Cache', + 'Are you sure you want to flush ALL Redis cache?

' + + 'This will delete:
' + + '• All cached country data
' + + '• All cached configurations
' + + '• Force regeneration for future requests

' + + 'This action cannot be undone!' + ); + + if (!confirmed) return; + + try { + showFlushingIndicator(); + + const response = await fetch('/api/cache/flush', { + method: 'POST', + headers: {'Content-Type': 'application/json'} + }); + + const data = await response.json(); + + hideFlushingIndicator(); + + if (data.success) { + showToast('success', 'Cache Flushed', 'All Redis cache has been cleared successfully!'); + loadCacheStats(); + } else { + showToast('danger', 'Error', 'Failed to flush cache: ' + (data.error || 'Unknown error')); + } + } catch (error) { + hideFlushingIndicator(); + showToast('danger', 'Error', 'Network error: ' + error.message); + } +} + +function showConfirmModal(title, message) { + return new Promise((resolve) => { + const modalId = 'confirmModal_' + Date.now(); + const modalHtml = ` + + `; + + document.body.insertAdjacentHTML('beforeend', modalHtml); + const modalEl = document.getElementById(modalId); + const modal = new bootstrap.Modal(modalEl); + + modalEl.querySelector('#confirmBtn').addEventListener('click', () => { + modal.hide(); + resolve(true); + }); + + modalEl.addEventListener('hidden.bs.modal', () => { + modalEl.remove(); + resolve(false); + }); + + modal.show(); + }); +} + +function showFlushingIndicator() { + const indicator = document.createElement('div'); + indicator.id = 'flushingIndicator'; + indicator.className = 'position-fixed top-50 start-50 translate-middle'; + indicator.style.zIndex = '9999'; + indicator.innerHTML = ` +
+
+ +
Flushing Cache...
+

Please wait

+
+
+ `; + document.body.appendChild(indicator); +} + +function hideFlushingIndicator() { + const indicator = document.getElementById('flushingIndicator'); + if (indicator) { + indicator.remove(); + } +} + +function showToast(type, title, message) { + const toastId = 'toast_' + Date.now(); + const bgClass = type === 'success' ? 'bg-success' : type === 'danger' ? 'bg-danger' : 'bg-warning'; + + const toastHtml = ` +
+ +
+ `; + + document.body.insertAdjacentHTML('beforeend', toastHtml); + const toastEl = document.getElementById(toastId); + const toast = new bootstrap.Toast(toastEl, { delay: 5000 }); + + toast.show(); + + toastEl.addEventListener('hidden.bs.toast', () => { + toastEl.parentElement.remove(); + }); +} + +function showAlert(type, message) { + showToast(type, type === 'success' ? 'Success' : 'Error', message); +} + +document.addEventListener('DOMContentLoaded', function() { + const statsPanel = document.getElementById('cacheStatsPanel'); + if (statsPanel) { + statsPanel.addEventListener('shown.bs.collapse', function() { + loadCacheStats(); + }); + } +}); diff --git a/static/js/progress.js b/static/js/progress.js new file mode 100644 index 0000000..a62569d --- /dev/null +++ b/static/js/progress.js @@ -0,0 +1,197 @@ +let progressInterval = null; + +function startProgressPolling() { + if (progressInterval) { + clearInterval(progressInterval); + } + + progressInterval = setInterval(async () => { + try { + const response = await fetch('/api/progress'); + const data = await response.json(); + + if (data.active) { + updateProgressUI(data.message, data.progress, data.total); + } else { + stopProgressPolling(); + } + } catch (error) { + console.error('Progress polling error:', error); + } + }, 500); +} + +function stopProgressPolling() { + if (progressInterval) { + clearInterval(progressInterval); + progressInterval = null; + } +} + +function updateProgressUI(message, progress, total) { + const progressSection = document.getElementById('progressSection'); + const progressBar = progressSection.querySelector('.progress-bar'); + const progressMessage = document.getElementById('progressMessage'); + const progressPercentage = document.getElementById('progressPercentage'); + + const percentage = total > 0 ? Math.round((progress / total) * 100) : 0; + progressBar.style.width = percentage + '%'; + + if (progressPercentage) { + progressPercentage.textContent = percentage + '%'; + } + + progressBar.setAttribute('aria-valuenow', percentage); + + if (progressMessage) { + progressMessage.textContent = message; + } +} + +function showProgress() { + const progressSection = document.getElementById('progressSection'); + const progressBar = progressSection.querySelector('.progress-bar'); + const progressMessage = document.getElementById('progressMessage'); + const progressPercentage = document.getElementById('progressPercentage'); + + progressBar.style.width = '0%'; + if (progressPercentage) { + progressPercentage.textContent = '0%'; + } + if (progressMessage) { + progressMessage.textContent = 'Initializing...'; + } + + document.getElementById('resultSection').style.display = 'none'; + progressSection.style.display = 'block'; + document.getElementById('generateBtn').disabled = true; + + startProgressPolling(); +} + +function hideProgress() { + const progressSection = document.getElementById('progressSection'); + progressSection.style.display = 'none'; + document.getElementById('generateBtn').disabled = false; + + stopProgressPolling(); +} + +function showResult(message, type = 'danger') { + const resultSection = document.getElementById('resultSection'); + const resultMessage = document.getElementById('resultMessage'); + const alertDiv = resultSection.querySelector('.alert'); + + const iconMap = { + 'success': 'check-circle', + 'danger': 'exclamation-circle', + 'warning': 'exclamation-triangle', + 'info': 'info-circle' + }; + + const icon = iconMap[type] || 'info-circle'; + + alertDiv.className = `alert alert-${type}`; + resultMessage.innerHTML = `${message}`; + resultSection.style.display = 'block'; + + if (type === 'success') { + setTimeout(() => { + resultSection.style.display = 'none'; + }, 5000); + } +} + +document.addEventListener('DOMContentLoaded', function() { + const form = document.getElementById('generateForm'); + if (form) { + form.addEventListener('submit', async function(e) { + e.preventDefault(); + + const formData = getFormData(); + + if (!formData) { + showResult('Please select at least one country to continue', 'warning'); + return; + } + + await downloadConfiguration(formData); + }); + } +}); + +async function downloadConfiguration(formData) { + showProgress(); + + try { + const endpoint = formData.app_type === 'raw-cidr' + ? '/api/generate/raw' + : '/api/generate'; + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(formData) + }); + + hideProgress(); + + if (response.ok) { + const fromCache = response.headers.get('X-From-Cache') === 'true'; + const generatedAt = response.headers.get('X-Generated-At'); + + const blob = await response.blob(); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + + const contentDisposition = response.headers.get('Content-Disposition'); + let filename = 'geoblock_config.conf'; + if (contentDisposition) { + const filenameMatch = contentDisposition.match(/filename="?(.+)"?/); + if (filenameMatch) { + filename = filenameMatch[1]; + } + } + + a.download = filename; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + document.body.removeChild(a); + + if (fromCache) { + showResult(` Lightning fast! Downloaded from cache: ${filename}`, 'success'); + } else { + showResult(`Configuration downloaded successfully: ${filename}`, 'success'); + } + } else { + const error = await response.json(); + showResult(error.error || 'An error occurred during download', 'danger'); + } + } catch (error) { + hideProgress(); + showResult('Network error: ' + error.message, 'danger'); + } +} + +function getFormData() { + const countries = Array.from(document.querySelectorAll('input[name="countries"]:checked')) + .map(input => input.value); + + if (countries.length === 0) { + return null; + } + + const useCacheCheckbox = document.getElementById('useCache'); + + return { + countries: countries, + app_type: document.getElementById('appType').value, + app_variant: document.getElementById('appVariant').value, + aggregate: document.getElementById('aggregate').checked, + use_cache: useCacheCheckbox ? useCacheCheckbox.checked : true + }; +} diff --git a/systemd/geoip-ban.service b/systemd/geoip-ban.service new file mode 100644 index 0000000..2aaf810 --- /dev/null +++ b/systemd/geoip-ban.service @@ -0,0 +1,27 @@ +[Unit] +Description=GeoIP Ban Configuration Generator +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/opt/geoip_block_generator +ExecStart=/opt/geoip_block_generator/start.sh + +Restart=always +RestartSec=10 + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=geoip-ban + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/geoip_block_generator/geoip_db /var/log/geoip-ban + +[Install] +WantedBy=multi-user.target diff --git a/systemd/geoip-precache.service b/systemd/geoip-precache.service new file mode 100644 index 0000000..deb82c3 --- /dev/null +++ b/systemd/geoip-precache.service @@ -0,0 +1,31 @@ +[Unit] +Description=GeoIP Country Pre-Cache Daemon +After=network-online.target redis-server.service +Wants=network-online.target +Requires=redis-server.service + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/opt/geoip_block_generator + +EnvironmentFile=/opt/geoip_block_generator/.env + +ExecStart=/opt/geoip_block_generator/venv/bin/python3 /opt/geoip_block_generator/precache_daemon.py + +Restart=always +RestartSec=30 + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=geoip-precache + +PrivateTmp=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/geoip_block_generator/geoip_db + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/systemd/geoip-scheduler.service b/systemd/geoip-scheduler.service new file mode 100644 index 0000000..c4acfd5 --- /dev/null +++ b/systemd/geoip-scheduler.service @@ -0,0 +1,34 @@ +[Unit] +Description=GeoIP Country Pre-Scanner Daemon +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/opt/geoip_block_generator + +EnvironmentFile=/opt/geoip_block_generator/.env + +# Python executable +ExecStart=/opt/geoip_block_generator/venv/bin/python /opt/geoip_block_generator/scheduler.py + +# Restart policy +Restart=always +RestartSec=10 + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=geoip-scheduler + +# Security +PrivateTmp=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/geoip_block_generator/geoip_db + +[Install] +WantedBy=multi-user.target diff --git a/systemd/geoip-webapp@.service b/systemd/geoip-webapp@.service new file mode 100644 index 0000000..34316bd --- /dev/null +++ b/systemd/geoip-webapp@.service @@ -0,0 +1,31 @@ +[Unit] +Description=GeoIP Ban Generator WebApp (Instance %i) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/opt/geoip_block_generator + +# Pass instance port as argument +ExecStart=/opt/geoip_block_generator/start-instance.sh %i + +Restart=always +RestartSec=10 + +# Logging per instance +StandardOutput=append:/var/log/geoip-ban/webapp-%i.log +StandardError=append:/var/log/geoip-ban/webapp-%i-error.log +SyslogIdentifier=geoip-webapp-%i + +# Security +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/geoip_block_generator/geoip_db /var/log/geoip-ban + +[Install] +WantedBy=multi-user.target diff --git a/templates/api.html b/templates/api.html new file mode 100644 index 0000000..cd82a56 --- /dev/null +++ b/templates/api.html @@ -0,0 +1,625 @@ +{% extends "base.html" %} + +{% block title %}API Documentation - {{ app_name }}{% endblock %} + +{% block content %} +
+
+
+ +
+

API Documentation

+

RESTful API for programmatic access to geo-blocking configuration generation.

+
+ + Base URL: +
+
+ + +
+
+
+
+ GET + /api/countries + Get available countries +
+ +
+
+
+
Description
+

Returns a list of all available countries with their ISO codes and flag emojis.

+ +
Response Schema
+ 
{
+  "success": true,
+  "countries": [
+    {
+      "code": "CN",
+      "name": "China",
+      "flag": "🇨🇳"
+    }
+  ]
+}
+ +
Try it out
+ + + +
+
+ + +
+
+
+
+ GET + /api/database/status + Check database status +
+ +
+
+
+
Description
+

Returns the current status of the MaxMind GeoIP database, including last update time and whether an update is needed.

+ +
Response Schema
+ 
{
+  "success": true,
+  "exists": true,
+  "needs_update": false,
+  "last_update": "2026-02-10T08:00:00",
+  "file_size": 5242880,
+  "auto_update": true
+}
+ +
Try it out
+ + + +
+
+ + +
+
+
+
+ POST + /api/database/update + Update database manually +
+ +
+
+
+
Description
+

Manually triggers a download and update of the MaxMind GeoIP database from configured sources.

+ +
Response Schema
+ 
{
+  "success": true,
+  "url": "https://github.com/...",
+  "size": 5242880
+}
+ +
Try it out
+ + + +
+
+ + +
+
+
+
+ GET + /api/progress + Get current generation progress +
+ +
+
+
+
Description
+

Returns the current progress status of any active configuration generation process. Poll this endpoint to monitor long-running operations.

+ +
Response Schema
+ 
{
+  "active": true,
+  "message": "[1/3] CN: Scanning MaxMind: 234 networks found",
+  "progress": 30,
+  "total": 100
+}
+ +
Fields
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescription
activebooleanWhether a generation process is currently active
messagestringCurrent progress message with detailed status
progressintegerCurrent progress value (0-100)
totalintegerTotal progress value (always 100)
+ +
Try it out
+ + + + +
Polling Example
+ 
// Poll every 500ms during generation
+const pollProgress = setInterval(async () => {
+  const response = await fetch('/api/progress');
+  const data = await response.json();
+  
+  if (data.active) {
+    console.log(`Progress: ${data.progress}% - ${data.message}`);
+  } else {
+    clearInterval(pollProgress);
+    console.log('Generation complete!');
+  }
+}, 500);
+
+
+ + +
+
+
+
+ POST + /api/generate/preview + Preview configuration (JSON response) +
+ +
+
+
+
Description
+

Generates configuration and returns it as JSON (instead of file download). Perfect for previewing or integrating into other applications.

+ +
Request Body
+ 
{
+  "countries": ["CN", "RU"],
+  "app_type": "nginx",
+  "app_variant": "map",
+  "aggregate": true,
+  "use_cache": true
+}
+ +
Parameters
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeRequiredDescription
countriesarrayrequiredList of ISO 3166-1 alpha-2 country codes
app_typestringrequiredOne of: nginx, apache, haproxy, raw-cidr
app_variantstringrequiredConfiguration style (depends on app_type)
aggregatebooleanoptionalAggregate IP networks to reduce count (default: true)
use_cachebooleanoptionalUse Redis cache if available (default: true). Set to false to force fresh data from SQLite/MaxMind
+ +
Response Schema
+ 
{
+  "success": true,
+  "config": "# Nginx Map Module Configuration\n...",
+  "stats": {
+    "countries": 2,
+    "total_networks": 4567,
+    "per_country": {
+      "CN": 2834,
+      "RU": 1733
+    }
+  },
+  "from_cache": true,
+  "cache_type": "redis",
+  "generated_at": "2026-02-16T10:30:00"
+}
+ +
Response Fields
+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescription
from_cachebooleanWhether the config was served from cache or freshly generated
cache_typestringredis (from Redis cache) or sqlite (from SQLite/fresh scan)
generated_atstringISO 8601 timestamp when the config was generated
+ +
cURL Examples
+ +

With cache (default):

+ 
curl -X POST /api/generate/preview \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "nginx",
+    "app_variant": "map",
+    "aggregate": true,
+    "use_cache": true
+  }' | jq .
+ +

Force fresh data (bypass cache):

+ 
curl -X POST /api/generate/preview \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "nginx",
+    "app_variant": "map",
+    "aggregate": true,
+    "use_cache": false
+  }' | jq .
+
+
+ + +
+
+
+
+ POST + /api/generate/raw + Generate raw CIDR blocklist +
+ +
+
+
+
Description
+

Generates a raw CIDR blocklist without application-specific configuration. Perfect for iptables, fail2ban, or custom implementations.

+ +
Request Body
+ 
{
+  "countries": ["CN", "RU"],
+  "app_variant": "txt",
+  "aggregate": true,
+  "use_cache": true
+}
+ +
Parameters
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeRequiredDescription
countriesarrayrequiredList of ISO 3166-1 alpha-2 country codes
app_variantstringoptionalOutput format: txt (default) or csv
aggregatebooleanoptionalAggregate IP networks (default: true)
use_cachebooleanoptionalUse Redis cache if available (default: true)
+ +
Response
+

Returns plain text file with CIDR blocks (one per line) or CSV with CIDR and country columns.

+ +
Response Headers
+ + + + + + + + + + + + + + + + + + + + + +
HeaderDescription
X-From-Cachetrue or false - indicates if served from Redis
X-Cache-Typeredis or sqlite - data source type
X-Generated-AtTimestamp when config was generated
+ +
cURL Examples
+ +

With cache (faster):

+ 
curl -X POST /api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_variant": "txt",
+    "aggregate": true,
+    "use_cache": true
+  }' \
+  -o blocklist.txt
+ +

Force fresh data (slower but guaranteed up-to-date):

+ 
curl -X POST /api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_variant": "txt",
+    "aggregate": true,
+    "use_cache": false
+  }' \
+  -o blocklist_fresh.txt
+
+
+ + + +
+
+
+
+ POST + /api/generate + Generate application configuration +
+ +
+
+
+
Description
+

Generates application-specific geo-blocking configuration for Nginx, Apache, or HAProxy and returns it as a downloadable file.

+ +
Request Body
+ 
{
+  "countries": ["CN", "RU"],
+  "app_type": "nginx",
+  "app_variant": "map",
+  "aggregate": true,
+  "use_cache": true
+}
+ +
Parameters
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeRequiredDescription
countriesarrayrequiredList of ISO 3166-1 alpha-2 country codes
app_typestringrequiredOne of: nginx, apache, haproxy
app_variantstringrequiredConfiguration style (depends on app_type)
aggregatebooleanoptionalAggregate IP networks (default: true)
use_cachebooleanoptionalUse Redis cache if available (default: true)
+ +
Available Variants
+
    +
  • nginx: geo, map, deny
    • +
  • apache: 22 (Apache 2.2), 24 (Apache 2.4)
    • +
  • haproxy: acl, lua
    • +
+ +
Response
+

Returns configuration file as text/plain with Content-Disposition header for download.

+ +
Response Headers
+ + + + + + + + + + + + + + + + + + + + + +
HeaderDescription
X-From-Cachetrue or false
X-Cache-Typeredis or sqlite
X-Generated-AtISO 8601 timestamp
+ +
Cache Behavior
+
+ With Redis enabled: +
    +
  • use_cache: true - Check Redis first, return cached config if available (fast, <1s)
    • +
  • use_cache: false - Bypass Redis, fetch from SQLite cache or scan MaxMind (slower, 5-30s)
    • +
+
+ +
cURL Examples
+ +

With cache (recommended for production):

+ 
curl -X POST /api/generate \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "nginx",
+    "app_variant": "map",
+    "aggregate": true,
+    "use_cache": true
+  }' \
+  -o geoblock.conf
+
+# Check if it was cached:
+curl -I -X POST /api/generate \
+  -H "Content-Type: application/json" \
+  -d '{"countries":["CN"],"app_type":"nginx","app_variant":"map"}' \
+  | grep "X-From-Cache"
+ +

Force fresh scan (for testing or updates):

+ 
curl -X POST /api/generate \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "nginx",
+    "app_variant": "map",
+    "aggregate": true,
+    "use_cache": false
+  }' \
+  -o geoblock_fresh.conf
+
+
+ + +
+
+
+{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..daeda6e --- /dev/null +++ b/templates/base.html @@ -0,0 +1,47 @@ + + + + + + {% block title %}{{ app_name }}{% endblock %} + + + + + + + + +
+ {% block content %}{% endblock %} +
+ + + + + + {% block scripts %}{% endblock %} + + diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..d72c82b --- /dev/null +++ b/templates/index.html @@ -0,0 +1,237 @@ +{% extends "base.html" %} + +{% block content %} +
+
+
+ + +
+
+
+ + Checking database status... +
+
+
+ {% if redis_enabled %} + {% if redis_connected %} +
+ Redis Cache: Active +
+ {% else %} +
+ Redis: Offline +
+ {% endif %} + {% else %} +
+ Redis: Disabled +
+ {% endif %} +
+
+ + + {% if redis_enabled and redis_connected %} +
+
+
+ + Cache Statistics + + +
+
+
+
+
+
+ + Loading cache statistics... +
+
+
+ + +
+
+
+
+ {% endif %} + + +
+
+

+ Generate Geo-Blocking Configuration +

+
+
+
+ + +
+ +
+ {% for country in countries %} +
+
+ + +
+
+ {% endfor %} +
+
+ + +
+
+ + +
+ + +
+ + +
+ + + +
+ + +
+ +
+
+ + +
+ + {% if redis_enabled and redis_connected %} +
+ + +
+ {% endif %} +
+
+ + +
+
+ +
+
+ +
+
+
+ + + + + + +
+
+ + + + +
+
+
+{% endblock %} + +{% block scripts %} + + +{% endblock %}