From bf02af01923a63003f21f8d017c098ec018ddbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Mon, 23 Feb 2026 13:10:16 +0100 Subject: [PATCH] haproxy map --- api.py | 95 ++++++++----- blocklist.txt | 0 geoip_handler.py | 67 ++++++++- precache_daemon.py | 4 + static/js/app.js | 5 + static/js/progress.js | 10 +- templates/api.html | 309 ++++++++++++++++++++++++++++-------------- 7 files changed, 348 insertions(+), 142 deletions(-) delete mode 100644 blocklist.txt diff --git a/api.py b/api.py index da8b2cc..0c3528e 100644 --- a/api.py +++ b/api.py @@ -350,6 +350,8 @@ def generate_preview(): 'apache_24': ConfigGenerator.generate_apache_24, 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + "haproxy_map": ConfigGenerator.generate_haproxy_map, + } generator_key = f"{app_type}_{app_variant}" @@ -398,40 +400,56 @@ def generate_preview(): traceback.print_exc() return jsonify({'success': False, 'error': str(e)}), 500 + @api_blueprint.route('/api/generate/raw', methods=['POST']) def generate_raw_cidr(): try: clear_progress() data = request.get_json() - + countries = data.get('countries', []) aggregate = data.get('aggregate', True) app_type = data.get('app_type', 'raw-cidr_txt') use_cache = data.get('use_cache', True) - + + as_js = bool(data.get('as_js', False)) + js_var = data.get('js_var', 'geoipBlocklist') + if app_type == 'raw-cidr': app_type = 'raw-cidr_txt' - + if not countries: return jsonify({'success': False, 'error': 'No countries selected'}), 400 - + if use_cache and redis_cache: cached = redis_cache.get_cached_config(countries, app_type, aggregate) if cached: if 'json' in app_type: - extension = 'json' - mimetype = 'application/json' + if as_js: + extension = 'js' + mimetype = 'application/javascript' + filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" + body = f"const {js_var} = {cached['config']};\n" + else: + extension = 'json' + mimetype = 'application/json' + filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" + body = cached['config'] + elif 'csv' in app_type: extension = 'csv' mimetype = 'text/csv' + filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" + body = cached['config'] + else: extension = 'txt' mimetype = 'text/plain' - - filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" - + filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}" + body = cached['config'] + return Response( - cached['config'], + body, mimetype=mimetype, headers={ 'Content-Disposition': f'attachment; filename="{filename}"', @@ -443,22 +461,22 @@ def generate_raw_cidr(): 'Expires': '0' } ) - + if handler.needs_update(): handler.check_and_update() - + update_progress(f'Loading data for {len(countries)} countries...', 0, 100) - + country_networks = {} cache_sources = {} total_countries = len(countries) - + for idx, country in enumerate(countries, 1): base_progress = int((idx - 1) / total_countries * 80) update_progress(f'[{idx}/{total_countries}] Loading {country}...', base_progress, 100) - + networks, source = get_country_networks_cached(country, use_cache=use_cache) - + if networks: country_networks[country] = networks cache_sources[country] = source @@ -468,65 +486,72 @@ def generate_raw_cidr(): next_progress, 100 ) - + if not country_networks: clear_progress() return jsonify({'success': False, 'error': 'No networks found'}), 404 - + update_progress('Generating file...', 85, 100) - + if 'txt' in app_type or 'cidr' in app_type or 'newline' in app_type: config_text = ConfigGenerator.generate_raw_cidr(country_networks, aggregate=aggregate, redis_ips=None) filename = f"blocklist_{'_'.join(sorted(countries))}.txt" mimetype = 'text/plain' - + elif 'json' in app_type: all_networks = [] for nets in country_networks.values(): all_networks.extend(nets) - + if aggregate: - all_networks = ConfigGenerator.aggregate_networks(all_networks) + all_networks = ConfigGenerator._aggregate_networks(all_networks) else: all_networks = sorted(list(set(all_networks))) - - config_text = json.dumps({ + + json_text = json.dumps({ 'countries': countries, 'networks': all_networks, 'count': len(all_networks), 'aggregated': aggregate }, indent=2) - filename = f"blocklist_{'_'.join(sorted(countries))}.json" - mimetype = 'application/json' - + + if as_js: + config_text = f"const {js_var} = {json_text};\n" + filename = f"blocklist_{'_'.join(sorted(countries))}.js" + mimetype = 'application/javascript' + else: + config_text = json_text + filename = f"blocklist_{'_'.join(sorted(countries))}.json" + mimetype = 'application/json' + elif 'csv' in app_type: config_text = ConfigGenerator.generate_csv(country_networks, aggregate=aggregate, redis_ips=None) filename = f"blocklist_{'_'.join(sorted(countries))}.csv" mimetype = 'text/csv' - + else: clear_progress() return jsonify({'success': False, 'error': f'Unknown format: {app_type}'}), 400 - + total_networks = sum(len(nets) for nets in country_networks.values()) stats = { 'countries': len(country_networks), 'total_networks': total_networks, 'per_country': {cc: len(nets) for cc, nets in country_networks.items()} } - + if redis_cache: update_progress('Saving to Redis cache...', 95, 100) redis_cache.save_config(countries, app_type, aggregate, config_text, stats) - + update_progress('Complete!', 100, 100) clear_progress() - + cache_type = 'hybrid' if cache_sources: most_common = max(set(cache_sources.values()), key=list(cache_sources.values()).count) cache_type = most_common - + return Response( config_text, mimetype=mimetype, @@ -540,13 +565,14 @@ def generate_raw_cidr(): 'Expires': '0' } ) - + except Exception as e: clear_progress() import traceback traceback.print_exc() return jsonify({'success': False, 'error': str(e)}), 500 + @api_blueprint.route('/api/generate', methods=['POST']) def generate_config(): try: @@ -622,6 +648,7 @@ def generate_config(): 'apache_24': ConfigGenerator.generate_apache_24, 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + 'haproxy_map': ConfigGenerator.generate_haproxy_map, } generator_key = f"{app_type}_{app_variant}" diff --git a/blocklist.txt b/blocklist.txt deleted file mode 100644 index e69de29..0000000 diff --git a/geoip_handler.py b/geoip_handler.py index 1197d29..09e1115 100644 --- a/geoip_handler.py +++ b/geoip_handler.py @@ -874,7 +874,7 @@ class ConfigGenerator: config += "}\n" # Log conversion statistics - print(f"[INFO] Generated nginx map: {converted_count} regex patterns", flush=True) + #print(f"[INFO] Generated nginx map: {converted_count} regex patterns", flush=True) if failed_count > 0: print(f"[WARNING] Failed to convert {failed_count} networks to regex - check config file", flush=True) @@ -1143,7 +1143,69 @@ class ConfigGenerator: default_backend servers """ return config - + + @staticmethod + def generate_haproxy_map(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: + """ + Generate HAProxy MAP file (IP COUNTRY format) + """ + countries = sorted(country_networks.keys()) + + redisstats = None + if redis_ips: + redisstats = {"total": len(redis_ips), "unique": len(redis_ips), "deduped": 0} + + handler = GeoIPHandler() + metadata = generate_metadata(countries, country_networks, redisstats, handler) + + # Aggregate networks + all_networks = [] + for networks in country_networks.values(): + all_networks.extend(networks) + if redis_ips: + all_networks.extend(redis_ips) + + if aggregate: + all_networks = ConfigGenerator._aggregate_networks(all_networks) + else: + all_networks = sorted(list(set(all_networks))) + + # Generate header + config = "# " + "="*77 + "\n" + config += "# HAProxy MAP Configuration\n" + config += f"# Generated: {metadata['timestamp']}\n" + config += "# " + "="*77 + "\n" + config += "# \n" + config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n" + config += f"# Total networks: {len(all_networks):,}\n" + config += "# \n" + config += "# Data sources:\n" + config += metadata['sources_formatted'] + "\n" + config += "# \n" + + if metadata['redis']: + config += f"# {metadata['redis']['formatted']}\n" + config += "# \n" + + config += "# Cache settings:\n" + config += f"# Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n" + config += f"# Database: {metadata['cache_db_path']}\n" + config += "# \n" + config += "# Usage in HAProxy:\n" + config += "# acl banned_ips src -f /path/to/this_file.acl\n" + config += "# http-request deny if banned_ips\n" + config += "# \n" + config += "# " + "="*77 + "\n" + config += "\n" + + # MAP BODY + for network in all_networks: + country = next((c for c, nets in country_networks.items() if network in nets), 'XX') + config += f"{network} {country}\n" + + return config + + @staticmethod def generate_haproxy_lua(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: """Generate HAProxy Lua script with detailed metadata header""" @@ -1283,7 +1345,6 @@ class ConfigGenerator: return config - @staticmethod def generate_csv(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str: """Generate CSV format with detailed metadata header""" diff --git a/precache_daemon.py b/precache_daemon.py index e7b1a2f..4535fbf 100644 --- a/precache_daemon.py +++ b/precache_daemon.py @@ -35,7 +35,10 @@ APP_TYPES = [ 'nginx_map', 'nginx_deny', 'apache_24', + 'apache_22', 'haproxy_acl', + 'haproxy_lua', + 'haproxy_map', 'raw-cidr_txt', 'raw-newline_txt', 'raw-json', @@ -198,6 +201,7 @@ def process_country(country, networks_count, force=False): 'apache_24': ConfigGenerator.generate_apache_24, 'haproxy_acl': ConfigGenerator.generate_haproxy_acl, 'haproxy_lua': ConfigGenerator.generate_haproxy_lua, + 'haproxy_map': ConfigGenerator.generate_haproxy_map, } generator = generators.get(app_type) diff --git a/static/js/app.js b/static/js/app.js index 0827d0e..f930b8b 100644 --- a/static/js/app.js +++ b/static/js/app.js @@ -54,6 +54,11 @@ const APP_VARIANTS = { value: 'lua', text: 'Lua Script', description: 'Lua-based blocking script for advanced HAProxy setups' + }, + { + value: 'map', + text: 'Map File', + description: 'HAProxy map format (for use with -m ip / map files)' } ] }; diff --git a/static/js/progress.js b/static/js/progress.js index 0fb9be8..4bd21b5 100644 --- a/static/js/progress.js +++ b/static/js/progress.js @@ -76,15 +76,13 @@ function showProgress() { window.progressInitTimeout = setTimeout(() => { if (progressMessage && progressMessage.textContent === 'Initializing...') { progressMessage.innerHTML = ` -
Initializing...
+
Request is being processed for over 10 seconds.
- Taking longer than expected...
- All workers may be busy processing other requests.
- Please wait for the queue to clear. + The task is queued and will start automatically once current processing is finished.
`; } - }, 5000); + }, 10000); startProgressPolling(); } @@ -183,7 +181,7 @@ async function downloadConfiguration(formData) { document.body.removeChild(a); if (fromCache) { - showResult(` Lightning fast! Downloaded from cache: ${filename}`, 'success'); + showResult(` Ready! Downloaded from cache: ${filename}`, 'success'); } else { showResult(`Configuration downloaded successfully: ${filename}`, 'success'); } diff --git a/templates/api.html b/templates/api.html index cd82a56..ac2b5e0 100644 --- a/templates/api.html +++ b/templates/api.html @@ -352,120 +352,231 @@ const pollProgress = setInterval(async () => { - -
-
-
-
- POST - /api/generate/raw - Generate raw CIDR blocklist -
- -
-
-
-
Description
-

Generates a raw CIDR blocklist without application-specific configuration. Perfect for iptables, fail2ban, or custom implementations.

- -
Request Body
- 
{
+
+

+  

+    

+      

+        POST
+        /api/generate/raw
+        Generate raw blocklist (TXT/CSV/JSON/JS)
+      

+      
+    

+  

+
+  

+    
Description

+    

+      Generates a raw blocklist without application-specific configuration.
+      Use this endpoint for programmatic integrations when you need structured output (JSON) or a JS-wrapped JSON payload.
+    

+
+    
Request Body

+
{
   "countries": ["CN", "RU"],
-  "app_variant": "txt",
+  "app_type": "raw-json",
   "aggregate": true,
-  "use_cache": true
+  "use_cache": true,
+  "as_js": false,
+  "js_var": "geoipBlocklist"
 }

-                    
-                    
Parameters

-                    
-                        
-                            
-                                
-                                
-                                
-                                
-                            
-                        
-                        
-                            
-                                
-                                
-                                
-                                
-                            
-                            
-                                
-                                
-                                
-                                
-                            
-                            
-                                
-                                
-                                
-                                
-                            
-                            
-                                
-                                
-                                
-                                
-                            
-                        
-                    
NameTypeRequiredDescription
countriesarrayrequiredList of ISO 3166-1 alpha-2 country codes
app_variantstringoptionalOutput format: txt (default) or csv
aggregatebooleanoptionalAggregate IP networks (default: true)
use_cachebooleanoptionalUse Redis cache if available (default: true)

-                    
-                    
Response

-                    
Returns plain text file with CIDR blocks (one per line) or CSV with CIDR and country columns.

-                    
-                    
Response Headers

-                    
-                        
-                            
-                                
-                                
-                            
-                        
-                        
-                            
-                                
-                                
-                            
-                            
-                                
-                                
-                            
-                            
-                                
-                                
-                            
-                        
-                    
HeaderDescription
X-From-Cachetrue or false - indicates if served from Redis
X-Cache-Typeredis or sqlite - data source type
X-Generated-AtTimestamp when config was generated

-                    
-                    
cURL Examples

-                    
-                    
With cache (faster):

-                    
curl -X POST /api/generate/raw \
+
+    
Parameters

+    
+      
+        
+          
+          
+          
+          
+        
+      
+      
+        
+          
+          
+          
+          
+        
+        
+          
+          
+          
+            
+        
+        
+          
+          
+          
+          
+        
+        
+          
+          
+          
+          
+        
+        
+        
+        
+        
+          
+          
+          
+          
+        
+      
+    
NameTypeRequiredDescription
countriesarrayrequiredList of ISO 3166-1 alpha-2 country codes
app_typestringoptional
+            Output format selector. Examples:
+            raw-json (JSON), raw-json + as_js=true (JS wrapper).
+            
aggregatebooleanoptionalAggregate IP networks (default: true)
use_cachebooleanoptionalUse Redis cache if available (default: true)

+        Only for raw-json.
+        When true, wraps JSON into JavaScript and returns application/javascript as:
+        const <js_var> = {...};
+        
js_varstringoptional
+            Variable name used by as_js wrapper (default: geoipBlocklist).
+          

+
+    
Responses

+    
+      
+        
+          
+          
+          
+        
+      
+      
+        
+          
+          
+          
+        
+        
+          
+          
+          
+        
+        
+          
+          
+          
+        
+        
+          
+          
+          
+        
+      
+    
ModeContent-TypeBody
raw-cidr_txttext/plainOne CIDR per line
raw-cidr_csvtext/csvCSV export
raw-cidr_jsonapplication/jsonJSON object with countries, networks, count, aggregated
raw-cidr_json + as_js=trueapplication/javascriptconst <js_var> = {...};

+
+
Responses

+
+  
+    
+      
+      
+      
+    
+  
+  
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+    
+      
+      
+      
+    
+  
+
ModeContent-TypeBody
raw-cidr_txttext/plainOne CIDR per line
raw-cidr_csvtext/csvCSV export
raw-jsonapplication/jsonJSON object with generated_at, countries, networks, total_networks
raw-json + as_js=trueapplication/javascriptconst <js_var> = {...};

+
+
Response Headers

+
+  
+    
+      
+      
+    
+  
+  
+    
+      
+      
+    
+    
+      
+      
+    
+    
+      
+      
+    
+  
+
HeaderDescription
X-From-Cachetrue or false - indicates if served from Redis
X-Cache-Typeredis-full or computed cache type (e.g. hybrid)
X-Generated-AtTimestamp when config was generated

+
+
cURL Examples

+
+
TXT (default):

+
curl -X POST /api/generate/raw \
   -H "Content-Type: application/json" \
   -d '{
     "countries": ["CN", "RU"],
-    "app_variant": "txt",
+    "app_type": "raw-cidr_txt",
     "aggregate": true,
     "use_cache": true
   }' \
   -o blocklist.txt

 
-                    
Force fresh data (slower but guaranteed up-to-date):

-                    
curl -X POST /api/generate/raw \
+
CSV:

+
curl -X POST /api/generate/raw \
   -H "Content-Type: application/json" \
   -d '{
     "countries": ["CN", "RU"],
-    "app_variant": "txt",
+    "app_type": "raw-cidr_csv",
     "aggregate": true,
-    "use_cache": false
+    "use_cache": true
   }' \
-  -o blocklist_fresh.txt

-                

-            

+  -o blocklist.csv
+ +

JSON:

+
curl -X POST /api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "raw-json",
+    "aggregate": true,
+    "use_cache": true
+  }' \
+  -o blocklist.json
+ +

JS-wrapped JSON:

+
curl -X POST /api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "raw-json",
+    "aggregate": true,
+    "use_cache": true,
+    "as_js": true,
+    "js_var": "geoipBlocklist"
+  }' \
+  -o blocklist.js
+
+
@@ -541,7 +652,7 @@ const pollProgress = setInterval(async () => {
Response