haproxy map

2026-02-23 13:10:16 +01:00
parent d82926a4d1
commit bf02af0192
7 changed files with 348 additions and 142 deletions
--- a/api.py
+++ b/api.py
@@ -350,6 +350,8 @@ def generate_preview():
                'apache_24': ConfigGenerator.generate_apache_24,
                'haproxy_acl': ConfigGenerator.generate_haproxy_acl,
                'haproxy_lua': ConfigGenerator.generate_haproxy_lua,
+                "haproxy_map": ConfigGenerator.generate_haproxy_map,
+
            }
            
            generator_key = f"{app_type}_{app_variant}"
@@ -398,6 +400,7 @@ def generate_preview():
        traceback.print_exc()
        return jsonify({'success': False, 'error': str(e)}), 500

+
@api_blueprint.route('/api/generate/raw', methods=['POST'])
 def generate_raw_cidr():
    try:
@@ -409,6 +412,9 @@ def generate_raw_cidr():
        app_type = data.get('app_type', 'raw-cidr_txt')
        use_cache = data.get('use_cache', True)

+        as_js = bool(data.get('as_js', False))
+        js_var = data.get('js_var', 'geoipBlocklist')
+
        if app_type == 'raw-cidr':
            app_type = 'raw-cidr_txt'

@@ -419,19 +425,31 @@ def generate_raw_cidr():
            cached = redis_cache.get_cached_config(countries, app_type, aggregate)
            if cached:
                if 'json' in app_type:
+                    if as_js:
+                        extension = 'js'
+                        mimetype = 'application/javascript'
+                        filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
+                        body = f"const {js_var} = {cached['config']};\n"
+                    else:
                        extension = 'json'
                        mimetype = 'application/json'
+                        filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
+                        body = cached['config']
+
                elif 'csv' in app_type:
                    extension = 'csv'
                    mimetype = 'text/csv'
+                    filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
+                    body = cached['config']
+
                else:
                    extension = 'txt'
                    mimetype = 'text/plain'
-                
                    filename = f"blocklist_{'_'.join(sorted(countries))}.{extension}"
+                    body = cached['config']

                return Response(
-                    cached['config'],
+                    body,
                    mimetype=mimetype,
                    headers={
                        'Content-Disposition': f'attachment; filename="{filename}"',
@@ -486,16 +504,23 @@ def generate_raw_cidr():
                all_networks.extend(nets)

            if aggregate:
-                all_networks = ConfigGenerator.aggregate_networks(all_networks)
+                all_networks = ConfigGenerator._aggregate_networks(all_networks)
            else:
                all_networks = sorted(list(set(all_networks)))

-            config_text = json.dumps({
+            json_text = json.dumps({
                'countries': countries,
                'networks': all_networks,
                'count': len(all_networks),
                'aggregated': aggregate
            }, indent=2)
+
+            if as_js:
+                config_text = f"const {js_var} = {json_text};\n"
+                filename = f"blocklist_{'_'.join(sorted(countries))}.js"
+                mimetype = 'application/javascript'
+            else:
+                config_text = json_text
                filename = f"blocklist_{'_'.join(sorted(countries))}.json"
                mimetype = 'application/json'

@@ -547,6 +572,7 @@ def generate_raw_cidr():
        traceback.print_exc()
        return jsonify({'success': False, 'error': str(e)}), 500

+
@api_blueprint.route('/api/generate', methods=['POST'])
 def generate_config():
    try:
@@ -622,6 +648,7 @@ def generate_config():
            'apache_24': ConfigGenerator.generate_apache_24,
            'haproxy_acl': ConfigGenerator.generate_haproxy_acl,
            'haproxy_lua': ConfigGenerator.generate_haproxy_lua,
+            'haproxy_map': ConfigGenerator.generate_haproxy_map,
        }
        
        generator_key = f"{app_type}_{app_variant}"
--- a/blocklist.txt
+++ b/blocklist.txt
--- a/geoip_handler.py
+++ b/geoip_handler.py
@@ -874,7 +874,7 @@ class ConfigGenerator:
        config += "}\n"
        
        # Log conversion statistics
-        print(f"[INFO] Generated nginx map: {converted_count} regex patterns", flush=True)
+        #print(f"[INFO] Generated nginx map: {converted_count} regex patterns", flush=True)
        
        if failed_count > 0:
            print(f"[WARNING] Failed to convert {failed_count} networks to regex - check config file", flush=True)
@@ -1144,6 +1144,68 @@ class ConfigGenerator:
    """
        return config

+    @staticmethod
+    def generate_haproxy_map(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str:
+        """
+        Generate HAProxy MAP file (IP COUNTRY format)
+        """
+        countries = sorted(country_networks.keys())
+
+        redisstats = None
+        if redis_ips:
+            redisstats = {"total": len(redis_ips), "unique": len(redis_ips), "deduped": 0}
+
+        handler = GeoIPHandler()
+        metadata = generate_metadata(countries, country_networks, redisstats, handler)
+
+        # Aggregate networks
+        all_networks = []
+        for networks in country_networks.values():
+            all_networks.extend(networks)
+        if redis_ips:
+            all_networks.extend(redis_ips)
+
+        if aggregate:
+            all_networks = ConfigGenerator._aggregate_networks(all_networks)
+        else:
+            all_networks = sorted(list(set(all_networks)))
+
+        # Generate header
+        config = "# " + "="*77 + "\n"
+        config += "# HAProxy MAP Configuration\n"
+        config += f"# Generated: {metadata['timestamp']}\n"
+        config += "# " + "="*77 + "\n"
+        config += "# \n"
+        config += f"# Countries: {metadata['countries_string']} ({metadata['country_count']} countries)\n"
+        config += f"# Total networks: {len(all_networks):,}\n"
+        config += "# \n"
+        config += "# Data sources:\n"
+        config += metadata['sources_formatted'] + "\n"
+        config += "# \n"
+        
+        if metadata['redis']:
+            config += f"# {metadata['redis']['formatted']}\n"
+            config += "# \n"
+        
+        config += "# Cache settings:\n"
+        config += f"#   Max age: {metadata['cache_max_age_hours']} hours ({metadata['cache_max_age_days']:.1f} days)\n"
+        config += f"#   Database: {metadata['cache_db_path']}\n"
+        config += "# \n"
+        config += "# Usage in HAProxy:\n"
+        config += "#   acl banned_ips src -f /path/to/this_file.acl\n"
+        config += "#   http-request deny if banned_ips\n"
+        config += "# \n"
+        config += "# " + "="*77 + "\n"
+        config += "\n"
+
+        # MAP BODY
+        for network in all_networks:
+            country = next((c for c, nets in country_networks.items() if network in nets), 'XX')
+            config += f"{network} {country}\n"
+
+        return config
+
+
    @staticmethod
    def generate_haproxy_lua(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str:
        """Generate HAProxy Lua script with detailed metadata header"""
@@ -1283,7 +1345,6 @@ class ConfigGenerator:
        
        return config

-    
    @staticmethod
    def generate_csv(country_networks: dict, aggregate: bool = True, redis_ips: set = None) -> str:
        """Generate CSV format with detailed metadata header"""
--- a/precache_daemon.py
+++ b/precache_daemon.py
@@ -35,7 +35,10 @@ APP_TYPES = [
    'nginx_map',
    'nginx_deny',
    'apache_24',
+    'apache_22',
    'haproxy_acl',
+    'haproxy_lua',
+    'haproxy_map',
    'raw-cidr_txt',
    'raw-newline_txt',
    'raw-json',
@@ -198,6 +201,7 @@ def process_country(country, networks_count, force=False):
                        'apache_24': ConfigGenerator.generate_apache_24,
                        'haproxy_acl': ConfigGenerator.generate_haproxy_acl,
                        'haproxy_lua': ConfigGenerator.generate_haproxy_lua,
+                        'haproxy_map': ConfigGenerator.generate_haproxy_map,
                    }
                    
                    generator = generators.get(app_type)
--- a/static/js/app.js
+++ b/static/js/app.js
@@ -54,6 +54,11 @@ const APP_VARIANTS = {
            value: 'lua', 
            text: 'Lua Script',
            description: 'Lua-based blocking script for advanced HAProxy setups'
+        },
+        {
+            value: 'map',
+            text: 'Map File',
+            description: 'HAProxy map format (for use with -m ip / map files)'
        }
    ]
 };
--- a/static/js/progress.js
+++ b/static/js/progress.js
@@ -76,15 +76,13 @@ function showProgress() {
    window.progressInitTimeout = setTimeout(() => {
        if (progressMessage && progressMessage.textContent === 'Initializing...') {
            progressMessage.innerHTML = `
-                <div>Initializing...</div>
+                <div>Request is being processed for over 10 seconds.</div>
                <div style="margin-top: 10px; color: #856404; background: #fff3cd; padding: 10px; border-radius: 4px; font-size: 0.9em;">
-                    Taking longer than expected...<br>
-                    All workers may be busy processing other requests.<br>
-                    Please wait for the queue to clear.
+                    The task is queued and will start automatically once current processing is finished.
                </div>
            `;
        }
-    }, 5000);
+    }, 10000);
    
    startProgressPolling();
 }
@@ -183,7 +181,7 @@ async function downloadConfiguration(formData) {
            document.body.removeChild(a);
            
            if (fromCache) {
-                showResult(`<i class="fas fa-bolt"></i> <strong>Lightning fast!</strong> Downloaded from cache: ${filename}`, 'success');
+                showResult(`<i class="fas fa-bolt"></i> <strong>Ready!</strong> Downloaded from cache: ${filename}`, 'success');
            } else {
                showResult(`Configuration downloaded successfully: ${filename}`, 'success');
            }
--- a/templates/api.html
+++ b/templates/api.html
@@ -359,21 +359,27 @@ const pollProgress = setInterval(async () => {
      <div>
        <span class="badge bg-success me-2">POST</span>
        <code class="api-path">/api/generate/raw</code>
-                            <span class="ms-3 text-muted">Generate raw CIDR blocklist</span>
+        <span class="ms-3 text-muted">Generate raw blocklist (TXT/CSV/JSON/JS)</span>
      </div>
      <i class="fas fa-chevron-down"></i>
    </div>
  </div>
+
  <div class="card-body collapse" id="endpoint6">
    <h6 class="fw-bold">Description</h6>
-                    <p>Generates a raw CIDR blocklist without application-specific configuration. Perfect for iptables, fail2ban, or custom implementations.</p>
+    <p>
+      Generates a raw blocklist without application-specific configuration.
+      Use this endpoint for programmatic integrations when you need structured output (JSON) or a JS-wrapped JSON payload.
+    </p>

    <h6 class="fw-bold mt-3">Request Body</h6>
 <pre><code>{
  "countries": [<span class="text-warning">"CN"</span>, <span class="text-warning">"RU"</span>],
-  "app_variant": <span class="text-warning">"txt"</span>,
+  "app_type": <span class="text-warning">"raw-json"</span>,
  "aggregate": <span class="text-success">true</span>,
-  "use_cache": <span class="text-success">true</span>
+  "use_cache": <span class="text-success">true</span>,
+  "as_js": <span class="text-success">false</span>,
+  "js_var": <span class="text-warning">"geoipBlocklist"</span>
 }</code></pre>

    <h6 class="fw-bold mt-3">Parameters</h6>
@@ -394,10 +400,13 @@ const pollProgress = setInterval(async () => {
          <td>List of ISO 3166-1 alpha-2 country codes</td>
        </tr>
        <tr>
-                                <td><code>app_variant</code></td>
+          <td><code>app_type</code></td>
          <td>string</td>
          <td><span class="badge bg-secondary">optional</span></td>
-                                <td>Output format: <code>txt</code> (default) or <code>csv</code></td>
+            <td>
+            Output format selector. Examples:
+            <code>raw-json</code> (JSON), <code>raw-json</code> + <code>as_js=true</code> (JS wrapper).
+            </td>
        </tr>
        <tr>
          <td><code>aggregate</code></td>
@@ -411,11 +420,89 @@ const pollProgress = setInterval(async () => {
          <td><span class="badge bg-secondary">optional</span></td>
          <td>Use Redis cache if available (default: true)</td>
        </tr>
+        <tr>
+        <td>
+        Only for <code>raw-json</code>.
+        When <code>true</code>, wraps JSON into JavaScript and returns <code>application/javascript</code> as:
+        <code>const &lt;js_var&gt; = {...};</code>
+        </td>
+        </tr>
+        <tr>
+          <td><code>js_var</code></td>
+          <td>string</td>
+          <td><span class="badge bg-secondary">optional</span></td>
+          <td>
+            Variable name used by <code>as_js</code> wrapper (default: <code>geoipBlocklist</code>).
+          </td>
+        </tr>
      </tbody>
    </table>

-                    <h6 class="fw-bold mt-3">Response</h6>
-                    <p>Returns plain text file with CIDR blocks (one per line) or CSV with CIDR and country columns.</p>
+    <h6 class="fw-bold mt-3">Responses</h6>
+    <table class="table table-sm">
+      <thead>
+        <tr>
+          <th>Mode</th>
+          <th>Content-Type</th>
+          <th>Body</th>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td><code>raw-cidr_txt</code></td>
+          <td><code>text/plain</code></td>
+          <td>One CIDR per line</td>
+        </tr>
+        <tr>
+          <td><code>raw-cidr_csv</code></td>
+          <td><code>text/csv</code></td>
+          <td>CSV export</td>
+        </tr>
+        <tr>
+          <td><code>raw-cidr_json</code></td>
+          <td><code>application/json</code></td>
+          <td>JSON object with <code>countries</code>, <code>networks</code>, <code>count</code>, <code>aggregated</code></td>
+        </tr>
+        <tr>
+          <td><code>raw-cidr_json</code> + <code>as_js=true</code></td>
+          <td><code>application/javascript</code></td>
+          <td><code>const &lt;js_var&gt; = {...};</code></td>
+        </tr>
+      </tbody>
+    </table>
+
+<h6 class="fw-bold mt-3">Responses</h6>
+<table class="table table-sm">
+  <thead>
+    <tr>
+      <th>Mode</th>
+      <th>Content-Type</th>
+      <th>Body</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><code>raw-cidr_txt</code></td>
+      <td><code>text/plain</code></td>
+      <td>One CIDR per line</td>
+    </tr>
+    <tr>
+      <td><code>raw-cidr_csv</code></td>
+      <td><code>text/csv</code></td>
+      <td>CSV export</td>
+    </tr>
+    <tr>
+      <td><code>raw-json</code></td>
+      <td><code>application/json</code></td>
+      <td>JSON object with <code>generated_at</code>, <code>countries</code>, <code>networks</code>, <code>total_networks</code></td>
+    </tr>
+    <tr>
+      <td><code>raw-json</code> + <code>as_js=true</code></td>
+      <td><code>application/javascript</code></td>
+      <td><code>const &lt;js_var&gt; = {...};</code></td>
+    </tr>
+  </tbody>
+</table>

 <h6 class="fw-bold mt-3">Response Headers</h6>
 <table class="table table-sm">
@@ -432,7 +519,7 @@ const pollProgress = setInterval(async () => {
    </tr>
    <tr>
      <td><code>X-Cache-Type</code></td>
-                                <td><code>redis</code> or <code>sqlite</code> - data source type</td>
+      <td><code>redis-full</code> or computed cache type (e.g. <code>hybrid</code>)</td>
    </tr>
    <tr>
      <td><code>X-Generated-At</code></td>
@@ -443,27 +530,51 @@ const pollProgress = setInterval(async () => {

 <h6 class="fw-bold mt-3">cURL Examples</h6>

-                    <p class="mb-2"><strong>With cache (faster):</strong></p>
+<p class="mb-2"><strong>TXT (default):</strong></p>
 <pre><code>curl -X POST <span id="curlUrl2"></span>/api/generate/raw \
  -H "Content-Type: application/json" \
  -d '{
    "countries": ["CN", "RU"],
-    "app_variant": "txt",
+    "app_type": "raw-cidr_txt",
    "aggregate": true,
    "use_cache": true
  }' \
  -o blocklist.txt</code></pre>

-                    <p class="mb-2 mt-3"><strong>Force fresh data (slower but guaranteed up-to-date):</strong></p>
+<p class="mb-2 mt-3"><strong>CSV:</strong></p>
+<pre><code>curl -X POST <span id="curlUrl2csv"></span>/api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "raw-cidr_csv",
+    "aggregate": true,
+    "use_cache": true
+  }' \
+  -o blocklist.csv</code></pre>
+
+<p class="mb-2 mt-3"><strong>JSON:</strong></p>
 <pre><code>curl -X POST <span id="curlUrl2b"></span>/api/generate/raw \
  -H "Content-Type: application/json" \
  -d '{
    "countries": ["CN", "RU"],
-    "app_variant": "txt",
+    "app_type": "raw-json",
    "aggregate": true,
-    "use_cache": false
+    "use_cache": true
  }' \
-  -o blocklist_fresh.txt</code></pre>
+  -o blocklist.json</code></pre>
+
+<p class="mb-2 mt-3"><strong>JS-wrapped JSON:</strong></p>
+<pre><code>curl -X POST <span id="curlUrl2c"></span>/api/generate/raw \
+  -H "Content-Type: application/json" \
+  -d '{
+    "countries": ["CN", "RU"],
+    "app_type": "raw-json",
+    "aggregate": true,
+    "use_cache": true,
+    "as_js": true,
+    "js_var": "geoipBlocklist"
+  }' \
+  -o blocklist.js</code></pre>
  </div>
 </div>

@@ -541,7 +652,7 @@ const pollProgress = setInterval(async () => {
                    <ul>
                        <li><strong>nginx:</strong> <code>geo</code>, <code>map</code>, <code>deny</code></li>
                        <li><strong>apache:</strong> <code>22</code> (Apache 2.2), <code>24</code> (Apache 2.4)</li>
-                        <li><strong>haproxy:</strong> <code>acl</code>, <code>lua</code></li>
+                        <li><strong>haproxy:</strong> <code>acl</code>, <code>lua</code>, <code>map</code></li>
                    </ul>
                    
                    <h6 class="fw-bold mt-3">Response</h6>