small change in js

static/js/progress.js

@@ -29,6 +29,13 @@ function stopProgressPolling() {
 }
 
 function updateProgressUI(message, progress, total) {
+
+    if (window.progressInitTimeout) {
+        clearTimeout(window.progressInitTimeout);
+        window.progressInitTimeout = null;
+    }
+    
+
     const progressSection = document.getElementById('progressSection');
     const progressBar = progressSection.querySelector('.progress-bar');
     const progressMessage = document.getElementById('progressMessage');
@@ -66,6 +73,19 @@ function showProgress() {
     progressSection.style.display = 'block';
     document.getElementById('generateBtn').disabled = true;
     
+    window.progressInitTimeout = setTimeout(() => {
+        if (progressMessage && progressMessage.textContent === 'Initializing...') {
+            progressMessage.innerHTML = `
+                <div>Initializing...</div>
+                <div style="margin-top: 10px; color: #856404; background: #fff3cd; padding: 10px; border-radius: 4px; font-size: 0.9em;">
+                    Taking longer than expected...<br>
+                    All workers may be busy processing other requests.<br>
+                    Please wait for the queue to clear.
+                </div>
+            `;
+        }
+    }, 5000);
+    
     startProgressPolling();
 }
 

test_performance.py

@@ -14,12 +14,14 @@ from collections import defaultdict
 import sys
 import argparse
 import threading
+import json
 
 class PerformanceTest:
     def __init__(self, base_url):
         self.base_url = base_url
         self.results = defaultdict(list)
         self.errors = []
+        self.error_samples = []  # Store sample responses
         self.lock = threading.Lock()
         self.available_countries = self._fetch_available_countries()
         
@@ -58,23 +60,107 @@ class PerformanceTest:
             
             duration = time.time() - start
             
+            # Check status code
+            if resp.status_code != 200:
+                error_info = {
+                    'endpoint': endpoint,
+                    'error': f"HTTP {resp.status_code}",
+                    'duration': duration,
+                    'response_preview': resp.text[:300]
+                }
+                
+                with self.lock:
+                    self.errors.append(error_info)
+                    if len(self.error_samples) < 3:
+                        self.error_samples.append({
+                            'type': f'HTTP {resp.status_code}',
+                            'url': url,
+                            'status': resp.status_code,
+                            'headers': dict(resp.headers),
+                            'body': resp.text[:500]
+                        })
+                
+                return {
+                    'status': resp.status_code,
+                    'duration': duration,
+                    'size': 0,
+                    'success': False,
+                    'endpoint': endpoint,
+                    'error': f"HTTP {resp.status_code}"
+                }
+            
+            # Check if this is a RAW endpoint (returns plaintext, not JSON)
+            is_raw_endpoint = '/api/generate/raw' in endpoint
+            
+            if is_raw_endpoint:
+                # RAW endpoints return plaintext, not JSON
                 return {
                     'status': resp.status_code,
                     'duration': duration,
                     'size': len(resp.content),
-                'success': resp.status_code == 200,
+                    'success': True,
                     'endpoint': endpoint,
-                'cache_type': resp.json().get('cache_type') if resp.status_code == 200 else None
+                    'cache_type': None  # RAW doesn't have cache_type in response
+                }
+            
+            # Try to parse JSON for non-RAW endpoints
+            try:
+                json_data = resp.json()
+            except json.JSONDecodeError as e:
+                error_info = {
+                    'endpoint': endpoint,
+                    'error': f"Invalid JSON: {str(e)}",
+                    'duration': duration,
+                    'response_preview': resp.text[:300]
+                }
+                
+                with self.lock:
+                    self.errors.append(error_info)
+                    if len(self.error_samples) < 3:
+                        self.error_samples.append({
+                            'type': 'JSON Parse Error',
+                            'url': url,
+                            'status': resp.status_code,
+                            'headers': dict(resp.headers),
+                            'body': resp.text[:500],
+                            'error': str(e)
+                        })
+                
+                return {
+                    'status': resp.status_code,
+                    'duration': duration,
+                    'size': len(resp.content),
+                    'success': False,
+                    'endpoint': endpoint,
+                    'error': f"Invalid JSON: {str(e)}"
+                }
+            
+            return {
+                'status': resp.status_code,
+                'duration': duration,
+                'size': len(resp.content),
+                'success': True,
+                'endpoint': endpoint,
+                'cache_type': json_data.get('cache_type')
             }
             
         except Exception as e:
             duration = time.time() - start
-            with self.lock:
+            error_info = {
-                self.errors.append({
                 'endpoint': endpoint,
                 'error': str(e),
                 'duration': duration
+            }
+            
+            with self.lock:
+                self.errors.append(error_info)
+                if len(self.error_samples) < 3:
+                    self.error_samples.append({
+                        'type': 'Exception',
+                        'url': url,
+                        'error': str(e)
                     })
+            
             return {
                 'status': 0,
                 'duration': duration,
@@ -84,6 +170,7 @@ class PerformanceTest:
                 'error': str(e)
             }
 
+    
     def simulate_user(self, user_id, duration_seconds, think_time_range=(1, 5)):
         """Simulate single user behavior - ONLY single countries"""
         formats = [
@@ -273,8 +360,30 @@ class PerformanceTest:
             for error, count in sorted(error_types.items(), key=lambda x: -x[1])[:10]:
                 self.log(f"  {count:3d}x {error}")
             
+            # Show detailed error samples
+            if self.error_samples:
+                self.log(f"\n{'='*70}")
+                self.log("DETAILED ERROR SAMPLES")
+                self.log(f"{'='*70}")
                 
-def run_user_simulation(base_url, num_users, duration):
+                for idx, sample in enumerate(self.error_samples, 1):
+                    self.log(f"\nSample #{idx}: {sample['type']}")
+                    self.log(f"  URL: {sample['url']}")
+                    if 'status' in sample:
+                        self.log(f"  Status: {sample['status']}")
+                    if 'headers' in sample:
+                        self.log(f"  Content-Type: {sample['headers'].get('Content-Type', 'N/A')}")
+                        self.log(f"  Content-Length: {sample['headers'].get('Content-Length', 'N/A')}")
+                    if 'error' in sample:
+                        self.log(f"  Error: {sample['error']}")
+                    if 'body' in sample:
+                        self.log(f"  Response body preview:")
+                        self.log(f"  ---")
+                        for line in sample['body'].split('\n')[:10]:
+                            self.log(f"  {line}")
+                        self.log(f"  ---")
+
+def run_user_simulation(base_url, num_users, duration, think_time):
     """Run only user simulation"""
     tester = PerformanceTest(base_url)
     
@@ -282,13 +391,13 @@ def run_user_simulation(base_url, num_users, duration):
     print("GEOIP BAN API - USER SIMULATION")
     print(f"Target: {base_url}")
     print(f"Simulating {num_users} concurrent users for {duration}s")
+    print(f"Think time: {think_time[0]}-{think_time[1]}s")
     print("="*70 + "\n")
     
-    tester.test_concurrent_users(num_users, duration, think_time_range=(1, 5))
+    tester.test_concurrent_users(num_users, duration, think_time_range=think_time)
     tester.final_report()
 
-
+def run_quick_test(base_url, num_users, duration, think_time):
-def run_quick_test(base_url):
     """Quick performance test - single countries only"""
     tester = PerformanceTest(base_url)
     
@@ -317,10 +426,87 @@ def run_quick_test(base_url):
         count=50, concurrent=10
     )
     
-    tester.test_concurrent_users(num_users=10, duration_seconds=30, think_time_range=(1, 3))
+    tester.test_concurrent_users(num_users=num_users, duration_seconds=duration, think_time_range=think_time)
     
     tester.final_report()
 
+def run_full_test_suite(base_url, num_users, duration, think_time):
+    """Execute complete test suite - single countries only"""
+    tester = PerformanceTest(base_url)
+    
+    print("\n" + "="*70)
+    print("GEOIP BAN API - FULL PERFORMANCE TEST SUITE")
+    print(f"Target: {base_url}")
+    print("="*70 + "\n")
+    
+    if not tester.available_countries:
+        print("ERROR: No countries available from API")
+        return
+    
+    tester.test_scenario("Health Check", "GET", "/api/database/status", count=50, concurrent=10)
+    time.sleep(1)
+    
+    test_country = tester.available_countries[0]
+    
+    tester.test_scenario(
+        "Single Country (Cached)",
+        "POST", "/api/generate/preview",
+        data={'countries': [test_country], 'app_type': 'nginx', 'app_variant': 'geo', 'aggregate': True},
+        count=100, concurrent=20
+    )
+    time.sleep(1)
+    
+    tester.test_scenario(
+        "Heavy Load (50 concurrent)",
+        "POST", "/api/generate/preview",
+        data={'countries': [test_country], 'app_type': 'nginx', 'app_variant': 'map', 'aggregate': True},
+        count=200, concurrent=50
+    )
+    time.sleep(2)
+    
+    tester.log("\nSPIKE TEST - Sudden burst of 100 requests")
+    start = time.time()
+    tester.test_scenario(
+        "Spike Test",
+        "POST", "/api/generate/preview",
+        data={'countries': [test_country], 'app_type': 'apache', 'app_variant': '24', 'aggregate': True},
+        count=100, concurrent=100
+    )
+    spike_duration = time.time() - start
+    tester.log(f"  Spike handled in: {spike_duration:.2f}s")
+    time.sleep(2)
+    
+    tester.test_concurrent_users(num_users=num_users, duration_seconds=duration, think_time_range=think_time)
+    time.sleep(2)
+    tester.test_concurrent_users(num_users=num_users*5, duration_seconds=duration//2, think_time_range=(think_time[0]/2, think_time[1]/2))
+    
+    tester.final_report()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Performance testing for GeoIP Ban API')
+    parser.add_argument('--url', default='http://127.0.0.1:5000', help='API base URL')
+    parser.add_argument('--mode', choices=['quick', 'full', 'users'], default='quick',
+                       help='Test mode: quick (5min), full (15min), users (simulation only)')
+    parser.add_argument('--users', type=int, default=10, help='Number of concurrent users')
+    parser.add_argument('--duration', type=int, default=60, help='Test duration in seconds')
+    parser.add_argument('--think-min', type=float, default=1.0, help='Minimum think time between requests (seconds)')
+    parser.add_argument('--think-max', type=float, default=5.0, help='Maximum think time between requests (seconds)')
+    
+    args = parser.parse_args()
+    
+    think_time = (args.think_min, args.think_max)
+    
+    try:
+        if args.mode == 'quick':
+            run_quick_test(args.url, args.users, args.duration, think_time)
+        elif args.mode == 'full':
+            run_full_test_suite(args.url, args.users, args.duration, think_time)
+        elif args.mode == 'users':
+            run_user_simulation(args.url, args.users, args.duration, think_time)
+    except KeyboardInterrupt:
+        print("\n\nTest interrupted by user")
+        sys.exit(1)
+
 
 def run_full_test_suite(base_url):
     """Execute complete test suite - single countries only"""
@@ -374,24 +560,27 @@ def run_full_test_suite(base_url):
     
     tester.final_report()
 
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Performance testing for GeoIP Ban API')
     parser.add_argument('--url', default='http://127.0.0.1:5000', help='API base URL')
     parser.add_argument('--mode', choices=['quick', 'full', 'users'], default='quick',
                        help='Test mode: quick (5min), full (15min), users (simulation only)')
-    parser.add_argument('--users', type=int, default=10, help='Number of concurrent users (for users mode)')
+    parser.add_argument('--users', type=int, default=10, help='Number of concurrent users')
-    parser.add_argument('--duration', type=int, default=60, help='Test duration in seconds (for users mode)')
+    parser.add_argument('--duration', type=int, default=60, help='Test duration in seconds')
+    parser.add_argument('--think-min', type=float, default=1.0, help='Minimum think time between requests (seconds)')
+    parser.add_argument('--think-max', type=float, default=5.0, help='Maximum think time between requests (seconds)')
     
     args = parser.parse_args()
     
+    think_time = (args.think_min, args.think_max)
+    
     try:
         if args.mode == 'quick':
-            run_quick_test(args.url)
+            run_quick_test(args.url, args.users, args.duration, think_time)
         elif args.mode == 'full':
-            run_full_test_suite(args.url)
+            run_full_test_suite(args.url, args.users, args.duration, think_time)
         elif args.mode == 'users':
-            run_user_simulation(args.url, args.users, args.duration)
+            run_user_simulation(args.url, args.users, args.duration, think_time)
     except KeyboardInterrupt:
         print("\n\nTest interrupted by user")
         sys.exit(1)