app

app.py

@@ -17,6 +17,7 @@ import config
 from cve_handler import CVEHandler, update_all_vendors
 import api
 
+
 def setup_logging():
     log_level = getattr(logging, config.LOG_LEVEL.upper(), logging.INFO)
     
@@ -60,16 +61,19 @@ if not config.DEBUG:
 app.register_blueprint(api.api_bp, url_prefix='/api')
 cve_handler = CVEHandler()
 
+
 @app.context_processor
 def inject_config():
     return {'config': config}
 
+
 def add_security_headers(response):
     if config.ENABLE_SECURITY_HEADERS:
         for header, value in config.SECURITY_HEADERS.items():
             response.headers[header] = value
     return response
 
+
 def gzip_response(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
@@ -99,6 +103,7 @@ def gzip_response(f):
         return response
     return decorated_function
 
+
 def etag_support(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
@@ -121,12 +126,14 @@ def etag_support(f):
     
     return decorated_function
 
+
 @app.route('/')
 @gzip_response
 @etag_support
 def index():
     return render_template('index.html', vendors=config.VENDORS)
 
+
 @app.route('/health')
 def health():
     try:
@@ -166,10 +173,12 @@ def health():
             'error': str(e)
         }), 503
 
+
 @app.route('/favicon.ico')
 def favicon():
     return '', 204
 
+
 @app.route('/version')
 def version():
     return jsonify({
@@ -179,12 +188,14 @@ def version():
         'flask_version': '3.0.2'
     })
 
+
 @app.errorhandler(404)
 def not_found(error):
     if request.path.startswith('/api/'):
         return jsonify({'error': 'Endpoint not found'}), 404
     return render_template('404.html'), 404
 
+
 @app.errorhandler(500)
 def internal_error(error):
     logger.error(f"Internal error: {error}")
@@ -192,6 +203,7 @@ def internal_error(error):
         return jsonify({'error': 'Internal server error'}), 500
     return render_template('500.html'), 500
 
+
 @app.after_request
 def after_request(response):
     response = add_security_headers(response)
@@ -204,6 +216,7 @@ def after_request(response):
     
     return response
 
+
 def background_update_task():
     logger.info("Background update task started")
     time.sleep(60)
@@ -225,31 +238,68 @@ def background_update_task():
 
 
 def start_background_tasks():
-    if config.ENABLE_AUTO_UPDATE:
-        update_thread = threading.Thread(
-            target=background_update_task,
-            daemon=True,
-            name="CVE-Update-Thread"
-        )
-        update_thread.start()
-        logger.info("Auto-update enabled: background task started")
-    else:
-        logger.info("Auto-update disabled")
+    is_gunicorn = (
+        'gunicorn' in os.environ.get('SERVER_SOFTWARE', '').lower() or
+        os.environ.get('GUNICORN_CMD_ARGS') is not None
+    )
     
-    if config.DISCORD_BOT_ENABLED:
-        try:
-            from discord_bot import start_discord_bot
-            discord_thread = threading.Thread(
-                target=start_discord_bot,
-                daemon=True,
-                name="Discord-Bot-Thread"
-            )
-            discord_thread.start()
-            logger.info("Discord bot started")
-        except ImportError:
-            logger.warning("discord.py not installed, Discord bot disabled")
-        except Exception as e:
-            logger.error(f"Failed to start Discord bot: {e}", exc_info=True)
+    is_dev_mode = __name__ == '__main__'
+    
+    if is_gunicorn:
+        logger.info("=" * 70)
+        logger.info("PRODUCTION MODE (Gunicorn)")
+        logger.info("=" * 70)
+        logger.info("Background tasks handled by separate containers:")
+        logger.info("  Scheduler:    cve-monitor-scheduler")
+        logger.info("  Discord Bot:  cve-monitor-discord")
+        logger.info("=" * 70)
+        return
+    
+    if is_dev_mode:
+        logger.info("=" * 70)
+        logger.info("DEVELOPMENT MODE (Standalone Flask)")
+        logger.info("=" * 70)
+        logger.info("Starting background tasks in threads...")
+        logger.info("")
+        
+        if config.ENABLE_AUTO_UPDATE:
+            try:
+                update_thread = threading.Thread(
+                    target=background_update_task,
+                    daemon=True,
+                    name="CVE-Update-Thread"
+                )
+                update_thread.start()
+                logger.info("  Scheduler thread:  STARTED")
+            except Exception as e:
+                logger.error(f"  Scheduler thread:  FAILED - {e}")
+        else:
+            logger.info("  Scheduler thread:  DISABLED")
+        
+        if config.DISCORD_BOT_ENABLED:
+            try:
+                from discord_bot import start_discord_bot
+                
+                discord_thread = threading.Thread(
+                    target=start_discord_bot,
+                    daemon=True,
+                    name="Discord-Bot-Thread"
+                )
+                discord_thread.start()
+                logger.info("  Discord bot thread: STARTED")
+                
+            except ImportError:
+                logger.warning("  Discord bot thread: FAILED - discord.py not installed")
+            except Exception as e:
+                logger.error(f"  Discord bot thread: FAILED - {e}")
+        else:
+            logger.info("  Discord bot thread: DISABLED")
+        
+        logger.info("")
+        logger.info("=" * 70)
+    else:
+        logger.info("Production mode: background tasks in separate containers")
+
 
 def initialize_app():
     logger.info(f"{'='*60}")
@@ -282,6 +332,7 @@ def initialize_app():
     logger.info(f"{'='*60}")
     start_background_tasks()
 
+
 _initialized = False
 
 if __name__ == '__main__':
@@ -300,4 +351,4 @@ if __name__ == '__main__':
 else:
     if not _initialized:
         initialize_app()
-        _initialized = True
+        _initialized = True

scheduler.py

@@ -2,8 +2,8 @@
 
 import time
 import logging
-from datetime import datetime
-from cve_handler import update_all_vendors
+from datetime import datetime, timedelta
+from cve_handler import CVEHandler
 import config
 
 logging.basicConfig(
@@ -12,29 +12,153 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-def run_scheduler():
-    logger.info("CVE Update Scheduler started")
-    logger.info(f"Update interval: {config.UPDATE_INTERVAL_HOURS} hours")
-    logger.info(f"Auto-update enabled: {config.ENABLE_AUTO_UPDATE}")
+
+def incremental_update():
+
+    logger.info("=" * 80)
+    logger.info(f"Starting incremental update at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    logger.info("=" * 80)
     
+    try:
+        handler = CVEHandler()
+        days_back = getattr(config, 'SCHEDULER_UPDATE_DAYS', 7)
+        
+        updated_count = 0
+        failed_count = 0
+        total_new_cves = 0
+        
+        for vendor in config.VENDORS:
+            if not vendor.get('use_cpe'):
+                logger.debug(f"Skipping {vendor['name']} (CPE disabled)")
+                continue
+            
+            try:
+                logger.info(f"Updating {vendor['name']}...")
+                
+                new_cves = handler.fetch_cve_from_nvd(
+                    keywords=vendor.get('keywords', []),
+                    vendor_config=vendor,
+                    days_back=days_back
+                )
+                
+                if new_cves:
+                    handler.save_cves_to_cache(new_cves, vendor['code'])
+                    logger.info(f"  ✓ {vendor['name']}: Added {len(new_cves)} new CVEs")
+                    total_new_cves += len(new_cves)
+                    updated_count += 1
+                else:
+                    logger.info(f"  ✓ {vendor['name']}: No new CVEs")
+                    updated_count += 1
+                
+            except Exception as e:
+                logger.error(f"  ✗ {vendor['name']}: {e}")
+                failed_count += 1
+        
+        logger.info("=" * 80)
+        logger.info(f"✓ Incremental update completed")
+        logger.info(f"  Vendors updated: {updated_count}")
+        logger.info(f"  Vendors failed: {failed_count}")
+        logger.info(f"  New CVEs added: {total_new_cves}")
+        logger.info("=" * 80)
+        
+        return updated_count, failed_count
+        
+    except Exception as e:
+        logger.error(f"✗ Incremental update failed: {e}", exc_info=True)
+        return 0, len(config.VENDORS)
+
+
+def initial_full_scan():
+
+    logger.info("=" * 80)
+    logger.info("INITIAL FULL SCAN")
+    logger.info("=" * 80)
+    logger.info(f"This will scan {config.INITIAL_LOOKBACK_DAYS} days back for all vendors")
+    
+    try:
+        from cve_handler import update_all_vendors
+        updated, failed = update_all_vendors(force=True)
+        
+        logger.info("=" * 80)
+        logger.info(f"✓ Initial scan completed: {updated} updated, {failed} failed")
+        logger.info("=" * 80)
+        
+        return updated, failed
+        
+    except Exception as e:
+        logger.error(f"✗ Initial scan failed: {e}", exc_info=True)
+        return 0, len(config.VENDORS)
+
+
+def run_scheduler():
+    logger.info("=" * 80)
+    logger.info("CVE Monitor Scheduler Starting")
+    logger.info("=" * 80)
+    
+    update_interval_hours = config.UPDATE_INTERVAL_HOURS
+    scheduler_update_days = getattr(config, 'SCHEDULER_UPDATE_DAYS', 7)
+    
+    logger.info(f"Configuration:")
+    logger.info(f"  Update interval: {update_interval_hours} hours")
+    logger.info(f"  Incremental lookback: {scheduler_update_days} days")
+    logger.info(f"  Initial lookback: {config.INITIAL_LOOKBACK_DAYS} days")
+    logger.info(f"  Auto-update enabled: {config.ENABLE_AUTO_UPDATE}")
+    
+    if config.ENABLE_AUTO_UPDATE:
+        logger.info("\nChecking database status...")
+        
+        try:
+            handler = CVEHandler()
+            
+            with handler.get_db_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT COUNT(*) FROM cve_cache")
+                total_cves = cursor.fetchone()[0]
+            
+            if total_cves == 0:
+                logger.info(f"  Database is EMPTY")
+                logger.info(f"Starting initial full scan ({config.INITIAL_LOOKBACK_DAYS} days)...")
+                logger.info(f"This may take 30-60 minutes depending on API rate limits\n")
+                
+                initial_full_scan()
+                
+            else:
+                logger.info(f"✓ Database has {total_cves} CVEs")
+                logger.info(f"Skipping initial scan, starting incremental updates\n")
+                logger.info("Performing startup incremental update...")
+                incremental_update()
+                
+        except Exception as e:
+            logger.error(f"Error checking database: {e}", exc_info=True)
+    
+    logger.info(f"\nWaiting 2 minutes before starting scheduled updates...")
     time.sleep(120)
     
     while True:
         if config.ENABLE_AUTO_UPDATE:
             try:
-                logger.info(f"Starting scheduled update at {datetime.now()}")
-                updated, failed = update_all_vendors()
-                logger.info(f"Update completed: {updated} successful, {failed} failed")
+                updated, failed = incremental_update()
+                
             except Exception as e:
                 logger.error(f"Error in scheduled update: {e}", exc_info=True)
         else:
             logger.debug("Auto-update is disabled, skipping")
         
-        sleep_seconds = config.UPDATE_INTERVAL_HOURS * 3600
-        next_update = datetime.now().timestamp() + sleep_seconds
-        logger.info(f"Next update scheduled at {datetime.fromtimestamp(next_update)}")
+        sleep_seconds = update_interval_hours * 3600
+        next_update = datetime.now() + timedelta(seconds=sleep_seconds)
+        
+        logger.info(f"\n{'=' * 80}")
+        logger.info(f"Next incremental update at: {next_update.strftime('%Y-%m-%d %H:%M:%S')}")
+        logger.info(f"Sleeping for {update_interval_hours} hours...")
+        logger.info(f"{'=' * 80}\n")
+        
         time.sleep(sleep_seconds)
 
 
 if __name__ == '__main__':
-    run_scheduler()
+    try:
+        run_scheduler()
+    except KeyboardInterrupt:
+        logger.info("\n\nScheduler stopped by user")
+    except Exception as e:
+        logger.error(f"Scheduler crashed: {e}", exc_info=True)