app

app.py

@@ -17,6 +17,7 @@ import config
 from cve_handler import CVEHandler, update_all_vendors
 import api
 
+
 def setup_logging():
     log_level = getattr(logging, config.LOG_LEVEL.upper(), logging.INFO)
     
@@ -60,16 +61,19 @@ if not config.DEBUG:
 app.register_blueprint(api.api_bp, url_prefix='/api')
 cve_handler = CVEHandler()
 
+
 @app.context_processor
 def inject_config():
     return {'config': config}
 
+
 def add_security_headers(response):
     if config.ENABLE_SECURITY_HEADERS:
         for header, value in config.SECURITY_HEADERS.items():
             response.headers[header] = value
     return response
 
+
 def gzip_response(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
@@ -99,6 +103,7 @@ def gzip_response(f):
         return response
     return decorated_function
 
+
 def etag_support(f):
     @wraps(f)
     def decorated_function(*args, **kwargs):
@@ -121,12 +126,14 @@ def etag_support(f):
     
     return decorated_function
 
+
 @app.route('/')
 @gzip_response
 @etag_support
 def index():
     return render_template('index.html', vendors=config.VENDORS)
 
+
 @app.route('/health')
 def health():
     try:
@@ -166,10 +173,12 @@ def health():
             'error': str(e)
         }), 503
 
+
 @app.route('/favicon.ico')
 def favicon():
     return '', 204
 
+
 @app.route('/version')
 def version():
     return jsonify({
@@ -179,12 +188,14 @@ def version():
         'flask_version': '3.0.2'
     })
 
+
 @app.errorhandler(404)
 def not_found(error):
     if request.path.startswith('/api/'):
         return jsonify({'error': 'Endpoint not found'}), 404
     return render_template('404.html'), 404
 
+
 @app.errorhandler(500)
 def internal_error(error):
     logger.error(f"Internal error: {error}")
@@ -192,6 +203,7 @@ def internal_error(error):
         return jsonify({'error': 'Internal server error'}), 500
     return render_template('500.html'), 500
 
+
 @app.after_request
 def after_request(response):
     response = add_security_headers(response)
@@ -204,6 +216,7 @@ def after_request(response):
     
     return response
 
+
 def background_update_task():
     logger.info("Background update task started")
     time.sleep(60)
@@ -225,31 +238,68 @@ def background_update_task():
 
 
 def start_background_tasks():
-    if config.ENABLE_AUTO_UPDATE:
+    is_gunicorn = (
-        update_thread = threading.Thread(
+        'gunicorn' in os.environ.get('SERVER_SOFTWARE', '').lower() or
-            target=background_update_task,
+        os.environ.get('GUNICORN_CMD_ARGS') is not None
-            daemon=True,
+    )
-            name="CVE-Update-Thread"
+    
-        )
+    is_dev_mode = __name__ == '__main__'
-        update_thread.start()
+    
-        logger.info("Auto-update enabled: background task started")
+    if is_gunicorn:
-    else:
+        logger.info("=" * 70)
-        logger.info("Auto-update disabled")
+        logger.info("PRODUCTION MODE (Gunicorn)")
+        logger.info("=" * 70)
+        logger.info("Background tasks handled by separate containers:")
+        logger.info("  Scheduler:    cve-monitor-scheduler")
+        logger.info("  Discord Bot:  cve-monitor-discord")
+        logger.info("=" * 70)
+        return
+    
+    if is_dev_mode:
+        logger.info("=" * 70)
+        logger.info("DEVELOPMENT MODE (Standalone Flask)")
+        logger.info("=" * 70)
+        logger.info("Starting background tasks in threads...")
+        logger.info("")
+        
+        if config.ENABLE_AUTO_UPDATE:
+            try:
+                update_thread = threading.Thread(
+                    target=background_update_task,
+                    daemon=True,
+                    name="CVE-Update-Thread"
+                )
+                update_thread.start()
+                logger.info("  Scheduler thread:  STARTED")
+            except Exception as e:
+                logger.error(f"  Scheduler thread:  FAILED - {e}")
+        else:
+            logger.info("  Scheduler thread:  DISABLED")
+        
+        if config.DISCORD_BOT_ENABLED:
+            try:
+                from discord_bot import start_discord_bot
+                
+                discord_thread = threading.Thread(
+                    target=start_discord_bot,
+                    daemon=True,
+                    name="Discord-Bot-Thread"
+                )
+                discord_thread.start()
+                logger.info("  Discord bot thread: STARTED")
+                
+            except ImportError:
+                logger.warning("  Discord bot thread: FAILED - discord.py not installed")
+            except Exception as e:
+                logger.error(f"  Discord bot thread: FAILED - {e}")
+        else:
+            logger.info("  Discord bot thread: DISABLED")
+        
+        logger.info("")
+        logger.info("=" * 70)
+    else:
+        logger.info("Production mode: background tasks in separate containers")
 
-    if config.DISCORD_BOT_ENABLED:
-        try:
-            from discord_bot import start_discord_bot
-            discord_thread = threading.Thread(
-                target=start_discord_bot,
-                daemon=True,
-                name="Discord-Bot-Thread"
-            )
-            discord_thread.start()
-            logger.info("Discord bot started")
-        except ImportError:
-            logger.warning("discord.py not installed, Discord bot disabled")
-        except Exception as e:
-            logger.error(f"Failed to start Discord bot: {e}", exc_info=True)
 
 def initialize_app():
     logger.info(f"{'='*60}")
@@ -282,6 +332,7 @@ def initialize_app():
     logger.info(f"{'='*60}")
     start_background_tasks()
 
+
 _initialized = False
 
 if __name__ == '__main__':

cve_handler.py

@@ -505,30 +505,77 @@ class CVEHandler:
             
             logger.info(f"Total unique CVEs after deduplication: {len(unique_cves)}")
             
+            new_count = 0
+            updated_count = 0
+            
             with self.get_db_connection() as conn:
                 cursor = conn.cursor()
                 
                 for cve_id, cve in unique_cves.items():
                     cursor.execute("""
-                        INSERT OR REPLACE INTO cve_cache 
+                        SELECT discord_notified FROM cve_cache WHERE cve_id = ?
-                        (cve_id, vendor_code, description, published_date, last_modified, 
+                    """, (cve_id,))
-                        cvss_score, cvss_vector, severity, refs, cwe_ids, 
+                    
-                        affected_products, raw_data, updated_at)
+                    existing = cursor.fetchone()
-                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
+                    
-                    """, (
+                    if not existing:
-                        cve_id,
+                        cursor.execute("""
-                        vendor_code,
+                            INSERT INTO cve_cache 
-                        cve.get('description'),
+                            (cve_id, vendor_code, description, published_date, last_modified, 
-                        cve.get('published_date'),
+                            cvss_score, cvss_vector, severity, refs, cwe_ids, 
-                        cve.get('last_modified'),
+                            affected_products, raw_data, discord_notified, 
-                        cve.get('cvss_score'),
+                            created_at, updated_at)
-                        cve.get('cvss_vector'),
+                            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-                        cve.get('severity'),
+                        """, (
-                        cve.get('references'),
+                            cve_id,
-                        cve.get('cwe_ids'),
+                            vendor_code,
-                        None,
+                            cve.get('description'),
-                        cve.get('raw_data')
+                            cve.get('published_date'),
-                    ))
+                            cve.get('last_modified'),
+                            cve.get('cvss_score'),
+                            cve.get('cvss_vector'),
+                            cve.get('severity'),
+                            cve.get('references'),
+                            cve.get('cwe_ids'),
+                            None,
+                            cve.get('raw_data')
+                        ))
+                        new_count += 1
+                    else:
+                        old_notified = existing[0]
+                        
+                        cursor.execute("""
+                            UPDATE cve_cache SET
+                                vendor_code = ?,
+                                description = ?,
+                                published_date = ?,
+                                last_modified = ?,
+                                cvss_score = ?,
+                                cvss_vector = ?,
+                                severity = ?,
+                                refs = ?,
+                                cwe_ids = ?,
+                                affected_products = ?,
+                                raw_data = ?,
+                                discord_notified = ?,
+                                updated_at = CURRENT_TIMESTAMP
+                            WHERE cve_id = ?
+                        """, (
+                            vendor_code,
+                            cve.get('description'),
+                            cve.get('published_date'),
+                            cve.get('last_modified'),
+                            cve.get('cvss_score'),
+                            cve.get('cvss_vector'),
+                            cve.get('severity'),
+                            cve.get('references'),
+                            cve.get('cwe_ids'),
+                            None,
+                            cve.get('raw_data'),
+                            old_notified,
+                            cve_id
+                        ))
+                        updated_count += 1
                 
                 cursor.execute("""
                     INSERT OR REPLACE INTO cve_metadata 
@@ -540,11 +587,16 @@ class CVEHandler:
                     list(unique_cves.keys())[0] if unique_cves else None
                 ))
             
-            logger.info(f"✓ Successfully updated {len(unique_cves)} CVEs for {vendor['name']}")
+            if new_count > 0:
+                logger.info(f"Added {new_count} new CVEs for {vendor['name']}")
+            if updated_count > 0:
+                logger.info(f"Updated {updated_count} existing CVEs for {vendor['name']}")
+            
+            logger.info(f"Successfully processed {len(unique_cves)} CVEs for {vendor['name']}")
             return True
             
         except Exception as e:
-            logger.error(f"✗ Error updating vendor cache for {vendor_code}: {e}", exc_info=True)
+            logger.error(f"Error updating vendor cache for {vendor_code}: {e}", exc_info=True)
             
             try:
                 with self.get_db_connection() as conn:
@@ -559,6 +611,7 @@ class CVEHandler:
             
             return False
 
+
     def get_vendor_cves(self, vendor_code: str, limit: int = None, offset: int = 0,
                        severity: str = None, year: int = None) -> List[Dict]:
         with self.get_db_connection() as conn:

scheduler.py

@@ -2,8 +2,8 @@
 
 import time
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
-from cve_handler import update_all_vendors
+from cve_handler import CVEHandler
 import config
 
 logging.basicConfig(
@@ -12,29 +12,153 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-def run_scheduler():
-    logger.info("CVE Update Scheduler started")
-    logger.info(f"Update interval: {config.UPDATE_INTERVAL_HOURS} hours")
-    logger.info(f"Auto-update enabled: {config.ENABLE_AUTO_UPDATE}")
 
+def incremental_update():
+
+    logger.info("=" * 80)
+    logger.info(f"Starting incremental update at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    logger.info("=" * 80)
+    
+    try:
+        handler = CVEHandler()
+        days_back = getattr(config, 'SCHEDULER_UPDATE_DAYS', 7)
+        
+        updated_count = 0
+        failed_count = 0
+        total_new_cves = 0
+        
+        for vendor in config.VENDORS:
+            if not vendor.get('use_cpe'):
+                logger.debug(f"Skipping {vendor['name']} (CPE disabled)")
+                continue
+            
+            try:
+                logger.info(f"Updating {vendor['name']}...")
+                
+                new_cves = handler.fetch_cve_from_nvd(
+                    keywords=vendor.get('keywords', []),
+                    vendor_config=vendor,
+                    days_back=days_back
+                )
+                
+                if new_cves:
+                    handler.save_cves_to_cache(new_cves, vendor['code'])
+                    logger.info(f"  ✓ {vendor['name']}: Added {len(new_cves)} new CVEs")
+                    total_new_cves += len(new_cves)
+                    updated_count += 1
+                else:
+                    logger.info(f"  ✓ {vendor['name']}: No new CVEs")
+                    updated_count += 1
+                
+            except Exception as e:
+                logger.error(f"  ✗ {vendor['name']}: {e}")
+                failed_count += 1
+        
+        logger.info("=" * 80)
+        logger.info(f"✓ Incremental update completed")
+        logger.info(f"  Vendors updated: {updated_count}")
+        logger.info(f"  Vendors failed: {failed_count}")
+        logger.info(f"  New CVEs added: {total_new_cves}")
+        logger.info("=" * 80)
+        
+        return updated_count, failed_count
+        
+    except Exception as e:
+        logger.error(f"✗ Incremental update failed: {e}", exc_info=True)
+        return 0, len(config.VENDORS)
+
+
+def initial_full_scan():
+
+    logger.info("=" * 80)
+    logger.info("INITIAL FULL SCAN")
+    logger.info("=" * 80)
+    logger.info(f"This will scan {config.INITIAL_LOOKBACK_DAYS} days back for all vendors")
+    
+    try:
+        from cve_handler import update_all_vendors
+        updated, failed = update_all_vendors(force=True)
+        
+        logger.info("=" * 80)
+        logger.info(f"✓ Initial scan completed: {updated} updated, {failed} failed")
+        logger.info("=" * 80)
+        
+        return updated, failed
+        
+    except Exception as e:
+        logger.error(f"✗ Initial scan failed: {e}", exc_info=True)
+        return 0, len(config.VENDORS)
+
+
+def run_scheduler():
+    logger.info("=" * 80)
+    logger.info("CVE Monitor Scheduler Starting")
+    logger.info("=" * 80)
+    
+    update_interval_hours = config.UPDATE_INTERVAL_HOURS
+    scheduler_update_days = getattr(config, 'SCHEDULER_UPDATE_DAYS', 7)
+    
+    logger.info(f"Configuration:")
+    logger.info(f"  Update interval: {update_interval_hours} hours")
+    logger.info(f"  Incremental lookback: {scheduler_update_days} days")
+    logger.info(f"  Initial lookback: {config.INITIAL_LOOKBACK_DAYS} days")
+    logger.info(f"  Auto-update enabled: {config.ENABLE_AUTO_UPDATE}")
+    
+    if config.ENABLE_AUTO_UPDATE:
+        logger.info("\nChecking database status...")
+        
+        try:
+            handler = CVEHandler()
+            
+            with handler.get_db_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT COUNT(*) FROM cve_cache")
+                total_cves = cursor.fetchone()[0]
+            
+            if total_cves == 0:
+                logger.info(f"  Database is EMPTY")
+                logger.info(f"Starting initial full scan ({config.INITIAL_LOOKBACK_DAYS} days)...")
+                logger.info(f"This may take 30-60 minutes depending on API rate limits\n")
+                
+                initial_full_scan()
+                
+            else:
+                logger.info(f"✓ Database has {total_cves} CVEs")
+                logger.info(f"Skipping initial scan, starting incremental updates\n")
+                logger.info("Performing startup incremental update...")
+                incremental_update()
+                
+        except Exception as e:
+            logger.error(f"Error checking database: {e}", exc_info=True)
+    
+    logger.info(f"\nWaiting 2 minutes before starting scheduled updates...")
     time.sleep(120)
     
     while True:
         if config.ENABLE_AUTO_UPDATE:
             try:
-                logger.info(f"Starting scheduled update at {datetime.now()}")
+                updated, failed = incremental_update()
-                updated, failed = update_all_vendors()
+                
-                logger.info(f"Update completed: {updated} successful, {failed} failed")
             except Exception as e:
                 logger.error(f"Error in scheduled update: {e}", exc_info=True)
         else:
             logger.debug("Auto-update is disabled, skipping")
         
-        sleep_seconds = config.UPDATE_INTERVAL_HOURS * 3600
+        sleep_seconds = update_interval_hours * 3600
-        next_update = datetime.now().timestamp() + sleep_seconds
+        next_update = datetime.now() + timedelta(seconds=sleep_seconds)
-        logger.info(f"Next update scheduled at {datetime.fromtimestamp(next_update)}")
+        
+        logger.info(f"\n{'=' * 80}")
+        logger.info(f"Next incremental update at: {next_update.strftime('%Y-%m-%d %H:%M:%S')}")
+        logger.info(f"Sleeping for {update_interval_hours} hours...")
+        logger.info(f"{'=' * 80}\n")
+        
         time.sleep(sleep_seconds)
 
 
 if __name__ == '__main__':
-    run_scheduler()
+    try:
+        run_scheduler()
+    except KeyboardInterrupt:
+        logger.info("\n\nScheduler stopped by user")
+    except Exception as e:
+        logger.error(f"Scheduler crashed: {e}", exc_info=True)