250 lines
9.6 KiB
Python
250 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
|
|
from app.core.config import settings as app_settings
|
|
from app.core.cron_utils import CronValidationError, describe_cron_expression, parse_cron_expression, preview_next_runs
|
|
from app.db.session import SessionLocal
|
|
from app.models.router import Router
|
|
from app.services.backup_service import backup_service
|
|
from app.services.log_service import log_service
|
|
from app.services.router_service import router_service
|
|
from app.services.settings_service import settings_service
|
|
|
|
|
|
class SchedulerService:
|
|
def __init__(self):
|
|
self.scheduler = BackgroundScheduler(timezone=app_settings.timezone)
|
|
self.started = False
|
|
|
|
def start(self):
|
|
if self.started:
|
|
return
|
|
self.reschedule()
|
|
self.scheduler.start()
|
|
self.started = True
|
|
|
|
def shutdown(self):
|
|
if self.started:
|
|
self.scheduler.shutdown(wait=False)
|
|
self.started = False
|
|
|
|
def _parse_cron(self, expr: str):
|
|
return parse_cron_expression(expr, app_settings.timezone)
|
|
|
|
def validate_cron(self, expr: str):
|
|
return self._parse_cron(expr)
|
|
|
|
def _interval_next_runs(self, minutes: int, count: int = 3):
|
|
now = datetime.now()
|
|
return [now + timedelta(minutes=minutes * index) for index in range(1, count + 1)]
|
|
|
|
def scheduler_status(self):
|
|
with SessionLocal() as db:
|
|
settings = settings_service.get_or_create(db)
|
|
return {
|
|
'timezone': app_settings.timezone,
|
|
'running': self.started,
|
|
'jobs': [
|
|
self._describe_job(
|
|
key='auto_export',
|
|
label='settings.schedulerAutoExportLabel',
|
|
enabled=settings.enable_auto_export,
|
|
cron=settings.export_cron,
|
|
),
|
|
self._describe_job(
|
|
key='auto_binary',
|
|
label='settings.schedulerBinaryLabel',
|
|
enabled=bool(settings.binary_cron),
|
|
cron=settings.binary_cron,
|
|
),
|
|
self._describe_job(
|
|
key='retention',
|
|
label='settings.schedulerRetentionLabel',
|
|
enabled=bool(settings.retention_cron),
|
|
cron=settings.retention_cron,
|
|
),
|
|
self._describe_interval_job(
|
|
key='connection_probe',
|
|
label='settings.schedulerConnectionLabel',
|
|
minutes=settings.connection_test_interval_minutes,
|
|
),
|
|
{
|
|
'key': 'log_cleanup',
|
|
'label': 'settings.schedulerLogsLabel',
|
|
'enabled': True,
|
|
'cron': None,
|
|
'description': 'settings.schedulerLogsDescription',
|
|
'description_params': None,
|
|
'valid': True,
|
|
'next_runs': [],
|
|
'error': None,
|
|
},
|
|
],
|
|
}
|
|
|
|
def _describe_job(self, key: str, label: str, enabled: bool, cron: str | None):
|
|
cron = (cron or '').strip()
|
|
if not enabled or not cron:
|
|
return {
|
|
'key': key,
|
|
'label': label,
|
|
'enabled': False,
|
|
'cron': cron or None,
|
|
'description': 'settings.scheduleDisabledHint',
|
|
'description_params': None,
|
|
'valid': True,
|
|
'next_runs': [],
|
|
'error': None,
|
|
}
|
|
try:
|
|
next_runs = preview_next_runs(cron, app_settings.timezone, count=3)
|
|
return {
|
|
'key': key,
|
|
'label': label,
|
|
'enabled': True,
|
|
'cron': cron,
|
|
'description': 'settings.schedulerCronDescription',
|
|
'description_params': {'description': describe_cron_expression(cron)},
|
|
'valid': True,
|
|
'next_runs': next_runs,
|
|
'error': None,
|
|
}
|
|
except CronValidationError as exc:
|
|
return {
|
|
'key': key,
|
|
'label': label,
|
|
'enabled': True,
|
|
'cron': cron,
|
|
'description': 'settings.schedulerInvalidCron',
|
|
'description_params': None,
|
|
'valid': False,
|
|
'next_runs': [],
|
|
'error': str(exc),
|
|
}
|
|
|
|
def _describe_interval_job(self, key: str, label: str, minutes: int):
|
|
minutes = int(minutes or 0)
|
|
if minutes <= 0:
|
|
return {
|
|
'key': key,
|
|
'label': label,
|
|
'enabled': False,
|
|
'cron': None,
|
|
'description': 'settings.connectionTestsDisabledHint',
|
|
'description_params': None,
|
|
'valid': True,
|
|
'next_runs': [],
|
|
'error': None,
|
|
}
|
|
return {
|
|
'key': key,
|
|
'label': label,
|
|
'enabled': True,
|
|
'cron': None,
|
|
'description': 'settings.connectionTestsEverySummary',
|
|
'description_params': {'minutes': minutes},
|
|
'valid': True,
|
|
'next_runs': self._interval_next_runs(minutes),
|
|
'error': None,
|
|
}
|
|
|
|
def reschedule(self):
|
|
self.scheduler.remove_all_jobs()
|
|
with SessionLocal() as db:
|
|
settings = settings_service.get_or_create(db)
|
|
job_definitions = [
|
|
('auto_export', settings.enable_auto_export, settings.export_cron, self._run_auto_export, 'auto export'),
|
|
('auto_binary', bool(settings.binary_cron), settings.binary_cron, self._run_binary_backup, 'binary backup'),
|
|
('retention', bool(settings.retention_cron), settings.retention_cron, self._run_retention, 'retention cleanup'),
|
|
]
|
|
|
|
pending_logs: list[str] = []
|
|
for job_id, enabled, cron, callback, label in job_definitions:
|
|
cron = (cron or '').strip()
|
|
if not enabled or not cron:
|
|
continue
|
|
try:
|
|
trigger = self._parse_cron(cron)
|
|
self.scheduler.add_job(
|
|
callback,
|
|
trigger=trigger,
|
|
id=job_id,
|
|
replace_existing=True,
|
|
coalesce=True,
|
|
max_instances=1,
|
|
misfire_grace_time=300,
|
|
)
|
|
except Exception as exc:
|
|
pending_logs.append(f'Scheduler skipped invalid {label} cron ({cron}): {exc}')
|
|
|
|
if int(settings.connection_test_interval_minutes or 0) > 0:
|
|
self.scheduler.add_job(
|
|
self._run_connection_probes,
|
|
trigger='interval',
|
|
minutes=int(settings.connection_test_interval_minutes),
|
|
id='connection_probe',
|
|
replace_existing=True,
|
|
coalesce=True,
|
|
max_instances=1,
|
|
misfire_grace_time=300,
|
|
)
|
|
|
|
self.scheduler.add_job(
|
|
self._run_log_cleanup,
|
|
trigger='interval',
|
|
days=1,
|
|
id='log_cleanup',
|
|
replace_existing=True,
|
|
coalesce=True,
|
|
max_instances=1,
|
|
misfire_grace_time=300,
|
|
)
|
|
|
|
for message in pending_logs:
|
|
log_service.add(db, message, commit=False)
|
|
if pending_logs:
|
|
db.commit()
|
|
|
|
def _run_auto_export(self):
|
|
with SessionLocal() as db:
|
|
routers = db.query(Router).all()
|
|
for router in routers:
|
|
try:
|
|
backup_service.export_router(db, type('U', (), {'id': router.owner_id})(), router.id)
|
|
except Exception as exc:
|
|
log_service.add(db, f'Scheduled export failed for {router.name}: {exc}')
|
|
|
|
def _run_binary_backup(self):
|
|
with SessionLocal() as db:
|
|
routers = db.query(Router).all()
|
|
for router in routers:
|
|
try:
|
|
backup_service.binary_backup(db, type('U', (), {'id': router.owner_id})(), router.id)
|
|
except Exception as exc:
|
|
log_service.add(db, f'Scheduled binary backup failed for {router.name}: {exc}')
|
|
|
|
def _run_retention(self):
|
|
with SessionLocal() as db:
|
|
backup_service.cleanup_old_backups(db)
|
|
|
|
def _run_connection_probes(self):
|
|
with SessionLocal() as db:
|
|
settings = settings_service.get_or_create(db)
|
|
routers = db.query(Router).all()
|
|
for router in routers:
|
|
result = router_service.test_connection(db, router, settings.global_ssh_key)
|
|
if not result['success']:
|
|
log_service.add(db, f'Scheduled connection test failed for {router.name}: {result.get("error") or "Unknown error"}')
|
|
|
|
def _run_log_cleanup(self):
|
|
with SessionLocal() as db:
|
|
settings = settings_service.get_or_create(db)
|
|
deleted = log_service.delete_older_than(db, settings.log_retention_days)
|
|
log_service.add(db, f'Log retention cleanup removed {deleted} entries older than {settings.log_retention_days} days')
|
|
|
|
|
|
scheduler_service = SchedulerService()
|