789 lines
39 KiB
Python
789 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""Audyt, wyszukiwanie wpisow i bezpieczna optymalizacja obrazow Drupal 6.
|
|
|
|
Tryb domyslny jest wylacznie do odczytu. Zapisy sa mozliwe tylko z --apply.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from urllib.parse import quote
|
|
|
|
from PIL import Image
|
|
|
|
from drupal_lookup import DrupalReferenceLookup, node_url, search_url
|
|
from image_optimizer import create_optimized_candidate
|
|
from image_safety import DEFAULT_MAX_IMAGE_PIXELS, safe_image_open
|
|
|
|
|
|
def build_parser():
|
|
parser = argparse.ArgumentParser(
|
|
description='Drupal 6: duze pliki, linki do tematow/komentarzy, bezpieczna optymalizacja',
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
)
|
|
analysis = parser.add_argument_group('Analiza i filtrowanie')
|
|
analysis.add_argument('--top', type=int, default=20, help='Ile najwiekszych plikow wyswietlic')
|
|
analysis.add_argument('--extensions', default='jpg,jpeg,png,gif', help='Rozszerzenia do analizy')
|
|
analysis.add_argument('--exclude-folders', nargs='*', default=[], help='Podfoldery pomijane w files/')
|
|
analysis.add_argument('--path-contains', default='', help='Filtr fragmentu sciezki, np. skan')
|
|
analysis.add_argument('--fid', type=int, action='append', help='Analizuj tylko podane fid (mozna powtorzyc)')
|
|
analysis.add_argument('--min-size-mb', type=float, default=0, help='Pokaz pliki od wskazanej liczby MB')
|
|
analysis.add_argument('--find-duplicates', action='store_true', help='Sprawdz duplikaty po SHA-256')
|
|
analysis.add_argument('--show-deleted', action='store_true', help='Pokaz pliki na dysku spoza tabeli files')
|
|
analysis.add_argument('--debug', action='store_true', help='Szczegolowy log')
|
|
|
|
lookup = parser.add_argument_group('Wskazanie tematu / posta / komentarza')
|
|
lookup.add_argument('--find-usage', '--where-used', dest='find_usage', action='store_true',
|
|
help='Szukaj powiazan plikow z wezłami i komentarzami')
|
|
lookup.add_argument('--usage-scope', choices=('top', 'optimizable', 'all'), default='top',
|
|
help='W ktorych plikach szukac powiazan')
|
|
lookup.add_argument('--site-url', help='Adres Drupal, np. https://forum.example.org')
|
|
lookup.add_argument('--no-clean-urls', action='store_true', help='Linki w formacie ?q=node/ID')
|
|
lookup.add_argument('--comment-permalinks', action='store_true',
|
|
help='Linki comment/CID (TYLKO z zainstalowanym backportem Permalink w Drupal 6)')
|
|
lookup.add_argument('--no-text-scan', action='store_true', help='Nie szukaj w tresci wpisow/komentarzy')
|
|
lookup.add_argument('--scan-cck-text', action='store_true', help='Szukaj takze w polach tekstowych CCK (wolniejsze)')
|
|
lookup.add_argument('--no-basename-match', action='store_true', help='W tekscie wymagaj sciezki, nie samej nazwy')
|
|
lookup.add_argument('--table-prefix', help='Prefix tabel Drupala, np. drupal_')
|
|
lookup.add_argument('--public-files-path', help='Sciezka publiczna w Drupal, np. sites/default/files')
|
|
lookup.add_argument('--drupal-root', help='Katalog glowny instalacji Drupal (pomaga rozwiazac sciezki)')
|
|
|
|
change = parser.add_argument_group('Optymalizacja i kontrola zapisow')
|
|
change.add_argument('--optimize', action='store_true', help='Zaplanuj optymalizacje lub wykonaj ja z --apply')
|
|
change.add_argument('--update-db', action='store_true', help='Wyrownaj pole files.filesize (z --apply)')
|
|
mode = change.add_mutually_exclusive_group()
|
|
mode.add_argument('--apply', action='store_true', help='Zezwol na realny zapis obrazow/DB')
|
|
mode.add_argument('--dry-run', action='store_true', help='Jawny tryb podgladu (domyslny)')
|
|
change.add_argument('--backup-dir', help='Katalog dla kopii oryginalow (wymagany z --optimize --apply)')
|
|
change.add_argument('--max-changes', type=int, default=0, help='Maksymalna liczba realnie optymalizowanych plikow (0=bez limitu)')
|
|
change.add_argument('--optimize-top', type=int, default=0, help='Optymalizuj najwyzej N najwiekszych kandydatow (0=wszystkie)')
|
|
change.add_argument('--max-width', type=int, default=2400, help='Maksymalna szerokosc po optymalizacji')
|
|
change.add_argument('--max-height', type=int, default=3200, help='Maksymalna wysokosc po optymalizacji')
|
|
change.add_argument('--max-file-mb', type=float, default=2.0, help='Docelowy limit MB; nie kosztem duzej utraty jakosci')
|
|
change.add_argument('--jpeg-quality', type=int, default=92, help='Poczatkowa jakosc JPEG')
|
|
change.add_argument('--min-jpeg-quality', type=int, default=85, help='Minimalna dopuszczalna jakosc JPEG')
|
|
change.add_argument('--min-gain-percent', type=float, default=5.0, help='Minimalne zmniejszenie objetosci w procentach')
|
|
change.add_argument('--max-image-mpixels', type=float, default=DEFAULT_MAX_IMAGE_PIXELS / 1_000_000,
|
|
help='Limit megapikseli oryginalnego obrazu; powyzej tego progu plik bedzie pominiety (ochrona RAM)')
|
|
|
|
report = parser.add_argument_group('Raporty')
|
|
report.add_argument('--report-csv', help='CSV z plikami i linkami (UTF-8 BOM, otwiera sie w Excelu)')
|
|
report.add_argument('--report-json', help='Raport JSON z lokalizacjami, rozmiarami i statusami')
|
|
return parser
|
|
|
|
|
|
def validate_args(args, parser):
|
|
if args.top <= 0 or args.min_size_mb < 0 or args.max_file_mb <= 0:
|
|
parser.error('--top > 0, --min-size-mb >= 0 i --max-file-mb > 0 sa wymagane')
|
|
if min(args.max_width, args.max_height) <= 0:
|
|
parser.error('--max-width i --max-height musza byc dodatnie')
|
|
if not (1 <= args.min_jpeg_quality <= args.jpeg_quality <= 95):
|
|
parser.error('Jakosc JPEG: 1 <= --min-jpeg-quality <= --jpeg-quality <= 95')
|
|
if not (0 <= args.min_gain_percent <= 100):
|
|
parser.error('--min-gain-percent musi byc w przedziale 0..100')
|
|
if args.max_changes < 0 or args.optimize_top < 0:
|
|
parser.error('--max-changes i --optimize-top nie moga byc ujemne')
|
|
if not (0 < args.max_image_mpixels <= 500):
|
|
parser.error('--max-image-mpixels musi byc w zakresie (0, 500]')
|
|
if args.apply and not (args.optimize or args.update_db):
|
|
parser.error('--apply wymaga --optimize lub --update-db')
|
|
if args.apply and args.optimize and not args.backup_dir:
|
|
parser.error('Dla --optimize --apply podaj --backup-dir poza folderem z plikami')
|
|
if args.scan_cck_text and args.no_text_scan:
|
|
parser.error('--scan-cck-text wyklucza sie z --no-text-scan')
|
|
if args.site_url and not re.match(r'^https?://[^/]+', args.site_url, flags=re.I):
|
|
parser.error('--site-url musi byc adresem https://... lub http://...')
|
|
if args.table_prefix and not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', args.table_prefix):
|
|
parser.error('Niedozwolony prefix tabel')
|
|
|
|
|
|
def load_config():
|
|
try:
|
|
config = importlib.import_module('config')
|
|
except ModuleNotFoundError as exc:
|
|
if exc.name == 'config':
|
|
raise RuntimeError('Brak config.py - skopiuj config.example.py do config.py i ustaw dane') from exc
|
|
raise
|
|
if not hasattr(config, 'DB_CONFIG') or not hasattr(config, 'FILES_BASE_PATH'):
|
|
raise RuntimeError('config.py musi definiowac DB_CONFIG oraz FILES_BASE_PATH')
|
|
return config
|
|
|
|
|
|
def get_connection(config):
|
|
import pymysql
|
|
db = dict(config.DB_CONFIG)
|
|
connection_kwargs = {
|
|
'host': db['host'],
|
|
'user': db['user'],
|
|
'password': db['password'],
|
|
'database': db['database'],
|
|
'charset': db.get('charset', 'utf8'),
|
|
'port': int(db.get('port', 3306)),
|
|
'cursorclass': pymysql.cursors.DictCursor,
|
|
'autocommit': False,
|
|
'connect_timeout': 15,
|
|
}
|
|
for optional in ('unix_socket', 'ssl'):
|
|
if optional in db:
|
|
connection_kwargs[optional] = db[optional]
|
|
return pymysql.connect(**connection_kwargs)
|
|
|
|
|
|
def _within(path, root):
|
|
path, root = os.path.realpath(path), os.path.realpath(root)
|
|
try:
|
|
return os.path.commonpath([path, root]) == root
|
|
except ValueError:
|
|
return False
|
|
|
|
|
|
def resolve_filepath(db_filepath, base_path, public_path='sites/default/files', drupal_root=None):
|
|
"""Rozpoznaj Drupal filepath: relative/to/files albo sites/default/files/x.jpg.
|
|
|
|
NIGDY nie zezwalaj na wyjscie poza FILES_BASE_PATH (takze przez symlink).
|
|
Zwraca (full_path, relative_to_files) lub (None, None).
|
|
"""
|
|
if not isinstance(db_filepath, str) or '\x00' in db_filepath or '://' in db_filepath:
|
|
return None, None
|
|
db_filepath = db_filepath.replace('\\', '/').strip()
|
|
base = os.path.realpath(base_path)
|
|
public = (public_path or '').replace('\\', '/').strip('/')
|
|
if not db_filepath:
|
|
return None, None
|
|
|
|
candidates = []
|
|
if os.path.isabs(db_filepath):
|
|
if _within(db_filepath, base):
|
|
candidates.append(db_filepath)
|
|
elif not (public and db_filepath.lstrip('/').startswith(public + '/')) and not (
|
|
db_filepath.lstrip('/').startswith(os.path.basename(base) + '/')
|
|
):
|
|
# /sites/default/files/x jest sciezka URL; /etc/passwd nie.
|
|
return None, None
|
|
|
|
relative = db_filepath.lstrip('/')
|
|
if '..' in relative.split('/'):
|
|
return None, None
|
|
normalized = posixpath.normpath(relative)
|
|
if normalized in ('', '.', '..'):
|
|
return None, None
|
|
if public and normalized.startswith(public + '/'):
|
|
candidates.append(os.path.join(base, normalized[len(public) + 1:]))
|
|
# Czeste przy przenoszeniu Drupala: config podaje katalog "files",
|
|
# a DB przechowuje "files/name.jpg".
|
|
base_folder = os.path.basename(base)
|
|
if normalized.startswith(base_folder + '/'):
|
|
candidates.append(os.path.join(base, normalized[len(base_folder) + 1:]))
|
|
if drupal_root:
|
|
candidates.append(os.path.join(drupal_root, normalized))
|
|
candidates.append(os.path.join(base, normalized))
|
|
|
|
# Zachowuj oryginalna sciezke: realpath ukrylby symlink i mogl ominac
|
|
# zabezpieczenie blokujace jego modyfikacje. _within sprawdza realpath.
|
|
allowed = [os.path.abspath(c) for c in candidates if _within(c, base)]
|
|
if not allowed:
|
|
return None, None
|
|
found = next((path for path in allowed if os.path.isfile(path)), allowed[0])
|
|
return found, os.path.relpath(found, base).replace(os.sep, '/')
|
|
|
|
|
|
def sha256_file(path):
|
|
digest = hashlib.sha256()
|
|
with open(path, 'rb') as src:
|
|
for chunk in iter(lambda: src.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def file_resolution(path, *, max_pixels=DEFAULT_MAX_IMAGE_PIXELS):
|
|
try:
|
|
# W metadanych mozemy odczytac wymiary nawet wtedy, gdy uzytkownik
|
|
# ustawi niski prog optymalizacji. Nie dekodujemy pikseli.
|
|
with safe_image_open(path, max_pixels=max(max_pixels, DEFAULT_MAX_IMAGE_PIXELS),
|
|
check_pixels=False) as im:
|
|
return im.size
|
|
except Exception:
|
|
return (0, 0)
|
|
|
|
|
|
def _excluded(rel_path, folders):
|
|
rel_path = rel_path.replace('\\', '/').lstrip('/')
|
|
for folder in folders:
|
|
item = folder.replace('\\', '/').strip('/')
|
|
if item and (rel_path == item or rel_path.startswith(item + '/')):
|
|
return True
|
|
return False
|
|
|
|
|
|
def collect_images(connection, config, args):
|
|
base = os.path.realpath(config.FILES_BASE_PATH)
|
|
table_prefix = args.table_prefix if args.table_prefix is not None else getattr(config, 'DB_TABLE_PREFIX', '')
|
|
if table_prefix and not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', table_prefix):
|
|
raise ValueError('Nieprawidlowy prefix tabel w config.py')
|
|
tables_files = f'`{table_prefix}files`'
|
|
with connection.cursor() as cursor:
|
|
cursor.execute(f'SELECT fid, filepath, filesize FROM {tables_files}')
|
|
file_rows = cursor.fetchall()
|
|
|
|
public = args.public_files_path if args.public_files_path is not None else getattr(config, 'PUBLIC_FILES_PATH', 'sites/default/files')
|
|
root = args.drupal_root if args.drupal_root is not None else getattr(config, 'DRUPAL_ROOT', None)
|
|
extensions = {'.' + x.strip().lower().lstrip('.') for x in args.extensions.split(',') if x.strip()}
|
|
groups = {}
|
|
missing = []
|
|
for row in file_rows:
|
|
fid = int(row['fid'])
|
|
if args.fid and fid not in args.fid:
|
|
continue
|
|
db_path = str(row['filepath'] or '')
|
|
if not db_path.lower().endswith(tuple(extensions)):
|
|
continue
|
|
if args.path_contains and args.path_contains.casefold() not in db_path.casefold():
|
|
continue
|
|
full_path, rel_path = resolve_filepath(db_path, base, public, root)
|
|
if not full_path:
|
|
missing.append({'fid': fid, 'db_filepath': db_path, 'reason': 'niebezpieczna lub nierozpoznana sciezka'})
|
|
continue
|
|
if _excluded(rel_path, args.exclude_folders):
|
|
continue
|
|
if not os.path.isfile(full_path):
|
|
missing.append({'fid': fid, 'db_filepath': db_path, 'path': full_path, 'reason': 'brak na dysku'})
|
|
continue
|
|
stat_result = os.stat(full_path)
|
|
if stat_result.st_size < args.min_size_mb * 1024 * 1024:
|
|
continue
|
|
if full_path not in groups:
|
|
width, height = file_resolution(full_path, max_pixels=int(args.max_image_mpixels * 1_000_000))
|
|
groups[full_path] = {
|
|
'path': full_path,
|
|
'rel_path': rel_path,
|
|
'actual_size': stat_result.st_size,
|
|
'before_size': stat_result.st_size,
|
|
'original_ino': stat_result.st_ino,
|
|
'original_mtime_ns': stat_result.st_mtime_ns,
|
|
'after_size': stat_result.st_size,
|
|
'width': width, 'height': height,
|
|
'db_rows': [], 'fids': [], 'references': [],
|
|
'optimize_status': 'not_requested', 'optimization_reason': '',
|
|
'quality': None,
|
|
}
|
|
img = groups[full_path]
|
|
img['db_rows'].append({'fid': fid, 'db_filepath': db_path, 'filesize': int(row['filesize'] or 0)})
|
|
img['fids'].append(fid)
|
|
|
|
images = list(groups.values())
|
|
for image in images:
|
|
image['fid'] = image['fids'][0]
|
|
image['db_filepath'] = image['db_rows'][0]['db_filepath']
|
|
image['db_size'] = image['db_rows'][0]['filesize']
|
|
images.sort(key=lambda img: img['before_size'], reverse=True)
|
|
return images, missing, table_prefix
|
|
|
|
|
|
def is_candidate(image, args):
|
|
return (
|
|
not image['width'] or not image['height'] or
|
|
image['width'] > args.max_width or
|
|
image['height'] > args.max_height or
|
|
image['before_size'] > args.max_file_mb * 1024 * 1024
|
|
)
|
|
|
|
|
|
def _update_sizes(connection, table_prefix, image, filesize):
|
|
table = f'`{table_prefix}files`'
|
|
for db_row in image['db_rows']:
|
|
fid = db_row['fid']
|
|
db_path = db_row['db_filepath']
|
|
with connection.cursor() as cursor:
|
|
cursor.execute(
|
|
f'UPDATE {table} SET filesize = %s WHERE fid = %s AND filepath = %s',
|
|
(int(filesize), fid, db_path),
|
|
)
|
|
# Dla rowcount=0 sprawdzamy, czy row nadal istnieje i ma oczekiwany rozmiar.
|
|
if cursor.rowcount == 0:
|
|
cursor.execute(f'SELECT filesize, filepath FROM {table} WHERE fid = %s', (fid,))
|
|
row = cursor.fetchone()
|
|
if not row or row['filepath'] != db_path or int(row['filesize']) != int(filesize):
|
|
raise RuntimeError(f'Nie udalo sie zsynchronizowac files.fid={fid}')
|
|
|
|
|
|
def _restore_database_sizes(connection, table_prefix, image):
|
|
# Kompensacja takze dla starego MySQL/MyISAM (ROLLBACK moze nie dzialac).
|
|
table = f'`{table_prefix}files`'
|
|
with connection.cursor() as cursor:
|
|
for row in image['db_rows']:
|
|
cursor.execute(
|
|
f'UPDATE {table} SET filesize = %s WHERE fid = %s AND filepath = %s',
|
|
(row['filesize'], row['fid'], row['db_filepath'])
|
|
)
|
|
connection.commit()
|
|
|
|
|
|
def _atomic_restore(backup_file, destination):
|
|
fd, restore_tmp = tempfile.mkstemp(prefix='.d6-restore-', dir=os.path.dirname(destination))
|
|
os.close(fd)
|
|
try:
|
|
shutil.copy2(backup_file, restore_tmp)
|
|
os.replace(restore_tmp, destination)
|
|
finally:
|
|
if os.path.exists(restore_tmp):
|
|
os.unlink(restore_tmp)
|
|
|
|
|
|
def _safe_backup_dir(requested, base, drupal_root=None):
|
|
backup_root = os.path.realpath(os.path.abspath(requested))
|
|
if _within(backup_root, base):
|
|
raise ValueError('Backup nie moze znajdowac sie wewnatrz FILES_BASE_PATH')
|
|
if drupal_root and _within(backup_root, drupal_root):
|
|
raise ValueError('Backup nie moze znajdowac sie wewnatrz katalogu WWW Drupala')
|
|
os.makedirs(backup_root, exist_ok=True)
|
|
run_id = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ') + f'-{os.getpid()}'
|
|
run_dir = os.path.join(backup_root, run_id)
|
|
os.mkdir(run_dir)
|
|
return run_dir
|
|
|
|
|
|
def apply_candidate(connection, table_prefix, image, candidate, *, backup_run_dir, base_dir):
|
|
"""Backup + atomowe podmienienie pliku + SQL (rollback i restore przy bledzie)."""
|
|
source = image['path']
|
|
tmp = candidate['temp_path']
|
|
if not tmp:
|
|
raise RuntimeError('Brak pliku kandydata')
|
|
stat_before = os.stat(source)
|
|
if os.path.islink(source) or stat_before.st_nlink != 1:
|
|
raise RuntimeError('Plik jest dowiazaniem symbolicznym/twardym; pomijam')
|
|
if (stat_before.st_size != image['before_size'] or
|
|
stat_before.st_ino != image['original_ino'] or
|
|
stat_before.st_mtime_ns != image['original_mtime_ns']):
|
|
raise RuntimeError('Plik zostal zmieniony od poczatku skanowania')
|
|
|
|
# Kopia w zewnetrznym folderze z zachowaniem struktury i uprawnien.
|
|
relative_path = os.path.relpath(source, base_dir)
|
|
backup_file = os.path.join(backup_run_dir, relative_path)
|
|
os.makedirs(os.path.dirname(backup_file), exist_ok=True)
|
|
if os.path.lexists(backup_file):
|
|
raise RuntimeError(f'Kopia zapasowa juz istnieje: {backup_file}')
|
|
shutil.copy2(source, backup_file)
|
|
if os.path.getsize(backup_file) != stat_before.st_size:
|
|
raise RuntimeError('Nieudane kopiowanie backupu (niezgodny rozmiar)')
|
|
|
|
# Zachowanie uprawnien, czasu oraz wlasciciela jesli jest mozliwosc.
|
|
shutil.copystat(source, tmp)
|
|
if hasattr(os, 'chown'):
|
|
try:
|
|
os.chown(tmp, stat_before.st_uid, stat_before.st_gid)
|
|
except PermissionError:
|
|
# Uruchomienie z uprawnieniami wlasciciela daje i tak ten sam uid.
|
|
if os.stat(tmp).st_uid != stat_before.st_uid:
|
|
raise
|
|
|
|
stat_now = os.stat(source)
|
|
if (stat_now.st_ino, stat_now.st_size, stat_now.st_mtime_ns) != (
|
|
stat_before.st_ino, stat_before.st_size, stat_before.st_mtime_ns
|
|
):
|
|
raise RuntimeError('Plik zostal zmodyfikowany rownolegle; przerwano zapis')
|
|
|
|
os.replace(tmp, source) # w obrebie tego samego filesystemu
|
|
candidate['temp_path'] = None
|
|
try:
|
|
_update_sizes(connection, table_prefix, image, candidate['after'])
|
|
connection.commit()
|
|
except Exception as exc:
|
|
connection.rollback()
|
|
try:
|
|
_atomic_restore(backup_file, source)
|
|
except Exception as restore_error:
|
|
raise RuntimeError(
|
|
f'Blad DB ({exc}); AUTO-RESTORE NIEUDANY ({restore_error}), kopia: {backup_file}'
|
|
) from exc
|
|
try:
|
|
_restore_database_sizes(connection, table_prefix, image)
|
|
except Exception as restore_db_error:
|
|
raise RuntimeError(
|
|
f'Plik przywrocony z backupu, ale nie udalo sie odtworzyc rozmiarow w DB: '
|
|
f'{restore_db_error}; kopia: {backup_file}'
|
|
) from exc
|
|
raise RuntimeError(f'Blad DB, przywrocono plik i rozmiary w bazie: {exc}') from exc
|
|
return backup_file
|
|
|
|
|
|
def process_changes(connection, config, args, images, table_prefix):
|
|
summary = {'candidates': 0, 'planned': 0, 'optimized': 0, 'saved_bytes': 0,
|
|
'db_mismatches': 0, 'db_updated': 0, 'errors': 0, 'backups_dir': ''}
|
|
base = os.path.realpath(config.FILES_BASE_PATH)
|
|
if args.apply and args.optimize:
|
|
root = args.drupal_root if args.drupal_root is not None else getattr(config, 'DRUPAL_ROOT', None)
|
|
summary['backups_dir'] = _safe_backup_dir(args.backup_dir, base, root)
|
|
print(f'[BACKUP] Kopie oryginalow: {summary["backups_dir"]}')
|
|
|
|
candidates = [img for img in images if is_candidate(img, args)]
|
|
summary['candidates'] = len(candidates)
|
|
if args.optimize_top:
|
|
candidates = candidates[:args.optimize_top]
|
|
candidate_ids = {img['path'] for img in candidates}
|
|
max_bytes = int(args.max_file_mb * 1024 * 1024)
|
|
|
|
for image in images:
|
|
mismatched_rows = [db for db in image['db_rows'] if db['filesize'] != image['before_size']]
|
|
summary['db_mismatches'] += len(mismatched_rows)
|
|
if not args.optimize or image['path'] not in candidate_ids:
|
|
if args.update_db and mismatched_rows:
|
|
if args.apply:
|
|
try:
|
|
_update_sizes(connection, table_prefix, image, image['before_size'])
|
|
connection.commit()
|
|
summary['db_updated'] += len(mismatched_rows)
|
|
print(f'[DB-UPDATE] fid={image["fids"]} -> {image["before_size"]} B')
|
|
except Exception as exc:
|
|
connection.rollback()
|
|
try:
|
|
_restore_database_sizes(connection, table_prefix, image)
|
|
except Exception as restore_error:
|
|
print(f'[CRITICAL] Nie udalo sie przywrocic metadanych DB: {restore_error}')
|
|
summary['errors'] += 1
|
|
print(f'[ERROR DB] {image["rel_path"]}: {exc}')
|
|
else:
|
|
print(f'[DB-PLAN] fid={image["fids"]}: dysk {image["before_size"]} B, DB: {[r["filesize"] for r in image["db_rows"]]}')
|
|
continue
|
|
|
|
if os.path.islink(image['path']) or os.stat(image['path']).st_nlink != 1:
|
|
image['optimize_status'] = 'skip'
|
|
image['optimization_reason'] = 'dowiazanie symboliczne/twarde (pomijam)'
|
|
print(f'[SKIP] {image["rel_path"]}: {image["optimization_reason"]}')
|
|
continue
|
|
if args.apply and args.max_changes and summary['optimized'] >= args.max_changes:
|
|
image['optimize_status'] = 'limit'
|
|
image['optimization_reason'] = 'osiagnieto limit --max-changes'
|
|
continue
|
|
# W dry-run obliczamy prawdziwy rozmiar po zapisie na pliku tymczasowym.
|
|
result = create_optimized_candidate(
|
|
image['path'], max_width=args.max_width, max_height=args.max_height,
|
|
max_bytes=max_bytes, jpeg_quality=args.jpeg_quality,
|
|
min_jpeg_quality=args.min_jpeg_quality,
|
|
min_gain_percent=args.min_gain_percent,
|
|
temp_dir=os.path.dirname(image['path']) if args.apply else None,
|
|
max_pixels=int(args.max_image_mpixels * 1_000_000),
|
|
)
|
|
image['optimization_reason'] = result['reason']
|
|
image['optimize_status'] = result['status']
|
|
image['quality'] = result.get('quality')
|
|
image['preview_size'] = result['after']
|
|
if result['status'] == 'error':
|
|
summary['errors'] += 1
|
|
print(f'[ERROR] {image["rel_path"]}: {result["reason"]}')
|
|
continue
|
|
if result['status'] == 'skip':
|
|
if 'przekracza --max-image-mpixels' in result['reason'] or 'Pillow odrzucil' in result['reason']:
|
|
print(f'[SKIP-LIMIT] {image["rel_path"]}: {result["reason"]}')
|
|
elif args.debug:
|
|
print(f'[SKIP] {image["rel_path"]}: {result["reason"]}')
|
|
# Opcjonalny sync DB dla plikow, ktorych optymalizacja sie nie udala.
|
|
if args.update_db and mismatched_rows:
|
|
if args.apply:
|
|
try:
|
|
_update_sizes(connection, table_prefix, image, image['before_size'])
|
|
connection.commit()
|
|
summary['db_updated'] += len(mismatched_rows)
|
|
except Exception as exc:
|
|
connection.rollback()
|
|
try:
|
|
_restore_database_sizes(connection, table_prefix, image)
|
|
except Exception as restore_error:
|
|
print(f'[CRITICAL] Nie udalo sie przywrocic metadanych DB: {restore_error}')
|
|
summary['errors'] += 1
|
|
print(f'[ERROR DB] {image["rel_path"]}: {exc}')
|
|
else:
|
|
print(f'[DB-PLAN] fid={image["fids"]}: {image["before_size"]} B')
|
|
continue
|
|
summary['planned'] += 1
|
|
gain = result['before'] - result['after']
|
|
desc = (
|
|
f'{image["rel_path"]} | {result["before"] / 1024**2:.2f} -> '
|
|
f'{result["after"] / 1024**2:.2f} MB | '
|
|
f'{result["width"]}x{result["height"]} | -{gain / 1024**2:.2f} MB'
|
|
)
|
|
try:
|
|
if args.apply:
|
|
backup_file = apply_candidate(connection, table_prefix, image, result,
|
|
backup_run_dir=summary['backups_dir'], base_dir=base)
|
|
image['backup_file'] = backup_file
|
|
image['after_size'] = result['after']
|
|
image['optimize_status'] = 'done'
|
|
summary['optimized'] += 1
|
|
summary['saved_bytes'] += gain
|
|
summary['db_updated'] += len(image['db_rows'])
|
|
print(f'[ZMIENIONO] {desc} | backup: {backup_file}')
|
|
else:
|
|
image['optimize_status'] = 'planned'
|
|
image['after_size'] = result['after']
|
|
summary['saved_bytes'] += gain
|
|
print(f'[DRY-RUN] {desc} (bez zapisu, DB bez zmian)')
|
|
except Exception as exc:
|
|
image['optimize_status'] = 'error'
|
|
image['optimization_reason'] = str(exc)
|
|
summary['errors'] += 1
|
|
print(f'[ERROR] {image["rel_path"]}: {exc}')
|
|
finally:
|
|
if result.get('temp_path') and os.path.isfile(result['temp_path']):
|
|
os.unlink(result['temp_path'])
|
|
summary['apply'] = bool(args.apply)
|
|
return summary
|
|
|
|
|
|
def find_references(connection, args, config, images, table_prefix):
|
|
if not args.find_usage:
|
|
return
|
|
if args.usage_scope == 'top':
|
|
selected = images[:args.top]
|
|
elif args.usage_scope == 'optimizable':
|
|
selected = [image for image in images if is_candidate(image, args)]
|
|
else:
|
|
selected = images
|
|
|
|
print(f'[INFO] Szukam tematu/posta dla {len(selected)} plikow (scope={args.usage_scope})')
|
|
if len(selected) > 1000:
|
|
print('[WARN] Duza liczba plikow do wyszukania; skan tekstu moze trwac dlugo i zuzyc duzo RAM')
|
|
lookup_files = []
|
|
for image in selected:
|
|
for db in image['db_rows']:
|
|
lookup_files.append({'fid': db['fid'], 'db_filepath': db['db_filepath'], 'rel_path': image['rel_path']})
|
|
if not lookup_files:
|
|
return
|
|
base_url = args.site_url if args.site_url is not None else getattr(config, 'DRUPAL_BASE_URL', '')
|
|
finder = DrupalReferenceLookup(connection, table_prefix=table_prefix,
|
|
base_url=base_url, clean_urls=not args.no_clean_urls,
|
|
comment_permalinks=args.comment_permalinks)
|
|
found = finder.lookup(lookup_files, scan_text=not args.no_text_scan,
|
|
scan_cck_text=args.scan_cck_text, basename_match=not args.no_basename_match)
|
|
for image in selected:
|
|
image['usage_checked'] = True
|
|
# Laczymy referencje do tego samego pliku z kilku rekordow fid.
|
|
combined = {}
|
|
for fid in image['fids']:
|
|
for ref in found.get(fid, []):
|
|
key = (ref['nid'], ref['cid'])
|
|
if key not in combined:
|
|
combined[key] = dict(ref)
|
|
combined[key]['sources'] = list(ref['sources'])
|
|
combined[key]['fids'] = [fid]
|
|
else:
|
|
if fid not in combined[key]['fids']:
|
|
combined[key]['fids'].append(fid)
|
|
for source in ref['sources']:
|
|
if source not in combined[key]['sources']:
|
|
combined[key]['sources'].append(source)
|
|
if ref['confidence'] == 'pewne':
|
|
combined[key]['confidence'] = 'pewne'
|
|
image['references'] = sorted(combined.values(), key=lambda r: (r['nid'], r['cid'] or 0))
|
|
return
|
|
|
|
|
|
def disk_only_files(images, config, args):
|
|
base = os.path.realpath(config.FILES_BASE_PATH)
|
|
known = {img['rel_path'] for img in images}
|
|
allowed = {'.' + x.strip().lower().lstrip('.') for x in args.extensions.split(',') if x.strip()}
|
|
seen = []
|
|
for root, directories, files in os.walk(base, followlinks=False):
|
|
directories[:] = [folder for folder in directories if not _excluded(
|
|
os.path.relpath(os.path.join(root, folder), base).replace(os.sep, '/'), args.exclude_folders
|
|
)]
|
|
for filename in files:
|
|
if os.path.splitext(filename)[1].lower() not in allowed:
|
|
continue
|
|
full_path = os.path.join(root, filename)
|
|
if not _within(full_path, base):
|
|
continue
|
|
rel = os.path.relpath(full_path, base).replace(os.sep, '/')
|
|
if rel not in known and not _excluded(rel, args.exclude_folders):
|
|
seen.append(rel)
|
|
return seen
|
|
|
|
|
|
def image_url(base_url, public_files_path, rel_path):
|
|
if not base_url or not public_files_path:
|
|
return ''
|
|
return (f'{base_url.rstrip("/")}/' +
|
|
quote(public_files_path.strip('/') + '/' + rel_path.lstrip('/'), safe='/'))
|
|
|
|
|
|
def display_summary(images, missing, process_stats, args, config, *, deleted=None, duplicates=None):
|
|
print('\n=== PODSUMOWANIE ===')
|
|
print(f'Znalezione pliki: {len(images)} | Nieodnalezione: {len(missing)} | '
|
|
f'Niezgodne rozmiary DB: {process_stats["db_mismatches"]}')
|
|
print(f'Kandydaci do zmniejszenia: {process_stats["candidates"]} | '
|
|
f'Zmniejszone: {process_stats["optimized"]} | Bledy: {process_stats["errors"]}')
|
|
prefix = 'Przewidywane' if not args.apply else 'Uzyskane'
|
|
print(f'{prefix} oszczednosci: {process_stats["saved_bytes"] / 1024**2:.2f} MB')
|
|
if process_stats['backups_dir']:
|
|
print(f'Kopie zapasowe: {process_stats["backups_dir"]}')
|
|
print('\n=== NAJWIEKSZE OBRAZY ===')
|
|
base_url = args.site_url if args.site_url is not None else getattr(config, 'DRUPAL_BASE_URL', '')
|
|
for rank, img in enumerate(images[:args.top], 1):
|
|
print(f'{rank:3d}. {img["before_size"] / 1024**2:8.2f} MB '
|
|
f'{img["width"]:5d}x{img["height"]:<5d} '
|
|
f'fid={",".join(map(str, img["fids"]))} {img["rel_path"]}')
|
|
if base_url:
|
|
public_path = (args.public_files_path if args.public_files_path is not None
|
|
else getattr(config, 'PUBLIC_FILES_PATH', 'sites/default/files'))
|
|
print(f' PLIK (jesli publiczny): {image_url(base_url, public_path, img["rel_path"])}')
|
|
if args.find_usage and img.get('usage_checked'):
|
|
if img['references']:
|
|
for ref in img['references']:
|
|
label = f'{ref["title"]} ({ref["node_type"]}, nid={ref["nid"]}'
|
|
if ref['cid']:
|
|
label += f', cid={ref["cid"]}'
|
|
label += f', {ref["confidence"]})'
|
|
if not ref['published']:
|
|
label += ' [WPIS NIEOPUBLIKOWANY]'
|
|
if ref.get('comment_published') is False:
|
|
label += ' [KOMENTARZ NIEOPUBLIKOWANY]'
|
|
print(f' -> {label}')
|
|
print(f' LINK: {ref["url"]}')
|
|
print(f' EDYCJA: {ref["edit_url"]}')
|
|
print(f' SKAD: {", ".join(ref["sources"])}')
|
|
else:
|
|
print(' -> [BRAK POWIAZANIA] Nie znaleziono w obslugiwanych polach')
|
|
name = os.path.basename(img['rel_path'])
|
|
maybe_search_url = search_url(base_url, name, clean_urls=not args.no_clean_urls)
|
|
if maybe_search_url:
|
|
print(f' SZUKAJ (Drupal Search, jesli aktywny): {maybe_search_url}')
|
|
if args.find_usage and args.usage_scope != 'top':
|
|
shown = {img['path'] for img in images[:args.top]}
|
|
additional = [img for img in images if img['path'] not in shown and img['references']]
|
|
if additional:
|
|
print(f'\n=== POWIAZANIA POZA TOP {args.top} ===')
|
|
for img in additional:
|
|
print(f'{img["rel_path"]} ({img["before_size"] / 1024**2:.2f} MB)')
|
|
for ref in img['references']:
|
|
print(f' -> {ref["title"]}: {ref["url"]} [{ref["confidence"]}]')
|
|
print(f'\n=== NAJWYZSZA ROZDZIELCZOSC (TOP {args.top}) ===')
|
|
for img in sorted(images, key=lambda f: f['width'] * f['height'], reverse=True)[:args.top]:
|
|
print(f'{img["width"]:5d}x{img["height"]:<5d} {img["before_size"] / 1024**2:8.2f} MB {img["rel_path"]}')
|
|
if missing:
|
|
print(f'\n=== PLIKI W BAZIE BEZ PLIKU NA DYSKU (pierwsze 30) ===')
|
|
for item in missing[:30]:
|
|
print(f'fid={item["fid"]} {item["db_filepath"]}: {item["reason"]}')
|
|
if duplicates:
|
|
print(f'\n=== DUPLIKATY SHA-256: {len(duplicates)} grup ===')
|
|
for d in duplicates:
|
|
print(f'Hash {d["hash"][:16]}... | {d["size"] / 1024**2:.2f} MB')
|
|
for filepath in d['paths']:
|
|
print(f' {filepath}')
|
|
if deleted is not None:
|
|
print(f'\n=== NA DYSKU, ALE NIE W ZESTAWIENIU: {len(deleted)} (pierwsze 60) ===')
|
|
for rel in deleted[:60]:
|
|
print(f' {rel}')
|
|
if args.fid or args.path_contains or args.min_size_mb:
|
|
print(' [UWAGA] Przy aktywnych filtrach lista zawiera takze pliki odfiltrowane.')
|
|
if not args.apply:
|
|
print('\n[DRY-RUN] Nie zmieniono plikow ani bazy. Zapis wymaga --apply.')
|
|
|
|
|
|
def duplicate_groups(images):
|
|
by_hash = defaultdict(list)
|
|
for image in images:
|
|
digest = sha256_file(image['path'])
|
|
by_hash[digest].append(image)
|
|
return [{'hash': digest, 'size': items[0]['before_size'],
|
|
'paths': [img['rel_path'] for img in items]}
|
|
for digest, items in by_hash.items() if len(items) > 1]
|
|
|
|
|
|
def write_csv(csv_path, images, config, args):
|
|
fieldnames = [
|
|
'fid', 'filepath_db', 'filepath', 'rozmiar_przed_B', 'rozmiar_po_lub_plan_B',
|
|
'rozmiar_db_B', 'szerokosc', 'wysokosc', 'status_optymalizacji',
|
|
'powod', 'nid', 'typ_wpisu', 'tytul', 'cid', 'link_do_wpisu',
|
|
'link_edycji', 'link_do_pliku', 'zrodlo_powiazania', 'pewnosc', 'opublikowany', 'komentarz_opublikowany', 'szukaj_w_drupal',
|
|
]
|
|
base_url = args.site_url if args.site_url is not None else getattr(config, 'DRUPAL_BASE_URL', '')
|
|
public = (args.public_files_path if args.public_files_path is not None
|
|
else getattr(config, 'PUBLIC_FILES_PATH', 'sites/default/files'))
|
|
with open(csv_path, 'w', encoding='utf-8-sig', newline='') as target:
|
|
writer = csv.DictWriter(target, fieldnames=fieldnames, delimiter=';')
|
|
writer.writeheader()
|
|
for image in images:
|
|
for db_row in image['db_rows']:
|
|
refs = [ref for ref in image['references']
|
|
if not ref.get('fids') or db_row['fid'] in ref['fids']] or [None]
|
|
for ref in refs:
|
|
row = {
|
|
'fid': db_row['fid'], 'filepath_db': db_row['db_filepath'],
|
|
'filepath': image['rel_path'], 'rozmiar_przed_B': image['before_size'],
|
|
'rozmiar_po_lub_plan_B': image['after_size'],
|
|
'rozmiar_db_B': db_row['filesize'], 'szerokosc': image['width'],
|
|
'wysokosc': image['height'], 'status_optymalizacji': image['optimize_status'],
|
|
'powod': image['optimization_reason'],
|
|
'link_do_pliku': image_url(base_url, public, image['rel_path']),
|
|
'szukaj_w_drupal': search_url(base_url, os.path.basename(image['rel_path']),
|
|
clean_urls=not args.no_clean_urls),
|
|
}
|
|
if ref:
|
|
row.update({
|
|
'nid': ref['nid'], 'typ_wpisu': ref['node_type'], 'tytul': ref['title'],
|
|
'cid': ref['cid'] or '', 'link_do_wpisu': ref['url'],
|
|
'link_edycji': ref['edit_url'], 'zrodlo_powiazania': ', '.join(ref['sources']),
|
|
'pewnosc': ref['confidence'], 'opublikowany': 'tak' if ref['published'] else 'nie',
|
|
'komentarz_opublikowany': ('tak' if ref['comment_published'] else 'nie')
|
|
if ref.get('comment_published') is not None else '',
|
|
})
|
|
writer.writerow(row)
|
|
|
|
|
|
def write_json(path, images, missing, stats, duplicates=None, deleted=None):
|
|
output = {
|
|
'mode': 'apply' if stats.get('apply') else 'dry_run_or_read_only',
|
|
'stats': stats, 'missing': missing,
|
|
'images': images, 'duplicate_groups': duplicates or [], 'disk_only': deleted or [],
|
|
}
|
|
with open(path, 'w', encoding='utf-8') as target:
|
|
json.dump(output, target, ensure_ascii=False, indent=2)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = build_parser()
|
|
args = parser.parse_args(argv)
|
|
validate_args(args, parser)
|
|
try:
|
|
config = load_config()
|
|
if not os.path.isdir(config.FILES_BASE_PATH):
|
|
raise RuntimeError(f'Brak katalogu FILES_BASE_PATH: {config.FILES_BASE_PATH}')
|
|
with get_connection(config) as connection:
|
|
images, missing, prefix = collect_images(connection, config, args)
|
|
print(f'[INFO] Zidentyfikowano {len(images)} obrazow w bazie; '
|
|
f'{len(missing)} wpisow bez pliku. Tryb: {"APPLY" if args.apply else "DRY-RUN"}.')
|
|
stats = process_changes(connection, config, args, images, prefix)
|
|
find_references(connection, args, config, images, prefix)
|
|
duplicates = duplicate_groups(images) if args.find_duplicates else None
|
|
deleted = disk_only_files(images, config, args) if args.show_deleted else None
|
|
display_summary(images, missing, stats, args, config, deleted=deleted, duplicates=duplicates)
|
|
if args.report_csv:
|
|
write_csv(args.report_csv, images, config, args)
|
|
print(f'[RAPORT] CSV: {args.report_csv}')
|
|
if args.report_json:
|
|
write_json(args.report_json, images, missing, stats, duplicates=duplicates, deleted=deleted)
|
|
print(f'[RAPORT] JSON: {args.report_json}')
|
|
return 1 if stats['errors'] else 0
|
|
except Exception as exc:
|
|
print(f'[ERROR] {type(exc).__name__}: {exc}', file=sys.stderr)
|
|
if args.debug:
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 2
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|