first commit
This commit is contained in:
commit
58cfd95600
35 files changed
+5357
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,204 @@
|
||||
"""Internet Archive user session and restricted download relay.
|
||||
|
||||
The relay keeps IA cookies inside this process. aria2 only receives a signed,
|
||||
local endpoint, and never sees an IA session cookie or the account password.
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi.responses import PlainTextResponse, Response, StreamingResponse
|
||||
|
||||
from . import store
|
||||
|
||||
|
||||
class ArchiveAuthError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def session_path():
|
||||
return store.DATA / 'private' / 'archive-session.json'
|
||||
|
||||
|
||||
def _read_session():
|
||||
try:
|
||||
value = json.loads(session_path().read_text(encoding='utf-8'))
|
||||
cookies = value['cookies']
|
||||
if not all(isinstance(cookies.get(k), str) and cookies[k] for k in ('logged-in-user', 'logged-in-sig')):
|
||||
return None
|
||||
return value
|
||||
except (FileNotFoundError, KeyError, TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def session_status():
|
||||
value = _read_session()
|
||||
return {'authenticated': bool(value), 'email': value.get('email', '') if value else ''}
|
||||
|
||||
|
||||
def session_cookies():
|
||||
cookies = httpx.Cookies()
|
||||
value = _read_session()
|
||||
if value:
|
||||
for key in ('logged-in-user', 'logged-in-sig'):
|
||||
cookies.set(key, value['cookies'][key], domain='.archive.org', path='/')
|
||||
return cookies
|
||||
|
||||
|
||||
def _cookie_value(value):
|
||||
# The official xauthn API includes cookie attributes after the value.
|
||||
if not isinstance(value, str):
|
||||
raise ArchiveAuthError('Brak danych sesji w odpowiedzi Archive.org')
|
||||
value = value.split(';', 1)[0].strip()
|
||||
if not value or len(value) > 4096 or any(x in value for x in ('\n', '\r', '\0')):
|
||||
raise ArchiveAuthError('Niepoprawne dane sesji Archive.org')
|
||||
return value
|
||||
|
||||
|
||||
def _save_session(email, cookies):
|
||||
path = session_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(path.parent, 0o700)
|
||||
fd, name = tempfile.mkstemp(dir=path.parent, prefix='.archive-session-', suffix='.tmp')
|
||||
try:
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||||
json.dump({'email': email, 'cookies': cookies}, f)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.chmod(name, 0o600)
|
||||
os.replace(name, path)
|
||||
finally:
|
||||
if os.path.exists(name):
|
||||
os.unlink(name)
|
||||
|
||||
|
||||
async def login(email, password):
|
||||
"""Authenticate using IA's documented xauthn endpoint (no password stored)."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=25, follow_redirects=False) as client:
|
||||
result = await client.post('https://archive.org/services/xauthn/', params={'op': 'login'},
|
||||
data={'email': email, 'password': password})
|
||||
result.raise_for_status()
|
||||
data = result.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
raise ArchiveAuthError('Błąd połączenia z logowaniem Archive.org') from exc
|
||||
|
||||
if not data.get('success'):
|
||||
reason = str((data.get('values') or {}).get('reason') or data.get('error') or '')
|
||||
if reason in ('account_bad_password', 'account_not_found'):
|
||||
raise ArchiveAuthError('Nieprawidłowy adres e-mail lub hasło')
|
||||
raise ArchiveAuthError('Archive.org odrzuciło logowanie: ' + (reason[:120] or 'brak szczegółów'))
|
||||
try:
|
||||
raw = data['values']['cookies']
|
||||
cookies = {key: _cookie_value(raw[key]) for key in ('logged-in-user', 'logged-in-sig')}
|
||||
except (KeyError, TypeError) as exc:
|
||||
raise ArchiveAuthError('Logowanie udane, ale brak ciasteczek sesji w odpowiedzi') from exc
|
||||
_save_session(email, cookies)
|
||||
return session_status()
|
||||
|
||||
|
||||
def logout():
|
||||
session_path().unlink(missing_ok=True)
|
||||
|
||||
|
||||
def is_archive_download(url):
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
return parsed.scheme == 'https' and parsed.hostname in ('archive.org', 'www.archive.org') and parsed.path.startswith('/download/')
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def internal_url(job_id, index):
|
||||
return f'http://127.0.0.1:8000/internal/archive/{job_id}/{index}'
|
||||
|
||||
|
||||
def signed_header(job_id, index):
|
||||
secret = os.environ['ARIA2_SECRET'].encode('utf-8')
|
||||
message = f'archive:{job_id}:{index}'.encode('ascii')
|
||||
digest = hmac.new(secret, message, hashlib.sha256).hexdigest()
|
||||
return 'X-X360-Proxy-Token: ' + digest
|
||||
|
||||
|
||||
def verify_token(job_id, index, token):
|
||||
try:
|
||||
expected = signed_header(job_id, index).split(': ', 1)[1]
|
||||
except UnicodeEncodeError:
|
||||
return False
|
||||
return secrets.compare_digest(expected, token or '')
|
||||
|
||||
|
||||
def allowed_redirect(url):
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
name = (parts.hostname or '').lower()
|
||||
return parts.scheme == 'https' and parts.port in (None, 443) and (name == 'archive.org' or name.endswith('.archive.org'))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
async def stream_archive(url, method, headers):
|
||||
"""Stream authorized bytes and preserve HTTP Range for aria2 resumes.
|
||||
|
||||
Only HTTPS Archive.org redirects are followed. Cookie domain scoping is
|
||||
handled by httpx, unlike forwarding a raw Cookie header through redirects.
|
||||
"""
|
||||
outbound_headers = {'Accept-Encoding': 'identity', 'User-Agent': 'X360-Queue/1.1'}
|
||||
for key in ('range', 'if-range', 'if-modified-since', 'if-none-match'):
|
||||
if key in headers:
|
||||
outbound_headers[key] = headers[key]
|
||||
client = httpx.AsyncClient(timeout=httpx.Timeout(120, connect=25), cookies=session_cookies(),
|
||||
follow_redirects=False, trust_env=False)
|
||||
current = url
|
||||
try:
|
||||
for _ in range(7):
|
||||
if not allowed_redirect(current):
|
||||
return PlainTextResponse('Nieprawidłowe przekierowanie Archive.org', status_code=502)
|
||||
request = client.build_request(method, current, headers=outbound_headers)
|
||||
remote = await client.send(request, stream=True)
|
||||
if remote.status_code in (301, 302, 303, 307, 308) and remote.headers.get('location'):
|
||||
new_url = urljoin(current, remote.headers['location'])
|
||||
await remote.aclose()
|
||||
current = new_url
|
||||
continue
|
||||
break
|
||||
else:
|
||||
return PlainTextResponse('Zbyt wiele przekierowań Archive.org', status_code=502)
|
||||
|
||||
if remote.status_code not in (200, 206, 304, 416):
|
||||
status = remote.status_code
|
||||
await remote.aclose()
|
||||
if status in (401, 403):
|
||||
return PlainTextResponse('Archive.org odmówiło dostępu. Zaloguj się i sprawdź uprawnienia do pliku.', status_code=403)
|
||||
return PlainTextResponse(f'Archive.org HTTP {status}', status_code=status if 400 <= status < 600 else 502)
|
||||
if remote.status_code == 200 and 'text/html' in remote.headers.get('content-type', '').lower():
|
||||
await remote.aclose()
|
||||
return PlainTextResponse('Archive.org zwróciło stronę HTML zamiast pliku. Sprawdź logowanie.', status_code=403)
|
||||
safe_headers = {k: v for k, v in remote.headers.items() if k.lower() in (
|
||||
'content-length', 'content-range', 'accept-ranges', 'content-disposition',
|
||||
'content-type', 'etag', 'last-modified')}
|
||||
if method == 'HEAD' or remote.status_code in (304, 416):
|
||||
await remote.aclose()
|
||||
return Response(status_code=remote.status_code, headers=safe_headers)
|
||||
|
||||
async def chunks():
|
||||
try:
|
||||
async for chunk in remote.aiter_raw(chunk_size=131072):
|
||||
yield chunk
|
||||
finally:
|
||||
await remote.aclose()
|
||||
await client.aclose()
|
||||
return StreamingResponse(chunks(), status_code=remote.status_code, headers=safe_headers,
|
||||
media_type=safe_headers.get('content-type'))
|
||||
except httpx.HTTPError:
|
||||
return PlainTextResponse('Nie można połączyć się z Archive.org', status_code=502)
|
||||
finally:
|
||||
# A streamed response owns client until the last chunk is sent.
|
||||
if 'remote' not in locals() or remote.is_closed:
|
||||
await client.aclose()
|
||||
@@ -0,0 +1,760 @@
|
||||
"""Direct, authenticated Internet Archive downloader.
|
||||
|
||||
Avoids the old aria2 -> localhost HTTP relay (which hid origin errors behind
|
||||
HTTP 500). Supports Range resumes, transient 5xx/429 retries and pause/cancel.
|
||||
Only forwards IA cookies via httpx's domain-scoped cookie jar.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from contextlib import closing
|
||||
from threading import Event, Lock
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from . import archive_auth, store
|
||||
from .download import source_urls, target_filename
|
||||
|
||||
RETRY_CODES = {408, 425, 429, 500, 502, 503, 504}
|
||||
MAX_ATTEMPTS = 5
|
||||
MAX_CONNECTIONS = 50
|
||||
MIN_SEGMENT_BYTES = 8 * 1024 * 1024
|
||||
CHUNK_BYTES = 256 * 1024
|
||||
RANGE = re.compile(r'^bytes (\d+)-(\d+)/(\d+|\*)$', re.I)
|
||||
FINISH_RANGE = re.compile(r'^bytes \*/(\d+)$', re.I)
|
||||
LOG_LOCK = Lock()
|
||||
|
||||
|
||||
class ArchiveDownloadError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ArchiveStopped(Exception):
|
||||
"""Pause or cancellation was requested, not a transfer error."""
|
||||
|
||||
|
||||
def _client():
|
||||
return httpx.Client(timeout=httpx.Timeout(90, connect=25), cookies=archive_auth.session_cookies(),
|
||||
follow_redirects=False, trust_env=True)
|
||||
|
||||
|
||||
def _check_state(jid, stop=None):
|
||||
if stop is not None and stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
current = store.get(jid)
|
||||
if not current or current['state'] in ('paused', 'cancelled', 'error'):
|
||||
raise ArchiveStopped()
|
||||
|
||||
|
||||
def _response(client, url, headers):
|
||||
current = url
|
||||
for _ in range(8):
|
||||
if not archive_auth.allowed_redirect(current):
|
||||
raise ArchiveDownloadError('Archive.org: nieprawidłowe przekierowanie poza domenę Archive.org')
|
||||
request = client.build_request('GET', current, headers=headers)
|
||||
response = client.send(request, stream=True)
|
||||
if response.status_code in (301, 302, 303, 307, 308) and 'location' in response.headers:
|
||||
next_url = urljoin(current, response.headers['location'])
|
||||
response.close()
|
||||
current = next_url
|
||||
continue
|
||||
return response
|
||||
raise ArchiveDownloadError('Archive.org: zbyt wiele przekierowań')
|
||||
|
||||
|
||||
def _log(job_id, message):
|
||||
target = store.DATA / 'logs' / (job_id + '.download.log')
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with LOG_LOCK:
|
||||
with target.open('a', encoding='utf-8') as fp:
|
||||
fp.write(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + message + '\n')
|
||||
|
||||
|
||||
def _transfer_one(job, *, batch=None, connections=None, stop=None):
|
||||
jid = job['id']
|
||||
ix = job['part_index']
|
||||
urls = source_urls(job)
|
||||
if ix >= len(urls):
|
||||
store.update(jid, state='preparing')
|
||||
return
|
||||
uri = urls[ix]
|
||||
if not archive_auth.is_archive_download(uri):
|
||||
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
|
||||
root = store.DATA / 'raw' / jid
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
filename = target_filename(uri)
|
||||
if not filename:
|
||||
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
|
||||
destination = root / filename
|
||||
partial = root / (filename + '.part')
|
||||
_check_state(jid, stop)
|
||||
if batch is None:
|
||||
store.update(jid, state='downloading', gid='', note=f'Archive.org: plik {ix+1}/{len(urls)}')
|
||||
_log(jid, f'Rozpoczęto plik {ix+1}/{len(urls)}: {filename}')
|
||||
|
||||
if connections is None:
|
||||
connections = _connection_limit()
|
||||
# An existing manifest may contain many segments even if the new budget
|
||||
# only allows one active connection. Keep those downloaded bytes.
|
||||
if (connections > 1 or _segment_dir(partial).exists()) and _parallel_transfer(
|
||||
job, ix, urls, uri, filename, partial, destination, connections, batch=batch, stop=stop):
|
||||
return
|
||||
# If the server cannot serve ranges (or parallel transfer was disabled),
|
||||
# keep the verified single-stream prefix, not an unfinished assembly.
|
||||
_discard_parallel_state(partial)
|
||||
# Existing aria2 control files are not compatible with this downloader.
|
||||
# Start from our own .part file rather than assuming an aria2 file is complete.
|
||||
with _client() as client:
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
_check_state(jid, stop)
|
||||
start = partial.stat().st_size if partial.exists() else 0
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.0', 'Accept-Encoding': 'identity'}
|
||||
if start:
|
||||
headers['Range'] = f'bytes={start}-'
|
||||
try:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(jid, f'HTTP {code} z Archive.org, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
reason = f'Archive.org HTTP {code}'
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'{reason} po {MAX_ATTEMPTS} próbach; możliwy problem serwera Archive.org')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia do pliku.')
|
||||
elif code == 404:
|
||||
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono wybranego pliku')
|
||||
elif code == 416 and start:
|
||||
m = FINISH_RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if m and int(m.group(1)) == start:
|
||||
_check_state(jid, stop)
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename} (HTTP 416 potwierdziło rozmiar)')
|
||||
return
|
||||
partial.unlink(missing_ok=True)
|
||||
continue
|
||||
elif code not in (200, 206):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
|
||||
else:
|
||||
if 'text/html' in response.headers.get('Content-Type', '').lower():
|
||||
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
|
||||
if code == 206:
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or int(match.group(1)) != start:
|
||||
raise ArchiveDownloadError('Archive.org zwróciło niezgodny zakres HTTP Range')
|
||||
total = int(match.group(3)) if match.group(3) != '*' else 0
|
||||
else:
|
||||
# Server ignored Range: do not append duplicate bytes.
|
||||
start = 0
|
||||
total = int(response.headers.get('Content-Length') or 0)
|
||||
written = start
|
||||
last_update = 0.0
|
||||
prev = written
|
||||
prev_at = time.monotonic()
|
||||
mode = 'ab' if start else 'wb'
|
||||
with partial.open(mode) as fp:
|
||||
for chunk in response.iter_raw(chunk_size=256 * 1024):
|
||||
_check_state(jid, stop)
|
||||
if not chunk:
|
||||
continue
|
||||
fp.write(chunk)
|
||||
written += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.8:
|
||||
elapsed = max(0.001, now - prev_at)
|
||||
speed = int((written - prev) / elapsed)
|
||||
prev, prev_at, last_update = written, now, now
|
||||
portion = written / total if total else 0
|
||||
if batch is not None:
|
||||
batch.report(ix, written, total)
|
||||
else:
|
||||
store.update(jid, progress=round((ix + min(portion, 1)) / len(urls) * 100, 2),
|
||||
speed=speed, total=total, completed=written,
|
||||
note=f'Archive.org: plik {ix+1}/{len(urls)}, próba {attempt+1}')
|
||||
if total and written != total:
|
||||
raise httpx.ReadError(f'Przerwany transfer: pobrano {written} z {total} bajtów')
|
||||
_check_state(jid, stop)
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename}: {written} bajtów')
|
||||
return
|
||||
except ArchiveStopped:
|
||||
raise
|
||||
except ArchiveDownloadError:
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(jid, f'Błąd transferu {type(exc).__name__}: {str(exc)[:180]}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org: przerwane połączenie po {MAX_ATTEMPTS} próbach: {type(exc).__name__}: {exc}') from exc
|
||||
# Applies to transient HTTP codes and connection exceptions.
|
||||
if batch is None:
|
||||
store.update(jid, note=f'Archive.org: ponawianie po HTTP/błędzie sieci ({attempt+2}/{MAX_ATTEMPTS})', speed=0)
|
||||
for _ in range(min(2 ** attempt, 12) * 2):
|
||||
_check_state(jid, stop)
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _finish(job, ix, urls, *, batch=None):
|
||||
jid = job['id']
|
||||
if batch is not None:
|
||||
batch.complete(ix)
|
||||
return
|
||||
if ix + 1 < len(urls):
|
||||
store.update(jid, part_index=ix+1, state='queued', gid='', speed=0,
|
||||
progress=round((ix+1)/len(urls) * 100, 2), note='Pobrano część archiwum')
|
||||
else:
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie Archive.org ukończone')
|
||||
|
||||
|
||||
|
||||
class _RangeUnsupported(Exception):
|
||||
"""A server stopped honoring byte ranges; caller can use the serial path."""
|
||||
def __init__(self, message, reset_prefix=False):
|
||||
super().__init__(message)
|
||||
self.reset_prefix = reset_prefix
|
||||
|
||||
|
||||
def _connection_limit():
|
||||
try:
|
||||
return max(1, min(MAX_CONNECTIONS, int(os.getenv('ARCHIVE_CONNECTIONS', '50'))))
|
||||
except (ValueError, OverflowError):
|
||||
return 50
|
||||
|
||||
|
||||
def _segment_dir(partial):
|
||||
return Path(str(partial) + '.segments')
|
||||
|
||||
|
||||
def _read_manifest(folder):
|
||||
try:
|
||||
item = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
return item
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _write_manifest(folder, manifest):
|
||||
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
temp = folder / 'manifest.json.tmp'
|
||||
with temp.open('w', encoding='utf-8') as f:
|
||||
json.dump(manifest, f, sort_keys=True)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
temp.replace(folder / 'manifest.json')
|
||||
|
||||
|
||||
def _discard_parallel_state(partial, reset_prefix=False):
|
||||
"""Discard stale segment files without interpreting an assembled prefix as a resume."""
|
||||
folder = _segment_dir(partial)
|
||||
if not folder.exists():
|
||||
return
|
||||
manifest = _read_manifest(folder)
|
||||
if reset_prefix:
|
||||
partial.unlink(missing_ok=True)
|
||||
elif manifest and manifest.get('assembling') and partial.exists():
|
||||
prefix = manifest.get('prefix')
|
||||
if isinstance(prefix, int) and 0 <= prefix <= partial.stat().st_size:
|
||||
with partial.open('r+b') as f:
|
||||
f.truncate(prefix)
|
||||
else:
|
||||
partial.unlink(missing_ok=True)
|
||||
shutil.rmtree(folder)
|
||||
|
||||
|
||||
def _validate_ranges(ranges, prefix, total):
|
||||
if not isinstance(ranges, list) or len(ranges) < 2 or len(ranges) > MAX_CONNECTIONS:
|
||||
return False
|
||||
cursor = prefix
|
||||
for item in ranges:
|
||||
if (not isinstance(item, list) or len(item) != 2 or
|
||||
not all(isinstance(x, int) and not isinstance(x, bool) for x in item)):
|
||||
return False
|
||||
start, end = item
|
||||
if start != cursor or end < start or end >= total:
|
||||
return False
|
||||
cursor = end + 1
|
||||
return cursor == total
|
||||
|
||||
|
||||
def _prepare_segments(partial, uri, total, validator, connections):
|
||||
folder = _segment_dir(partial)
|
||||
source_hash = hashlib.sha256(uri.encode('utf-8')).hexdigest()
|
||||
manifest = _read_manifest(folder)
|
||||
valid = False
|
||||
if manifest:
|
||||
prefix = manifest.get('prefix')
|
||||
numeric_prefix = isinstance(prefix, int) and not isinstance(prefix, bool)
|
||||
prefix_present = numeric_prefix and (prefix == 0 or
|
||||
(partial.exists() and partial.stat().st_size >= prefix))
|
||||
valid = (manifest.get('version') == 1 and manifest.get('source_hash') == source_hash and
|
||||
manifest.get('total') == total and manifest.get('validator') == validator and
|
||||
isinstance(prefix, int) and not isinstance(prefix, bool) and 0 <= prefix < total and
|
||||
_validate_ranges(manifest.get('ranges'), prefix, total) and prefix_present)
|
||||
if valid and partial.exists() and partial.stat().st_size != prefix:
|
||||
# A crash during assembly is safe to recover from: parts remain intact.
|
||||
if manifest.get('assembling') and partial.stat().st_size > prefix:
|
||||
with partial.open('r+b') as f:
|
||||
f.truncate(prefix)
|
||||
else:
|
||||
valid = False
|
||||
if valid:
|
||||
return folder, manifest
|
||||
|
||||
if folder.exists():
|
||||
# Invalid or mismatched metadata cannot establish which bytes are safe.
|
||||
_discard_parallel_state(partial, reset_prefix=True)
|
||||
prefix = partial.stat().st_size if partial.exists() else 0
|
||||
if prefix > total:
|
||||
partial.unlink(missing_ok=True)
|
||||
prefix = 0
|
||||
if prefix >= total:
|
||||
return None, {'prefix': prefix, 'total': total}
|
||||
remaining = total - prefix
|
||||
n = min(connections, max(1, remaining // MIN_SEGMENT_BYTES))
|
||||
if n < 2:
|
||||
return None, None
|
||||
quotient, extra = divmod(remaining, n)
|
||||
cursor = prefix
|
||||
ranges = []
|
||||
for i in range(n):
|
||||
end = cursor + quotient + (1 if i < extra else 0) - 1
|
||||
ranges.append([cursor, end])
|
||||
cursor = end + 1
|
||||
manifest = {'version': 1, 'source_hash': source_hash, 'total': total,
|
||||
'validator': validator, 'prefix': prefix, 'ranges': ranges,
|
||||
'assembling': False}
|
||||
_write_manifest(folder, manifest)
|
||||
return folder, manifest
|
||||
|
||||
|
||||
def _retry_wait(job_id, attempt, stop, retry_after=None, external_stop=None):
|
||||
delay = min(2 ** attempt, 16)
|
||||
if retry_after and retry_after.isdigit():
|
||||
delay = min(max(1, int(retry_after)), 60)
|
||||
for _ in range(delay * 2):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
_check_state(job_id, external_stop)
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _check_content(response):
|
||||
if 'text/html' in response.headers.get('Content-Type', '').lower():
|
||||
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
|
||||
if response.headers.get('Content-Encoding', 'identity').lower() not in ('', 'identity'):
|
||||
raise ArchiveDownloadError('Archive.org zignorowało Accept-Encoding: identity; transfer w tym kodowaniu jest niebezpieczny')
|
||||
|
||||
|
||||
def _probe_range(job_id, uri, stop=None):
|
||||
"""Only a real 206 with a numeric total enables parallel mode."""
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity', 'Range': 'bytes=0-0'}
|
||||
with _client() as client:
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
_check_state(job_id, stop)
|
||||
try:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(job_id, f'Archive.org: sprawdzenie Range HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code} podczas sprawdzania Range po {MAX_ATTEMPTS} próbach')
|
||||
delay = response.headers.get('Retry-After')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia.')
|
||||
elif code == 404:
|
||||
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono pliku')
|
||||
elif code == 200:
|
||||
_check_content(response)
|
||||
return None
|
||||
elif code == 206:
|
||||
_check_content(response)
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or (int(match.group(1)), int(match.group(2))) != (0, 0) or match.group(3) == '*':
|
||||
return None
|
||||
total = int(match.group(3))
|
||||
if total <= 0:
|
||||
return None
|
||||
etag = response.headers.get('ETag', '').strip()
|
||||
if etag.startswith('W/'):
|
||||
etag = ''
|
||||
validator = etag or response.headers.get('Last-Modified', '').strip()
|
||||
if not validator or len(validator) > 256 or '\n' in validator:
|
||||
validator = ''
|
||||
return total, validator
|
||||
else:
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
|
||||
except ArchiveDownloadError:
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(job_id, f'Archive.org: sprawdzenie Range - {type(exc).__name__}: {str(exc)[:150]}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org: brak odpowiedzi na żądanie Range po {MAX_ATTEMPTS} próbach') from exc
|
||||
delay = None
|
||||
_retry_wait(job_id, attempt, Event(), delay, stop)
|
||||
return None
|
||||
|
||||
|
||||
def _segment_worker(job_id, uri, folder, manifest, index, stop, external_stop=None):
|
||||
begin, end = manifest['ranges'][index]
|
||||
total = manifest['total']
|
||||
limit = end - begin + 1
|
||||
target = folder / f'{index:02d}.part'
|
||||
if target.exists() and target.stat().st_size > limit:
|
||||
target.unlink()
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
_check_state(job_id, external_stop)
|
||||
present = target.stat().st_size if target.exists() else 0
|
||||
if present == limit:
|
||||
return
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity',
|
||||
'Range': f'bytes={begin + present}-{end}'}
|
||||
if manifest['validator']:
|
||||
headers['If-Range'] = manifest['validator']
|
||||
try:
|
||||
with _client() as client:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(job_id, f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code} po {MAX_ATTEMPTS} próbach')
|
||||
delay = response.headers.get('Retry-After')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}: brak dostępu')
|
||||
elif code == 200:
|
||||
raise _RangeUnsupported('Archive.org zignorowało zakres HTTP Range / If-Range', reset_prefix=True)
|
||||
elif code != 206:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}')
|
||||
else:
|
||||
_check_content(response)
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or match.group(3) == '*' or (int(match.group(1)), int(match.group(2)), int(match.group(3))) != (begin + present, end, total):
|
||||
raise _RangeUnsupported('Archive.org zwróciło niezgodny Content-Range', reset_prefix=True)
|
||||
if manifest['validator']:
|
||||
header = 'ETag' if manifest['validator'].startswith('"') else 'Last-Modified'
|
||||
received_validator = response.headers.get(header, '').strip()
|
||||
if received_validator and received_validator != manifest['validator']:
|
||||
raise _RangeUnsupported('Archive.org zmieniło plik podczas pobierania', reset_prefix=True)
|
||||
last_check = time.monotonic()
|
||||
with target.open('ab') as f:
|
||||
for chunk in response.iter_raw(chunk_size=CHUNK_BYTES):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
if time.monotonic() - last_check >= 1:
|
||||
_check_state(job_id, external_stop)
|
||||
last_check = time.monotonic()
|
||||
if not chunk:
|
||||
continue
|
||||
if f.tell() + len(chunk) > limit:
|
||||
raise _RangeUnsupported('Archive.org wysłało zbyt dużo danych w zakresie', reset_prefix=True)
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
if target.stat().st_size != limit:
|
||||
raise httpx.ReadError('Przerwana transmisja segmentu')
|
||||
return
|
||||
except (ArchiveStopped, ArchiveDownloadError, _RangeUnsupported):
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(job_id, f'Segment {index+1}: {type(exc).__name__}: {str(exc)[:150]}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: przerwany transfer po {MAX_ATTEMPTS} próbach: {type(exc).__name__}') from exc
|
||||
delay = None
|
||||
_retry_wait(job_id, attempt, stop, delay, external_stop)
|
||||
|
||||
|
||||
def _report_parallel(job_id, ix, url_count, partial, folder, manifest, last, batch=None):
|
||||
prefix = manifest['prefix']
|
||||
downloaded = prefix
|
||||
for index, (start, end) in enumerate(manifest['ranges']):
|
||||
path = folder / f'{index:02d}.part'
|
||||
if path.exists():
|
||||
downloaded += min(end - start + 1, path.stat().st_size)
|
||||
now = time.monotonic()
|
||||
prev_size, prev_at = last
|
||||
speed = max(0, int((downloaded - prev_size) / max(0.001, now - prev_at)))
|
||||
total = manifest['total']
|
||||
if batch is None:
|
||||
store.update(job_id, completed=downloaded, total=total, speed=speed,
|
||||
progress=round((ix + min(downloaded / total, 1)) / url_count * 100, 2),
|
||||
note=f'Archive.org: plik {ix+1}/{url_count}, {len(manifest["ranges"])} połączeń')
|
||||
else:
|
||||
batch.report(ix, downloaded, total)
|
||||
return downloaded, now
|
||||
|
||||
|
||||
def _assemble_segments(job_id, partial, destination, folder, manifest, stop=None):
|
||||
prefix = manifest['prefix']
|
||||
total = manifest['total']
|
||||
for index, (start, end) in enumerate(manifest['ranges']):
|
||||
part = folder / f'{index:02d}.part'
|
||||
if not part.is_file() or part.stat().st_size != end - start + 1:
|
||||
raise ArchiveDownloadError('Niekompletne segmenty; nie można złożyć pliku')
|
||||
manifest['assembling'] = True
|
||||
_write_manifest(folder, manifest)
|
||||
with partial.open('r+b' if partial.exists() else 'w+b') as output:
|
||||
output.truncate(prefix)
|
||||
output.seek(prefix)
|
||||
for index in range(len(manifest['ranges'])):
|
||||
with (folder / f'{index:02d}.part').open('rb') as source:
|
||||
while True:
|
||||
_check_state(job_id, stop)
|
||||
block = source.read(1024 * 1024)
|
||||
if not block:
|
||||
break
|
||||
output.write(block)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
if partial.stat().st_size != total:
|
||||
raise ArchiveDownloadError('Błąd scalania pliku: niezgodny rozmiar końcowy')
|
||||
partial.replace(destination)
|
||||
shutil.rmtree(folder)
|
||||
|
||||
|
||||
def _parallel_transfer(job, ix, urls, uri, filename, partial, destination, connections, *, batch=None, stop=None):
|
||||
jid = job['id']
|
||||
batch_stop = stop
|
||||
probe = _probe_range(jid, uri, stop)
|
||||
if probe is None:
|
||||
_log(jid, 'Archive.org: serwer nie obsługuje Range; pobieranie pojedynczym połączeniem')
|
||||
return False
|
||||
total, validator = probe
|
||||
folder, manifest = _prepare_segments(partial, uri, total, validator, connections)
|
||||
if folder is None and manifest is None:
|
||||
_log(jid, 'Archive.org: mały plik lub końcówka; pobieranie pojedynczym połączeniem')
|
||||
return False
|
||||
if folder is None and manifest['prefix'] == total:
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
return True
|
||||
_log(jid, f'Archive.org: {len(manifest["ranges"])} połączeń HTTP Range, rozmiar {total} B, wznowienie od {manifest["prefix"]} B')
|
||||
local_stop = Event()
|
||||
current = (manifest['prefix'] + sum(min((folder / f'{idx:02d}.part').stat().st_size, end - start + 1)
|
||||
for idx, (start, end) in enumerate(manifest['ranges'])
|
||||
if (folder / f'{idx:02d}.part').exists()), time.monotonic())
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=min(connections, len(manifest['ranges']))) as pool:
|
||||
pending = {pool.submit(_segment_worker, jid, uri, folder, manifest, i, local_stop, batch_stop)
|
||||
for i in range(len(manifest['ranges']))}
|
||||
try:
|
||||
while pending:
|
||||
_check_state(jid, batch_stop)
|
||||
done, pending = wait(pending, timeout=0.8, return_when=FIRST_COMPLETED)
|
||||
current = _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
|
||||
for task in done:
|
||||
task.result()
|
||||
finally:
|
||||
local_stop.set()
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
except _RangeUnsupported as exc:
|
||||
_log(jid, 'Archive.org: niezgodna obsługa Range, przełączam na jedno połączenie')
|
||||
_discard_parallel_state(partial, reset_prefix=exc.reset_prefix)
|
||||
return False
|
||||
_check_state(jid, batch_stop)
|
||||
_report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
|
||||
_log(jid, 'Archive.org: scalanie segmentów')
|
||||
if batch is None:
|
||||
store.update(jid, note='Archive.org: scalanie segmentów', speed=0)
|
||||
if batch_stop is None:
|
||||
_assemble_segments(jid, partial, destination, folder, manifest)
|
||||
else:
|
||||
_assemble_segments(jid, partial, destination, folder, manifest, batch_stop)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename}: {total} bajtów, {len(manifest["ranges"])} połączeń')
|
||||
return True
|
||||
|
||||
|
||||
class _BatchProgress:
|
||||
"""Collect progress from file threads; only the scheduler updates SQLite.
|
||||
|
||||
A single stream/segment never overwrites the total progress or total speed
|
||||
from another file. No extra database table is needed for old installations.
|
||||
"""
|
||||
|
||||
def __init__(self, job_id, filenames, limit):
|
||||
self.job_id = job_id
|
||||
self.filenames = filenames
|
||||
self.limit = limit
|
||||
self.lock = Lock()
|
||||
self.sizes = [0] * len(filenames)
|
||||
self.totals = [0] * len(filenames)
|
||||
self.seen = [False] * len(filenames)
|
||||
self.done = set()
|
||||
self.active = set()
|
||||
self.transferred = 0
|
||||
self.last_publish = time.monotonic()
|
||||
|
||||
def start(self, index):
|
||||
with self.lock:
|
||||
self.active.add(index)
|
||||
|
||||
def report(self, index, downloaded, total):
|
||||
with self.lock:
|
||||
downloaded = max(0, int(downloaded))
|
||||
if self.seen[index]:
|
||||
self.transferred += max(0, downloaded - self.sizes[index])
|
||||
else:
|
||||
# Existing .part and .segments from an earlier run are not
|
||||
# counted as freshly transferred bytes per second.
|
||||
self.seen[index] = True
|
||||
self.sizes[index] = downloaded
|
||||
self.totals[index] = max(0, int(total))
|
||||
|
||||
def complete(self, index, size=None):
|
||||
if size is None:
|
||||
size = (store.DATA / 'raw' / self.job_id / self.filenames[index]).stat().st_size
|
||||
with self.lock:
|
||||
if self.seen[index]:
|
||||
self.transferred += max(0, size - self.sizes[index])
|
||||
self.seen[index] = True
|
||||
self.sizes[index] = size
|
||||
self.totals[index] = size
|
||||
self.done.add(index)
|
||||
self.active.discard(index)
|
||||
|
||||
def publish(self):
|
||||
now = time.monotonic()
|
||||
with self.lock:
|
||||
elapsed = max(0.001, now - self.last_publish)
|
||||
speed = max(0, int(self.transferred / elapsed))
|
||||
self.transferred = 0
|
||||
self.last_publish = now
|
||||
total_files = len(self.filenames)
|
||||
fractions = []
|
||||
for index in range(total_files):
|
||||
if index in self.done:
|
||||
fractions.append(1.0)
|
||||
elif self.totals[index]:
|
||||
fractions.append(min(self.sizes[index] / self.totals[index], 1))
|
||||
else:
|
||||
fractions.append(0.0)
|
||||
completed_bytes = sum(self.sizes)
|
||||
total_bytes = sum(self.totals)
|
||||
if total_bytes and all(self.totals):
|
||||
# A 20 KiB NFO must not count the same as an 8 GiB ISO.
|
||||
progress = round(100 * min(completed_bytes, total_bytes) / total_bytes, 2)
|
||||
else:
|
||||
# Until all content lengths are known, show the fraction of
|
||||
# completed files/known segments as a provisional estimate.
|
||||
progress = round(sum(fractions) / total_files * 100, 2)
|
||||
finished = len(self.done)
|
||||
active = len(self.active)
|
||||
contiguous = next((i for i in range(total_files) if i not in self.done), total_files)
|
||||
store.update(self.job_id, part_index=contiguous, progress=progress,
|
||||
completed=completed_bytes, total=total_bytes, speed=speed,
|
||||
note=f'Archive.org: {finished}/{total_files} gotowe, {active} równolegle, limit {self.limit} połączeń')
|
||||
|
||||
|
||||
def _transfer_many(job):
|
||||
"""Download multiple Archive.org files at once with ONE connection budget.
|
||||
|
||||
Each slot owns a fixed connection allowance. Reusing the slot for the next
|
||||
file preserves the global cap even with 50-segment old resume manifests.
|
||||
"""
|
||||
jid = job['id']
|
||||
urls = source_urls(job)
|
||||
if not urls:
|
||||
raise ArchiveDownloadError('Brak plików Archive.org do pobrania')
|
||||
names = []
|
||||
for uri in urls:
|
||||
if not archive_auth.is_archive_download(uri):
|
||||
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
|
||||
name = target_filename(uri)
|
||||
if not name:
|
||||
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
|
||||
names.append(name)
|
||||
if len({name.casefold() for name in names}) != len(names):
|
||||
raise ArchiveDownloadError('Wybrane pliki mają powtarzające się nazwy. Wybierz pliki o różnych nazwach.')
|
||||
|
||||
root = store.DATA / 'raw' / jid
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
_check_state(jid)
|
||||
limit = _connection_limit()
|
||||
tracker = _BatchProgress(jid, names, limit)
|
||||
waiting = deque()
|
||||
for ix, name in enumerate(names):
|
||||
dest = root / name
|
||||
if dest.is_file() and not Path(str(dest) + '.aria2').exists():
|
||||
# A completed destination was atomically renamed by the old or
|
||||
# new downloader. A legacy aria2 control file means its sibling
|
||||
# destination may only contain an incomplete preallocated file.
|
||||
tracker.complete(ix, dest.stat().st_size)
|
||||
_log(jid, f'Plik {ix+1}/{len(urls)} już gotowy: {name}')
|
||||
else:
|
||||
waiting.append(ix)
|
||||
|
||||
store.update(jid, state='downloading', gid='', note=f'Archive.org: start równoległego pobierania {len(waiting)} plików')
|
||||
tracker.publish()
|
||||
if not waiting:
|
||||
_check_state(jid)
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie Archive.org ukończone')
|
||||
return
|
||||
|
||||
slots = min(limit, len(waiting))
|
||||
budgets = [limit // slots + (1 if i < limit % slots else 0) for i in range(slots)]
|
||||
_log(jid, f'Archive.org: start {slots} plików równolegle; budżet {limit} połączeń HTTP '
|
||||
f'(przydział na plik: {", ".join(map(str, budgets))})')
|
||||
stopped = Event()
|
||||
first_error = None
|
||||
|
||||
def launch(pool, slot):
|
||||
ix = waiting.popleft()
|
||||
item = dict(job, part_index=ix)
|
||||
tracker.start(ix)
|
||||
return pool.submit(_transfer_one, item, batch=tracker, connections=budgets[slot], stop=stopped)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=slots, thread_name_prefix='archive-file') as pool:
|
||||
running = {launch(pool, slot): slot for slot in range(slots)}
|
||||
try:
|
||||
while running:
|
||||
_check_state(jid, stopped)
|
||||
finished, _ = wait(running, timeout=0.8, return_when=FIRST_COMPLETED)
|
||||
tracker.publish()
|
||||
for future in finished:
|
||||
slot = running.pop(future)
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
first_error = first_error or exc
|
||||
stopped.set()
|
||||
else:
|
||||
if waiting and not stopped.is_set():
|
||||
running[launch(pool, slot)] = slot
|
||||
if first_error is not None:
|
||||
break
|
||||
finally:
|
||||
stopped.set()
|
||||
for future in running:
|
||||
future.cancel()
|
||||
|
||||
if first_error is not None:
|
||||
tracker.publish()
|
||||
raise first_error
|
||||
_check_state(jid) # Especially important if a pause arrives at the last byte.
|
||||
tracker.publish()
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
part_index=len(urls), note='Pobieranie Archive.org ukończone')
|
||||
_log(jid, f'Archive.org: pobrano wszystkie {len(urls)} pliki')
|
||||
|
||||
|
||||
async def run_one(job):
|
||||
import asyncio
|
||||
urls = source_urls(job)
|
||||
if job['source_type'] == 'archive_many' and len(urls) > 1:
|
||||
await asyncio.to_thread(_transfer_many, job)
|
||||
else:
|
||||
await asyncio.to_thread(_transfer_one, job)
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Supervise anacrolix-based BitTorrent subprocesses (no aria2 for torrents).
|
||||
|
||||
Torrent payload data starts only after file selection. All subprocess state is
|
||||
written as atomic JSON so a restart can detect interrupted work and resume.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import store
|
||||
|
||||
BT_HELPER = '/usr/local/bin/x360-bt'
|
||||
_INSPECTIONS = {}
|
||||
_DOWNLOADS = {}
|
||||
|
||||
|
||||
class BitTorrentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def exe():
|
||||
candidate = os.getenv('BT_HELPER', BT_HELPER)
|
||||
if not Path(candidate).is_file():
|
||||
raise BitTorrentError(f'Brak silnika BitTorrent: {candidate} (przebuduj obraz Docker)')
|
||||
return candidate
|
||||
|
||||
|
||||
def _read(path):
|
||||
try:
|
||||
value = json.loads(Path(path).read_text(encoding='utf-8'))
|
||||
return value if isinstance(value, dict) else {}
|
||||
except (FileNotFoundError, ValueError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def inspect_state(preview_id):
|
||||
state = _read(store.DATA / 'metadata' / preview_id / 'state.json')
|
||||
process = _INSPECTIONS.get(preview_id)
|
||||
if not state and (not process or process.returncode is not None):
|
||||
return {'status': 'error', 'error': 'Klient BitTorrent przerwał odczyt metadanych. Sprawdź engine.log podglądu lub ponów.'}
|
||||
return state
|
||||
|
||||
|
||||
def job_state(job_id):
|
||||
return _read(store.DATA / 'bt' / (job_id + '.json'))
|
||||
|
||||
|
||||
async def _spawn(args, log_path):
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with log_path.open('ab') as log:
|
||||
try:
|
||||
return await asyncio.create_subprocess_exec(*args, stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=log, stderr=asyncio.subprocess.STDOUT)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise BitTorrentError('Nie można uruchomić silnika BT: ' + str(exc)) from exc
|
||||
|
||||
|
||||
async def start_inspection(preview_id, magnet):
|
||||
from . import torrent_meta
|
||||
folder = torrent_meta.inspect_dir(preview_id)
|
||||
args = [exe(), 'inspect', '-magnet', magnet, '-out', str(folder / 'metadata.torrent'),
|
||||
'-status', str(folder / 'state.json'), '-timeout', '5m', '-port', '0']
|
||||
process = await _spawn(args, folder / 'engine.log')
|
||||
_INSPECTIONS[preview_id] = process
|
||||
return process
|
||||
|
||||
|
||||
async def _stop(process):
|
||||
if process and process.returncode is None:
|
||||
process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), 6)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def stop_inspection(preview_id):
|
||||
await _stop(_INSPECTIONS.pop(preview_id, None))
|
||||
|
||||
|
||||
async def start_download(job):
|
||||
jid = job['id']
|
||||
old = _DOWNLOADS.get(jid)
|
||||
if old and old.returncode is None:
|
||||
return
|
||||
path = store.DATA / 'torrents' / (jid + '.torrent')
|
||||
if not path.is_file():
|
||||
raise BitTorrentError('Brak zapisanego pliku .torrent dla zadania')
|
||||
try:
|
||||
spec = json.loads(job['source'])
|
||||
selection = spec['selection']
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise BitTorrentError('Uszkodzony zapis wyboru plików torrenta') from exc
|
||||
output = store.DATA / 'raw' / jid
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
status_path = store.DATA / 'bt' / (jid + '.json')
|
||||
status_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
status_path.unlink(missing_ok=True)
|
||||
args = [exe(), 'fetch', '-torrent', str(path), '-select', selection, '-out', str(output),
|
||||
'-status', str(status_path), '-port', os.getenv('BT_PORT', '6881')]
|
||||
_DOWNLOADS[jid] = await _spawn(args, store.DATA / 'logs' / (jid + '.bt.log'))
|
||||
store.update(jid, gid='', state='downloading', note='BitTorrent: uruchomiono klienta')
|
||||
|
||||
|
||||
async def poll_download(job):
|
||||
jid = job['id']
|
||||
process = _DOWNLOADS.get(jid)
|
||||
if process is None:
|
||||
# A container restart loses the subprocess; persistently downloaded
|
||||
# pieces in data/raw are verified and resumed by the helper.
|
||||
await start_download(job)
|
||||
return
|
||||
state = job_state(jid)
|
||||
done, total = int(state.get('downloaded', 0) or 0), int(state.get('total', 0) or 0)
|
||||
speed = int(state.get('speed', 0) or 0)
|
||||
peers = int(state.get('connections', 0) or 0)
|
||||
known = int(state.get('known_peers', 0) or 0)
|
||||
if state.get('status') == 'done' and process.returncode == 0:
|
||||
await process.wait()
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='BitTorrent ukończony, przygotowanie plików')
|
||||
return
|
||||
if state.get('status') == 'error' or process.returncode not in (None, 0):
|
||||
await process.wait()
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
raise BitTorrentError(state.get('error') or 'Silnik BitTorrent zakończył się niepowodzeniem; sprawdź log BT')
|
||||
if process.returncode == 0 and state.get('status') != 'done':
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
raise BitTorrentError('Silnik BitTorrent zakończył pracę bez potwierdzenia ukończenia danych')
|
||||
percent = round(done / total * 100, 2) if total else 0
|
||||
store.update(jid, progress=percent, speed=speed, total=total, completed=done,
|
||||
note=f'BitTorrent: {peers} połączeń, {known} znanych peerów')
|
||||
|
||||
|
||||
async def stop_download(job_id):
|
||||
await _stop(_DOWNLOADS.pop(job_id, None))
|
||||
|
||||
|
||||
async def shutdown():
|
||||
jobs = list(_INSPECTIONS.values()) + list(_DOWNLOADS.values())
|
||||
_INSPECTIONS.clear()
|
||||
_DOWNLOADS.clear()
|
||||
await asyncio.gather(*(_stop(p) for p in jobs))
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""Validation, safe paths and Internet Archive integration."""
|
||||
import html
|
||||
import re
|
||||
import socket
|
||||
import ipaddress
|
||||
from urllib.parse import urlparse, quote, unquote, parse_qs
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
FORMATS = {'xex', 'god', 'raw'}
|
||||
ARCHIVE_EXTS = ('.zip', '.7z', '.rar', '.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')
|
||||
SAFE_ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,150}$')
|
||||
VIMM_PAGE = re.compile(r'^/vault/[0-9]+/?$')
|
||||
|
||||
|
||||
def safe_title(name):
|
||||
value = re.sub(r'[^\w .()\[\]-]+', '_', name, flags=re.UNICODE).strip(' ._')[:78]
|
||||
return value or 'Gra'
|
||||
|
||||
|
||||
def normalize_magnet(value):
|
||||
# Pasted magnet URLs often contain HTML entities, hard wraps or Markdown.
|
||||
url = html.unescape(value).strip()
|
||||
url = re.sub(r"(?i)^['\"`]*\*{0,2}magnet\*{0,2}:\?", 'magnet:?', url)
|
||||
if not url.lower().startswith('magnet:?'):
|
||||
return url
|
||||
url = re.sub(r"\\\s*", '', url)
|
||||
url = re.sub(r"\s+", '', url)
|
||||
return html.unescape(url.strip("'\"`"))
|
||||
|
||||
|
||||
def validate_url(value):
|
||||
url = normalize_magnet(value)
|
||||
if url.lower().startswith('magnet:?'):
|
||||
if len(url) > 12000:
|
||||
raise ValueError('Link magnet jest zbyt dlugi')
|
||||
params = parse_qs(urlparse(url).query, keep_blank_values=True)
|
||||
xt = params.get('xt', [])
|
||||
if not any(re.fullmatch(r'urn:btih:(?:[0-9a-fA-F]{40}|[A-Za-z2-7]{32})', item, re.I)
|
||||
or re.fullmatch(r'urn:btmh:1220[0-9a-fA-F]{64}', item, re.I)
|
||||
for item in xt):
|
||||
raise ValueError('Nieprawidlowy hash xt w magnet (BTIH: 40 znakow HEX lub 32 Base32)')
|
||||
return 'magnet', url
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValueError('Podaj adres HTTP/HTTPS albo link magnet')
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError('Niepoprawny numer portu w URL') from exc
|
||||
if port not in (None, 80, 443):
|
||||
raise ValueError('Niestandardowy port w URL jest niedozwolony')
|
||||
host = parsed.hostname.lower().strip('.')
|
||||
if host in ('localhost',) or host.endswith('.localhost') or host.endswith('.local'):
|
||||
raise ValueError('Adres lokalny nie jest dozwolony')
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
ip = None
|
||||
if ip is not None and not ip.is_global:
|
||||
raise ValueError('Adres prywatny lub zarezerwowany jest niedozwolony')
|
||||
if host in ('vimm.net', 'www.vimm.net') and VIMM_PAGE.fullmatch(parsed.path):
|
||||
return 'vimm', url
|
||||
return 'url', url
|
||||
|
||||
|
||||
def parse_archive_id(value):
|
||||
value = value.strip()
|
||||
u = urlparse(value)
|
||||
if u.scheme in ('http', 'https') and u.hostname in ('archive.org', 'www.archive.org'):
|
||||
pieces = [unquote(x) for x in u.path.split('/') if x]
|
||||
if len(pieces) >= 2 and pieces[0] in ('details', 'download', 'metadata'):
|
||||
value = pieces[1]
|
||||
else:
|
||||
raise ValueError('Link musi wskazywac element archive.org/details/<identyfikator>')
|
||||
if not SAFE_ID.fullmatch(value):
|
||||
raise ValueError('Bledny identyfikator elementu Archive.org')
|
||||
return value
|
||||
|
||||
|
||||
def archive_download_url(identifier, filename):
|
||||
identifier = parse_archive_id(identifier)
|
||||
if (not filename or filename.startswith('/') or '\\' in filename or '\x00' in filename
|
||||
or any(p in ('..', '.') for p in PurePosixPath(filename).parts)):
|
||||
raise ValueError('Niedozwolona sciezka pliku Archive.org')
|
||||
return f'https://archive.org/download/{quote(identifier)}/{quote(filename, safe="/")}'
|
||||
|
||||
|
||||
def safe_import_file(filename, root=Path('/data/imports')):
|
||||
# User selects from server-side imports (including subdirectories).
|
||||
if not filename or '\\' in filename or filename.startswith('/') or '\x00' in filename:
|
||||
raise ValueError('Niepoprawna sciezka pliku')
|
||||
item = root.joinpath(*PurePosixPath(filename).parts)
|
||||
if not item.resolve().is_relative_to(root.resolve()):
|
||||
raise ValueError('Plik spoza katalogu imports')
|
||||
if not item.is_file() or item.is_symlink():
|
||||
raise ValueError('Wskazany plik nie istnieje')
|
||||
return item
|
||||
|
||||
|
||||
def is_archive(filename):
|
||||
name = filename.lower()
|
||||
return name.endswith(ARCHIVE_EXTS) or bool(re.search(r'\.(?:7z|zip)\.001$', name)) or bool(re.search(r'\.part0*1\.rar$', name))
|
||||
|
||||
|
||||
def is_additional_volume(filename):
|
||||
name = filename.lower()
|
||||
return bool(re.search(r'\.(?:7z|zip)\.0*0?[2-9][0-9]*$', name) or
|
||||
re.search(r'\.part0*[2-9][0-9]*\.rar$', name) or
|
||||
re.search(r'\.r\d\d$', name))
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
"""aria2 JSON-RPC. aria2 is started alongside the API server, not exposed publicly."""
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from . import archive_auth, store
|
||||
|
||||
RPC = 'http://127.0.0.1:6800/jsonrpc'
|
||||
|
||||
|
||||
async def rpc(method, *params):
|
||||
payload = {
|
||||
'jsonrpc': '2.0', 'id': 'x360', 'method': 'aria2.' + method,
|
||||
'params': ['token:' + os.environ['ARIA2_SECRET'], *params],
|
||||
}
|
||||
# On a container restart the API may start slightly before aria2 RPC.
|
||||
# Retry the connection instead of permanently failing restored jobs.
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for attempt in range(15):
|
||||
try:
|
||||
r = await client.post(RPC, json=payload)
|
||||
r.raise_for_status()
|
||||
answer = r.json()
|
||||
if 'error' in answer:
|
||||
raise RuntimeError(answer['error'].get('message', 'Bledna odpowiedz aria2'))
|
||||
return answer['result']
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout):
|
||||
if attempt == 14:
|
||||
raise
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
def source_urls(job):
|
||||
if job['source_type'] == 'archive_many':
|
||||
return json.loads(job['source'])
|
||||
return [job['source']]
|
||||
|
||||
|
||||
def target_filename(uri):
|
||||
if uri.lower().startswith('magnet:'):
|
||||
return None
|
||||
name = urllib.parse.unquote(Path(urllib.parse.urlsplit(uri).path).name)
|
||||
if not name or name in ('.', '..'):
|
||||
return 'pobrany-plik.bin'
|
||||
# Do not allow URI-provided paths to escape the download directory.
|
||||
return ''.join(c if c not in '\\/:*?"<>|\x00\n\r' else '_' for c in name)[:200]
|
||||
|
||||
|
||||
async def begin(job):
|
||||
base = store.DATA / 'raw' / job['id']
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
urls = source_urls(job)
|
||||
ix = job['part_index']
|
||||
if ix >= len(urls):
|
||||
store.update(job['id'], state='preparing')
|
||||
return
|
||||
uri = urls[ix]
|
||||
opts = {
|
||||
'dir': str(base), 'continue': 'true', 'max-connection-per-server': '2',
|
||||
'split': '2', 'seed-time': '0', 'auto-file-renaming': 'false',
|
||||
'allow-overwrite': 'false', 'max-tries': '8', 'retry-wait': '30',
|
||||
}
|
||||
if job['source_type'] != 'torrent_selected':
|
||||
name = target_filename(uri)
|
||||
if name and '.' in name:
|
||||
opts['out'] = name
|
||||
elif name:
|
||||
# Let the server's Content-Disposition supply the real filename.
|
||||
opts['content-disposition'] = 'true'
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
torrent = store.DATA / 'torrents' / (job['id'] + '.torrent')
|
||||
encoded = base64.b64encode(torrent.read_bytes()).decode('ascii')
|
||||
select = json.loads(job['source'])['selection']
|
||||
opts['select-file'] = select
|
||||
opts['bt-remove-unselected-file'] = 'true'
|
||||
gid = await rpc('addTorrent', encoded, [], opts)
|
||||
note = 'Pobieranie wybranych plików torrenta'
|
||||
else:
|
||||
if archive_auth.is_archive_download(uri):
|
||||
# aria2 downloads from the local proxy. It never receives IA cookies.
|
||||
opts['header'] = [archive_auth.signed_header(job['id'], ix)]
|
||||
uri = archive_auth.internal_url(job['id'], ix)
|
||||
opts['split'] = '1'
|
||||
opts['max-connection-per-server'] = '1'
|
||||
gid = await rpc('addUri', [uri], opts)
|
||||
note = f'Pobieranie pliku {ix+1}/{len(urls)}'
|
||||
store.update(job['id'], gid=gid, state='downloading', note=note)
|
||||
|
||||
|
||||
async def poll(job):
|
||||
urls = source_urls(job)
|
||||
gid = job['gid']
|
||||
if not gid:
|
||||
await begin(job)
|
||||
return
|
||||
try:
|
||||
response = await rpc('tellStatus', gid, ['gid','status','totalLength','completedLength',
|
||||
'downloadSpeed','followedBy','errorMessage'])
|
||||
except RuntimeError as ex:
|
||||
# On restart aria2 may lose an old GID. Re-add to resume available .aria2 files.
|
||||
if 'not found' in str(ex).lower():
|
||||
store.update(job['id'], gid='', note='Odzyskiwanie transferu po restarcie')
|
||||
return
|
||||
raise
|
||||
|
||||
followed = response.get('followedBy', [])
|
||||
if followed:
|
||||
store.update(job['id'], gid=followed[0], note='Magnet: metadane gotowe')
|
||||
return
|
||||
state = response.get('status')
|
||||
length = int(response.get('totalLength', 0) or 0)
|
||||
completed = int(response.get('completedLength', 0) or 0)
|
||||
speed = int(response.get('downloadSpeed', 0) or 0)
|
||||
part_progress = (completed / length) if length else 0
|
||||
ix = job['part_index']
|
||||
pct = (ix + part_progress) / len(urls) * 100
|
||||
store.update(job['id'], progress=round(pct, 2), speed=speed, total=length,
|
||||
completed=completed, note=f'Pobieranie {ix+1}/{len(urls)}: {state}')
|
||||
if state == 'complete':
|
||||
if ix + 1 < len(urls):
|
||||
store.update(job['id'], gid='', part_index=ix+1)
|
||||
else:
|
||||
store.update(job['id'], state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie ukonczone, przygotowuje pliki')
|
||||
elif state in ('error','removed'):
|
||||
message = response.get('errorMessage') or state
|
||||
if job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(urls[ix])):
|
||||
if any(x in message.lower() for x in ('authorization', '403', '401', 'forbidden')):
|
||||
message += ' — Internet Archive: sprawdź logowanie i dostęp do pliku.'
|
||||
raise RuntimeError('aria2: ' + message)
|
||||
elif state == 'paused':
|
||||
store.update(job['id'], state='paused', speed=0)
|
||||
|
||||
|
||||
async def cancel_download(gid):
|
||||
if not gid:
|
||||
return
|
||||
try:
|
||||
await rpc('forceRemove', gid)
|
||||
except (RuntimeError, httpx.HTTPError):
|
||||
pass
|
||||
|
||||
|
||||
async def pause_download(gid):
|
||||
if gid:
|
||||
await rpc('forcePause', gid)
|
||||
|
||||
|
||||
async def resume_download(gid):
|
||||
if gid:
|
||||
await rpc('unpause', gid)
|
||||
+515
@@ -0,0 +1,515 @@
|
||||
"""Self-hosted X360 Web Queue: authenticated FastAPI + one resilient worker."""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, unquote
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import archive_auth, archive_download, bt, download, pipeline, store, torrent_meta, vimm
|
||||
from .common import (FORMATS, archive_download_url, parse_archive_id, safe_import_file,
|
||||
safe_title, validate_url)
|
||||
|
||||
|
||||
basic = HTTPBasic()
|
||||
|
||||
|
||||
def authorize(credentials: HTTPBasicCredentials = Depends(basic)):
|
||||
u = os.getenv('WEB_USER', 'admin').encode()
|
||||
p = os.getenv('WEB_PASSWORD', '').encode()
|
||||
if not p or not (secrets.compare_digest(credentials.username.encode(), u)
|
||||
and secrets.compare_digest(credentials.password.encode(), p)):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='Nieprawidlowy login lub haslo',
|
||||
headers={'WWW-Authenticate': 'Basic'})
|
||||
|
||||
|
||||
class AddJob(BaseModel):
|
||||
url: str = Field(min_length=1, max_length=4000)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class AddImports(BaseModel):
|
||||
filename: str = Field(min_length=1, max_length=300)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class AddArchive(BaseModel):
|
||||
identifier: str = Field(min_length=1, max_length=250)
|
||||
files: list[str] = Field(min_length=1, max_length=150)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class ItemLookup(BaseModel):
|
||||
identifier: str = Field(min_length=1, max_length=250)
|
||||
|
||||
|
||||
class ArchiveLogin(BaseModel):
|
||||
email: str = Field(min_length=3, max_length=254)
|
||||
password: str = Field(min_length=1, max_length=1024)
|
||||
|
||||
|
||||
class MagnetInspect(BaseModel):
|
||||
magnet: str = Field(min_length=20, max_length=12000)
|
||||
|
||||
|
||||
class MagnetJob(BaseModel):
|
||||
preview_id: str = Field(min_length=32, max_length=32)
|
||||
files: list[int] = Field(min_length=1, max_length=20000)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
def new_job(source_type, source, title, fmt):
|
||||
if fmt not in FORMATS:
|
||||
raise HTTPException(status_code=400, detail='Format: xex, god albo raw')
|
||||
jid = str(uuid.uuid4())
|
||||
store.create({'id': jid, 'title': safe_title(title or 'Xbox 360'),
|
||||
'source_type': source_type, 'source': source, 'format': fmt})
|
||||
return store.get(jid)
|
||||
|
||||
|
||||
async def worker():
|
||||
while True:
|
||||
job = store.next_job()
|
||||
if not job:
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
jid = job['id']
|
||||
try:
|
||||
if job['state'] == 'preparing':
|
||||
await pipeline.process_job(job)
|
||||
elif job['source_type'] == 'import':
|
||||
# Local import is copied into persistent raw dir, then converted.
|
||||
src = safe_import_file(job['source'])
|
||||
target = store.DATA / 'raw' / jid
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(shutil.copy2, src, target / src.name)
|
||||
store.update(jid, state='preparing', progress=100)
|
||||
elif job['source_type'] == 'vimm':
|
||||
await vimm.browser_download(job)
|
||||
elif job['source_type'] == 'torrent_selected':
|
||||
if job['state'] == 'queued':
|
||||
await bt.start_download(job)
|
||||
else:
|
||||
await bt.poll_download(job)
|
||||
elif job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(job['source'])):
|
||||
if job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.update(jid, gid='')
|
||||
await archive_download.run_one(job)
|
||||
elif job['state'] == 'queued':
|
||||
# Resume if there is an aria2 GID instead of accidentally
|
||||
# starting a duplicate download.
|
||||
if job['gid']:
|
||||
await download.resume_download(job['gid'])
|
||||
store.update(jid, state='downloading')
|
||||
else:
|
||||
await download.begin(job)
|
||||
else:
|
||||
await download.poll(job)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except archive_download.ArchiveStopped:
|
||||
pass
|
||||
except Exception as ex:
|
||||
store.update(jid, state='error', speed=0, error=str(ex)[:1800],
|
||||
note='Błąd zadania: otwórz Log')
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
for subdir in ('raw', 'ready', 'work', 'imports', 'logs', 'metadata', 'torrents'):
|
||||
(store.DATA / subdir).mkdir(parents=True, exist_ok=True)
|
||||
store.init_db()
|
||||
task = asyncio.create_task(worker())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
await bt.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(title='X360 Web Queue', version='0.1.0', lifespan=lifespan,
|
||||
docs_url=None, redoc_url=None, openapi_url=None)
|
||||
api = APIRouter(prefix='/api', dependencies=[Depends(authorize)])
|
||||
MAX_INSPECT_SECONDS = 300
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def guard_cross_origin_requests(request: Request, call_next):
|
||||
if request.method in ('POST', 'PUT', 'PATCH', 'DELETE'):
|
||||
origin = request.headers.get('origin')
|
||||
if origin and urlparse(origin).netloc != request.headers.get('host'):
|
||||
return PlainTextResponse('Nieprawidlowa domena zrodla zadania', status_code=403)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
@app.get('/')
|
||||
async def index(_: None = Depends(authorize)):
|
||||
return FileResponse(Path(__file__).parent / 'static' / 'index.html')
|
||||
|
||||
|
||||
@app.get('/healthz')
|
||||
async def healthz():
|
||||
return {'ok': True}
|
||||
|
||||
|
||||
@api.get('/jobs')
|
||||
async def jobs():
|
||||
return store.list_jobs()
|
||||
|
||||
|
||||
@api.post('/jobs')
|
||||
async def add_download(body: AddJob):
|
||||
try:
|
||||
kind, url = validate_url(body.url)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(status_code=400, detail=str(ex)) from ex
|
||||
parsed = urlparse(url)
|
||||
guessed = unquote(Path(parsed.path).name) if kind != 'magnet' else 'Torrent'
|
||||
if kind == 'vimm':
|
||||
guessed = 'Vimm-' + parsed.path.rstrip('/').split('/')[-1]
|
||||
if kind == 'magnet':
|
||||
raise HTTPException(400, 'Magnet wymaga najpierw wyboru plików (/api/magnet/inspect)')
|
||||
return new_job(kind, url, body.title or guessed, body.format)
|
||||
|
||||
|
||||
@api.get('/archive/session')
|
||||
async def archive_session():
|
||||
return archive_auth.session_status()
|
||||
|
||||
|
||||
@api.post('/archive/session')
|
||||
async def archive_login(body: ArchiveLogin):
|
||||
if '@' not in body.email:
|
||||
raise HTTPException(400, 'Niepoprawny adres e-mail')
|
||||
try:
|
||||
return await archive_auth.login(body.email.strip(), body.password)
|
||||
except archive_auth.ArchiveAuthError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
@api.delete('/archive/session')
|
||||
async def archive_logout():
|
||||
archive_auth.logout()
|
||||
return archive_auth.session_status()
|
||||
|
||||
|
||||
@app.api_route('/internal/archive/{job_id}/{part_index}', methods=['GET', 'HEAD'])
|
||||
async def archive_proxy(job_id: str, part_index: int, request: Request):
|
||||
if (not 0 <= part_index <= 149 or
|
||||
not archive_auth.verify_token(job_id, part_index, request.headers.get('x-x360-proxy-token'))):
|
||||
raise HTTPException(403, 'Odmowa dostępu')
|
||||
job = store.get(job_id)
|
||||
if not job or job['source_type'] not in ('url', 'archive_many'):
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
urls = download.source_urls(job)
|
||||
if part_index >= len(urls) or not archive_auth.is_archive_download(urls[part_index]):
|
||||
raise HTTPException(404, 'Nie znaleziono pliku')
|
||||
return await archive_auth.stream_archive(urls[part_index], request.method, request.headers)
|
||||
|
||||
|
||||
@api.post('/magnet/inspect')
|
||||
async def magnet_inspect(body: MagnetInspect):
|
||||
try:
|
||||
kind, magnet = validate_url(body.magnet)
|
||||
if kind != 'magnet':
|
||||
raise ValueError('Wklej link magnet')
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
await asyncio.to_thread(torrent_meta.cleanup_old)
|
||||
root = store.DATA / 'metadata'
|
||||
active_previews = sum(1 for item in root.iterdir() if item.is_dir() and not item.is_symlink() and
|
||||
(item / 'request.json').is_file() and
|
||||
not list(item.glob('*.torrent')) and
|
||||
time.time() - item.stat().st_mtime < MAX_INSPECT_SECONDS)
|
||||
if active_previews >= 3:
|
||||
raise HTTPException(429, 'Trzy podglądy już pobierają metadane. Poczekaj na zakończenie.')
|
||||
preview_id = uuid.uuid4().hex
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
directory.mkdir(parents=True, mode=0o700)
|
||||
try:
|
||||
await bt.start_inspection(preview_id, magnet)
|
||||
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': magnet, 'engine': 'anacrolix'})
|
||||
except (RuntimeError, httpx.HTTPError, OSError) as exc:
|
||||
await bt.stop_inspection(preview_id)
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
raise HTTPException(502, f'Nie można rozpocząć odczytu metadanych: {exc}') from exc
|
||||
return {'preview_id': preview_id, 'status': 'loading'}
|
||||
|
||||
|
||||
@api.post('/magnet/upload')
|
||||
async def torrent_upload(request: Request):
|
||||
# No multipart dependency. Transfer raw .torrent bytes with a fixed size limit.
|
||||
if request.headers.get('content-type', '').split(';', 1)[0].strip() != 'application/x-bittorrent':
|
||||
raise HTTPException(415, 'Wymagany plik .torrent')
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(raw) + len(chunk) > torrent_meta.MAX_METADATA:
|
||||
raise HTTPException(413, 'Plik .torrent przekracza limit 32 MiB')
|
||||
raw.extend(chunk)
|
||||
if not raw:
|
||||
raise HTTPException(400, 'Pusty plik .torrent')
|
||||
preview_id = uuid.uuid4().hex
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
directory.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
target = directory / 'uploaded.torrent'
|
||||
try:
|
||||
# Validate before storing the metadata. No game payload is transferred.
|
||||
decoded = torrent_meta.decode_bencode(bytes(raw))
|
||||
if b'info' not in decoded:
|
||||
raise torrent_meta.MetadataError('Plik nie zawiera pola info')
|
||||
target.write_bytes(raw)
|
||||
torrent_meta.read_torrent(target)
|
||||
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': ''})
|
||||
except (torrent_meta.MetadataError, OSError) as exc:
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
return {'preview_id': preview_id, 'status': 'ready'}
|
||||
|
||||
|
||||
@api.get('/magnet/inspect/{preview_id}')
|
||||
async def magnet_inspect_status(preview_id: str):
|
||||
try:
|
||||
req = torrent_meta.load_request(preview_id)
|
||||
path = torrent_meta.torrent_file(preview_id)
|
||||
if path:
|
||||
info = torrent_meta.read_torrent(path)
|
||||
await bt.stop_inspection(preview_id)
|
||||
info.update({'status': 'ready', 'suggested': torrent_meta.suggested_indices(req['magnet'], len(info['files']))})
|
||||
return info
|
||||
elapsed = max(0, int(time.time() - req['created']))
|
||||
status = bt.inspect_state(preview_id)
|
||||
if status.get('status') == 'error':
|
||||
return {'status': 'error', 'error': status.get('error', 'Silnik BT zgłosił błąd')[:500]}
|
||||
if elapsed > MAX_INSPECT_SECONDS:
|
||||
await bt.stop_inspection(preview_id)
|
||||
return {'status': 'error', 'error': 'Brak metadanych po 5 minutach. Sprawdź dostępność peerów i DHT/UDP.'}
|
||||
return {'status': 'loading', 'elapsed': elapsed,
|
||||
'connections': int(status.get('connections') or 0),
|
||||
'seeders': 0, 'known_peers': int(status.get('known_peers') or 0),
|
||||
'aria_status': 'anacrolix'}
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
except (RuntimeError, httpx.HTTPError) as exc:
|
||||
raise HTTPException(502, 'aria2: ' + str(exc)[:200]) from exc
|
||||
|
||||
|
||||
@api.post('/magnet/inspect/{preview_id}/cancel')
|
||||
async def magnet_inspect_cancel(preview_id: str):
|
||||
try:
|
||||
req = torrent_meta.load_request(preview_id)
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
if req.get('engine') == 'anacrolix':
|
||||
await bt.stop_inspection(preview_id)
|
||||
else:
|
||||
await download.cancel_download(req['gid'])
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
return {'ok': True}
|
||||
|
||||
|
||||
@api.post('/magnet/jobs')
|
||||
async def magnet_selected_job(body: MagnetJob):
|
||||
if body.format not in FORMATS:
|
||||
raise HTTPException(400, 'Format: xex, god albo raw')
|
||||
try:
|
||||
torrent_meta.load_request(body.preview_id)
|
||||
path = torrent_meta.torrent_file(body.preview_id)
|
||||
if path is None:
|
||||
raise torrent_meta.MetadataError('Metadane jeszcze nie są gotowe')
|
||||
info = torrent_meta.read_torrent(path)
|
||||
await bt.stop_inspection(body.preview_id)
|
||||
valid = {item['index'] for item in info['files'] if not item['padding']}
|
||||
if len(body.files) != len(set(body.files)) or not set(body.files).issubset(valid):
|
||||
raise torrent_meta.MetadataError('Wybierz poprawne indeksy plików')
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
jid = str(uuid.uuid4())
|
||||
dest = store.DATA / 'torrents' / (jid + '.torrent')
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(shutil.copyfile, path, dest)
|
||||
selection = ','.join(str(i) for i in sorted(body.files))
|
||||
data = json.dumps({'selection': selection, 'count': len(body.files), 'name': info['name']})
|
||||
try:
|
||||
store.create({'id': jid, 'title': safe_title(body.title or info['name']),
|
||||
'source_type': 'torrent_selected', 'source': data, 'format': body.format})
|
||||
except Exception:
|
||||
dest.unlink(missing_ok=True)
|
||||
raise
|
||||
return store.get(jid)
|
||||
|
||||
|
||||
@api.post('/archive/lookup')
|
||||
async def archive_lookup(body: ItemLookup):
|
||||
try:
|
||||
identifier = parse_archive_id(body.identifier)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=25, follow_redirects=True, cookies=archive_auth.session_cookies()) as client:
|
||||
res = await client.get('https://archive.org/metadata/' + identifier)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
except (httpx.HTTPError, ValueError) as ex:
|
||||
raise HTTPException(502, 'Archive.org metadata: ' + str(ex)) from ex
|
||||
result = []
|
||||
for entry in data.get('files', []):
|
||||
name = entry.get('name', '')
|
||||
if not name or entry.get('source') == 'metadata' or name.startswith('__'):
|
||||
continue
|
||||
# Show all file types with an extension; user chooses exact archive parts.
|
||||
if '.' not in Path(name).name:
|
||||
continue
|
||||
try:
|
||||
archive_download_url(identifier, name)
|
||||
except ValueError:
|
||||
continue
|
||||
result.append({'name': name, 'size': int(entry.get('size') or 0),
|
||||
'type': entry.get('format', '')})
|
||||
return {'identifier': identifier, 'title': data.get('metadata', {}).get('title', identifier),
|
||||
'files': result[:1000]}
|
||||
|
||||
|
||||
@api.post('/archive/jobs')
|
||||
async def add_archive(body: AddArchive):
|
||||
try:
|
||||
identifier = parse_archive_id(body.identifier)
|
||||
if len(set(body.files)) != len(body.files):
|
||||
raise ValueError('Pliki nie moga sie powtarzac')
|
||||
urls = [archive_download_url(identifier, file) for file in body.files]
|
||||
if len({Path(unquote(urlparse(u).path)).name for u in urls}) != len(urls):
|
||||
raise ValueError('Pliki maja identyczne nazwy; wybierz jeden folder na zadanie')
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
# Preserving filename order is important for multipart archives.
|
||||
return new_job('archive_many', json.dumps(urls), body.title or identifier, body.format)
|
||||
|
||||
|
||||
@api.get('/imports')
|
||||
async def imports():
|
||||
root = store.DATA / 'imports'
|
||||
result = []
|
||||
for item in root.rglob('*'):
|
||||
if len(result) >= 200:
|
||||
break
|
||||
if item.is_file() and not item.is_symlink() and item.resolve().is_relative_to(root.resolve()):
|
||||
result.append({'name': str(item.relative_to(root)), 'size': item.stat().st_size})
|
||||
return result
|
||||
|
||||
|
||||
@api.post('/imports/jobs')
|
||||
async def import_job(body: AddImports):
|
||||
try:
|
||||
path = safe_import_file(body.filename)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
return new_job('import', body.filename, path.stem, body.format)
|
||||
|
||||
|
||||
@api.post('/jobs/{job_id}/{action}')
|
||||
async def manage(job_id: str, action: str):
|
||||
job = store.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
state = job['state']
|
||||
if action == 'pause' and state in ('queued', 'downloading'):
|
||||
if state == 'downloading':
|
||||
try:
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
else:
|
||||
await download.pause_download(job['gid'])
|
||||
except (RuntimeError, httpx.HTTPError) as ex:
|
||||
raise HTTPException(502, str(ex)) from ex
|
||||
store.update(job_id, state='paused', speed=0, note='Wstrzymano')
|
||||
elif action == 'resume' and state == 'paused':
|
||||
store.update(job_id, state='queued', note='Wznowienie zadania')
|
||||
elif action == 'cancel' and state in ('queued', 'paused', 'downloading'):
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
elif job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.update(job_id, state='cancelled', speed=0, note='Anulowano')
|
||||
elif action == 'retry' and state in ('error', 'cancelled'):
|
||||
# For already-downloaded input, retry conversion without redownloading.
|
||||
root = store.DATA / 'raw' / job_id
|
||||
good_files = ([p for p in root.rglob('*') if p.is_file() and not p.name.endswith('.aria2')]
|
||||
if root.exists() else [])
|
||||
complete = job['progress'] >= 99.9 and bool(good_files)
|
||||
if job['source_type'] == 'archive_many':
|
||||
# One finished file (or a .part segment) never means the whole
|
||||
# multi-file archive is ready to convert.
|
||||
complete = all((root / download.target_filename(url)).is_file()
|
||||
for url in download.source_urls(job))
|
||||
store.update(job_id, state='preparing' if complete else 'queued',
|
||||
gid='' if not complete else job['gid'], error='',
|
||||
note='Ponowienie zadania')
|
||||
elif action == 'remove' and state in ('error','cancelled','done','paused'):
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
elif state == 'paused' and job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.erase(job_id)
|
||||
return {'ok': True, 'note': 'Usunieto wpis z kolejki, pliki pozostaly na dysku'}
|
||||
else:
|
||||
raise HTTPException(409, 'Ta akcja nie jest dostepna dla obecnego stanu zadania')
|
||||
return store.get(job_id)
|
||||
|
||||
|
||||
@api.get('/jobs/{job_id}/log')
|
||||
async def job_log(job_id: str):
|
||||
if not store.get(job_id):
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
path = store.DATA / 'logs' / f'{job_id}.log'
|
||||
job = store.get(job_id)
|
||||
lines = [f"Stan: {job['state']}", f"Etap: {job['note'] or 'brak'}"]
|
||||
if job['error']:
|
||||
lines.append('Błąd pobierania / zadania: ' + job['error'])
|
||||
bt_log = store.DATA / 'logs' / f'{job_id}.bt.log'
|
||||
if bt_log.is_file():
|
||||
lines.append('BitTorrent (ostatnie 12 KiB):')
|
||||
lines.append(bt_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
|
||||
download_log = store.DATA / 'logs' / f'{job_id}.download.log'
|
||||
if download_log.is_file():
|
||||
lines.append('Pobieranie (ostatnie 12 KiB):')
|
||||
lines.append(download_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
|
||||
lines.append('')
|
||||
lines.append(path.read_bytes()[-24000:].decode('utf-8', errors='replace')
|
||||
if path.is_file() else 'Konwersja jeszcze nie została uruchomiona.')
|
||||
return PlainTextResponse('\n'.join(lines))
|
||||
|
||||
|
||||
@api.get('/system')
|
||||
async def system():
|
||||
usage = shutil.disk_usage(store.DATA / 'raw')
|
||||
host_dir = os.getenv('XBOX_HOST_DIR', '/xbox').rstrip('/') or '/xbox'
|
||||
return {'free': usage.free, 'total': usage.total,
|
||||
'host_data_dir': host_dir,
|
||||
'formats': ['xex','god','raw'],
|
||||
'vimm_enabled': os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() in ('1','yes','true'),
|
||||
'bt_engine': 'anacrolix (Go)', 'bt_port': os.getenv('BT_PORT', '6881')}
|
||||
|
||||
|
||||
app.include_router(api)
|
||||
app.mount('/static', StaticFiles(directory=str(Path(__file__).parent / 'static')), name='static')
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
"""Automatic ISO -> XEX/GOD pipeline; no console-side patching of executables."""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from . import store
|
||||
from .common import is_archive, is_additional_volume, safe_title
|
||||
|
||||
READY = store.DATA / 'ready'
|
||||
WORK = store.DATA / 'work'
|
||||
RAW = store.DATA / 'raw'
|
||||
LOGS = store.DATA / 'logs'
|
||||
|
||||
|
||||
class ProcessingError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def safe_archive_path(name):
|
||||
# Archive members are always relative, independent of platform.
|
||||
norm = name.replace('\\', '/')
|
||||
if (not norm or norm.startswith('/') or '\x00' in norm or
|
||||
re.match(r'^[A-Za-z]:', norm) or
|
||||
any(x == '..' for x in PurePosixPath(norm).parts)):
|
||||
raise ProcessingError(f'Archiwum zawiera niebezpieczna sciezke: {name[:150]}')
|
||||
|
||||
|
||||
def verify_no_symlinks(directory):
|
||||
for root, dirs, files in os.walk(directory, followlinks=False):
|
||||
for name in dirs + files:
|
||||
path = Path(root) / name
|
||||
if path.is_symlink():
|
||||
raise ProcessingError('Pliki-symboliczne linki nie sa dopuszczalne: ' + name)
|
||||
|
||||
|
||||
async def command(job_id, *args):
|
||||
LOGS.mkdir(parents=True, exist_ok=True)
|
||||
log_path = LOGS / f'{job_id}.log'
|
||||
with log_path.open('ab') as log:
|
||||
log.write(('\n>>> ' + ' '.join(str(x) for x in args) + '\n').encode('utf-8'))
|
||||
process = await asyncio.create_subprocess_exec(*map(str, args), stdout=log, stderr=log,
|
||||
stdin=asyncio.subprocess.DEVNULL)
|
||||
try:
|
||||
code = await process.wait()
|
||||
except asyncio.CancelledError:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
raise
|
||||
if code:
|
||||
data = log_path.read_bytes()[-2500:].decode('utf-8', errors='replace')
|
||||
raise ProcessingError(f'Narzedzie {args[0]} zakonczone bledem {code}. Ostatni log:\n{data[-1400:]}')
|
||||
|
||||
|
||||
def list_primary_archives(source):
|
||||
files = [p for p in source.rglob('*') if p.is_file() and not p.is_symlink()]
|
||||
archives = [p for p in files if is_archive(p.name) and not is_additional_volume(p.name)]
|
||||
# Multipart .rar is detected by the first volume, subsequent parts are ignored.
|
||||
# A single .zip with .z01 volumes is also supported by 7z.
|
||||
return archives
|
||||
|
||||
|
||||
async def unpack_archive(job_id, archive, destination):
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
lower = archive.name.lower()
|
||||
store.update(job_id, state='extracting', note='Rozpakowywanie ' + archive.name[:100])
|
||||
if lower.endswith('.zip') and not any(archive.parent.glob(archive.stem + '.z[0-9][0-9]')):
|
||||
# Standard ZIP: validate entries to prevent writes outside destination.
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
for member in z.infolist():
|
||||
safe_archive_path(member.filename)
|
||||
if (member.external_attr >> 16) & 0o170000 == 0o120000:
|
||||
raise ProcessingError('Archiwum ZIP zawiera link symboliczny')
|
||||
await asyncio.to_thread(z.extractall, destination)
|
||||
elif lower.endswith(('.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')):
|
||||
# tarfile's filters are version-dependent; reject links/special entries ourselves.
|
||||
with tarfile.open(archive) as tf:
|
||||
for member in tf.getmembers():
|
||||
safe_archive_path(member.name)
|
||||
if not (member.isfile() or member.isdir()):
|
||||
raise ProcessingError('Archiwum TAR zawiera link lub plik specjalny')
|
||||
await asyncio.to_thread(tf.extractall, destination, filter='data')
|
||||
else:
|
||||
# 7z/RAR/multipart. Validate file paths *before* invoking extraction.
|
||||
listing = await asyncio.create_subprocess_exec('7z','l','-slt',str(archive),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
listing_out, listing_err = await listing.communicate()
|
||||
if listing.returncode:
|
||||
if not lower.endswith('.rar'):
|
||||
raise ProcessingError('7z nie rozpoznaje archiwum: ' + listing_err[-300:].decode(errors='replace'))
|
||||
# RAR5 fallback: libunarr (unar) understands many archives missing
|
||||
# from Debian's p7zip package. Validate lsar entries first.
|
||||
lsar = await asyncio.create_subprocess_exec('lsar', '-json', '-no-recursion', str(archive),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
stdout, stderr = await lsar.communicate()
|
||||
if lsar.returncode:
|
||||
raise ProcessingError('7z/unar nie rozpoznaje RAR: ' + stderr[-400:].decode(errors='replace'))
|
||||
try:
|
||||
items = json.loads(stdout).get('lsarContents', [])
|
||||
if not items:
|
||||
raise ProcessingError('Nie mozna odczytac zawartosci RAR')
|
||||
for item in items:
|
||||
safe_archive_path(item['XADFileName'])
|
||||
if any(item.get(x) for x in ('XADIsLink', 'XADIsSymbolicLink', 'XADLinkDestination')):
|
||||
raise ProcessingError('RAR zawiera link symboliczny')
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise ProcessingError('Niepoprawna lista plikow RAR') from exc
|
||||
await command(job_id, 'unar', '-no-directory', '-force-overwrite',
|
||||
'-output-directory', str(destination), str(archive))
|
||||
verify_no_symlinks(destination)
|
||||
return
|
||||
data = listing_out.decode('utf-8', errors='replace')
|
||||
entries = data.split('----------', 1)
|
||||
if len(entries) != 2:
|
||||
raise ProcessingError('Nie mozna zweryfikowac listy plikow archiwum')
|
||||
for line in entries[1].splitlines():
|
||||
if line.startswith('Path = '):
|
||||
safe_archive_path(line[7:])
|
||||
if line.startswith(('Symbolic Link = ', 'Hard Link = ')):
|
||||
raise ProcessingError('Archiwum zawiera link symboliczny lub twardy')
|
||||
await command(job_id, '7z', 'x', '-y', '-bd', str(archive), f'-o{destination}')
|
||||
verify_no_symlinks(destination)
|
||||
|
||||
|
||||
async def locate_payload(job_id, root, tmp):
|
||||
"""Find X360 ISOs or default.xex, recursively extracting up to 2 archive layers."""
|
||||
current = root
|
||||
for depth in range(3):
|
||||
verify_no_symlinks(current)
|
||||
isos = sorted(p for p in current.rglob('*') if p.is_file() and p.suffix.lower() == '.iso')
|
||||
xex_files = sorted(p for p in current.rglob('*') if p.is_file() and p.name.lower() == 'default.xex')
|
||||
if isos or xex_files:
|
||||
return isos, xex_files
|
||||
archives = list_primary_archives(current)
|
||||
if not archives:
|
||||
break
|
||||
if len(archives) != 1:
|
||||
raise ProcessingError('Znaleziono kilka niezaleznych archiwow. Dodaj jedno na zadanie lub polacz czesci gry w jedno archiwum.')
|
||||
if depth >= 2:
|
||||
raise ProcessingError('Archiwa zagniezdzone glebiej niz 3 poziomy nie sa obsługiwane')
|
||||
out = tmp / f'unpacked-{depth}'
|
||||
await unpack_archive(job_id, archives[0], out)
|
||||
current = out
|
||||
raise ProcessingError('Brak obrazu .iso ani default.xex. Sprawdz czy pobrano wszystkie czesci archiwum.')
|
||||
|
||||
|
||||
async def process_job(job):
|
||||
jid = job['id']
|
||||
fmt = job['format']
|
||||
source = RAW / jid
|
||||
temp = WORK / jid
|
||||
slug = safe_title(job['title'])[:55] + '-' + jid[:8]
|
||||
dest = READY / fmt / slug
|
||||
atomic_dest = READY / fmt / ('.' + slug + '.working')
|
||||
if not source.exists():
|
||||
raise ProcessingError('Brak pobranych plikow w katalogu raw')
|
||||
temp.mkdir(parents=True, exist_ok=True)
|
||||
if atomic_dest.exists():
|
||||
shutil.rmtree(atomic_dest)
|
||||
atomic_dest.mkdir(parents=True, exist_ok=True)
|
||||
store.update(jid, state='preparing', note='Analiza pobranych plikow', speed=0)
|
||||
try:
|
||||
verify_no_symlinks(source)
|
||||
if fmt == 'raw':
|
||||
await asyncio.to_thread(shutil.copytree, source, atomic_dest / 'files')
|
||||
else:
|
||||
isos, xex_files = await locate_payload(jid, source, temp)
|
||||
if fmt == 'xex':
|
||||
store.update(jid, state='converting', note='Tworzenie folderu XEX')
|
||||
if xex_files:
|
||||
if len(xex_files) > 1:
|
||||
raise ProcessingError('Kilka plikow default.xex; wybierz jeden obraz ISO na zadanie')
|
||||
await asyncio.to_thread(shutil.copytree, xex_files[0].parent, atomic_dest / 'Gra')
|
||||
else:
|
||||
for n, iso in enumerate(isos, 1):
|
||||
dest_dir = atomic_dest / ('Gra' if len(isos) == 1 else f'Disc_{n}')
|
||||
await command(jid, 'xdvdfs', 'unpack', str(iso), str(dest_dir))
|
||||
if not any(p.name.lower() == 'default.xex' for p in dest_dir.rglob('*') if p.is_file()):
|
||||
raise ProcessingError(f'Obraz ISO nie zawiera default.xex: {iso.name}')
|
||||
else:
|
||||
store.update(jid, state='converting', note='Konwersja ISO do GOD')
|
||||
if not isos and xex_files:
|
||||
if len(xex_files) > 1:
|
||||
raise ProcessingError('Kilka folderow default.xex; GOD wymaga jednego obrazu')
|
||||
packed = temp / 'packed.xiso.iso'
|
||||
await command(jid, 'xdvdfs', 'pack', str(xex_files[0].parent), str(packed))
|
||||
isos = [packed]
|
||||
for n, iso in enumerate(isos, 1):
|
||||
# iso2god creates <TitleID>/<ContentType>/<MediaID> inside provided root.
|
||||
# Prefix these folders with Content/0000000000000000 for Xbox internal HDD.
|
||||
disc_dir = atomic_dest if len(isos) == 1 else atomic_dest / f'Disc_{n}'
|
||||
content = disc_dir / 'Content' / '0000000000000000'
|
||||
content.mkdir(parents=True, exist_ok=True)
|
||||
await command(jid, 'iso2god', str(iso), str(content))
|
||||
if not any(p.is_dir() and re.fullmatch(r'[0-9A-Fa-f]{8}', p.name)
|
||||
for p in content.iterdir()):
|
||||
raise ProcessingError(f'Brak katalogu TitleID w wyniku ISO2GOD: {iso.name}')
|
||||
if dest.exists():
|
||||
raise ProcessingError('Folder wynikowy juz istnieje: ' + str(dest))
|
||||
atomic_dest.rename(dest)
|
||||
store.update(jid, state='done', progress=100, output=str(dest),
|
||||
note='Gotowe do skopiowania na Xbox 360')
|
||||
finally:
|
||||
# Full download is preserved; only temporary extraction is deleted.
|
||||
shutil.rmtree(temp, ignore_errors=True)
|
||||
if atomic_dest.exists():
|
||||
shutil.rmtree(atomic_dest, ignore_errors=True)
|
||||
@@ -0,0 +1,577 @@
|
||||
'use strict';
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const NS = 'http://www.w3.org/2000/svg';
|
||||
const UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
const ACTIVE = new Set(['downloading', 'preparing', 'extracting', 'converting', 'vimm_browser']);
|
||||
const LABELS = {
|
||||
queued: 'W kolejce', downloading: 'Pobieranie', paused: 'Wstrzymano',
|
||||
preparing: 'Przygotowanie', extracting: 'Rozpakowanie', converting: 'Konwersja',
|
||||
done: 'Gotowe', error: 'Błąd', cancelled: 'Anulowano', vimm_browser: 'Vimm'
|
||||
};
|
||||
const S = { archiveId: '', archiveTitle: '', archiveFiles: [], selected: new Set(),
|
||||
jobs: [], filter: 'all', connected: false, hostRoot: '/xbox', magnetPreview: null,
|
||||
magnetFiles: [], magnetSelected: new Set(), magnetName: '', magnetTitle: '', magnetFormat: 'xex' };
|
||||
|
||||
function normalizeMagnet(raw) {
|
||||
return raw.trim()
|
||||
.replace(/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i, 'magnet:?')
|
||||
.replace(/&|&|&/gi, '&')
|
||||
.replace(/\\\s*/g, '')
|
||||
.replace(/\s+/g, '')
|
||||
.replace(/[\'"`]+$/g, '');
|
||||
}
|
||||
|
||||
function bytes(n) {
|
||||
n = Math.max(0, Number(n) || 0);
|
||||
let i = 0;
|
||||
while (n >= 1024 && i < UNITS.length - 1) { n /= 1024; i++; }
|
||||
return (i ? n.toFixed(1) : n.toFixed(0)) + ' ' + UNITS[i];
|
||||
}
|
||||
|
||||
function el(tag, cls, text) {
|
||||
const node = document.createElement(tag);
|
||||
if (cls) node.className = cls;
|
||||
if (text !== undefined && text !== null) node.textContent = text;
|
||||
return node;
|
||||
}
|
||||
|
||||
function icon(name) {
|
||||
const svg = document.createElementNS(NS, 'svg');
|
||||
svg.setAttribute('class', 'icon');
|
||||
svg.setAttribute('aria-hidden', 'true');
|
||||
const use = document.createElementNS(NS, 'use');
|
||||
use.setAttribute('href', '#i-' + name);
|
||||
svg.append(use);
|
||||
return svg;
|
||||
}
|
||||
|
||||
function makeButton(label, glyph, cls = 'secondary') {
|
||||
const button = el('button', cls);
|
||||
button.type = 'button';
|
||||
button.append(icon(glyph), document.createTextNode(label));
|
||||
return button;
|
||||
}
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const response = await fetch('/api' + path, {
|
||||
...options,
|
||||
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
|
||||
});
|
||||
let result;
|
||||
try { result = await response.json(); }
|
||||
catch { result = { detail: 'Nieprawidłowa odpowiedź serwera' }; }
|
||||
if (!response.ok) {
|
||||
throw new Error(typeof result.detail === 'string' ? result.detail : JSON.stringify(result.detail || result));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function connection(online) {
|
||||
S.connected = online;
|
||||
$('connectionState').classList.toggle('online', online);
|
||||
$('connectionState').classList.toggle('offline', !online);
|
||||
$('connectionText').textContent = online ? 'Połączono' : 'Brak połączenia';
|
||||
}
|
||||
|
||||
function flash(message, bad = false) {
|
||||
const t = $('toast');
|
||||
t.textContent = message;
|
||||
t.classList.toggle('bad', bad);
|
||||
t.style.display = 'block';
|
||||
clearTimeout(window._toastTimeout);
|
||||
window._toastTimeout = setTimeout(() => { t.style.display = 'none'; }, 5000);
|
||||
}
|
||||
|
||||
function filterJobs(jobs) {
|
||||
if (S.filter === 'active') return jobs.filter(j => !['done', 'error', 'cancelled'].includes(j.state));
|
||||
if (S.filter === 'done') return jobs.filter(j => j.state === 'done');
|
||||
if (S.filter === 'error') return jobs.filter(j => j.state === 'error');
|
||||
return jobs;
|
||||
}
|
||||
|
||||
function sourceDescription(job) {
|
||||
if (job.source_type === 'archive_many') {
|
||||
try { return 'Internet Archive · ' + JSON.parse(job.source).length + ' plików'; }
|
||||
catch { return 'Internet Archive'; }
|
||||
}
|
||||
if (job.source_type === 'import') return 'Import · ' + job.source;
|
||||
if (job.source_type === 'magnet') return 'Magnet';
|
||||
if (job.source_type === 'torrent_selected') {
|
||||
try { return 'Torrent · ' + JSON.parse(job.source).count + ' plików'; }
|
||||
catch { return 'Torrent'; }
|
||||
}
|
||||
if (job.source_type === 'vimm') return 'Vimm · ' + job.source;
|
||||
return job.source;
|
||||
}
|
||||
|
||||
function jobAction(label, glyph, action, job, danger = false) {
|
||||
const button = makeButton(label, glyph, 'action' + (danger ? ' danger' : ''));
|
||||
button.addEventListener('click', async () => {
|
||||
if (action === 'remove' && !confirm('Usunąć wpis? Pobrane pliki zostaną na dysku.')) return;
|
||||
if (action === 'cancel' && !confirm('Anulować to zadanie?')) return;
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/jobs/' + encodeURIComponent(job.id) + '/' + action, { method: 'POST' });
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
return button;
|
||||
}
|
||||
|
||||
async function showLog(job) {
|
||||
try {
|
||||
const response = await fetch('/api/jobs/' + encodeURIComponent(job.id) + '/log');
|
||||
if (!response.ok) throw new Error('HTTP ' + response.status);
|
||||
$('logContent').textContent = await response.text();
|
||||
$('logDialog').showModal();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
}
|
||||
|
||||
function hostPath(path) {
|
||||
if (typeof path !== 'string') return '';
|
||||
if (path === '/data/raw') return S.hostRoot + '/downloads';
|
||||
if (path.startsWith('/data/raw/')) return S.hostRoot + '/downloads/' + path.slice('/data/raw/'.length);
|
||||
if (path === '/data') return S.hostRoot;
|
||||
if (path.startsWith('/data/')) return S.hostRoot + '/' + path.slice('/data/'.length);
|
||||
return path;
|
||||
}
|
||||
|
||||
function jobRow(job) {
|
||||
const card = el('article', 'job');
|
||||
const header = el('div', 'job-top');
|
||||
const heading = el('div', 'job-heading');
|
||||
heading.append(el('div', 'job-name', job.title), el('div', 'job-sub', sourceDescription(job)));
|
||||
header.append(heading);
|
||||
const statusClass = job.state === 'done' ? 'done' : job.state === 'error' ? 'error' : ACTIVE.has(job.state) ? 'running' : '';
|
||||
header.append(el('span', 'job-status ' + statusClass, LABELS[job.state] || job.state));
|
||||
card.append(header);
|
||||
|
||||
const pct = Math.max(0, Math.min(100, Number(job.progress) || 0));
|
||||
const unknown = job.state === 'vimm_browser';
|
||||
const progress = el('div', 'job-progress');
|
||||
const track = el('div', 'progressbar' + (job.state === 'error' ? ' is-error' : ''));
|
||||
const fill = el('span');
|
||||
fill.style.width = pct + '%';
|
||||
track.append(fill);
|
||||
progress.append(track, el('span', 'progress-text', unknown ? '—' : pct.toFixed(1) + '%'));
|
||||
card.append(progress);
|
||||
|
||||
const meta = el('div', 'job-meta');
|
||||
meta.append(el('span', 'note-text', job.note || 'Oczekiwanie'));
|
||||
meta.append(el('span', 'speed', Number(job.speed) > 0 ? bytes(job.speed) + '/s' : ''));
|
||||
card.append(meta);
|
||||
|
||||
if (job.state === 'error' && job.error) card.append(el('div', 'job-error', job.error));
|
||||
if (job.output) card.append(el('div', 'job-output', 'Zapisano: ' + hostPath(job.output)));
|
||||
|
||||
const bottom = el('div', 'job-bottom');
|
||||
const chips = el('div', 'job-chips');
|
||||
chips.append(el('span', 'job-chip', (job.format || 'raw').toUpperCase()));
|
||||
bottom.append(chips);
|
||||
const actions = el('div', 'job-actions');
|
||||
if (['queued', 'downloading'].includes(job.state)) {
|
||||
actions.append(jobAction('Pauza', 'pause', 'pause', job), jobAction('Anuluj', 'x', 'cancel', job, true));
|
||||
}
|
||||
if (job.state === 'paused') {
|
||||
actions.append(jobAction('Wznów', 'play', 'resume', job), jobAction('Anuluj', 'x', 'cancel', job, true));
|
||||
}
|
||||
if (['error', 'cancelled'].includes(job.state)) actions.append(jobAction('Ponów', 'retry', 'retry', job));
|
||||
if (['error', 'done', 'cancelled', 'paused'].includes(job.state)) actions.append(jobAction('Usuń', 'trash', 'remove', job, true));
|
||||
const log = makeButton('Log', 'file', 'action');
|
||||
log.addEventListener('click', () => showLog(job));
|
||||
actions.append(log);
|
||||
bottom.append(actions);
|
||||
card.append(bottom);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderJobs() {
|
||||
$('jobsCount').textContent = String(S.jobs.length);
|
||||
$('countQueued').textContent = String(S.jobs.filter(j => j.state === 'queued' || j.state === 'paused').length);
|
||||
$('countActive').textContent = String(S.jobs.filter(j => ACTIVE.has(j.state)).length);
|
||||
$('countDone').textContent = String(S.jobs.filter(j => j.state === 'done').length);
|
||||
const list = $('jobsList');
|
||||
list.replaceChildren();
|
||||
const filtered = filterJobs(S.jobs);
|
||||
if (!filtered.length) {
|
||||
list.append(el('div', 'empty-state', S.filter === 'all' ? 'Brak zadań. Dodaj link powyżej.' : 'Brak zadań w tym widoku.'));
|
||||
return;
|
||||
}
|
||||
for (const job of filtered) list.append(jobRow(job));
|
||||
}
|
||||
|
||||
async function refreshJobs(showError = false) {
|
||||
try {
|
||||
S.jobs = await api('/jobs');
|
||||
renderJobs();
|
||||
connection(true);
|
||||
} catch (e) {
|
||||
connection(false);
|
||||
if (showError) flash('Nie można odczytać kolejki: ' + e.message, true);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshSystem() {
|
||||
try {
|
||||
const info = await api('/system');
|
||||
$('storageNum').textContent = bytes(info.free);
|
||||
const percent = info.total > 0 ? (100 * (1 - info.free / info.total)) : 0;
|
||||
$('storageFill').style.width = Math.max(0, Math.min(100, percent)) + '%';
|
||||
$('storageText').textContent = bytes(info.total) + ' łącznie';
|
||||
const hostRoot = String(info.host_data_dir || '/xbox').replace(/\/+$/, '');
|
||||
S.hostRoot = hostRoot;
|
||||
$('importHostPath').textContent = hostRoot + '/imports/';
|
||||
$('readyXexPath').textContent = hostRoot + '/ready/xex/';
|
||||
$('readyGodPath').textContent = hostRoot + '/ready/god/';
|
||||
$('downloadHostPath').textContent = hostRoot + '/downloads/';
|
||||
$('workHostPath').textContent = hostRoot + '/work/';
|
||||
connection(true);
|
||||
} catch {
|
||||
$('storageText').textContent = 'Nie udało się odczytać dysku';
|
||||
connection(false);
|
||||
}
|
||||
}
|
||||
|
||||
for (const tab of document.querySelectorAll('.tab')) {
|
||||
tab.addEventListener('click', () => {
|
||||
for (const other of document.querySelectorAll('.tab')) {
|
||||
const selected = other === tab;
|
||||
other.classList.toggle('active', selected);
|
||||
other.setAttribute('aria-selected', String(selected));
|
||||
}
|
||||
for (const name of ['links', 'archive', 'import']) {
|
||||
$('pane-' + name).classList.toggle('hidden', name !== tab.dataset.tab);
|
||||
}
|
||||
if (tab.dataset.tab === 'import') refreshImports();
|
||||
});
|
||||
}
|
||||
|
||||
for (const filter of document.querySelectorAll('.filter')) {
|
||||
filter.addEventListener('click', () => {
|
||||
S.filter = filter.dataset.filter;
|
||||
for (const other of document.querySelectorAll('.filter')) {
|
||||
const selected = other === filter;
|
||||
other.classList.toggle('active', selected);
|
||||
other.setAttribute('aria-pressed', String(selected));
|
||||
}
|
||||
renderJobs();
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshArchiveSession() {
|
||||
try {
|
||||
const result = await api('/archive/session');
|
||||
$('archiveSessionText').textContent = result.authenticated ? ('Sesja zapisana · ' + result.email) : 'Niezalogowano';
|
||||
$('archiveLoginForm').classList.toggle('hidden', result.authenticated);
|
||||
$('archiveLogout').classList.toggle('hidden', !result.authenticated);
|
||||
} catch (e) { $('archiveSessionText').textContent = 'Błąd sprawdzania sesji'; }
|
||||
}
|
||||
|
||||
$('archiveLoginForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const button = $('archiveLoginForm').querySelector('[type=submit]');
|
||||
button.disabled = true;
|
||||
try {
|
||||
const result = await api('/archive/session', { method: 'POST', body: JSON.stringify({
|
||||
email: $('archiveEmail').value.trim(), password: $('archivePassword').value
|
||||
}) });
|
||||
$('archivePassword').value = '';
|
||||
flash('Zapisano sesję Archive.org: ' + result.email);
|
||||
await refreshArchiveSession();
|
||||
} catch (e) { $('archivePassword').value = ''; flash('Archive.org: ' + e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('archiveLogout').addEventListener('click', async () => {
|
||||
try {
|
||||
await api('/archive/session', { method: 'DELETE' });
|
||||
await refreshArchiveSession();
|
||||
flash('Usunięto zapisaną sesję Archive.org.');
|
||||
} catch (e) { flash(e.message, true); }
|
||||
});
|
||||
|
||||
function renderMagnet() {
|
||||
const list = $('magnetFiles');
|
||||
list.replaceChildren();
|
||||
const text = $('magnetFilter').value.toLowerCase();
|
||||
let displayed = 0;
|
||||
for (const file of S.magnetFiles) {
|
||||
if (!file.name.toLowerCase().includes(text)) continue;
|
||||
if (displayed >= 500) break;
|
||||
displayed++;
|
||||
const row = el('label', 'file-row');
|
||||
const box = el('input');
|
||||
box.type = 'checkbox';
|
||||
box.checked = S.magnetSelected.has(file.index);
|
||||
box.disabled = Boolean(file.padding);
|
||||
box.addEventListener('change', () => {
|
||||
if (box.checked) S.magnetSelected.add(file.index);
|
||||
else S.magnetSelected.delete(file.index);
|
||||
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
|
||||
});
|
||||
row.append(box, el('span', 'file-name', file.name),
|
||||
el('span', 'size', file.padding ? 'Systemowy' : bytes(file.size)));
|
||||
list.append(row);
|
||||
}
|
||||
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
|
||||
if (displayed >= 500) list.append(el('div', 'empty-state', 'Pierwsze 500 plików. Zawęź filtr, aby wyszukać kolejne.'));
|
||||
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
|
||||
}
|
||||
|
||||
async function pollMagnet(previewId) {
|
||||
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
|
||||
try {
|
||||
const data = await api('/magnet/inspect/' + encodeURIComponent(previewId));
|
||||
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
|
||||
if (data.status === 'ready') {
|
||||
S.magnetName = data.name;
|
||||
S.magnetFiles = data.files;
|
||||
S.magnetSelected = new Set((data.suggested || []).filter(i => !data.files.find(f => f.index === i)?.padding));
|
||||
$('magnetName').textContent = data.name + ' (' + data.files.length + ' plików)';
|
||||
$('magnetStatus').textContent = 'Metadane pobrane. Wybierz pliki.';
|
||||
$('magnetResults').classList.remove('hidden');
|
||||
renderMagnet();
|
||||
return;
|
||||
}
|
||||
if (data.status === 'loading') {
|
||||
const seconds = Number(data.elapsed) || 0;
|
||||
const connections = Number(data.connections) || 0;
|
||||
const known = Number(data.known_peers) || 0;
|
||||
let notice = 'Metadane: ' + seconds + ' s | połączenia: ' + connections + ' | znani peerzy: ' + known;
|
||||
if (seconds >= 25 && connections === 0)
|
||||
notice += ' | Brak połączeń. Sprawdź UDP/DHT lub wczytaj .torrent.';
|
||||
$('magnetStatus').textContent = notice;
|
||||
}
|
||||
if (data.status === 'error') {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + data.error;
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + e.message;
|
||||
return;
|
||||
}
|
||||
setTimeout(() => pollMagnet(previewId), 3000);
|
||||
}
|
||||
|
||||
async function inspectMagnet(magnet, title, format) {
|
||||
S.magnetPreview = null;
|
||||
S.magnetTitle = title;
|
||||
S.magnetFormat = format;
|
||||
S.magnetFiles = [];
|
||||
S.magnetSelected.clear();
|
||||
$('magnetStatus').textContent = 'Oczekiwanie na metadane z sieci BitTorrent…';
|
||||
$('magnetResults').classList.add('hidden');
|
||||
$('magnetFilter').value = '';
|
||||
$('magnetDialog').showModal();
|
||||
try {
|
||||
const data = await api('/magnet/inspect', { method: 'POST', body: JSON.stringify({ magnet }) });
|
||||
if (!$('magnetDialog').open) return;
|
||||
S.magnetPreview = data.preview_id;
|
||||
pollMagnet(data.preview_id);
|
||||
} catch (e) { $('magnetStatus').textContent = 'Błąd: ' + e.message; }
|
||||
}
|
||||
|
||||
async function uploadTorrent(file, fromLinks) {
|
||||
if (!file) return;
|
||||
if (file.size > 32 * 1024 * 1024) return flash('Limit pliku .torrent: 32 MiB.', true);
|
||||
if (fromLinks) {
|
||||
S.magnetTitle = $('linkTitle').value.trim();
|
||||
S.magnetFormat = $('linkFormat').value;
|
||||
}
|
||||
const previous = S.magnetPreview;
|
||||
S.magnetPreview = null;
|
||||
S.magnetFiles = [];
|
||||
S.magnetSelected.clear();
|
||||
$('magnetResults').classList.add('hidden');
|
||||
$('magnetFilter').value = '';
|
||||
$('magnetStatus').textContent = 'Wczytywanie pliku .torrent...';
|
||||
if (!$('magnetDialog').open) $('magnetDialog').showModal();
|
||||
if (previous) await api('/magnet/inspect/' + encodeURIComponent(previous) + '/cancel', { method: 'POST' }).catch(() => {});
|
||||
try {
|
||||
const response = await fetch('/api/magnet/upload', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/x-bittorrent' }, body: file
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.detail || 'HTTP ' + response.status);
|
||||
if (!$('magnetDialog').open) return;
|
||||
S.magnetPreview = data.preview_id;
|
||||
await pollMagnet(data.preview_id);
|
||||
} catch (error) {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + error.message;
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of ['torrentFileLinks', 'torrentFileDialog']) {
|
||||
$(id).addEventListener('change', async (event) => {
|
||||
const file = event.target.files?.[0];
|
||||
event.target.value = '';
|
||||
await uploadTorrent(file, id === 'torrentFileLinks');
|
||||
});
|
||||
}
|
||||
|
||||
$('magnetFilter').addEventListener('input', renderMagnet);
|
||||
$('magnetToggleAll').addEventListener('click', () => {
|
||||
const visible = S.magnetFiles.filter(f => !f.padding && f.name.toLowerCase().includes($('magnetFilter').value.toLowerCase())).slice(0, 500);
|
||||
const allSelected = visible.every(f => S.magnetSelected.has(f.index));
|
||||
for (const file of visible) {
|
||||
if (allSelected) S.magnetSelected.delete(file.index);
|
||||
else S.magnetSelected.add(file.index);
|
||||
}
|
||||
renderMagnet();
|
||||
});
|
||||
$('closeMagnet').addEventListener('click', () => $('magnetDialog').close());
|
||||
$('magnetDialog').addEventListener('close', () => {
|
||||
const preview = S.magnetPreview;
|
||||
S.magnetPreview = null;
|
||||
if (preview) api('/magnet/inspect/' + encodeURIComponent(preview) + '/cancel', { method: 'POST' }).catch(() => {});
|
||||
});
|
||||
$('magnetAdd').addEventListener('click', async () => {
|
||||
if (!S.magnetSelected.size) return flash('Zaznacz co najmniej jeden plik.', true);
|
||||
const button = $('magnetAdd');
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/magnet/jobs', { method: 'POST', body: JSON.stringify({
|
||||
preview_id: S.magnetPreview, files: [...S.magnetSelected],
|
||||
title: S.magnetTitle, format: S.magnetFormat
|
||||
}) });
|
||||
$('urls').value = '';
|
||||
S.magnetPreview = null;
|
||||
$('magnetDialog').close();
|
||||
flash('Wybrane pliki dodano do kolejki.');
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('linksForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const raw = $('urls').value.trim();
|
||||
if (!raw) return flash('Podaj co najmniej jeden link.', true);
|
||||
const button = $('linksForm').querySelector('[type=submit]');
|
||||
const title = $('linkTitle').value.trim();
|
||||
const format = $('linkFormat').value;
|
||||
if (/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i.test(raw)) {
|
||||
const magnet = normalizeMagnet(raw);
|
||||
if ((magnet.match(/magnet:\?/gi) || []).length !== 1)
|
||||
return flash('Magnet wklej pojedynczo, aby wybrać pliki.', true);
|
||||
await inspectMagnet(magnet, title, format);
|
||||
return;
|
||||
}
|
||||
const items = raw.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
|
||||
let ok = 0;
|
||||
const errors = [];
|
||||
button.disabled = true;
|
||||
try {
|
||||
for (const url of items) {
|
||||
try { await api('/jobs', { method: 'POST', body: JSON.stringify({ url, title, format }) }); ok++; }
|
||||
catch (e) { errors.push(url.slice(0, 65) + ': ' + e.message); }
|
||||
}
|
||||
if (ok) {
|
||||
$('urls').value = '';
|
||||
flash('Dodano zadań: ' + ok);
|
||||
await refreshJobs();
|
||||
}
|
||||
if (errors.length) flash(errors.join('\n'), true);
|
||||
} finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('archiveForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const button = $('archiveForm').querySelector('[type=submit]');
|
||||
button.disabled = true;
|
||||
$('archiveResults').classList.add('hidden');
|
||||
try {
|
||||
const data = await api('/archive/lookup', { method: 'POST', body: JSON.stringify({ identifier: $('archiveId').value }) });
|
||||
S.archiveId = data.identifier;
|
||||
S.archiveTitle = data.title;
|
||||
S.archiveFiles = data.files;
|
||||
S.selected.clear();
|
||||
$('archiveName').textContent = data.title + ' (' + data.files.length + ')';
|
||||
$('archiveResults').classList.remove('hidden');
|
||||
renderArchive();
|
||||
} catch (e) { flash('Internet Archive: ' + e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
function renderArchive() {
|
||||
const list = $('archiveFiles');
|
||||
list.replaceChildren();
|
||||
const text = $('archiveFilter').value.toLowerCase();
|
||||
let displayed = 0;
|
||||
for (const file of S.archiveFiles) {
|
||||
if (!file.name.toLowerCase().includes(text)) continue;
|
||||
if (displayed >= 400) break;
|
||||
displayed++;
|
||||
const row = el('label', 'file-row');
|
||||
const checkbox = el('input');
|
||||
checkbox.type = 'checkbox';
|
||||
checkbox.checked = S.selected.has(file.name);
|
||||
checkbox.addEventListener('change', () => {
|
||||
if (checkbox.checked) S.selected.add(file.name);
|
||||
else S.selected.delete(file.name);
|
||||
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
|
||||
});
|
||||
row.append(checkbox, el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
|
||||
list.append(row);
|
||||
}
|
||||
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
|
||||
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
|
||||
}
|
||||
|
||||
$('archiveFilter').addEventListener('input', renderArchive);
|
||||
$('archiveAdd').addEventListener('click', async () => {
|
||||
if (!S.selected.size) return flash('Zaznacz pliki.', true);
|
||||
const button = $('archiveAdd');
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/archive/jobs', { method: 'POST', body: JSON.stringify({
|
||||
identifier: S.archiveId, files: [...S.selected], title: S.archiveTitle, format: $('archiveFormat').value
|
||||
}) });
|
||||
S.selected.clear();
|
||||
renderArchive();
|
||||
flash('Dodano do kolejki.');
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
async function refreshImports() {
|
||||
try {
|
||||
const files = await api('/imports');
|
||||
const list = $('importList');
|
||||
list.replaceChildren();
|
||||
if (!files.length) {
|
||||
list.append(el('div', 'empty-state', 'Katalog data/imports/ jest pusty.'));
|
||||
return;
|
||||
}
|
||||
for (const file of files) {
|
||||
const row = el('div', 'file-row');
|
||||
row.append(icon('file'), el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
|
||||
const button = makeButton('Przetwórz', 'play');
|
||||
button.addEventListener('click', async () => {
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/imports/jobs', { method: 'POST', body: JSON.stringify({
|
||||
filename: file.name, format: $('importFormat').value
|
||||
}) });
|
||||
flash('Dodano: ' + file.name);
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
row.append(button);
|
||||
list.append(row);
|
||||
}
|
||||
} catch (e) { flash('Import: ' + e.message, true); }
|
||||
}
|
||||
|
||||
$('refreshImports').addEventListener('click', refreshImports);
|
||||
$('refreshJobs').addEventListener('click', () => refreshJobs(true));
|
||||
$('closeLog').addEventListener('click', () => $('logDialog').close());
|
||||
refreshJobs();
|
||||
refreshSystem();
|
||||
refreshArchiveSession();
|
||||
setInterval(() => refreshJobs(), 3000);
|
||||
setInterval(refreshSystem, 30000);
|
||||
@@ -0,0 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||
<rect width="64" height="64" rx="10" fill="#1b1e21"/>
|
||||
<path d="M17 16 32 32 47 16M17 48 32 32 47 48" fill="none" stroke="#91c99a" stroke-width="6" stroke-linecap="square"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 245 B |
@@ -0,0 +1,151 @@
|
||||
<!doctype html>
|
||||
<html lang="pl">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="dark">
|
||||
<meta name="theme-color" content="#141619">
|
||||
<title>X360 Queue</title>
|
||||
<link rel="icon" href="/static/icon.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<script src="/static/app.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<svg class="svg-defs" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
|
||||
<symbol id="i-download" viewBox="0 0 24 24"><path d="M12 3v12m-5-5 5 5 5-5M5 17v3h14v-3"/></symbol>
|
||||
<symbol id="i-link" viewBox="0 0 24 24"><path d="M10 13a5 5 0 0 0 7.4.4l2-2a5 5 0 0 0-7.1-7.1l-1.1 1.1M14 11a5 5 0 0 0-7.4-.4l-2 2a5 5 0 0 0 7.1 7.1l1.1-1.1"/></symbol>
|
||||
<symbol id="i-archive" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="4" rx="1"/><path d="M5 8v12h14V8M10 12h4"/></symbol>
|
||||
<symbol id="i-folder" viewBox="0 0 24 24"><path d="M3 7a2 2 0 0 1 2-2h5l2 2h7a2 2 0 0 1 2 2v10H3z"/></symbol>
|
||||
<symbol id="i-plus" viewBox="0 0 24 24"><path d="M12 5v14M5 12h14"/></symbol>
|
||||
<symbol id="i-refresh" viewBox="0 0 24 24"><path d="M20 7v5h-5M4 17v-5h5"/><path d="M5.6 9A7 7 0 0 1 18 6l2 6M4 12l2 6a7 7 0 0 0 12.4-3"/></symbol>
|
||||
<symbol id="i-pause" viewBox="0 0 24 24"><path d="M8 5v14M16 5v14"/></symbol>
|
||||
<symbol id="i-play" viewBox="0 0 24 24"><path d="m8 5 11 7-11 7z"/></symbol>
|
||||
<symbol id="i-x" viewBox="0 0 24 24"><path d="M6 6l12 12M18 6 6 18"/></symbol>
|
||||
<symbol id="i-retry" viewBox="0 0 24 24"><path d="M3 12a9 9 0 1 0 3-6.7M3 4v5h5"/></symbol>
|
||||
<symbol id="i-trash" viewBox="0 0 24 24"><path d="M4 7h16M10 4h4M6 7l1 13h10l1-13M10 11v5m4-5v5"/></symbol>
|
||||
<symbol id="i-file" viewBox="0 0 24 24"><path d="M6 3h8l4 4v14H6zM14 3v5h5M9 12h6M9 16h6"/></symbol>
|
||||
<symbol id="i-harddrive" viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="14" rx="2"/><path d="M3 13h18M7 16h.01M11 16h.01"/></symbol>
|
||||
<symbol id="i-activity" viewBox="0 0 24 24"><path d="M3 12h4l3-7 4 14 3-7h4"/></symbol>
|
||||
<symbol id="i-check" viewBox="0 0 24 24"><path d="m5 12 5 5L19 7"/></symbol>
|
||||
<symbol id="i-clock" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6l4 2"/></symbol>
|
||||
<symbol id="i-alert" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6m0 4v.1"/></symbol>
|
||||
<symbol id="i-list" viewBox="0 0 24 24"><path d="M9 6h12M9 12h12M9 18h12M4 6h.01M4 12h.01M4 18h.01"/></symbol>
|
||||
</svg>
|
||||
|
||||
<div class="shell">
|
||||
<header class="topbar">
|
||||
<div class="brand">
|
||||
<div class="brand-mark" aria-hidden="true">X</div>
|
||||
<div class="brand-text"><strong>X360 Queue</strong><span>Linux / RGH</span></div>
|
||||
</div>
|
||||
<div class="topbar-right">
|
||||
<span class="connection" id="connectionState"><span class="connection-dot"></span><span id="connectionText">Łączenie…</span></span>
|
||||
<span class="version">LOCAL</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-title">
|
||||
<div><h1>Pobieranie</h1><p>Dodawanie plików i konwersja XEX / GOD.</p></div>
|
||||
</div>
|
||||
|
||||
<section class="metrics" aria-label="Stan aplikacji">
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-clock"/></svg> W kolejce</span><strong id="countQueued">0</strong></div>
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-activity"/></svg> W toku</span><strong id="countActive">0</strong></div>
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-check"/></svg> Gotowe</span><strong id="countDone">0</strong></div>
|
||||
<div class="metric disk-metric">
|
||||
<span class="metric-label"><svg class="icon"><use href="#i-harddrive"/></svg> Wolne miejsce</span>
|
||||
<strong id="storageNum">—</strong>
|
||||
<div class="meter"><span id="storageFill" style="width:0%"></span></div>
|
||||
<small id="storageText">—</small>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" aria-labelledby="addHeading">
|
||||
<div class="panel-heading"><h2 id="addHeading">Dodaj zadanie</h2><span class="muted-info">Zadania działają po zamknięciu przeglądarki</span></div>
|
||||
<nav class="tabs" aria-label="Wybierz źródło">
|
||||
<button class="tab active" type="button" data-tab="links" aria-selected="true"><svg class="icon"><use href="#i-link"/></svg>Link / magnet</button>
|
||||
<button class="tab" type="button" data-tab="archive" aria-selected="false"><svg class="icon"><use href="#i-archive"/></svg>Internet Archive</button>
|
||||
<button class="tab" type="button" data-tab="import" aria-selected="false"><svg class="icon"><use href="#i-folder"/></svg>Import</button>
|
||||
</nav>
|
||||
|
||||
<div class="tab-body" id="pane-links">
|
||||
<form id="linksForm">
|
||||
<div class="field">
|
||||
<label for="urls">Linki <span>— jeden adres w wierszu</span></label>
|
||||
<textarea id="urls" rows="3" spellcheck="false" placeholder="https://example.org/obraz.iso magnet:?xt=urn:btih:..."></textarea>
|
||||
</div>
|
||||
<div class="input-row">
|
||||
<div class="field title-field"><label for="linkTitle">Nazwa <span>(opcjonalnie)</span></label><input id="linkTitle" placeholder="Automatyczna" autocomplete="off"></div>
|
||||
<div class="field format-field"><label for="linkFormat">Format wyjściowy</label><select id="linkFormat"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select></div>
|
||||
<button class="primary submit-button" type="submit"><svg class="icon"><use href="#i-plus"/></svg>Dodaj do kolejki</button>
|
||||
</div>
|
||||
</form>
|
||||
<div class="torrent-upload-inline"><label for="torrentFileLinks" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileLinks" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Alternatywa, gdy magnet nie zwraca metadanych.</span></div>
|
||||
<p class="note">Magnet: najpierw lista plików, potem wybór i pobieranie. Vimm: tylko dostępny przycisk pobierania. Limity i zabezpieczenia serwisu pozostają aktywne.</p>
|
||||
</div>
|
||||
|
||||
<div class="tab-body hidden" id="pane-archive">
|
||||
<section class="archive-auth" aria-label="Konto Internet Archive">
|
||||
<div class="auth-heading"><strong>Konto Archive.org</strong><span id="archiveSessionText">Sprawdzanie sesji…</span></div>
|
||||
<form id="archiveLoginForm" class="auth-login hidden" autocomplete="on">
|
||||
<div class="field"><label for="archiveEmail">E-mail</label><input id="archiveEmail" type="email" autocomplete="username" required placeholder="konto@example.org"></div>
|
||||
<div class="field"><label for="archivePassword">Hasło</label><input id="archivePassword" type="password" autocomplete="current-password" required></div>
|
||||
<button type="submit" class="secondary"><svg class="icon"><use href="#i-link"/></svg>Zaloguj</button>
|
||||
</form>
|
||||
<button id="archiveLogout" class="secondary hidden" type="button">Wyloguj</button>
|
||||
<p class="note">Hasło nie jest zapisywane. Pobieranie używa sesji Archive.org; konto nie odblokowuje plików niedostępnych dla użytkownika. Panel udostępniaj przez HTTPS lub VPN.</p>
|
||||
</section>
|
||||
<form id="archiveForm">
|
||||
<div class="field"><label for="archiveId">Adres lub identyfikator</label>
|
||||
<div class="inline-input"><input id="archiveId" placeholder="archive.org/details/identyfikator" spellcheck="false"><button class="secondary" type="submit"><svg class="icon"><use href="#i-list"/></svg>Pokaż pliki</button></div>
|
||||
</div>
|
||||
</form>
|
||||
<div class="archive-results hidden" id="archiveResults">
|
||||
<div class="file-heading"><h3 id="archiveName"></h3><input id="archiveFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
|
||||
<div class="file-table" id="archiveFiles"></div>
|
||||
<div class="file-bottom"><span id="selectedCount">Wybrano: 0</span><div class="selection-controls"><select id="archiveFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="archiveAdd" class="primary" type="button"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
|
||||
<p class="note">Części jednego archiwum zaznacz razem.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tab-body hidden" id="pane-import">
|
||||
<div class="import-toolbar"><div><p class="import-hint">Skopiuj pliki do <code id="importHostPath">/xbox/imports/</code>.</p></div><div class="selection-controls"><select id="importFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="refreshImports" class="secondary" type="button"><svg class="icon"><use href="#i-refresh"/></svg>Odśwież</button></div></div>
|
||||
<div id="importList" class="file-table"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="queue-section" aria-labelledby="queueHeading">
|
||||
<div class="queue-header">
|
||||
<div class="queue-title"><h2 id="queueHeading">Kolejka <span id="jobsCount" class="count">0</span></h2><span class="muted-info">Odświeżanie co 3 s</span></div>
|
||||
<div class="queue-controls">
|
||||
<div class="filters" aria-label="Filtruj zadania">
|
||||
<button type="button" class="filter active" data-filter="all" aria-pressed="true">Wszystkie</button>
|
||||
<button type="button" class="filter" data-filter="active" aria-pressed="false">W toku</button>
|
||||
<button type="button" class="filter" data-filter="done" aria-pressed="false">Gotowe</button>
|
||||
<button type="button" class="filter" data-filter="error" aria-pressed="false">Błędy</button>
|
||||
</div>
|
||||
<button id="refreshJobs" class="secondary icon-button" type="button" title="Odśwież kolejkę" aria-label="Odśwież kolejkę"><svg class="icon"><use href="#i-refresh"/></svg></button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="jobsList" class="jobs"><div class="empty-state">Brak zadań.</div></div>
|
||||
</section>
|
||||
<footer class="footer"><span>Wyniki: <code id="readyXexPath">/xbox/ready/xex/</code> lub <code id="readyGodPath">/xbox/ready/god/</code></span><span>Pobieranie: <code id="downloadHostPath">/xbox/downloads/</code> · Robocze: <code id="workHostPath">/xbox/work/</code></span></footer>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<div id="toast" role="status" aria-live="polite"></div>
|
||||
<dialog id="magnetDialog" aria-labelledby="magnetHeading">
|
||||
<div class="dialog-heading"><h2 id="magnetHeading">Pliki w torrencie</h2><button type="button" class="secondary" id="closeMagnet"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div>
|
||||
<p id="magnetStatus" class="preview-status">Pobieranie metadanych torrenta…</p>
|
||||
<div class="torrent-upload-inline"><label for="torrentFileDialog" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileDialog" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Nie wymaga peerów ani oczekiwania na DHT.</span></div>
|
||||
<div id="magnetResults" class="hidden">
|
||||
<div class="file-heading"><h3 id="magnetName"></h3><input id="magnetFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
|
||||
<div class="file-table" id="magnetFiles"></div>
|
||||
<div class="file-bottom"><span id="magnetSelectedCount">Wybrano: 0</span><div class="selection-controls"><button type="button" id="magnetToggleAll" class="secondary">Zaznacz widoczne</button><button type="button" id="magnetAdd" class="primary"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
|
||||
<p class="note">Pobieranie rozpocznie się dopiero po wybraniu plików. Przy archiwach wieloczęściowych zaznacz wszystkie części.</p>
|
||||
</div>
|
||||
</dialog>
|
||||
<dialog id="logDialog" aria-labelledby="logHeading"><div class="dialog-heading"><h2 id="logHeading">Log zadania</h2><button type="button" class="secondary" id="closeLog"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div><pre id="logContent"></pre></dialog>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,214 @@
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
font-family: system-ui, -apple-system, "Segoe UI", Arial, sans-serif;
|
||||
font-size: 14px;
|
||||
font-synthesis: none;
|
||||
--bg: #121416;
|
||||
--panel: #1b1e21;
|
||||
--panel-hover: #21252a;
|
||||
--input: #151719;
|
||||
--line: #34383d;
|
||||
--line-soft: #2a2e32;
|
||||
--text: #e4e7e9;
|
||||
--muted: #a2a9af;
|
||||
--faint: #788189;
|
||||
--accent: #91c99a;
|
||||
--accent-hover: #afe0b5;
|
||||
--success-bg: #203427;
|
||||
--danger: #e8a09b;
|
||||
--danger-bg: #362324;
|
||||
}
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--text); }
|
||||
button, input, textarea, select { font: inherit; }
|
||||
button { cursor: pointer; }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
.hidden, .svg-defs { display: none !important; }
|
||||
.icon { width: 16px; height: 16px; flex: 0 0 auto; fill: none; stroke: currentColor; stroke-width: 1.7; stroke-linecap: round; stroke-linejoin: round; }
|
||||
.shell { max-width: 1120px; margin: auto; padding: 0 24px; }
|
||||
.topbar { height: 66px; display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid var(--line); }
|
||||
.brand { display: flex; align-items: center; gap: 11px; }
|
||||
.brand-mark { display: grid; place-items: center; width: 33px; height: 33px; border: 1px solid #72977c; color: #a9d6ad; background: #1d2a21; border-radius: 5px; font-weight: 800; font-size: 19px; line-height: 1; }
|
||||
.brand-text { display: flex; flex-direction: column; gap: 2px; }
|
||||
.brand-text strong { font-size: 14px; font-weight: 700; letter-spacing: .1px; }
|
||||
.brand-text span { color: var(--faint); font-size: 11px; }
|
||||
.topbar-right { display: flex; align-items: center; gap: 17px; }
|
||||
.connection { display: inline-flex; align-items: center; gap: 7px; font-size: 12px; color: var(--muted); }
|
||||
.connection-dot { width: 7px; height: 7px; background: var(--faint); border-radius: 50%; }
|
||||
.connection.online .connection-dot { background: var(--accent); }
|
||||
.connection.offline .connection-dot { background: var(--danger); }
|
||||
.version { border: 1px solid var(--line); color: var(--faint); font: 600 10px ui-monospace, SFMono-Regular, Consolas, monospace; padding: 4px 7px; letter-spacing: .5px; border-radius: 3px; }
|
||||
.page-title { padding: 31px 0 22px; }
|
||||
h1, h2, h3, p { margin-top: 0; }
|
||||
h1 { margin-bottom: 5px; font-size: 25px; font-weight: 680; letter-spacing: -.65px; line-height: 1.3; }
|
||||
.page-title p { margin-bottom: 0; color: var(--muted); font-size: 13px; }
|
||||
.metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); border: 1px solid var(--line); border-radius: 5px; background: var(--panel); overflow: hidden; }
|
||||
.metric { display: flex; flex-direction: column; gap: 9px; padding: 16px 19px; min-height: 112px; }
|
||||
.metric + .metric { border-left: 1px solid var(--line); }
|
||||
.metric-label { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 12px; }
|
||||
.metric-label .icon { width: 14px; height: 14px; color: var(--faint); }
|
||||
.metric strong { color: var(--text); font-weight: 650; font-size: 25px; font-variant-numeric: tabular-nums; line-height: 1; }
|
||||
.disk-metric { gap: 7px; }
|
||||
.disk-metric strong { font-size: 20px; }
|
||||
.disk-metric small { font-size: 10px; color: var(--faint); }
|
||||
.meter { height: 4px; background: #353a3e; overflow: hidden; width: 100%; }
|
||||
.meter span { display: block; height: 100%; background: var(--accent); transition: width .2s linear; }
|
||||
.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; margin-top: 26px; }
|
||||
.panel-heading { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 18px 20px 15px; }
|
||||
h2 { margin: 0; font-size: 16px; font-weight: 670; letter-spacing: -.2px; }
|
||||
.muted-info { font-size: 11px; color: var(--faint); }
|
||||
.tabs { display: flex; gap: 3px; padding: 0 15px; border-bottom: 1px solid var(--line); }
|
||||
.tab { display: inline-flex; align-items: center; gap: 8px; background: transparent; border: none; border-bottom: 2px solid transparent; padding: 11px 13px 12px; color: var(--muted); font-size: 12px; font-weight: 550; }
|
||||
.tab:hover { color: var(--text); background: #25292d; }
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
.tab .icon { width: 15px; height: 15px; }
|
||||
.tab-body { padding: 21px 20px 20px; }
|
||||
.field { min-width: 0; }
|
||||
label { display: block; margin-bottom: 7px; font-size: 12px; color: #cbd0d4; font-weight: 600; }
|
||||
label span { color: var(--faint); font-weight: 400; }
|
||||
input, select, textarea { display: block; width: 100%; min-width: 0; color: var(--text); background: var(--input); border: 1px solid #42474c; border-radius: 4px; padding: 9px 11px; outline: none; font-size: 12px; }
|
||||
input, select { height: 38px; }
|
||||
textarea { min-height: 89px; resize: vertical; line-height: 1.6; }
|
||||
input::placeholder, textarea::placeholder { color: #737c84; }
|
||||
select { appearance: auto; }
|
||||
.input-row { display: flex; gap: 12px; align-items: flex-end; margin-top: 13px; }
|
||||
.input-row .title-field { flex: 1 1 auto; }
|
||||
.input-row .format-field { flex: 0 0 172px; }
|
||||
button.primary, button.secondary, button.action { border-radius: 4px; display: inline-flex; align-items: center; justify-content: center; gap: 8px; font-size: 12px; font-weight: 620; white-space: nowrap; min-height: 38px; padding: 9px 12px; }
|
||||
button.primary { border: 1px solid #9ccc9e; color: #18231b; background: var(--accent); }
|
||||
button.primary:hover:not(:disabled) { background: var(--accent-hover); }
|
||||
button.secondary, button.action { border: 1px solid #444a50; background: #292d31; color: #d9dee1; }
|
||||
button.secondary:hover:not(:disabled), button.action:hover:not(:disabled) { background: #34393f; }
|
||||
button.action { min-height: 29px; padding: 5px 9px; font-size: 11px; }
|
||||
button.action.danger { color: #e3a6a4; }
|
||||
button.action.danger:hover { background: #3a2829; }
|
||||
button.icon-button { min-width: 36px; width: 36px; padding: 8px; }
|
||||
.note { color: var(--faint); font-size: 11px; line-height: 1.55; margin: 13px 0 0; }
|
||||
.inline-input { display: flex; align-items: center; gap: 9px; }
|
||||
.inline-input input { flex: 1 1 auto; }
|
||||
.file-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 24px; }
|
||||
.file-heading h3 { font-size: 13px; font-weight: 600; margin: 0; max-width: 62%; overflow-wrap: anywhere; }
|
||||
.file-heading input { max-width: 250px; }
|
||||
.file-table { max-height: 320px; overflow: auto; margin-top: 12px; border: 1px solid var(--line); border-radius: 4px; }
|
||||
.file-row { display: flex; gap: 12px; align-items: center; min-height: 44px; padding: 9px 11px; margin: 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; color: #d2d7da; font-weight: 400; }
|
||||
.file-row:last-child { border-bottom: 0; }
|
||||
.file-row:hover { background: var(--panel-hover); }
|
||||
.file-row input[type=checkbox] { margin: 0; min-height: 0; width: 15px; height: 15px; accent-color: var(--accent); flex-shrink: 0; }
|
||||
.file-name { flex: 1; min-width: 0; overflow-wrap: anywhere; }
|
||||
.size { color: var(--faint); font-size: 11px; white-space: nowrap; }
|
||||
.file-bottom { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 13px; color: var(--muted); font-size: 12px; }
|
||||
.selection-controls { display: flex; align-items: center; gap: 9px; }
|
||||
.selection-controls select { width: 145px; }
|
||||
.import-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 15px; }
|
||||
.import-hint { font-size: 12px; color: var(--muted); margin: 0; }
|
||||
code { color: #b7d1bd; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 11px; }
|
||||
.queue-section { padding-top: 29px; }
|
||||
.queue-header { display: flex; justify-content: space-between; align-items: center; gap: 14px; margin-bottom: 14px; }
|
||||
.queue-title { display: flex; align-items: baseline; gap: 13px; }
|
||||
.queue-title h2 { display: flex; align-items: center; gap: 9px; }
|
||||
.count { font-size: 11px; color: #bdc4c9; background: #2c3034; border: 1px solid #3d4247; min-width: 21px; padding: 2px 6px; text-align: center; border-radius: 3px; font-variant-numeric: tabular-nums; }
|
||||
.queue-controls { display: flex; gap: 9px; align-items: center; }
|
||||
.filters { display: flex; border: 1px solid var(--line); border-radius: 4px; overflow: hidden; }
|
||||
.filter { background: #1c1f22; color: var(--muted); border: 0; border-right: 1px solid var(--line); padding: 9px 13px; font-size: 11px; min-height: 36px; }
|
||||
.filter:last-child { border-right: 0; }
|
||||
.filter:hover { color: var(--text); background: #272b30; }
|
||||
.filter.active { color: var(--accent); background: #29342c; font-weight: 650; }
|
||||
.jobs { display: flex; flex-direction: column; gap: 8px; }
|
||||
.job { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; padding: 14px 17px; }
|
||||
.job-top { display: flex; justify-content: space-between; gap: 20px; align-items: flex-start; }
|
||||
.job-heading { min-width: 0; flex: 1; }
|
||||
.job-name { font-weight: 650; font-size: 13px; line-height: 1.45; overflow-wrap: anywhere; }
|
||||
.job-sub { color: var(--faint); font-size: 11px; margin-top: 3px; overflow-wrap: anywhere; max-height: 2.8em; overflow: hidden; }
|
||||
.job-status { flex-shrink: 0; display: inline-flex; align-items: center; gap: 6px; border: 1px solid #4a5156; color: #bbc4ca; background: #272c30; border-radius: 3px; font-size: 10px; line-height: 1; padding: 6px 8px; white-space: nowrap; }
|
||||
.job-status::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: #9ca5ac; }
|
||||
.job-status.running { background: #233126; color: #add7af; border-color: #39563e; }
|
||||
.job-status.running::before { background: var(--accent); }
|
||||
.job-status.done { background: var(--success-bg); color: #a8d9ac; border-color: #3b5942; }
|
||||
.job-status.done::before { background: #8fcd95; }
|
||||
.job-status.error { background: var(--danger-bg); color: #f1aaa3; border-color: #704446; }
|
||||
.job-status.error::before { background: var(--danger); }
|
||||
.job-progress { display: flex; align-items: center; gap: 12px; margin-top: 12px; }
|
||||
.progressbar { width: 100%; height: 5px; background: #373d41; overflow: hidden; }
|
||||
.progressbar span { display: block; height: 100%; background: #8aba94; transition: width .25s linear; }
|
||||
.progressbar.is-error span { background: #c47e78; }
|
||||
.progress-text { color: #c5cbd0; text-align: right; font-variant-numeric: tabular-nums; font-size: 11px; flex: 0 0 50px; }
|
||||
.job-meta { display: flex; align-items: center; justify-content: space-between; gap: 20px; margin-top: 8px; font-size: 11px; color: var(--faint); }
|
||||
.job-meta .note-text { overflow-wrap: anywhere; }
|
||||
.job-meta .speed { flex-shrink: 0; font-variant-numeric: tabular-nums; }
|
||||
.job-bottom { display: flex; justify-content: space-between; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 11px; }
|
||||
.job-chips { display: inline-flex; gap: 7px; align-items: center; }
|
||||
.job-chip { color: #aab3ba; font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; border: 1px solid #3c4449; padding: 3px 7px; border-radius: 3px; }
|
||||
.job-actions { display: flex; flex-wrap: wrap; gap: 6px; margin-left: auto; }
|
||||
.job-error { margin-top: 11px; padding: 9px 11px; border-left: 2px solid #c6857e; color: #ebaba6; background: #2b2122; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.5; }
|
||||
.job-output { color: #accdb5; overflow-wrap: anywhere; margin-top: 9px; font-size: 11px; }
|
||||
.empty-state { color: var(--faint); text-align: center; border: 1px dashed #42484e; background: var(--panel); padding: 34px 12px; border-radius: 4px; font-size: 12px; }
|
||||
.footer { display: flex; justify-content: space-between; gap: 20px; padding: 23px 0 36px; color: var(--faint); font-size: 11px; }
|
||||
#toast { display: none; position: fixed; right: 20px; bottom: 20px; max-width: min(420px, calc(100vw - 40px)); z-index: 20; padding: 13px 16px; border: 1px solid #6e9e75; border-radius: 4px; background: #24402a; color: #d3f1d6; font-size: 12px; line-height: 1.45; white-space: pre-line; }
|
||||
#toast.bad { background: #48282a; color: #ffd7d3; border-color: #955b60; }
|
||||
dialog { width: min(760px, calc(100vw - 30px)); max-height: 85vh; background: var(--panel); color: var(--text); border: 1px solid #545a60; border-radius: 5px; padding: 18px; }
|
||||
dialog::backdrop { background: rgba(0,0,0,.75); }
|
||||
.dialog-heading { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--line); }
|
||||
.dialog-heading h2 { font-size: 15px; }
|
||||
pre { overflow: auto; max-height: 66vh; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.65; color: #c8cdd0; }
|
||||
@media (max-width: 760px) {
|
||||
.shell { padding: 0 15px; }
|
||||
.topbar { height: 59px; }
|
||||
.version { display: none; }
|
||||
.page-title { padding: 22px 0 16px; }
|
||||
.metrics { grid-template-columns: repeat(2, minmax(0,1fr)); }
|
||||
.metric:nth-child(3) { border-left: 0; border-top: 1px solid var(--line); }
|
||||
.metric:nth-child(4) { border-top: 1px solid var(--line); }
|
||||
.metric { min-height: 99px; padding: 15px; }
|
||||
.panel { margin-top: 19px; }
|
||||
.panel-heading { padding: 16px 15px 13px; }
|
||||
.panel-heading .muted-info { display: none; }
|
||||
.tabs { padding: 0 6px; }
|
||||
.tab { padding: 11px 9px; font-size: 11px; gap: 6px; }
|
||||
.tab-body { padding: 16px 15px; }
|
||||
.input-row { flex-wrap: wrap; }
|
||||
.input-row .title-field { flex: 1 1 100%; }
|
||||
.input-row .format-field { flex: 1 1 110px; }
|
||||
.input-row .submit-button { flex: 1 1 auto; }
|
||||
.file-heading { align-items: stretch; flex-direction: column; }
|
||||
.file-heading h3 { max-width: 100%; }
|
||||
.file-heading input { max-width: none; }
|
||||
.queue-header { align-items: stretch; flex-direction: column; gap: 13px; }
|
||||
.queue-controls { justify-content: space-between; }
|
||||
.queue-title { justify-content: space-between; }
|
||||
.filter { padding: 9px; }
|
||||
.job { padding: 13px; }
|
||||
.footer { flex-direction: column; gap: 7px; }
|
||||
}
|
||||
@media (max-width: 440px) {
|
||||
.inline-input, .import-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.selection-controls { width: 100%; }
|
||||
.selection-controls select { flex: 1; min-width: 0; }
|
||||
.file-bottom { align-items: stretch; flex-direction: column; }
|
||||
.job-top { gap: 9px; }
|
||||
.job-status { max-width: 130px; white-space: normal; }
|
||||
.queue-title .muted-info { display: none; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*, *::before, *::after { transition-duration: 0ms !important; animation-duration: 0ms !important; }
|
||||
}
|
||||
|
||||
/* Archive login and torrent metadata selection: compact, flat controls. */
|
||||
.archive-auth { border: 1px solid var(--line); padding: 13px; margin-bottom: 20px; background: var(--input); border-radius: 4px; }
|
||||
.auth-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; font-size: 12px; }
|
||||
.auth-heading span { color: var(--muted); overflow-wrap: anywhere; font-size: 11px; }
|
||||
.auth-login { display: flex; align-items: end; gap: 10px; flex-wrap: wrap; margin-top: 13px; }
|
||||
.auth-login .field { flex: 1 1 190px; }
|
||||
.archive-auth .note { margin-top: 10px; }
|
||||
#archiveLogout { margin-top: 12px; }
|
||||
#magnetDialog { width: min(840px, calc(100vw - 24px)); }
|
||||
.preview-status { color: var(--muted); font-size: 12px; margin: 14px 0; }
|
||||
#magnetDialog .file-table { max-height: 45vh; }
|
||||
#magnetDialog .file-row input:disabled { opacity: .3; }
|
||||
@media(max-width: 540px) { .auth-login .field, .auth-login button { flex: 1 1 100%; } }
|
||||
|
||||
/* Flat torrent metadata fallback controls. */
|
||||
.torrent-upload-inline {display:flex; align-items:center; flex-wrap:wrap; gap:10px; margin:10px 0; color:var(--muted, #8d939d); font-size:12px;}
|
||||
.file-label {display:inline-flex; align-items:center; justify-content:center; cursor:pointer; border:1px solid #434a51; background:#292e33; color:#d7dce1; padding:7px 10px; border-radius:3px; font-size:12px; font-weight:600;}
|
||||
.file-label:hover {background:#32383e;}
|
||||
#torrentFileLinks, #torrentFileDialog {display:none !important;}
|
||||
@@ -0,0 +1,95 @@
|
||||
"""SQLite-backed durable job queue. Single-worker, multiple API connections."""
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
DATA = Path('/data')
|
||||
DB_PATH = DATA / 'queue.sqlite3'
|
||||
|
||||
FIELDS = {
|
||||
'title', 'source_type', 'source', 'format', 'state', 'progress', 'speed',
|
||||
'total', 'completed', 'part_index', 'gid', 'error', 'output', 'note', 'updated',
|
||||
}
|
||||
|
||||
|
||||
def connection():
|
||||
DATA.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(DB_PATH, timeout=30)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.execute('PRAGMA busy_timeout=30000')
|
||||
con.execute('PRAGMA journal_mode=WAL')
|
||||
return con
|
||||
|
||||
|
||||
def init_db():
|
||||
with connection() as con:
|
||||
con.execute('''CREATE TABLE IF NOT EXISTS jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
source_type TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
format TEXT NOT NULL,
|
||||
state TEXT NOT NULL DEFAULT 'queued',
|
||||
progress REAL NOT NULL DEFAULT 0,
|
||||
speed INTEGER NOT NULL DEFAULT 0,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
completed INTEGER NOT NULL DEFAULT 0,
|
||||
part_index INTEGER NOT NULL DEFAULT 0,
|
||||
gid TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created REAL NOT NULL,
|
||||
updated REAL NOT NULL
|
||||
)''')
|
||||
con.execute('CREATE INDEX IF NOT EXISTS jobs_created_idx ON jobs(created)')
|
||||
# Zadania przerwane w trakcie rozpakowania/konwersji moga byc ponowione.
|
||||
con.execute("UPDATE jobs SET state='preparing', note='Wznowiono konwersje po restarcie' "
|
||||
"WHERE state IN ('extracting','converting')")
|
||||
con.execute("UPDATE jobs SET state='queued', note='Przywrocono zadanie po restarcie' "
|
||||
"WHERE state='vimm_browser'")
|
||||
|
||||
|
||||
def create(job):
|
||||
now = time.time()
|
||||
with connection() as con:
|
||||
con.execute('''INSERT INTO jobs(id,title,source_type,source,format,state,created,updated)
|
||||
VALUES(?,?,?,?,?,'queued',?,?)''',
|
||||
(job['id'], job['title'], job['source_type'],
|
||||
job['source'], job['format'], now, now))
|
||||
|
||||
|
||||
def get(job_id):
|
||||
with connection() as con:
|
||||
row = con.execute('SELECT * FROM jobs WHERE id=?', (job_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def list_jobs():
|
||||
with connection() as con:
|
||||
rows = con.execute('SELECT * FROM jobs ORDER BY created DESC LIMIT 300').fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def next_job():
|
||||
with connection() as con:
|
||||
row = con.execute('''SELECT * FROM jobs
|
||||
WHERE state IN ('queued','downloading','preparing')
|
||||
ORDER BY created ASC LIMIT 1''').fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def update(job_id, **values):
|
||||
if not values:
|
||||
return
|
||||
if not set(values).issubset(FIELDS):
|
||||
raise ValueError('Niepoprawne pola aktualizacji bazy')
|
||||
values['updated'] = time.time()
|
||||
with connection() as con:
|
||||
con.execute('UPDATE jobs SET ' + ','.join(f'{k}=?' for k in values)
|
||||
+ ' WHERE id=?', (*values.values(), job_id))
|
||||
|
||||
|
||||
def erase(job_id):
|
||||
with connection() as con:
|
||||
con.execute('DELETE FROM jobs WHERE id=?', (job_id,))
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Safe, small bencode parser and ephemeral magnet metadata inspection state."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import store
|
||||
|
||||
ID = re.compile(r'^[a-f0-9]{32}$')
|
||||
MAX_METADATA = 32 * 1024 * 1024
|
||||
|
||||
|
||||
class MetadataError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def decode_bencode(raw):
|
||||
if len(raw) > MAX_METADATA:
|
||||
raise MetadataError('Metadane torrenta są zbyt duże')
|
||||
position = 0
|
||||
count = 0
|
||||
|
||||
def decode(depth=0):
|
||||
nonlocal position, count
|
||||
if depth > 24:
|
||||
raise MetadataError('Metadane mają zbyt duże zagnieżdżenie')
|
||||
count += 1
|
||||
if count > 600000 or position >= len(raw):
|
||||
raise MetadataError('Uszkodzone metadane torrenta')
|
||||
start = raw[position]
|
||||
if start == 105: # i<number>e
|
||||
end = raw.find(b'e', position + 1)
|
||||
if end < 0 or end - position > 25:
|
||||
raise MetadataError('Błędna liczba bencode')
|
||||
try:
|
||||
number = int(raw[position + 1:end])
|
||||
except ValueError as exc:
|
||||
raise MetadataError('Błędna liczba bencode') from exc
|
||||
position = end + 1
|
||||
return number
|
||||
if start in (108, 100): # l, d
|
||||
position += 1
|
||||
output = [] if start == 108 else {}
|
||||
while position < len(raw) and raw[position] != 101:
|
||||
if start == 108:
|
||||
output.append(decode(depth + 1))
|
||||
else:
|
||||
key = decode(depth + 1)
|
||||
if not isinstance(key, bytes):
|
||||
raise MetadataError('Niepoprawny klucz bencode')
|
||||
output[key] = decode(depth + 1)
|
||||
if position >= len(raw):
|
||||
raise MetadataError('Nieukończone metadane')
|
||||
position += 1
|
||||
return output
|
||||
if 48 <= start <= 57: # <len>:<bytes>
|
||||
colon = raw.find(b':', position, min(len(raw), position + 22))
|
||||
if colon < 0:
|
||||
raise MetadataError('Błędny ciąg bencode')
|
||||
try:
|
||||
length = int(raw[position:colon])
|
||||
except ValueError as exc:
|
||||
raise MetadataError('Błędna długość bencode') from exc
|
||||
position = colon + 1
|
||||
if length < 0 or length > len(raw) - position:
|
||||
raise MetadataError('Niepoprawna długość pola')
|
||||
result = raw[position:position + length]
|
||||
position += length
|
||||
return result
|
||||
raise MetadataError('Uszkodzone metadane torrenta')
|
||||
|
||||
result = decode()
|
||||
if position != len(raw) or not isinstance(result, dict):
|
||||
raise MetadataError('Niepoprawny format torrenta')
|
||||
return result
|
||||
|
||||
|
||||
def _segment(raw):
|
||||
if not isinstance(raw, bytes) or not raw or len(raw) > 255 or raw in (b'.', b'..'):
|
||||
raise MetadataError('Niedozwolona nazwa w torrencie')
|
||||
if any(char in raw for char in (b'/', b'\\', b'\0')):
|
||||
raise MetadataError('Niebezpieczna ścieżka w torrencie')
|
||||
return raw.decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
def read_torrent(path):
|
||||
try:
|
||||
data = decode_bencode(Path(path).read_bytes())
|
||||
info = data[b'info']
|
||||
name = _segment(info.get(b'name.utf-8') or info[b'name'])
|
||||
result = []
|
||||
if b'files' in info:
|
||||
files = info[b'files']
|
||||
if not isinstance(files, list) or len(files) > 50000:
|
||||
raise MetadataError('Nieprawidłowa lista plików torrenta')
|
||||
for index, file in enumerate(files, start=1):
|
||||
parts = file.get(b'path.utf-8') or file[b'path']
|
||||
if not isinstance(parts, list) or not parts:
|
||||
raise MetadataError('Błędna ścieżka pliku')
|
||||
length = file[b'length']
|
||||
if not isinstance(length, int) or length < 0:
|
||||
raise MetadataError('Nieprawidłowy rozmiar pliku')
|
||||
result.append({'index': index, 'name': '/'.join([name] + [_segment(x) for x in parts]),
|
||||
'size': length, 'padding': b'p' in file.get(b'attr', b'')})
|
||||
else:
|
||||
length = info[b'length']
|
||||
if not isinstance(length, int) or length < 0:
|
||||
raise MetadataError('Nieprawidłowy rozmiar pliku')
|
||||
result.append({'index': 1, 'name': name, 'size': length, 'padding': False})
|
||||
if not result:
|
||||
raise MetadataError('Torrent nie zawiera plików')
|
||||
return {'name': name, 'files': result}
|
||||
except (KeyError, TypeError, OSError, ValueError) as exc:
|
||||
if isinstance(exc, MetadataError):
|
||||
raise
|
||||
raise MetadataError('Nie można odczytać plików z torrenta') from exc
|
||||
|
||||
|
||||
def suggested_indices(magnet, max_index):
|
||||
"""magnet so indexes are zero-based; aria2 select-file indexes are one-based."""
|
||||
so = parse_qs(urlsplit(magnet).query).get('so', [''])[0]
|
||||
suggested = set()
|
||||
for chunk in so.split(',')[:1000]:
|
||||
try:
|
||||
if '-' in chunk:
|
||||
a, b = (int(x) for x in chunk.split('-', 1))
|
||||
if b < a or b - a > 1000:
|
||||
continue
|
||||
values = range(a, b + 1)
|
||||
else:
|
||||
values = [int(chunk)]
|
||||
suggested.update(n + 1 for n in values if 0 <= n < max_index)
|
||||
except ValueError:
|
||||
continue
|
||||
return sorted(suggested)
|
||||
|
||||
|
||||
def inspect_dir(preview_id):
|
||||
if not ID.fullmatch(preview_id):
|
||||
raise MetadataError('Nieprawidłowy identyfikator metadanych')
|
||||
return store.DATA / 'metadata' / preview_id
|
||||
|
||||
|
||||
def write_request(preview_id, data):
|
||||
path = inspect_dir(preview_id)
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
filename = path / 'request.json'
|
||||
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
|
||||
with os.fdopen(os.open(filename, flags, 0o600), 'w', encoding='utf-8') as f:
|
||||
json.dump({'created': time.time(), **data}, f)
|
||||
|
||||
|
||||
def load_request(preview_id):
|
||||
try:
|
||||
info = json.loads((inspect_dir(preview_id) / 'request.json').read_text(encoding='utf-8'))
|
||||
if not isinstance(info['gid'], str) or not isinstance(info['magnet'], str):
|
||||
raise MetadataError('Błędne dane podglądu')
|
||||
return info
|
||||
except FileNotFoundError as exc:
|
||||
raise MetadataError('Nie znaleziono podglądu magneta') from exc
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise MetadataError('Uszkodzone dane podglądu') from exc
|
||||
|
||||
|
||||
def torrent_file(preview_id):
|
||||
matches = list(inspect_dir(preview_id).glob('*.torrent'))
|
||||
return matches[0] if len(matches) == 1 and matches[0].stat().st_size <= MAX_METADATA else None
|
||||
|
||||
|
||||
def cleanup_old(max_age=86400):
|
||||
root = store.DATA / 'metadata'
|
||||
if not root.exists():
|
||||
return
|
||||
now = time.time()
|
||||
for child in root.iterdir():
|
||||
if (ID.fullmatch(child.name) and child.is_dir() and not child.is_symlink()
|
||||
and now - child.stat().st_mtime > max_age):
|
||||
shutil.rmtree(child)
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
import httpx
|
||||
|
||||
from . import store
|
||||
|
||||
|
||||
async def browser_download(job):
|
||||
if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
|
||||
raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
|
||||
url = job['source']
|
||||
u = urlparse(url)
|
||||
if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
|
||||
raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
|
||||
r = await c.get('https://vimm.net/robots.txt')
|
||||
if r.status_code == 200:
|
||||
parser = RobotFileParser()
|
||||
parser.parse(r.text.splitlines())
|
||||
if not parser.can_fetch('x360-web-queue', url):
|
||||
raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
|
||||
elif r.status_code not in (404, 410):
|
||||
raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
|
||||
|
||||
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
|
||||
|
||||
store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
|
||||
out = store.DATA / 'raw' / job['id']
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True, executable_path='/usr/bin/chromium',
|
||||
args=['--no-sandbox', '--disable-dev-shm-usage'])
|
||||
try:
|
||||
context = await browser.new_context(accept_downloads=True)
|
||||
page = await context.new_page()
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=60000)
|
||||
# Only the original, visible button is actionable. Missing/taken-down
|
||||
# buttons and challenges are not bypassed.
|
||||
controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
|
||||
usable = None
|
||||
for i in range(min(await controls.count(), 10)):
|
||||
c = controls.nth(i)
|
||||
if await c.is_visible() and await c.is_enabled():
|
||||
usable = c
|
||||
break
|
||||
if usable is None:
|
||||
raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
|
||||
store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
|
||||
timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
|
||||
try:
|
||||
async with page.expect_download(timeout=timeout) as d:
|
||||
await usable.click(timeout=15000)
|
||||
download = await d.value
|
||||
except PlaywrightTimeout as exc:
|
||||
raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
|
||||
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
|
||||
if not name:
|
||||
name = 'vimm-download.bin'
|
||||
store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
|
||||
await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
|
||||
failure = await download.failure()
|
||||
if failure:
|
||||
raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
|
||||
store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
|
||||
finally:
|
||||
await browser.close()
|
||||
Reference in new issue
Block a user