first commit

This commit is contained in:
Mateusz Gruszczyński committed 2026-10-15 14:15:10 +02:00
commit 58cfd95600
35 files changed
+5357

No files matched your search

View File
Whitespace-only changes.
+204
View File
@@ -0,0 +1,204 @@
"""Internet Archive user session and restricted download relay.
The relay keeps IA cookies inside this process. aria2 only receives a signed,
local endpoint, and never sees an IA session cookie or the account password.
"""
import hashlib
import hmac
import json
import os
import secrets
import tempfile
from pathlib import Path
from urllib.parse import urljoin, urlsplit
import httpx
from fastapi.responses import PlainTextResponse, Response, StreamingResponse
from . import store
class ArchiveAuthError(ValueError):
pass
def session_path():
return store.DATA / 'private' / 'archive-session.json'
def _read_session():
try:
value = json.loads(session_path().read_text(encoding='utf-8'))
cookies = value['cookies']
if not all(isinstance(cookies.get(k), str) and cookies[k] for k in ('logged-in-user', 'logged-in-sig')):
return None
return value
except (FileNotFoundError, KeyError, TypeError, ValueError, OSError):
return None
def session_status():
value = _read_session()
return {'authenticated': bool(value), 'email': value.get('email', '') if value else ''}
def session_cookies():
cookies = httpx.Cookies()
value = _read_session()
if value:
for key in ('logged-in-user', 'logged-in-sig'):
cookies.set(key, value['cookies'][key], domain='.archive.org', path='/')
return cookies
def _cookie_value(value):
# The official xauthn API includes cookie attributes after the value.
if not isinstance(value, str):
raise ArchiveAuthError('Brak danych sesji w odpowiedzi Archive.org')
value = value.split(';', 1)[0].strip()
if not value or len(value) > 4096 or any(x in value for x in ('\n', '\r', '\0')):
raise ArchiveAuthError('Niepoprawne dane sesji Archive.org')
return value
def _save_session(email, cookies):
path = session_path()
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(path.parent, 0o700)
fd, name = tempfile.mkstemp(dir=path.parent, prefix='.archive-session-', suffix='.tmp')
try:
with os.fdopen(fd, 'w', encoding='utf-8') as f:
json.dump({'email': email, 'cookies': cookies}, f)
f.flush()
os.fsync(f.fileno())
os.chmod(name, 0o600)
os.replace(name, path)
finally:
if os.path.exists(name):
os.unlink(name)
async def login(email, password):
"""Authenticate using IA's documented xauthn endpoint (no password stored)."""
try:
async with httpx.AsyncClient(timeout=25, follow_redirects=False) as client:
result = await client.post('https://archive.org/services/xauthn/', params={'op': 'login'},
data={'email': email, 'password': password})
result.raise_for_status()
data = result.json()
except (httpx.HTTPError, ValueError) as exc:
raise ArchiveAuthError('Błąd połączenia z logowaniem Archive.org') from exc
if not data.get('success'):
reason = str((data.get('values') or {}).get('reason') or data.get('error') or '')
if reason in ('account_bad_password', 'account_not_found'):
raise ArchiveAuthError('Nieprawidłowy adres e-mail lub hasło')
raise ArchiveAuthError('Archive.org odrzuciło logowanie: ' + (reason[:120] or 'brak szczegółów'))
try:
raw = data['values']['cookies']
cookies = {key: _cookie_value(raw[key]) for key in ('logged-in-user', 'logged-in-sig')}
except (KeyError, TypeError) as exc:
raise ArchiveAuthError('Logowanie udane, ale brak ciasteczek sesji w odpowiedzi') from exc
_save_session(email, cookies)
return session_status()
def logout():
session_path().unlink(missing_ok=True)
def is_archive_download(url):
try:
parsed = urlsplit(url)
return parsed.scheme == 'https' and parsed.hostname in ('archive.org', 'www.archive.org') and parsed.path.startswith('/download/')
except ValueError:
return False
def internal_url(job_id, index):
return f'http://127.0.0.1:8000/internal/archive/{job_id}/{index}'
def signed_header(job_id, index):
secret = os.environ['ARIA2_SECRET'].encode('utf-8')
message = f'archive:{job_id}:{index}'.encode('ascii')
digest = hmac.new(secret, message, hashlib.sha256).hexdigest()
return 'X-X360-Proxy-Token: ' + digest
def verify_token(job_id, index, token):
try:
expected = signed_header(job_id, index).split(': ', 1)[1]
except UnicodeEncodeError:
return False
return secrets.compare_digest(expected, token or '')
def allowed_redirect(url):
try:
parts = urlsplit(url)
name = (parts.hostname or '').lower()
return parts.scheme == 'https' and parts.port in (None, 443) and (name == 'archive.org' or name.endswith('.archive.org'))
except ValueError:
return False
async def stream_archive(url, method, headers):
"""Stream authorized bytes and preserve HTTP Range for aria2 resumes.
Only HTTPS Archive.org redirects are followed. Cookie domain scoping is
handled by httpx, unlike forwarding a raw Cookie header through redirects.
"""
outbound_headers = {'Accept-Encoding': 'identity', 'User-Agent': 'X360-Queue/1.1'}
for key in ('range', 'if-range', 'if-modified-since', 'if-none-match'):
if key in headers:
outbound_headers[key] = headers[key]
client = httpx.AsyncClient(timeout=httpx.Timeout(120, connect=25), cookies=session_cookies(),
follow_redirects=False, trust_env=False)
current = url
try:
for _ in range(7):
if not allowed_redirect(current):
return PlainTextResponse('Nieprawidłowe przekierowanie Archive.org', status_code=502)
request = client.build_request(method, current, headers=outbound_headers)
remote = await client.send(request, stream=True)
if remote.status_code in (301, 302, 303, 307, 308) and remote.headers.get('location'):
new_url = urljoin(current, remote.headers['location'])
await remote.aclose()
current = new_url
continue
break
else:
return PlainTextResponse('Zbyt wiele przekierowań Archive.org', status_code=502)
if remote.status_code not in (200, 206, 304, 416):
status = remote.status_code
await remote.aclose()
if status in (401, 403):
return PlainTextResponse('Archive.org odmówiło dostępu. Zaloguj się i sprawdź uprawnienia do pliku.', status_code=403)
return PlainTextResponse(f'Archive.org HTTP {status}', status_code=status if 400 <= status < 600 else 502)
if remote.status_code == 200 and 'text/html' in remote.headers.get('content-type', '').lower():
await remote.aclose()
return PlainTextResponse('Archive.org zwróciło stronę HTML zamiast pliku. Sprawdź logowanie.', status_code=403)
safe_headers = {k: v for k, v in remote.headers.items() if k.lower() in (
'content-length', 'content-range', 'accept-ranges', 'content-disposition',
'content-type', 'etag', 'last-modified')}
if method == 'HEAD' or remote.status_code in (304, 416):
await remote.aclose()
return Response(status_code=remote.status_code, headers=safe_headers)
async def chunks():
try:
async for chunk in remote.aiter_raw(chunk_size=131072):
yield chunk
finally:
await remote.aclose()
await client.aclose()
return StreamingResponse(chunks(), status_code=remote.status_code, headers=safe_headers,
media_type=safe_headers.get('content-type'))
except httpx.HTTPError:
return PlainTextResponse('Nie można połączyć się z Archive.org', status_code=502)
finally:
# A streamed response owns client until the last chunk is sent.
if 'remote' not in locals() or remote.is_closed:
await client.aclose()
+760
View File
@@ -0,0 +1,760 @@
"""Direct, authenticated Internet Archive downloader.
Avoids the old aria2 -> localhost HTTP relay (which hid origin errors behind
HTTP 500). Supports Range resumes, transient 5xx/429 retries and pause/cancel.
Only forwards IA cookies via httpx's domain-scoped cookie jar.
"""
import hashlib
import json
import os
import re
import shutil
import time
from collections import deque
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from contextlib import closing
from threading import Event, Lock
from pathlib import Path
from urllib.parse import urljoin
import httpx
from . import archive_auth, store
from .download import source_urls, target_filename
RETRY_CODES = {408, 425, 429, 500, 502, 503, 504}
MAX_ATTEMPTS = 5
MAX_CONNECTIONS = 50
MIN_SEGMENT_BYTES = 8 * 1024 * 1024
CHUNK_BYTES = 256 * 1024
RANGE = re.compile(r'^bytes (\d+)-(\d+)/(\d+|\*)$', re.I)
FINISH_RANGE = re.compile(r'^bytes \*/(\d+)$', re.I)
LOG_LOCK = Lock()
class ArchiveDownloadError(RuntimeError):
pass
class ArchiveStopped(Exception):
"""Pause or cancellation was requested, not a transfer error."""
def _client():
return httpx.Client(timeout=httpx.Timeout(90, connect=25), cookies=archive_auth.session_cookies(),
follow_redirects=False, trust_env=True)
def _check_state(jid, stop=None):
if stop is not None and stop.is_set():
raise ArchiveStopped()
current = store.get(jid)
if not current or current['state'] in ('paused', 'cancelled', 'error'):
raise ArchiveStopped()
def _response(client, url, headers):
current = url
for _ in range(8):
if not archive_auth.allowed_redirect(current):
raise ArchiveDownloadError('Archive.org: nieprawidłowe przekierowanie poza domenę Archive.org')
request = client.build_request('GET', current, headers=headers)
response = client.send(request, stream=True)
if response.status_code in (301, 302, 303, 307, 308) and 'location' in response.headers:
next_url = urljoin(current, response.headers['location'])
response.close()
current = next_url
continue
return response
raise ArchiveDownloadError('Archive.org: zbyt wiele przekierowań')
def _log(job_id, message):
target = store.DATA / 'logs' / (job_id + '.download.log')
target.parent.mkdir(parents=True, exist_ok=True)
with LOG_LOCK:
with target.open('a', encoding='utf-8') as fp:
fp.write(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + message + '\n')
def _transfer_one(job, *, batch=None, connections=None, stop=None):
jid = job['id']
ix = job['part_index']
urls = source_urls(job)
if ix >= len(urls):
store.update(jid, state='preparing')
return
uri = urls[ix]
if not archive_auth.is_archive_download(uri):
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
root = store.DATA / 'raw' / jid
root.mkdir(parents=True, exist_ok=True)
filename = target_filename(uri)
if not filename:
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
destination = root / filename
partial = root / (filename + '.part')
_check_state(jid, stop)
if batch is None:
store.update(jid, state='downloading', gid='', note=f'Archive.org: plik {ix+1}/{len(urls)}')
_log(jid, f'Rozpoczęto plik {ix+1}/{len(urls)}: {filename}')
if connections is None:
connections = _connection_limit()
# An existing manifest may contain many segments even if the new budget
# only allows one active connection. Keep those downloaded bytes.
if (connections > 1 or _segment_dir(partial).exists()) and _parallel_transfer(
job, ix, urls, uri, filename, partial, destination, connections, batch=batch, stop=stop):
return
# If the server cannot serve ranges (or parallel transfer was disabled),
# keep the verified single-stream prefix, not an unfinished assembly.
_discard_parallel_state(partial)
# Existing aria2 control files are not compatible with this downloader.
# Start from our own .part file rather than assuming an aria2 file is complete.
with _client() as client:
for attempt in range(MAX_ATTEMPTS):
_check_state(jid, stop)
start = partial.stat().st_size if partial.exists() else 0
headers = {'User-Agent': 'X360-Web-Queue/2.0', 'Accept-Encoding': 'identity'}
if start:
headers['Range'] = f'bytes={start}-'
try:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(jid, f'HTTP {code} z Archive.org, próba {attempt+1}/{MAX_ATTEMPTS}')
reason = f'Archive.org HTTP {code}'
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'{reason} po {MAX_ATTEMPTS} próbach; możliwy problem serwera Archive.org')
elif code in (401, 403):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia do pliku.')
elif code == 404:
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono wybranego pliku')
elif code == 416 and start:
m = FINISH_RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if m and int(m.group(1)) == start:
_check_state(jid, stop)
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename} (HTTP 416 potwierdziło rozmiar)')
return
partial.unlink(missing_ok=True)
continue
elif code not in (200, 206):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
else:
if 'text/html' in response.headers.get('Content-Type', '').lower():
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
if code == 206:
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or int(match.group(1)) != start:
raise ArchiveDownloadError('Archive.org zwróciło niezgodny zakres HTTP Range')
total = int(match.group(3)) if match.group(3) != '*' else 0
else:
# Server ignored Range: do not append duplicate bytes.
start = 0
total = int(response.headers.get('Content-Length') or 0)
written = start
last_update = 0.0
prev = written
prev_at = time.monotonic()
mode = 'ab' if start else 'wb'
with partial.open(mode) as fp:
for chunk in response.iter_raw(chunk_size=256 * 1024):
_check_state(jid, stop)
if not chunk:
continue
fp.write(chunk)
written += len(chunk)
now = time.monotonic()
if now - last_update >= 0.8:
elapsed = max(0.001, now - prev_at)
speed = int((written - prev) / elapsed)
prev, prev_at, last_update = written, now, now
portion = written / total if total else 0
if batch is not None:
batch.report(ix, written, total)
else:
store.update(jid, progress=round((ix + min(portion, 1)) / len(urls) * 100, 2),
speed=speed, total=total, completed=written,
note=f'Archive.org: plik {ix+1}/{len(urls)}, próba {attempt+1}')
if total and written != total:
raise httpx.ReadError(f'Przerwany transfer: pobrano {written} z {total} bajtów')
_check_state(jid, stop)
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename}: {written} bajtów')
return
except ArchiveStopped:
raise
except ArchiveDownloadError:
raise
except (httpx.HTTPError, OSError) as exc:
_log(jid, f'Błąd transferu {type(exc).__name__}: {str(exc)[:180]}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org: przerwane połączenie po {MAX_ATTEMPTS} próbach: {type(exc).__name__}: {exc}') from exc
# Applies to transient HTTP codes and connection exceptions.
if batch is None:
store.update(jid, note=f'Archive.org: ponawianie po HTTP/błędzie sieci ({attempt+2}/{MAX_ATTEMPTS})', speed=0)
for _ in range(min(2 ** attempt, 12) * 2):
_check_state(jid, stop)
time.sleep(0.5)
def _finish(job, ix, urls, *, batch=None):
jid = job['id']
if batch is not None:
batch.complete(ix)
return
if ix + 1 < len(urls):
store.update(jid, part_index=ix+1, state='queued', gid='', speed=0,
progress=round((ix+1)/len(urls) * 100, 2), note='Pobrano część archiwum')
else:
store.update(jid, state='preparing', progress=100, speed=0,
note='Pobieranie Archive.org ukończone')
class _RangeUnsupported(Exception):
"""A server stopped honoring byte ranges; caller can use the serial path."""
def __init__(self, message, reset_prefix=False):
super().__init__(message)
self.reset_prefix = reset_prefix
def _connection_limit():
try:
return max(1, min(MAX_CONNECTIONS, int(os.getenv('ARCHIVE_CONNECTIONS', '50'))))
except (ValueError, OverflowError):
return 50
def _segment_dir(partial):
return Path(str(partial) + '.segments')
def _read_manifest(folder):
try:
item = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
if not isinstance(item, dict):
return None
return item
except (OSError, ValueError):
return None
def _write_manifest(folder, manifest):
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
temp = folder / 'manifest.json.tmp'
with temp.open('w', encoding='utf-8') as f:
json.dump(manifest, f, sort_keys=True)
f.flush()
os.fsync(f.fileno())
temp.replace(folder / 'manifest.json')
def _discard_parallel_state(partial, reset_prefix=False):
"""Discard stale segment files without interpreting an assembled prefix as a resume."""
folder = _segment_dir(partial)
if not folder.exists():
return
manifest = _read_manifest(folder)
if reset_prefix:
partial.unlink(missing_ok=True)
elif manifest and manifest.get('assembling') and partial.exists():
prefix = manifest.get('prefix')
if isinstance(prefix, int) and 0 <= prefix <= partial.stat().st_size:
with partial.open('r+b') as f:
f.truncate(prefix)
else:
partial.unlink(missing_ok=True)
shutil.rmtree(folder)
def _validate_ranges(ranges, prefix, total):
if not isinstance(ranges, list) or len(ranges) < 2 or len(ranges) > MAX_CONNECTIONS:
return False
cursor = prefix
for item in ranges:
if (not isinstance(item, list) or len(item) != 2 or
not all(isinstance(x, int) and not isinstance(x, bool) for x in item)):
return False
start, end = item
if start != cursor or end < start or end >= total:
return False
cursor = end + 1
return cursor == total
def _prepare_segments(partial, uri, total, validator, connections):
folder = _segment_dir(partial)
source_hash = hashlib.sha256(uri.encode('utf-8')).hexdigest()
manifest = _read_manifest(folder)
valid = False
if manifest:
prefix = manifest.get('prefix')
numeric_prefix = isinstance(prefix, int) and not isinstance(prefix, bool)
prefix_present = numeric_prefix and (prefix == 0 or
(partial.exists() and partial.stat().st_size >= prefix))
valid = (manifest.get('version') == 1 and manifest.get('source_hash') == source_hash and
manifest.get('total') == total and manifest.get('validator') == validator and
isinstance(prefix, int) and not isinstance(prefix, bool) and 0 <= prefix < total and
_validate_ranges(manifest.get('ranges'), prefix, total) and prefix_present)
if valid and partial.exists() and partial.stat().st_size != prefix:
# A crash during assembly is safe to recover from: parts remain intact.
if manifest.get('assembling') and partial.stat().st_size > prefix:
with partial.open('r+b') as f:
f.truncate(prefix)
else:
valid = False
if valid:
return folder, manifest
if folder.exists():
# Invalid or mismatched metadata cannot establish which bytes are safe.
_discard_parallel_state(partial, reset_prefix=True)
prefix = partial.stat().st_size if partial.exists() else 0
if prefix > total:
partial.unlink(missing_ok=True)
prefix = 0
if prefix >= total:
return None, {'prefix': prefix, 'total': total}
remaining = total - prefix
n = min(connections, max(1, remaining // MIN_SEGMENT_BYTES))
if n < 2:
return None, None
quotient, extra = divmod(remaining, n)
cursor = prefix
ranges = []
for i in range(n):
end = cursor + quotient + (1 if i < extra else 0) - 1
ranges.append([cursor, end])
cursor = end + 1
manifest = {'version': 1, 'source_hash': source_hash, 'total': total,
'validator': validator, 'prefix': prefix, 'ranges': ranges,
'assembling': False}
_write_manifest(folder, manifest)
return folder, manifest
def _retry_wait(job_id, attempt, stop, retry_after=None, external_stop=None):
delay = min(2 ** attempt, 16)
if retry_after and retry_after.isdigit():
delay = min(max(1, int(retry_after)), 60)
for _ in range(delay * 2):
if stop.is_set():
raise ArchiveStopped()
_check_state(job_id, external_stop)
time.sleep(0.5)
def _check_content(response):
if 'text/html' in response.headers.get('Content-Type', '').lower():
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
if response.headers.get('Content-Encoding', 'identity').lower() not in ('', 'identity'):
raise ArchiveDownloadError('Archive.org zignorowało Accept-Encoding: identity; transfer w tym kodowaniu jest niebezpieczny')
def _probe_range(job_id, uri, stop=None):
"""Only a real 206 with a numeric total enables parallel mode."""
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity', 'Range': 'bytes=0-0'}
with _client() as client:
for attempt in range(MAX_ATTEMPTS):
_check_state(job_id, stop)
try:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(job_id, f'Archive.org: sprawdzenie Range HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org HTTP {code} podczas sprawdzania Range po {MAX_ATTEMPTS} próbach')
delay = response.headers.get('Retry-After')
elif code in (401, 403):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia.')
elif code == 404:
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono pliku')
elif code == 200:
_check_content(response)
return None
elif code == 206:
_check_content(response)
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or (int(match.group(1)), int(match.group(2))) != (0, 0) or match.group(3) == '*':
return None
total = int(match.group(3))
if total <= 0:
return None
etag = response.headers.get('ETag', '').strip()
if etag.startswith('W/'):
etag = ''
validator = etag or response.headers.get('Last-Modified', '').strip()
if not validator or len(validator) > 256 or '\n' in validator:
validator = ''
return total, validator
else:
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
except ArchiveDownloadError:
raise
except (httpx.HTTPError, OSError) as exc:
_log(job_id, f'Archive.org: sprawdzenie Range - {type(exc).__name__}: {str(exc)[:150]}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org: brak odpowiedzi na żądanie Range po {MAX_ATTEMPTS} próbach') from exc
delay = None
_retry_wait(job_id, attempt, Event(), delay, stop)
return None
def _segment_worker(job_id, uri, folder, manifest, index, stop, external_stop=None):
begin, end = manifest['ranges'][index]
total = manifest['total']
limit = end - begin + 1
target = folder / f'{index:02d}.part'
if target.exists() and target.stat().st_size > limit:
target.unlink()
for attempt in range(MAX_ATTEMPTS):
if stop.is_set():
raise ArchiveStopped()
_check_state(job_id, external_stop)
present = target.stat().st_size if target.exists() else 0
if present == limit:
return
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity',
'Range': f'bytes={begin + present}-{end}'}
if manifest['validator']:
headers['If-Range'] = manifest['validator']
try:
with _client() as client:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(job_id, f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code} po {MAX_ATTEMPTS} próbach')
delay = response.headers.get('Retry-After')
elif code in (401, 403):
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}: brak dostępu')
elif code == 200:
raise _RangeUnsupported('Archive.org zignorowało zakres HTTP Range / If-Range', reset_prefix=True)
elif code != 206:
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}')
else:
_check_content(response)
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or match.group(3) == '*' or (int(match.group(1)), int(match.group(2)), int(match.group(3))) != (begin + present, end, total):
raise _RangeUnsupported('Archive.org zwróciło niezgodny Content-Range', reset_prefix=True)
if manifest['validator']:
header = 'ETag' if manifest['validator'].startswith('"') else 'Last-Modified'
received_validator = response.headers.get(header, '').strip()
if received_validator and received_validator != manifest['validator']:
raise _RangeUnsupported('Archive.org zmieniło plik podczas pobierania', reset_prefix=True)
last_check = time.monotonic()
with target.open('ab') as f:
for chunk in response.iter_raw(chunk_size=CHUNK_BYTES):
if stop.is_set():
raise ArchiveStopped()
if time.monotonic() - last_check >= 1:
_check_state(job_id, external_stop)
last_check = time.monotonic()
if not chunk:
continue
if f.tell() + len(chunk) > limit:
raise _RangeUnsupported('Archive.org wysłało zbyt dużo danych w zakresie', reset_prefix=True)
f.write(chunk)
f.flush()
os.fsync(f.fileno())
if target.stat().st_size != limit:
raise httpx.ReadError('Przerwana transmisja segmentu')
return
except (ArchiveStopped, ArchiveDownloadError, _RangeUnsupported):
raise
except (httpx.HTTPError, OSError) as exc:
_log(job_id, f'Segment {index+1}: {type(exc).__name__}: {str(exc)[:150]}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: przerwany transfer po {MAX_ATTEMPTS} próbach: {type(exc).__name__}') from exc
delay = None
_retry_wait(job_id, attempt, stop, delay, external_stop)
def _report_parallel(job_id, ix, url_count, partial, folder, manifest, last, batch=None):
prefix = manifest['prefix']
downloaded = prefix
for index, (start, end) in enumerate(manifest['ranges']):
path = folder / f'{index:02d}.part'
if path.exists():
downloaded += min(end - start + 1, path.stat().st_size)
now = time.monotonic()
prev_size, prev_at = last
speed = max(0, int((downloaded - prev_size) / max(0.001, now - prev_at)))
total = manifest['total']
if batch is None:
store.update(job_id, completed=downloaded, total=total, speed=speed,
progress=round((ix + min(downloaded / total, 1)) / url_count * 100, 2),
note=f'Archive.org: plik {ix+1}/{url_count}, {len(manifest["ranges"])} połączeń')
else:
batch.report(ix, downloaded, total)
return downloaded, now
def _assemble_segments(job_id, partial, destination, folder, manifest, stop=None):
prefix = manifest['prefix']
total = manifest['total']
for index, (start, end) in enumerate(manifest['ranges']):
part = folder / f'{index:02d}.part'
if not part.is_file() or part.stat().st_size != end - start + 1:
raise ArchiveDownloadError('Niekompletne segmenty; nie można złożyć pliku')
manifest['assembling'] = True
_write_manifest(folder, manifest)
with partial.open('r+b' if partial.exists() else 'w+b') as output:
output.truncate(prefix)
output.seek(prefix)
for index in range(len(manifest['ranges'])):
with (folder / f'{index:02d}.part').open('rb') as source:
while True:
_check_state(job_id, stop)
block = source.read(1024 * 1024)
if not block:
break
output.write(block)
output.flush()
os.fsync(output.fileno())
if partial.stat().st_size != total:
raise ArchiveDownloadError('Błąd scalania pliku: niezgodny rozmiar końcowy')
partial.replace(destination)
shutil.rmtree(folder)
def _parallel_transfer(job, ix, urls, uri, filename, partial, destination, connections, *, batch=None, stop=None):
jid = job['id']
batch_stop = stop
probe = _probe_range(jid, uri, stop)
if probe is None:
_log(jid, 'Archive.org: serwer nie obsługuje Range; pobieranie pojedynczym połączeniem')
return False
total, validator = probe
folder, manifest = _prepare_segments(partial, uri, total, validator, connections)
if folder is None and manifest is None:
_log(jid, 'Archive.org: mały plik lub końcówka; pobieranie pojedynczym połączeniem')
return False
if folder is None and manifest['prefix'] == total:
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
return True
_log(jid, f'Archive.org: {len(manifest["ranges"])} połączeń HTTP Range, rozmiar {total} B, wznowienie od {manifest["prefix"]} B')
local_stop = Event()
current = (manifest['prefix'] + sum(min((folder / f'{idx:02d}.part').stat().st_size, end - start + 1)
for idx, (start, end) in enumerate(manifest['ranges'])
if (folder / f'{idx:02d}.part').exists()), time.monotonic())
try:
with ThreadPoolExecutor(max_workers=min(connections, len(manifest['ranges']))) as pool:
pending = {pool.submit(_segment_worker, jid, uri, folder, manifest, i, local_stop, batch_stop)
for i in range(len(manifest['ranges']))}
try:
while pending:
_check_state(jid, batch_stop)
done, pending = wait(pending, timeout=0.8, return_when=FIRST_COMPLETED)
current = _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
for task in done:
task.result()
finally:
local_stop.set()
for task in pending:
task.cancel()
except _RangeUnsupported as exc:
_log(jid, 'Archive.org: niezgodna obsługa Range, przełączam na jedno połączenie')
_discard_parallel_state(partial, reset_prefix=exc.reset_prefix)
return False
_check_state(jid, batch_stop)
_report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
_log(jid, 'Archive.org: scalanie segmentów')
if batch is None:
store.update(jid, note='Archive.org: scalanie segmentów', speed=0)
if batch_stop is None:
_assemble_segments(jid, partial, destination, folder, manifest)
else:
_assemble_segments(jid, partial, destination, folder, manifest, batch_stop)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename}: {total} bajtów, {len(manifest["ranges"])} połączeń')
return True
class _BatchProgress:
"""Collect progress from file threads; only the scheduler updates SQLite.
A single stream/segment never overwrites the total progress or total speed
from another file. No extra database table is needed for old installations.
"""
def __init__(self, job_id, filenames, limit):
self.job_id = job_id
self.filenames = filenames
self.limit = limit
self.lock = Lock()
self.sizes = [0] * len(filenames)
self.totals = [0] * len(filenames)
self.seen = [False] * len(filenames)
self.done = set()
self.active = set()
self.transferred = 0
self.last_publish = time.monotonic()
def start(self, index):
with self.lock:
self.active.add(index)
def report(self, index, downloaded, total):
with self.lock:
downloaded = max(0, int(downloaded))
if self.seen[index]:
self.transferred += max(0, downloaded - self.sizes[index])
else:
# Existing .part and .segments from an earlier run are not
# counted as freshly transferred bytes per second.
self.seen[index] = True
self.sizes[index] = downloaded
self.totals[index] = max(0, int(total))
def complete(self, index, size=None):
if size is None:
size = (store.DATA / 'raw' / self.job_id / self.filenames[index]).stat().st_size
with self.lock:
if self.seen[index]:
self.transferred += max(0, size - self.sizes[index])
self.seen[index] = True
self.sizes[index] = size
self.totals[index] = size
self.done.add(index)
self.active.discard(index)
def publish(self):
now = time.monotonic()
with self.lock:
elapsed = max(0.001, now - self.last_publish)
speed = max(0, int(self.transferred / elapsed))
self.transferred = 0
self.last_publish = now
total_files = len(self.filenames)
fractions = []
for index in range(total_files):
if index in self.done:
fractions.append(1.0)
elif self.totals[index]:
fractions.append(min(self.sizes[index] / self.totals[index], 1))
else:
fractions.append(0.0)
completed_bytes = sum(self.sizes)
total_bytes = sum(self.totals)
if total_bytes and all(self.totals):
# A 20 KiB NFO must not count the same as an 8 GiB ISO.
progress = round(100 * min(completed_bytes, total_bytes) / total_bytes, 2)
else:
# Until all content lengths are known, show the fraction of
# completed files/known segments as a provisional estimate.
progress = round(sum(fractions) / total_files * 100, 2)
finished = len(self.done)
active = len(self.active)
contiguous = next((i for i in range(total_files) if i not in self.done), total_files)
store.update(self.job_id, part_index=contiguous, progress=progress,
completed=completed_bytes, total=total_bytes, speed=speed,
note=f'Archive.org: {finished}/{total_files} gotowe, {active} równolegle, limit {self.limit} połączeń')
def _transfer_many(job):
"""Download multiple Archive.org files at once with ONE connection budget.
Each slot owns a fixed connection allowance. Reusing the slot for the next
file preserves the global cap even with 50-segment old resume manifests.
"""
jid = job['id']
urls = source_urls(job)
if not urls:
raise ArchiveDownloadError('Brak plików Archive.org do pobrania')
names = []
for uri in urls:
if not archive_auth.is_archive_download(uri):
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
name = target_filename(uri)
if not name:
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
names.append(name)
if len({name.casefold() for name in names}) != len(names):
raise ArchiveDownloadError('Wybrane pliki mają powtarzające się nazwy. Wybierz pliki o różnych nazwach.')
root = store.DATA / 'raw' / jid
root.mkdir(parents=True, exist_ok=True)
_check_state(jid)
limit = _connection_limit()
tracker = _BatchProgress(jid, names, limit)
waiting = deque()
for ix, name in enumerate(names):
dest = root / name
if dest.is_file() and not Path(str(dest) + '.aria2').exists():
# A completed destination was atomically renamed by the old or
# new downloader. A legacy aria2 control file means its sibling
# destination may only contain an incomplete preallocated file.
tracker.complete(ix, dest.stat().st_size)
_log(jid, f'Plik {ix+1}/{len(urls)} już gotowy: {name}')
else:
waiting.append(ix)
store.update(jid, state='downloading', gid='', note=f'Archive.org: start równoległego pobierania {len(waiting)} plików')
tracker.publish()
if not waiting:
_check_state(jid)
store.update(jid, state='preparing', progress=100, speed=0,
note='Pobieranie Archive.org ukończone')
return
slots = min(limit, len(waiting))
budgets = [limit // slots + (1 if i < limit % slots else 0) for i in range(slots)]
_log(jid, f'Archive.org: start {slots} plików równolegle; budżet {limit} połączeń HTTP '
f'(przydział na plik: {", ".join(map(str, budgets))})')
stopped = Event()
first_error = None
def launch(pool, slot):
ix = waiting.popleft()
item = dict(job, part_index=ix)
tracker.start(ix)
return pool.submit(_transfer_one, item, batch=tracker, connections=budgets[slot], stop=stopped)
with ThreadPoolExecutor(max_workers=slots, thread_name_prefix='archive-file') as pool:
running = {launch(pool, slot): slot for slot in range(slots)}
try:
while running:
_check_state(jid, stopped)
finished, _ = wait(running, timeout=0.8, return_when=FIRST_COMPLETED)
tracker.publish()
for future in finished:
slot = running.pop(future)
try:
future.result()
except Exception as exc:
first_error = first_error or exc
stopped.set()
else:
if waiting and not stopped.is_set():
running[launch(pool, slot)] = slot
if first_error is not None:
break
finally:
stopped.set()
for future in running:
future.cancel()
if first_error is not None:
tracker.publish()
raise first_error
_check_state(jid) # Especially important if a pause arrives at the last byte.
tracker.publish()
store.update(jid, state='preparing', progress=100, speed=0,
part_index=len(urls), note='Pobieranie Archive.org ukończone')
_log(jid, f'Archive.org: pobrano wszystkie {len(urls)} pliki')
async def run_one(job):
import asyncio
urls = source_urls(job)
if job['source_type'] == 'archive_many' and len(urls) > 1:
await asyncio.to_thread(_transfer_many, job)
else:
await asyncio.to_thread(_transfer_one, job)
+147
View File
@@ -0,0 +1,147 @@
"""Supervise anacrolix-based BitTorrent subprocesses (no aria2 for torrents).
Torrent payload data starts only after file selection. All subprocess state is
written as atomic JSON so a restart can detect interrupted work and resume.
"""
import asyncio
import json
import os
import time
from pathlib import Path
from . import store
BT_HELPER = '/usr/local/bin/x360-bt'
_INSPECTIONS = {}
_DOWNLOADS = {}
class BitTorrentError(RuntimeError):
pass
def exe():
candidate = os.getenv('BT_HELPER', BT_HELPER)
if not Path(candidate).is_file():
raise BitTorrentError(f'Brak silnika BitTorrent: {candidate} (przebuduj obraz Docker)')
return candidate
def _read(path):
try:
value = json.loads(Path(path).read_text(encoding='utf-8'))
return value if isinstance(value, dict) else {}
except (FileNotFoundError, ValueError, OSError):
return {}
def inspect_state(preview_id):
state = _read(store.DATA / 'metadata' / preview_id / 'state.json')
process = _INSPECTIONS.get(preview_id)
if not state and (not process or process.returncode is not None):
return {'status': 'error', 'error': 'Klient BitTorrent przerwał odczyt metadanych. Sprawdź engine.log podglądu lub ponów.'}
return state
def job_state(job_id):
return _read(store.DATA / 'bt' / (job_id + '.json'))
async def _spawn(args, log_path):
log_path.parent.mkdir(parents=True, exist_ok=True)
with log_path.open('ab') as log:
try:
return await asyncio.create_subprocess_exec(*args, stdin=asyncio.subprocess.DEVNULL,
stdout=log, stderr=asyncio.subprocess.STDOUT)
except (OSError, ValueError) as exc:
raise BitTorrentError('Nie można uruchomić silnika BT: ' + str(exc)) from exc
async def start_inspection(preview_id, magnet):
from . import torrent_meta
folder = torrent_meta.inspect_dir(preview_id)
args = [exe(), 'inspect', '-magnet', magnet, '-out', str(folder / 'metadata.torrent'),
'-status', str(folder / 'state.json'), '-timeout', '5m', '-port', '0']
process = await _spawn(args, folder / 'engine.log')
_INSPECTIONS[preview_id] = process
return process
async def _stop(process):
if process and process.returncode is None:
process.terminate()
try:
await asyncio.wait_for(process.wait(), 6)
except asyncio.TimeoutError:
process.kill()
await process.wait()
async def stop_inspection(preview_id):
await _stop(_INSPECTIONS.pop(preview_id, None))
async def start_download(job):
jid = job['id']
old = _DOWNLOADS.get(jid)
if old and old.returncode is None:
return
path = store.DATA / 'torrents' / (jid + '.torrent')
if not path.is_file():
raise BitTorrentError('Brak zapisanego pliku .torrent dla zadania')
try:
spec = json.loads(job['source'])
selection = spec['selection']
except (ValueError, KeyError, TypeError) as exc:
raise BitTorrentError('Uszkodzony zapis wyboru plików torrenta') from exc
output = store.DATA / 'raw' / jid
output.mkdir(parents=True, exist_ok=True)
status_path = store.DATA / 'bt' / (jid + '.json')
status_path.parent.mkdir(parents=True, exist_ok=True)
status_path.unlink(missing_ok=True)
args = [exe(), 'fetch', '-torrent', str(path), '-select', selection, '-out', str(output),
'-status', str(status_path), '-port', os.getenv('BT_PORT', '6881')]
_DOWNLOADS[jid] = await _spawn(args, store.DATA / 'logs' / (jid + '.bt.log'))
store.update(jid, gid='', state='downloading', note='BitTorrent: uruchomiono klienta')
async def poll_download(job):
jid = job['id']
process = _DOWNLOADS.get(jid)
if process is None:
# A container restart loses the subprocess; persistently downloaded
# pieces in data/raw are verified and resumed by the helper.
await start_download(job)
return
state = job_state(jid)
done, total = int(state.get('downloaded', 0) or 0), int(state.get('total', 0) or 0)
speed = int(state.get('speed', 0) or 0)
peers = int(state.get('connections', 0) or 0)
known = int(state.get('known_peers', 0) or 0)
if state.get('status') == 'done' and process.returncode == 0:
await process.wait()
_DOWNLOADS.pop(jid, None)
store.update(jid, state='preparing', progress=100, speed=0,
note='BitTorrent ukończony, przygotowanie plików')
return
if state.get('status') == 'error' or process.returncode not in (None, 0):
await process.wait()
_DOWNLOADS.pop(jid, None)
raise BitTorrentError(state.get('error') or 'Silnik BitTorrent zakończył się niepowodzeniem; sprawdź log BT')
if process.returncode == 0 and state.get('status') != 'done':
_DOWNLOADS.pop(jid, None)
raise BitTorrentError('Silnik BitTorrent zakończył pracę bez potwierdzenia ukończenia danych')
percent = round(done / total * 100, 2) if total else 0
store.update(jid, progress=percent, speed=speed, total=total, completed=done,
note=f'BitTorrent: {peers} połączeń, {known} znanych peerów')
async def stop_download(job_id):
await _stop(_DOWNLOADS.pop(job_id, None))
async def shutdown():
jobs = list(_INSPECTIONS.values()) + list(_DOWNLOADS.values())
_INSPECTIONS.clear()
_DOWNLOADS.clear()
await asyncio.gather(*(_stop(p) for p in jobs))
+109
View File
@@ -0,0 +1,109 @@
"""Validation, safe paths and Internet Archive integration."""
import html
import re
import socket
import ipaddress
from urllib.parse import urlparse, quote, unquote, parse_qs
from pathlib import Path, PurePosixPath
FORMATS = {'xex', 'god', 'raw'}
ARCHIVE_EXTS = ('.zip', '.7z', '.rar', '.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')
SAFE_ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,150}$')
VIMM_PAGE = re.compile(r'^/vault/[0-9]+/?$')
def safe_title(name):
value = re.sub(r'[^\w .()\[\]-]+', '_', name, flags=re.UNICODE).strip(' ._')[:78]
return value or 'Gra'
def normalize_magnet(value):
# Pasted magnet URLs often contain HTML entities, hard wraps or Markdown.
url = html.unescape(value).strip()
url = re.sub(r"(?i)^['\"`]*\*{0,2}magnet\*{0,2}:\?", 'magnet:?', url)
if not url.lower().startswith('magnet:?'):
return url
url = re.sub(r"\\\s*", '', url)
url = re.sub(r"\s+", '', url)
return html.unescape(url.strip("'\"`"))
def validate_url(value):
url = normalize_magnet(value)
if url.lower().startswith('magnet:?'):
if len(url) > 12000:
raise ValueError('Link magnet jest zbyt dlugi')
params = parse_qs(urlparse(url).query, keep_blank_values=True)
xt = params.get('xt', [])
if not any(re.fullmatch(r'urn:btih:(?:[0-9a-fA-F]{40}|[A-Za-z2-7]{32})', item, re.I)
or re.fullmatch(r'urn:btmh:1220[0-9a-fA-F]{64}', item, re.I)
for item in xt):
raise ValueError('Nieprawidlowy hash xt w magnet (BTIH: 40 znakow HEX lub 32 Base32)')
return 'magnet', url
parsed = urlparse(url)
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
raise ValueError('Podaj adres HTTP/HTTPS albo link magnet')
try:
port = parsed.port
except ValueError as exc:
raise ValueError('Niepoprawny numer portu w URL') from exc
if port not in (None, 80, 443):
raise ValueError('Niestandardowy port w URL jest niedozwolony')
host = parsed.hostname.lower().strip('.')
if host in ('localhost',) or host.endswith('.localhost') or host.endswith('.local'):
raise ValueError('Adres lokalny nie jest dozwolony')
try:
ip = ipaddress.ip_address(host)
except ValueError:
ip = None
if ip is not None and not ip.is_global:
raise ValueError('Adres prywatny lub zarezerwowany jest niedozwolony')
if host in ('vimm.net', 'www.vimm.net') and VIMM_PAGE.fullmatch(parsed.path):
return 'vimm', url
return 'url', url
def parse_archive_id(value):
value = value.strip()
u = urlparse(value)
if u.scheme in ('http', 'https') and u.hostname in ('archive.org', 'www.archive.org'):
pieces = [unquote(x) for x in u.path.split('/') if x]
if len(pieces) >= 2 and pieces[0] in ('details', 'download', 'metadata'):
value = pieces[1]
else:
raise ValueError('Link musi wskazywac element archive.org/details/<identyfikator>')
if not SAFE_ID.fullmatch(value):
raise ValueError('Bledny identyfikator elementu Archive.org')
return value
def archive_download_url(identifier, filename):
identifier = parse_archive_id(identifier)
if (not filename or filename.startswith('/') or '\\' in filename or '\x00' in filename
or any(p in ('..', '.') for p in PurePosixPath(filename).parts)):
raise ValueError('Niedozwolona sciezka pliku Archive.org')
return f'https://archive.org/download/{quote(identifier)}/{quote(filename, safe="/")}'
def safe_import_file(filename, root=Path('/data/imports')):
# User selects from server-side imports (including subdirectories).
if not filename or '\\' in filename or filename.startswith('/') or '\x00' in filename:
raise ValueError('Niepoprawna sciezka pliku')
item = root.joinpath(*PurePosixPath(filename).parts)
if not item.resolve().is_relative_to(root.resolve()):
raise ValueError('Plik spoza katalogu imports')
if not item.is_file() or item.is_symlink():
raise ValueError('Wskazany plik nie istnieje')
return item
def is_archive(filename):
name = filename.lower()
return name.endswith(ARCHIVE_EXTS) or bool(re.search(r'\.(?:7z|zip)\.001$', name)) or bool(re.search(r'\.part0*1\.rar$', name))
def is_additional_volume(filename):
name = filename.lower()
return bool(re.search(r'\.(?:7z|zip)\.0*0?[2-9][0-9]*$', name) or
re.search(r'\.part0*[2-9][0-9]*\.rar$', name) or
re.search(r'\.r\d\d$', name))
+156
View File
@@ -0,0 +1,156 @@
"""aria2 JSON-RPC. aria2 is started alongside the API server, not exposed publicly."""
import asyncio
import base64
import json
import os
import urllib.parse
from pathlib import Path
import httpx
from . import archive_auth, store
RPC = 'http://127.0.0.1:6800/jsonrpc'
async def rpc(method, *params):
payload = {
'jsonrpc': '2.0', 'id': 'x360', 'method': 'aria2.' + method,
'params': ['token:' + os.environ['ARIA2_SECRET'], *params],
}
# On a container restart the API may start slightly before aria2 RPC.
# Retry the connection instead of permanently failing restored jobs.
async with httpx.AsyncClient(timeout=15) as client:
for attempt in range(15):
try:
r = await client.post(RPC, json=payload)
r.raise_for_status()
answer = r.json()
if 'error' in answer:
raise RuntimeError(answer['error'].get('message', 'Bledna odpowiedz aria2'))
return answer['result']
except (httpx.ConnectError, httpx.ConnectTimeout):
if attempt == 14:
raise
await asyncio.sleep(1)
def source_urls(job):
if job['source_type'] == 'archive_many':
return json.loads(job['source'])
return [job['source']]
def target_filename(uri):
if uri.lower().startswith('magnet:'):
return None
name = urllib.parse.unquote(Path(urllib.parse.urlsplit(uri).path).name)
if not name or name in ('.', '..'):
return 'pobrany-plik.bin'
# Do not allow URI-provided paths to escape the download directory.
return ''.join(c if c not in '\\/:*?"<>|\x00\n\r' else '_' for c in name)[:200]
async def begin(job):
base = store.DATA / 'raw' / job['id']
base.mkdir(parents=True, exist_ok=True)
urls = source_urls(job)
ix = job['part_index']
if ix >= len(urls):
store.update(job['id'], state='preparing')
return
uri = urls[ix]
opts = {
'dir': str(base), 'continue': 'true', 'max-connection-per-server': '2',
'split': '2', 'seed-time': '0', 'auto-file-renaming': 'false',
'allow-overwrite': 'false', 'max-tries': '8', 'retry-wait': '30',
}
if job['source_type'] != 'torrent_selected':
name = target_filename(uri)
if name and '.' in name:
opts['out'] = name
elif name:
# Let the server's Content-Disposition supply the real filename.
opts['content-disposition'] = 'true'
if job['source_type'] == 'torrent_selected':
torrent = store.DATA / 'torrents' / (job['id'] + '.torrent')
encoded = base64.b64encode(torrent.read_bytes()).decode('ascii')
select = json.loads(job['source'])['selection']
opts['select-file'] = select
opts['bt-remove-unselected-file'] = 'true'
gid = await rpc('addTorrent', encoded, [], opts)
note = 'Pobieranie wybranych plików torrenta'
else:
if archive_auth.is_archive_download(uri):
# aria2 downloads from the local proxy. It never receives IA cookies.
opts['header'] = [archive_auth.signed_header(job['id'], ix)]
uri = archive_auth.internal_url(job['id'], ix)
opts['split'] = '1'
opts['max-connection-per-server'] = '1'
gid = await rpc('addUri', [uri], opts)
note = f'Pobieranie pliku {ix+1}/{len(urls)}'
store.update(job['id'], gid=gid, state='downloading', note=note)
async def poll(job):
urls = source_urls(job)
gid = job['gid']
if not gid:
await begin(job)
return
try:
response = await rpc('tellStatus', gid, ['gid','status','totalLength','completedLength',
'downloadSpeed','followedBy','errorMessage'])
except RuntimeError as ex:
# On restart aria2 may lose an old GID. Re-add to resume available .aria2 files.
if 'not found' in str(ex).lower():
store.update(job['id'], gid='', note='Odzyskiwanie transferu po restarcie')
return
raise
followed = response.get('followedBy', [])
if followed:
store.update(job['id'], gid=followed[0], note='Magnet: metadane gotowe')
return
state = response.get('status')
length = int(response.get('totalLength', 0) or 0)
completed = int(response.get('completedLength', 0) or 0)
speed = int(response.get('downloadSpeed', 0) or 0)
part_progress = (completed / length) if length else 0
ix = job['part_index']
pct = (ix + part_progress) / len(urls) * 100
store.update(job['id'], progress=round(pct, 2), speed=speed, total=length,
completed=completed, note=f'Pobieranie {ix+1}/{len(urls)}: {state}')
if state == 'complete':
if ix + 1 < len(urls):
store.update(job['id'], gid='', part_index=ix+1)
else:
store.update(job['id'], state='preparing', progress=100, speed=0,
note='Pobieranie ukonczone, przygotowuje pliki')
elif state in ('error','removed'):
message = response.get('errorMessage') or state
if job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(urls[ix])):
if any(x in message.lower() for x in ('authorization', '403', '401', 'forbidden')):
message += ' — Internet Archive: sprawdź logowanie i dostęp do pliku.'
raise RuntimeError('aria2: ' + message)
elif state == 'paused':
store.update(job['id'], state='paused', speed=0)
async def cancel_download(gid):
if not gid:
return
try:
await rpc('forceRemove', gid)
except (RuntimeError, httpx.HTTPError):
pass
async def pause_download(gid):
if gid:
await rpc('forcePause', gid)
async def resume_download(gid):
if gid:
await rpc('unpause', gid)
+515
View File
@@ -0,0 +1,515 @@
"""Self-hosted X360 Web Queue: authenticated FastAPI + one resilient worker."""
import asyncio
import contextlib
import json
import os
import re
import secrets
import shutil
import time
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse, unquote
import httpx
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status
from fastapi.responses import FileResponse, PlainTextResponse
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
from . import archive_auth, archive_download, bt, download, pipeline, store, torrent_meta, vimm
from .common import (FORMATS, archive_download_url, parse_archive_id, safe_import_file,
safe_title, validate_url)
basic = HTTPBasic()
def authorize(credentials: HTTPBasicCredentials = Depends(basic)):
u = os.getenv('WEB_USER', 'admin').encode()
p = os.getenv('WEB_PASSWORD', '').encode()
if not p or not (secrets.compare_digest(credentials.username.encode(), u)
and secrets.compare_digest(credentials.password.encode(), p)):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
detail='Nieprawidlowy login lub haslo',
headers={'WWW-Authenticate': 'Basic'})
class AddJob(BaseModel):
url: str = Field(min_length=1, max_length=4000)
title: str = Field(default='', max_length=160)
format: str = 'xex'
class AddImports(BaseModel):
filename: str = Field(min_length=1, max_length=300)
format: str = 'xex'
class AddArchive(BaseModel):
identifier: str = Field(min_length=1, max_length=250)
files: list[str] = Field(min_length=1, max_length=150)
title: str = Field(default='', max_length=160)
format: str = 'xex'
class ItemLookup(BaseModel):
identifier: str = Field(min_length=1, max_length=250)
class ArchiveLogin(BaseModel):
email: str = Field(min_length=3, max_length=254)
password: str = Field(min_length=1, max_length=1024)
class MagnetInspect(BaseModel):
magnet: str = Field(min_length=20, max_length=12000)
class MagnetJob(BaseModel):
preview_id: str = Field(min_length=32, max_length=32)
files: list[int] = Field(min_length=1, max_length=20000)
title: str = Field(default='', max_length=160)
format: str = 'xex'
def new_job(source_type, source, title, fmt):
if fmt not in FORMATS:
raise HTTPException(status_code=400, detail='Format: xex, god albo raw')
jid = str(uuid.uuid4())
store.create({'id': jid, 'title': safe_title(title or 'Xbox 360'),
'source_type': source_type, 'source': source, 'format': fmt})
return store.get(jid)
async def worker():
while True:
job = store.next_job()
if not job:
await asyncio.sleep(2)
continue
jid = job['id']
try:
if job['state'] == 'preparing':
await pipeline.process_job(job)
elif job['source_type'] == 'import':
# Local import is copied into persistent raw dir, then converted.
src = safe_import_file(job['source'])
target = store.DATA / 'raw' / jid
target.mkdir(parents=True, exist_ok=True)
await asyncio.to_thread(shutil.copy2, src, target / src.name)
store.update(jid, state='preparing', progress=100)
elif job['source_type'] == 'vimm':
await vimm.browser_download(job)
elif job['source_type'] == 'torrent_selected':
if job['state'] == 'queued':
await bt.start_download(job)
else:
await bt.poll_download(job)
elif job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(job['source'])):
if job['gid']:
await download.cancel_download(job['gid'])
store.update(jid, gid='')
await archive_download.run_one(job)
elif job['state'] == 'queued':
# Resume if there is an aria2 GID instead of accidentally
# starting a duplicate download.
if job['gid']:
await download.resume_download(job['gid'])
store.update(jid, state='downloading')
else:
await download.begin(job)
else:
await download.poll(job)
except asyncio.CancelledError:
raise
except archive_download.ArchiveStopped:
pass
except Exception as ex:
store.update(jid, state='error', speed=0, error=str(ex)[:1800],
note='Błąd zadania: otwórz Log')
await asyncio.sleep(1.5)
@asynccontextmanager
async def lifespan(app):
for subdir in ('raw', 'ready', 'work', 'imports', 'logs', 'metadata', 'torrents'):
(store.DATA / subdir).mkdir(parents=True, exist_ok=True)
store.init_db()
task = asyncio.create_task(worker())
try:
yield
finally:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
await bt.shutdown()
app = FastAPI(title='X360 Web Queue', version='0.1.0', lifespan=lifespan,
docs_url=None, redoc_url=None, openapi_url=None)
api = APIRouter(prefix='/api', dependencies=[Depends(authorize)])
MAX_INSPECT_SECONDS = 300
@app.middleware('http')
async def guard_cross_origin_requests(request: Request, call_next):
if request.method in ('POST', 'PUT', 'PATCH', 'DELETE'):
origin = request.headers.get('origin')
if origin and urlparse(origin).netloc != request.headers.get('host'):
return PlainTextResponse('Nieprawidlowa domena zrodla zadania', status_code=403)
return await call_next(request)
@app.get('/')
async def index(_: None = Depends(authorize)):
return FileResponse(Path(__file__).parent / 'static' / 'index.html')
@app.get('/healthz')
async def healthz():
return {'ok': True}
@api.get('/jobs')
async def jobs():
return store.list_jobs()
@api.post('/jobs')
async def add_download(body: AddJob):
try:
kind, url = validate_url(body.url)
except ValueError as ex:
raise HTTPException(status_code=400, detail=str(ex)) from ex
parsed = urlparse(url)
guessed = unquote(Path(parsed.path).name) if kind != 'magnet' else 'Torrent'
if kind == 'vimm':
guessed = 'Vimm-' + parsed.path.rstrip('/').split('/')[-1]
if kind == 'magnet':
raise HTTPException(400, 'Magnet wymaga najpierw wyboru plików (/api/magnet/inspect)')
return new_job(kind, url, body.title or guessed, body.format)
@api.get('/archive/session')
async def archive_session():
return archive_auth.session_status()
@api.post('/archive/session')
async def archive_login(body: ArchiveLogin):
if '@' not in body.email:
raise HTTPException(400, 'Niepoprawny adres e-mail')
try:
return await archive_auth.login(body.email.strip(), body.password)
except archive_auth.ArchiveAuthError as exc:
raise HTTPException(400, str(exc)) from exc
@api.delete('/archive/session')
async def archive_logout():
archive_auth.logout()
return archive_auth.session_status()
@app.api_route('/internal/archive/{job_id}/{part_index}', methods=['GET', 'HEAD'])
async def archive_proxy(job_id: str, part_index: int, request: Request):
if (not 0 <= part_index <= 149 or
not archive_auth.verify_token(job_id, part_index, request.headers.get('x-x360-proxy-token'))):
raise HTTPException(403, 'Odmowa dostępu')
job = store.get(job_id)
if not job or job['source_type'] not in ('url', 'archive_many'):
raise HTTPException(404, 'Nie znaleziono zadania')
urls = download.source_urls(job)
if part_index >= len(urls) or not archive_auth.is_archive_download(urls[part_index]):
raise HTTPException(404, 'Nie znaleziono pliku')
return await archive_auth.stream_archive(urls[part_index], request.method, request.headers)
@api.post('/magnet/inspect')
async def magnet_inspect(body: MagnetInspect):
try:
kind, magnet = validate_url(body.magnet)
if kind != 'magnet':
raise ValueError('Wklej link magnet')
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
await asyncio.to_thread(torrent_meta.cleanup_old)
root = store.DATA / 'metadata'
active_previews = sum(1 for item in root.iterdir() if item.is_dir() and not item.is_symlink() and
(item / 'request.json').is_file() and
not list(item.glob('*.torrent')) and
time.time() - item.stat().st_mtime < MAX_INSPECT_SECONDS)
if active_previews >= 3:
raise HTTPException(429, 'Trzy podglądy już pobierają metadane. Poczekaj na zakończenie.')
preview_id = uuid.uuid4().hex
directory = torrent_meta.inspect_dir(preview_id)
directory.mkdir(parents=True, mode=0o700)
try:
await bt.start_inspection(preview_id, magnet)
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': magnet, 'engine': 'anacrolix'})
except (RuntimeError, httpx.HTTPError, OSError) as exc:
await bt.stop_inspection(preview_id)
await asyncio.to_thread(shutil.rmtree, directory, True)
raise HTTPException(502, f'Nie można rozpocząć odczytu metadanych: {exc}') from exc
return {'preview_id': preview_id, 'status': 'loading'}
@api.post('/magnet/upload')
async def torrent_upload(request: Request):
# No multipart dependency. Transfer raw .torrent bytes with a fixed size limit.
if request.headers.get('content-type', '').split(';', 1)[0].strip() != 'application/x-bittorrent':
raise HTTPException(415, 'Wymagany plik .torrent')
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > torrent_meta.MAX_METADATA:
raise HTTPException(413, 'Plik .torrent przekracza limit 32 MiB')
raw.extend(chunk)
if not raw:
raise HTTPException(400, 'Pusty plik .torrent')
preview_id = uuid.uuid4().hex
directory = torrent_meta.inspect_dir(preview_id)
directory.mkdir(parents=True, exist_ok=False, mode=0o700)
target = directory / 'uploaded.torrent'
try:
# Validate before storing the metadata. No game payload is transferred.
decoded = torrent_meta.decode_bencode(bytes(raw))
if b'info' not in decoded:
raise torrent_meta.MetadataError('Plik nie zawiera pola info')
target.write_bytes(raw)
torrent_meta.read_torrent(target)
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': ''})
except (torrent_meta.MetadataError, OSError) as exc:
await asyncio.to_thread(shutil.rmtree, directory, True)
raise HTTPException(400, str(exc)) from exc
return {'preview_id': preview_id, 'status': 'ready'}
@api.get('/magnet/inspect/{preview_id}')
async def magnet_inspect_status(preview_id: str):
try:
req = torrent_meta.load_request(preview_id)
path = torrent_meta.torrent_file(preview_id)
if path:
info = torrent_meta.read_torrent(path)
await bt.stop_inspection(preview_id)
info.update({'status': 'ready', 'suggested': torrent_meta.suggested_indices(req['magnet'], len(info['files']))})
return info
elapsed = max(0, int(time.time() - req['created']))
status = bt.inspect_state(preview_id)
if status.get('status') == 'error':
return {'status': 'error', 'error': status.get('error', 'Silnik BT zgłosił błąd')[:500]}
if elapsed > MAX_INSPECT_SECONDS:
await bt.stop_inspection(preview_id)
return {'status': 'error', 'error': 'Brak metadanych po 5 minutach. Sprawdź dostępność peerów i DHT/UDP.'}
return {'status': 'loading', 'elapsed': elapsed,
'connections': int(status.get('connections') or 0),
'seeders': 0, 'known_peers': int(status.get('known_peers') or 0),
'aria_status': 'anacrolix'}
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
except (RuntimeError, httpx.HTTPError) as exc:
raise HTTPException(502, 'aria2: ' + str(exc)[:200]) from exc
@api.post('/magnet/inspect/{preview_id}/cancel')
async def magnet_inspect_cancel(preview_id: str):
try:
req = torrent_meta.load_request(preview_id)
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
if req.get('engine') == 'anacrolix':
await bt.stop_inspection(preview_id)
else:
await download.cancel_download(req['gid'])
directory = torrent_meta.inspect_dir(preview_id)
await asyncio.to_thread(shutil.rmtree, directory, True)
return {'ok': True}
@api.post('/magnet/jobs')
async def magnet_selected_job(body: MagnetJob):
if body.format not in FORMATS:
raise HTTPException(400, 'Format: xex, god albo raw')
try:
torrent_meta.load_request(body.preview_id)
path = torrent_meta.torrent_file(body.preview_id)
if path is None:
raise torrent_meta.MetadataError('Metadane jeszcze nie są gotowe')
info = torrent_meta.read_torrent(path)
await bt.stop_inspection(body.preview_id)
valid = {item['index'] for item in info['files'] if not item['padding']}
if len(body.files) != len(set(body.files)) or not set(body.files).issubset(valid):
raise torrent_meta.MetadataError('Wybierz poprawne indeksy plików')
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
jid = str(uuid.uuid4())
dest = store.DATA / 'torrents' / (jid + '.torrent')
dest.parent.mkdir(parents=True, exist_ok=True)
await asyncio.to_thread(shutil.copyfile, path, dest)
selection = ','.join(str(i) for i in sorted(body.files))
data = json.dumps({'selection': selection, 'count': len(body.files), 'name': info['name']})
try:
store.create({'id': jid, 'title': safe_title(body.title or info['name']),
'source_type': 'torrent_selected', 'source': data, 'format': body.format})
except Exception:
dest.unlink(missing_ok=True)
raise
return store.get(jid)
@api.post('/archive/lookup')
async def archive_lookup(body: ItemLookup):
try:
identifier = parse_archive_id(body.identifier)
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
try:
async with httpx.AsyncClient(timeout=25, follow_redirects=True, cookies=archive_auth.session_cookies()) as client:
res = await client.get('https://archive.org/metadata/' + identifier)
res.raise_for_status()
data = res.json()
except (httpx.HTTPError, ValueError) as ex:
raise HTTPException(502, 'Archive.org metadata: ' + str(ex)) from ex
result = []
for entry in data.get('files', []):
name = entry.get('name', '')
if not name or entry.get('source') == 'metadata' or name.startswith('__'):
continue
# Show all file types with an extension; user chooses exact archive parts.
if '.' not in Path(name).name:
continue
try:
archive_download_url(identifier, name)
except ValueError:
continue
result.append({'name': name, 'size': int(entry.get('size') or 0),
'type': entry.get('format', '')})
return {'identifier': identifier, 'title': data.get('metadata', {}).get('title', identifier),
'files': result[:1000]}
@api.post('/archive/jobs')
async def add_archive(body: AddArchive):
try:
identifier = parse_archive_id(body.identifier)
if len(set(body.files)) != len(body.files):
raise ValueError('Pliki nie moga sie powtarzac')
urls = [archive_download_url(identifier, file) for file in body.files]
if len({Path(unquote(urlparse(u).path)).name for u in urls}) != len(urls):
raise ValueError('Pliki maja identyczne nazwy; wybierz jeden folder na zadanie')
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
# Preserving filename order is important for multipart archives.
return new_job('archive_many', json.dumps(urls), body.title or identifier, body.format)
@api.get('/imports')
async def imports():
root = store.DATA / 'imports'
result = []
for item in root.rglob('*'):
if len(result) >= 200:
break
if item.is_file() and not item.is_symlink() and item.resolve().is_relative_to(root.resolve()):
result.append({'name': str(item.relative_to(root)), 'size': item.stat().st_size})
return result
@api.post('/imports/jobs')
async def import_job(body: AddImports):
try:
path = safe_import_file(body.filename)
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
return new_job('import', body.filename, path.stem, body.format)
@api.post('/jobs/{job_id}/{action}')
async def manage(job_id: str, action: str):
job = store.get(job_id)
if not job:
raise HTTPException(404, 'Nie znaleziono zadania')
state = job['state']
if action == 'pause' and state in ('queued', 'downloading'):
if state == 'downloading':
try:
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
else:
await download.pause_download(job['gid'])
except (RuntimeError, httpx.HTTPError) as ex:
raise HTTPException(502, str(ex)) from ex
store.update(job_id, state='paused', speed=0, note='Wstrzymano')
elif action == 'resume' and state == 'paused':
store.update(job_id, state='queued', note='Wznowienie zadania')
elif action == 'cancel' and state in ('queued', 'paused', 'downloading'):
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
elif job['gid']:
await download.cancel_download(job['gid'])
store.update(job_id, state='cancelled', speed=0, note='Anulowano')
elif action == 'retry' and state in ('error', 'cancelled'):
# For already-downloaded input, retry conversion without redownloading.
root = store.DATA / 'raw' / job_id
good_files = ([p for p in root.rglob('*') if p.is_file() and not p.name.endswith('.aria2')]
if root.exists() else [])
complete = job['progress'] >= 99.9 and bool(good_files)
if job['source_type'] == 'archive_many':
# One finished file (or a .part segment) never means the whole
# multi-file archive is ready to convert.
complete = all((root / download.target_filename(url)).is_file()
for url in download.source_urls(job))
store.update(job_id, state='preparing' if complete else 'queued',
gid='' if not complete else job['gid'], error='',
note='Ponowienie zadania')
elif action == 'remove' and state in ('error','cancelled','done','paused'):
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
elif state == 'paused' and job['gid']:
await download.cancel_download(job['gid'])
store.erase(job_id)
return {'ok': True, 'note': 'Usunieto wpis z kolejki, pliki pozostaly na dysku'}
else:
raise HTTPException(409, 'Ta akcja nie jest dostepna dla obecnego stanu zadania')
return store.get(job_id)
@api.get('/jobs/{job_id}/log')
async def job_log(job_id: str):
if not store.get(job_id):
raise HTTPException(404, 'Nie znaleziono zadania')
path = store.DATA / 'logs' / f'{job_id}.log'
job = store.get(job_id)
lines = [f"Stan: {job['state']}", f"Etap: {job['note'] or 'brak'}"]
if job['error']:
lines.append('Błąd pobierania / zadania: ' + job['error'])
bt_log = store.DATA / 'logs' / f'{job_id}.bt.log'
if bt_log.is_file():
lines.append('BitTorrent (ostatnie 12 KiB):')
lines.append(bt_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
download_log = store.DATA / 'logs' / f'{job_id}.download.log'
if download_log.is_file():
lines.append('Pobieranie (ostatnie 12 KiB):')
lines.append(download_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
lines.append('')
lines.append(path.read_bytes()[-24000:].decode('utf-8', errors='replace')
if path.is_file() else 'Konwersja jeszcze nie została uruchomiona.')
return PlainTextResponse('\n'.join(lines))
@api.get('/system')
async def system():
usage = shutil.disk_usage(store.DATA / 'raw')
host_dir = os.getenv('XBOX_HOST_DIR', '/xbox').rstrip('/') or '/xbox'
return {'free': usage.free, 'total': usage.total,
'host_data_dir': host_dir,
'formats': ['xex','god','raw'],
'vimm_enabled': os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() in ('1','yes','true'),
'bt_engine': 'anacrolix (Go)', 'bt_port': os.getenv('BT_PORT', '6881')}
app.include_router(api)
app.mount('/static', StaticFiles(directory=str(Path(__file__).parent / 'static')), name='static')
+211
View File
@@ -0,0 +1,211 @@
"""Automatic ISO -> XEX/GOD pipeline; no console-side patching of executables."""
import asyncio
import json
import os
import re
import shutil
import tarfile
import zipfile
from pathlib import Path, PurePosixPath
from . import store
from .common import is_archive, is_additional_volume, safe_title
READY = store.DATA / 'ready'
WORK = store.DATA / 'work'
RAW = store.DATA / 'raw'
LOGS = store.DATA / 'logs'
class ProcessingError(RuntimeError):
pass
def safe_archive_path(name):
# Archive members are always relative, independent of platform.
norm = name.replace('\\', '/')
if (not norm or norm.startswith('/') or '\x00' in norm or
re.match(r'^[A-Za-z]:', norm) or
any(x == '..' for x in PurePosixPath(norm).parts)):
raise ProcessingError(f'Archiwum zawiera niebezpieczna sciezke: {name[:150]}')
def verify_no_symlinks(directory):
for root, dirs, files in os.walk(directory, followlinks=False):
for name in dirs + files:
path = Path(root) / name
if path.is_symlink():
raise ProcessingError('Pliki-symboliczne linki nie sa dopuszczalne: ' + name)
async def command(job_id, *args):
LOGS.mkdir(parents=True, exist_ok=True)
log_path = LOGS / f'{job_id}.log'
with log_path.open('ab') as log:
log.write(('\n>>> ' + ' '.join(str(x) for x in args) + '\n').encode('utf-8'))
process = await asyncio.create_subprocess_exec(*map(str, args), stdout=log, stderr=log,
stdin=asyncio.subprocess.DEVNULL)
try:
code = await process.wait()
except asyncio.CancelledError:
process.terminate()
await process.wait()
raise
if code:
data = log_path.read_bytes()[-2500:].decode('utf-8', errors='replace')
raise ProcessingError(f'Narzedzie {args[0]} zakonczone bledem {code}. Ostatni log:\n{data[-1400:]}')
def list_primary_archives(source):
files = [p for p in source.rglob('*') if p.is_file() and not p.is_symlink()]
archives = [p for p in files if is_archive(p.name) and not is_additional_volume(p.name)]
# Multipart .rar is detected by the first volume, subsequent parts are ignored.
# A single .zip with .z01 volumes is also supported by 7z.
return archives
async def unpack_archive(job_id, archive, destination):
destination.mkdir(parents=True, exist_ok=True)
lower = archive.name.lower()
store.update(job_id, state='extracting', note='Rozpakowywanie ' + archive.name[:100])
if lower.endswith('.zip') and not any(archive.parent.glob(archive.stem + '.z[0-9][0-9]')):
# Standard ZIP: validate entries to prevent writes outside destination.
with zipfile.ZipFile(archive) as z:
for member in z.infolist():
safe_archive_path(member.filename)
if (member.external_attr >> 16) & 0o170000 == 0o120000:
raise ProcessingError('Archiwum ZIP zawiera link symboliczny')
await asyncio.to_thread(z.extractall, destination)
elif lower.endswith(('.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')):
# tarfile's filters are version-dependent; reject links/special entries ourselves.
with tarfile.open(archive) as tf:
for member in tf.getmembers():
safe_archive_path(member.name)
if not (member.isfile() or member.isdir()):
raise ProcessingError('Archiwum TAR zawiera link lub plik specjalny')
await asyncio.to_thread(tf.extractall, destination, filter='data')
else:
# 7z/RAR/multipart. Validate file paths *before* invoking extraction.
listing = await asyncio.create_subprocess_exec('7z','l','-slt',str(archive),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
listing_out, listing_err = await listing.communicate()
if listing.returncode:
if not lower.endswith('.rar'):
raise ProcessingError('7z nie rozpoznaje archiwum: ' + listing_err[-300:].decode(errors='replace'))
# RAR5 fallback: libunarr (unar) understands many archives missing
# from Debian's p7zip package. Validate lsar entries first.
lsar = await asyncio.create_subprocess_exec('lsar', '-json', '-no-recursion', str(archive),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await lsar.communicate()
if lsar.returncode:
raise ProcessingError('7z/unar nie rozpoznaje RAR: ' + stderr[-400:].decode(errors='replace'))
try:
items = json.loads(stdout).get('lsarContents', [])
if not items:
raise ProcessingError('Nie mozna odczytac zawartosci RAR')
for item in items:
safe_archive_path(item['XADFileName'])
if any(item.get(x) for x in ('XADIsLink', 'XADIsSymbolicLink', 'XADLinkDestination')):
raise ProcessingError('RAR zawiera link symboliczny')
except (ValueError, KeyError, TypeError) as exc:
raise ProcessingError('Niepoprawna lista plikow RAR') from exc
await command(job_id, 'unar', '-no-directory', '-force-overwrite',
'-output-directory', str(destination), str(archive))
verify_no_symlinks(destination)
return
data = listing_out.decode('utf-8', errors='replace')
entries = data.split('----------', 1)
if len(entries) != 2:
raise ProcessingError('Nie mozna zweryfikowac listy plikow archiwum')
for line in entries[1].splitlines():
if line.startswith('Path = '):
safe_archive_path(line[7:])
if line.startswith(('Symbolic Link = ', 'Hard Link = ')):
raise ProcessingError('Archiwum zawiera link symboliczny lub twardy')
await command(job_id, '7z', 'x', '-y', '-bd', str(archive), f'-o{destination}')
verify_no_symlinks(destination)
async def locate_payload(job_id, root, tmp):
"""Find X360 ISOs or default.xex, recursively extracting up to 2 archive layers."""
current = root
for depth in range(3):
verify_no_symlinks(current)
isos = sorted(p for p in current.rglob('*') if p.is_file() and p.suffix.lower() == '.iso')
xex_files = sorted(p for p in current.rglob('*') if p.is_file() and p.name.lower() == 'default.xex')
if isos or xex_files:
return isos, xex_files
archives = list_primary_archives(current)
if not archives:
break
if len(archives) != 1:
raise ProcessingError('Znaleziono kilka niezaleznych archiwow. Dodaj jedno na zadanie lub polacz czesci gry w jedno archiwum.')
if depth >= 2:
raise ProcessingError('Archiwa zagniezdzone glebiej niz 3 poziomy nie sa obsługiwane')
out = tmp / f'unpacked-{depth}'
await unpack_archive(job_id, archives[0], out)
current = out
raise ProcessingError('Brak obrazu .iso ani default.xex. Sprawdz czy pobrano wszystkie czesci archiwum.')
async def process_job(job):
jid = job['id']
fmt = job['format']
source = RAW / jid
temp = WORK / jid
slug = safe_title(job['title'])[:55] + '-' + jid[:8]
dest = READY / fmt / slug
atomic_dest = READY / fmt / ('.' + slug + '.working')
if not source.exists():
raise ProcessingError('Brak pobranych plikow w katalogu raw')
temp.mkdir(parents=True, exist_ok=True)
if atomic_dest.exists():
shutil.rmtree(atomic_dest)
atomic_dest.mkdir(parents=True, exist_ok=True)
store.update(jid, state='preparing', note='Analiza pobranych plikow', speed=0)
try:
verify_no_symlinks(source)
if fmt == 'raw':
await asyncio.to_thread(shutil.copytree, source, atomic_dest / 'files')
else:
isos, xex_files = await locate_payload(jid, source, temp)
if fmt == 'xex':
store.update(jid, state='converting', note='Tworzenie folderu XEX')
if xex_files:
if len(xex_files) > 1:
raise ProcessingError('Kilka plikow default.xex; wybierz jeden obraz ISO na zadanie')
await asyncio.to_thread(shutil.copytree, xex_files[0].parent, atomic_dest / 'Gra')
else:
for n, iso in enumerate(isos, 1):
dest_dir = atomic_dest / ('Gra' if len(isos) == 1 else f'Disc_{n}')
await command(jid, 'xdvdfs', 'unpack', str(iso), str(dest_dir))
if not any(p.name.lower() == 'default.xex' for p in dest_dir.rglob('*') if p.is_file()):
raise ProcessingError(f'Obraz ISO nie zawiera default.xex: {iso.name}')
else:
store.update(jid, state='converting', note='Konwersja ISO do GOD')
if not isos and xex_files:
if len(xex_files) > 1:
raise ProcessingError('Kilka folderow default.xex; GOD wymaga jednego obrazu')
packed = temp / 'packed.xiso.iso'
await command(jid, 'xdvdfs', 'pack', str(xex_files[0].parent), str(packed))
isos = [packed]
for n, iso in enumerate(isos, 1):
# iso2god creates <TitleID>/<ContentType>/<MediaID> inside provided root.
# Prefix these folders with Content/0000000000000000 for Xbox internal HDD.
disc_dir = atomic_dest if len(isos) == 1 else atomic_dest / f'Disc_{n}'
content = disc_dir / 'Content' / '0000000000000000'
content.mkdir(parents=True, exist_ok=True)
await command(jid, 'iso2god', str(iso), str(content))
if not any(p.is_dir() and re.fullmatch(r'[0-9A-Fa-f]{8}', p.name)
for p in content.iterdir()):
raise ProcessingError(f'Brak katalogu TitleID w wyniku ISO2GOD: {iso.name}')
if dest.exists():
raise ProcessingError('Folder wynikowy juz istnieje: ' + str(dest))
atomic_dest.rename(dest)
store.update(jid, state='done', progress=100, output=str(dest),
note='Gotowe do skopiowania na Xbox 360')
finally:
# Full download is preserved; only temporary extraction is deleted.
shutil.rmtree(temp, ignore_errors=True)
if atomic_dest.exists():
shutil.rmtree(atomic_dest, ignore_errors=True)
+577
View File
@@ -0,0 +1,577 @@
'use strict';
const $ = (id) => document.getElementById(id);
const NS = 'http://www.w3.org/2000/svg';
const UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
const ACTIVE = new Set(['downloading', 'preparing', 'extracting', 'converting', 'vimm_browser']);
const LABELS = {
queued: 'W kolejce', downloading: 'Pobieranie', paused: 'Wstrzymano',
preparing: 'Przygotowanie', extracting: 'Rozpakowanie', converting: 'Konwersja',
done: 'Gotowe', error: 'Błąd', cancelled: 'Anulowano', vimm_browser: 'Vimm'
};
const S = { archiveId: '', archiveTitle: '', archiveFiles: [], selected: new Set(),
jobs: [], filter: 'all', connected: false, hostRoot: '/xbox', magnetPreview: null,
magnetFiles: [], magnetSelected: new Set(), magnetName: '', magnetTitle: '', magnetFormat: 'xex' };
function normalizeMagnet(raw) {
return raw.trim()
.replace(/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i, 'magnet:?')
.replace(/&amp;|&#38;|&#x26;/gi, '&')
.replace(/\\\s*/g, '')
.replace(/\s+/g, '')
.replace(/[\'"`]+$/g, '');
}
function bytes(n) {
n = Math.max(0, Number(n) || 0);
let i = 0;
while (n >= 1024 && i < UNITS.length - 1) { n /= 1024; i++; }
return (i ? n.toFixed(1) : n.toFixed(0)) + ' ' + UNITS[i];
}
function el(tag, cls, text) {
const node = document.createElement(tag);
if (cls) node.className = cls;
if (text !== undefined && text !== null) node.textContent = text;
return node;
}
function icon(name) {
const svg = document.createElementNS(NS, 'svg');
svg.setAttribute('class', 'icon');
svg.setAttribute('aria-hidden', 'true');
const use = document.createElementNS(NS, 'use');
use.setAttribute('href', '#i-' + name);
svg.append(use);
return svg;
}
function makeButton(label, glyph, cls = 'secondary') {
const button = el('button', cls);
button.type = 'button';
button.append(icon(glyph), document.createTextNode(label));
return button;
}
async function api(path, options = {}) {
const response = await fetch('/api' + path, {
...options,
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
});
let result;
try { result = await response.json(); }
catch { result = { detail: 'Nieprawidłowa odpowiedź serwera' }; }
if (!response.ok) {
throw new Error(typeof result.detail === 'string' ? result.detail : JSON.stringify(result.detail || result));
}
return result;
}
function connection(online) {
S.connected = online;
$('connectionState').classList.toggle('online', online);
$('connectionState').classList.toggle('offline', !online);
$('connectionText').textContent = online ? 'Połączono' : 'Brak połączenia';
}
function flash(message, bad = false) {
const t = $('toast');
t.textContent = message;
t.classList.toggle('bad', bad);
t.style.display = 'block';
clearTimeout(window._toastTimeout);
window._toastTimeout = setTimeout(() => { t.style.display = 'none'; }, 5000);
}
function filterJobs(jobs) {
if (S.filter === 'active') return jobs.filter(j => !['done', 'error', 'cancelled'].includes(j.state));
if (S.filter === 'done') return jobs.filter(j => j.state === 'done');
if (S.filter === 'error') return jobs.filter(j => j.state === 'error');
return jobs;
}
function sourceDescription(job) {
if (job.source_type === 'archive_many') {
try { return 'Internet Archive · ' + JSON.parse(job.source).length + ' plików'; }
catch { return 'Internet Archive'; }
}
if (job.source_type === 'import') return 'Import · ' + job.source;
if (job.source_type === 'magnet') return 'Magnet';
if (job.source_type === 'torrent_selected') {
try { return 'Torrent · ' + JSON.parse(job.source).count + ' plików'; }
catch { return 'Torrent'; }
}
if (job.source_type === 'vimm') return 'Vimm · ' + job.source;
return job.source;
}
function jobAction(label, glyph, action, job, danger = false) {
const button = makeButton(label, glyph, 'action' + (danger ? ' danger' : ''));
button.addEventListener('click', async () => {
if (action === 'remove' && !confirm('Usunąć wpis? Pobrane pliki zostaną na dysku.')) return;
if (action === 'cancel' && !confirm('Anulować to zadanie?')) return;
button.disabled = true;
try {
await api('/jobs/' + encodeURIComponent(job.id) + '/' + action, { method: 'POST' });
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
return button;
}
async function showLog(job) {
try {
const response = await fetch('/api/jobs/' + encodeURIComponent(job.id) + '/log');
if (!response.ok) throw new Error('HTTP ' + response.status);
$('logContent').textContent = await response.text();
$('logDialog').showModal();
} catch (e) { flash(e.message, true); }
}
function hostPath(path) {
if (typeof path !== 'string') return '';
if (path === '/data/raw') return S.hostRoot + '/downloads';
if (path.startsWith('/data/raw/')) return S.hostRoot + '/downloads/' + path.slice('/data/raw/'.length);
if (path === '/data') return S.hostRoot;
if (path.startsWith('/data/')) return S.hostRoot + '/' + path.slice('/data/'.length);
return path;
}
function jobRow(job) {
const card = el('article', 'job');
const header = el('div', 'job-top');
const heading = el('div', 'job-heading');
heading.append(el('div', 'job-name', job.title), el('div', 'job-sub', sourceDescription(job)));
header.append(heading);
const statusClass = job.state === 'done' ? 'done' : job.state === 'error' ? 'error' : ACTIVE.has(job.state) ? 'running' : '';
header.append(el('span', 'job-status ' + statusClass, LABELS[job.state] || job.state));
card.append(header);
const pct = Math.max(0, Math.min(100, Number(job.progress) || 0));
const unknown = job.state === 'vimm_browser';
const progress = el('div', 'job-progress');
const track = el('div', 'progressbar' + (job.state === 'error' ? ' is-error' : ''));
const fill = el('span');
fill.style.width = pct + '%';
track.append(fill);
progress.append(track, el('span', 'progress-text', unknown ? '—' : pct.toFixed(1) + '%'));
card.append(progress);
const meta = el('div', 'job-meta');
meta.append(el('span', 'note-text', job.note || 'Oczekiwanie'));
meta.append(el('span', 'speed', Number(job.speed) > 0 ? bytes(job.speed) + '/s' : ''));
card.append(meta);
if (job.state === 'error' && job.error) card.append(el('div', 'job-error', job.error));
if (job.output) card.append(el('div', 'job-output', 'Zapisano: ' + hostPath(job.output)));
const bottom = el('div', 'job-bottom');
const chips = el('div', 'job-chips');
chips.append(el('span', 'job-chip', (job.format || 'raw').toUpperCase()));
bottom.append(chips);
const actions = el('div', 'job-actions');
if (['queued', 'downloading'].includes(job.state)) {
actions.append(jobAction('Pauza', 'pause', 'pause', job), jobAction('Anuluj', 'x', 'cancel', job, true));
}
if (job.state === 'paused') {
actions.append(jobAction('Wznów', 'play', 'resume', job), jobAction('Anuluj', 'x', 'cancel', job, true));
}
if (['error', 'cancelled'].includes(job.state)) actions.append(jobAction('Ponów', 'retry', 'retry', job));
if (['error', 'done', 'cancelled', 'paused'].includes(job.state)) actions.append(jobAction('Usuń', 'trash', 'remove', job, true));
const log = makeButton('Log', 'file', 'action');
log.addEventListener('click', () => showLog(job));
actions.append(log);
bottom.append(actions);
card.append(bottom);
return card;
}
function renderJobs() {
$('jobsCount').textContent = String(S.jobs.length);
$('countQueued').textContent = String(S.jobs.filter(j => j.state === 'queued' || j.state === 'paused').length);
$('countActive').textContent = String(S.jobs.filter(j => ACTIVE.has(j.state)).length);
$('countDone').textContent = String(S.jobs.filter(j => j.state === 'done').length);
const list = $('jobsList');
list.replaceChildren();
const filtered = filterJobs(S.jobs);
if (!filtered.length) {
list.append(el('div', 'empty-state', S.filter === 'all' ? 'Brak zadań. Dodaj link powyżej.' : 'Brak zadań w tym widoku.'));
return;
}
for (const job of filtered) list.append(jobRow(job));
}
async function refreshJobs(showError = false) {
try {
S.jobs = await api('/jobs');
renderJobs();
connection(true);
} catch (e) {
connection(false);
if (showError) flash('Nie można odczytać kolejki: ' + e.message, true);
}
}
async function refreshSystem() {
try {
const info = await api('/system');
$('storageNum').textContent = bytes(info.free);
const percent = info.total > 0 ? (100 * (1 - info.free / info.total)) : 0;
$('storageFill').style.width = Math.max(0, Math.min(100, percent)) + '%';
$('storageText').textContent = bytes(info.total) + ' łącznie';
const hostRoot = String(info.host_data_dir || '/xbox').replace(/\/+$/, '');
S.hostRoot = hostRoot;
$('importHostPath').textContent = hostRoot + '/imports/';
$('readyXexPath').textContent = hostRoot + '/ready/xex/';
$('readyGodPath').textContent = hostRoot + '/ready/god/';
$('downloadHostPath').textContent = hostRoot + '/downloads/';
$('workHostPath').textContent = hostRoot + '/work/';
connection(true);
} catch {
$('storageText').textContent = 'Nie udało się odczytać dysku';
connection(false);
}
}
for (const tab of document.querySelectorAll('.tab')) {
tab.addEventListener('click', () => {
for (const other of document.querySelectorAll('.tab')) {
const selected = other === tab;
other.classList.toggle('active', selected);
other.setAttribute('aria-selected', String(selected));
}
for (const name of ['links', 'archive', 'import']) {
$('pane-' + name).classList.toggle('hidden', name !== tab.dataset.tab);
}
if (tab.dataset.tab === 'import') refreshImports();
});
}
for (const filter of document.querySelectorAll('.filter')) {
filter.addEventListener('click', () => {
S.filter = filter.dataset.filter;
for (const other of document.querySelectorAll('.filter')) {
const selected = other === filter;
other.classList.toggle('active', selected);
other.setAttribute('aria-pressed', String(selected));
}
renderJobs();
});
}
async function refreshArchiveSession() {
try {
const result = await api('/archive/session');
$('archiveSessionText').textContent = result.authenticated ? ('Sesja zapisana · ' + result.email) : 'Niezalogowano';
$('archiveLoginForm').classList.toggle('hidden', result.authenticated);
$('archiveLogout').classList.toggle('hidden', !result.authenticated);
} catch (e) { $('archiveSessionText').textContent = 'Błąd sprawdzania sesji'; }
}
$('archiveLoginForm').addEventListener('submit', async (event) => {
event.preventDefault();
const button = $('archiveLoginForm').querySelector('[type=submit]');
button.disabled = true;
try {
const result = await api('/archive/session', { method: 'POST', body: JSON.stringify({
email: $('archiveEmail').value.trim(), password: $('archivePassword').value
}) });
$('archivePassword').value = '';
flash('Zapisano sesję Archive.org: ' + result.email);
await refreshArchiveSession();
} catch (e) { $('archivePassword').value = ''; flash('Archive.org: ' + e.message, true); }
finally { button.disabled = false; }
});
$('archiveLogout').addEventListener('click', async () => {
try {
await api('/archive/session', { method: 'DELETE' });
await refreshArchiveSession();
flash('Usunięto zapisaną sesję Archive.org.');
} catch (e) { flash(e.message, true); }
});
function renderMagnet() {
const list = $('magnetFiles');
list.replaceChildren();
const text = $('magnetFilter').value.toLowerCase();
let displayed = 0;
for (const file of S.magnetFiles) {
if (!file.name.toLowerCase().includes(text)) continue;
if (displayed >= 500) break;
displayed++;
const row = el('label', 'file-row');
const box = el('input');
box.type = 'checkbox';
box.checked = S.magnetSelected.has(file.index);
box.disabled = Boolean(file.padding);
box.addEventListener('change', () => {
if (box.checked) S.magnetSelected.add(file.index);
else S.magnetSelected.delete(file.index);
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
});
row.append(box, el('span', 'file-name', file.name),
el('span', 'size', file.padding ? 'Systemowy' : bytes(file.size)));
list.append(row);
}
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
if (displayed >= 500) list.append(el('div', 'empty-state', 'Pierwsze 500 plików. Zawęź filtr, aby wyszukać kolejne.'));
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
}
async function pollMagnet(previewId) {
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
try {
const data = await api('/magnet/inspect/' + encodeURIComponent(previewId));
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
if (data.status === 'ready') {
S.magnetName = data.name;
S.magnetFiles = data.files;
S.magnetSelected = new Set((data.suggested || []).filter(i => !data.files.find(f => f.index === i)?.padding));
$('magnetName').textContent = data.name + ' (' + data.files.length + ' plików)';
$('magnetStatus').textContent = 'Metadane pobrane. Wybierz pliki.';
$('magnetResults').classList.remove('hidden');
renderMagnet();
return;
}
if (data.status === 'loading') {
const seconds = Number(data.elapsed) || 0;
const connections = Number(data.connections) || 0;
const known = Number(data.known_peers) || 0;
let notice = 'Metadane: ' + seconds + ' s | połączenia: ' + connections + ' | znani peerzy: ' + known;
if (seconds >= 25 && connections === 0)
notice += ' | Brak połączeń. Sprawdź UDP/DHT lub wczytaj .torrent.';
$('magnetStatus').textContent = notice;
}
if (data.status === 'error') {
$('magnetStatus').textContent = 'Błąd: ' + data.error;
return;
}
} catch (e) {
$('magnetStatus').textContent = 'Błąd: ' + e.message;
return;
}
setTimeout(() => pollMagnet(previewId), 3000);
}
async function inspectMagnet(magnet, title, format) {
S.magnetPreview = null;
S.magnetTitle = title;
S.magnetFormat = format;
S.magnetFiles = [];
S.magnetSelected.clear();
$('magnetStatus').textContent = 'Oczekiwanie na metadane z sieci BitTorrent…';
$('magnetResults').classList.add('hidden');
$('magnetFilter').value = '';
$('magnetDialog').showModal();
try {
const data = await api('/magnet/inspect', { method: 'POST', body: JSON.stringify({ magnet }) });
if (!$('magnetDialog').open) return;
S.magnetPreview = data.preview_id;
pollMagnet(data.preview_id);
} catch (e) { $('magnetStatus').textContent = 'Błąd: ' + e.message; }
}
async function uploadTorrent(file, fromLinks) {
if (!file) return;
if (file.size > 32 * 1024 * 1024) return flash('Limit pliku .torrent: 32 MiB.', true);
if (fromLinks) {
S.magnetTitle = $('linkTitle').value.trim();
S.magnetFormat = $('linkFormat').value;
}
const previous = S.magnetPreview;
S.magnetPreview = null;
S.magnetFiles = [];
S.magnetSelected.clear();
$('magnetResults').classList.add('hidden');
$('magnetFilter').value = '';
$('magnetStatus').textContent = 'Wczytywanie pliku .torrent...';
if (!$('magnetDialog').open) $('magnetDialog').showModal();
if (previous) await api('/magnet/inspect/' + encodeURIComponent(previous) + '/cancel', { method: 'POST' }).catch(() => {});
try {
const response = await fetch('/api/magnet/upload', {
method: 'POST', headers: { 'Content-Type': 'application/x-bittorrent' }, body: file
});
const data = await response.json();
if (!response.ok) throw new Error(data.detail || 'HTTP ' + response.status);
if (!$('magnetDialog').open) return;
S.magnetPreview = data.preview_id;
await pollMagnet(data.preview_id);
} catch (error) {
$('magnetStatus').textContent = 'Błąd: ' + error.message;
}
}
for (const id of ['torrentFileLinks', 'torrentFileDialog']) {
$(id).addEventListener('change', async (event) => {
const file = event.target.files?.[0];
event.target.value = '';
await uploadTorrent(file, id === 'torrentFileLinks');
});
}
$('magnetFilter').addEventListener('input', renderMagnet);
$('magnetToggleAll').addEventListener('click', () => {
const visible = S.magnetFiles.filter(f => !f.padding && f.name.toLowerCase().includes($('magnetFilter').value.toLowerCase())).slice(0, 500);
const allSelected = visible.every(f => S.magnetSelected.has(f.index));
for (const file of visible) {
if (allSelected) S.magnetSelected.delete(file.index);
else S.magnetSelected.add(file.index);
}
renderMagnet();
});
$('closeMagnet').addEventListener('click', () => $('magnetDialog').close());
$('magnetDialog').addEventListener('close', () => {
const preview = S.magnetPreview;
S.magnetPreview = null;
if (preview) api('/magnet/inspect/' + encodeURIComponent(preview) + '/cancel', { method: 'POST' }).catch(() => {});
});
$('magnetAdd').addEventListener('click', async () => {
if (!S.magnetSelected.size) return flash('Zaznacz co najmniej jeden plik.', true);
const button = $('magnetAdd');
button.disabled = true;
try {
await api('/magnet/jobs', { method: 'POST', body: JSON.stringify({
preview_id: S.magnetPreview, files: [...S.magnetSelected],
title: S.magnetTitle, format: S.magnetFormat
}) });
$('urls').value = '';
S.magnetPreview = null;
$('magnetDialog').close();
flash('Wybrane pliki dodano do kolejki.');
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
$('linksForm').addEventListener('submit', async (event) => {
event.preventDefault();
const raw = $('urls').value.trim();
if (!raw) return flash('Podaj co najmniej jeden link.', true);
const button = $('linksForm').querySelector('[type=submit]');
const title = $('linkTitle').value.trim();
const format = $('linkFormat').value;
if (/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i.test(raw)) {
const magnet = normalizeMagnet(raw);
if ((magnet.match(/magnet:\?/gi) || []).length !== 1)
return flash('Magnet wklej pojedynczo, aby wybrać pliki.', true);
await inspectMagnet(magnet, title, format);
return;
}
const items = raw.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
let ok = 0;
const errors = [];
button.disabled = true;
try {
for (const url of items) {
try { await api('/jobs', { method: 'POST', body: JSON.stringify({ url, title, format }) }); ok++; }
catch (e) { errors.push(url.slice(0, 65) + ': ' + e.message); }
}
if (ok) {
$('urls').value = '';
flash('Dodano zadań: ' + ok);
await refreshJobs();
}
if (errors.length) flash(errors.join('\n'), true);
} finally { button.disabled = false; }
});
$('archiveForm').addEventListener('submit', async (event) => {
event.preventDefault();
const button = $('archiveForm').querySelector('[type=submit]');
button.disabled = true;
$('archiveResults').classList.add('hidden');
try {
const data = await api('/archive/lookup', { method: 'POST', body: JSON.stringify({ identifier: $('archiveId').value }) });
S.archiveId = data.identifier;
S.archiveTitle = data.title;
S.archiveFiles = data.files;
S.selected.clear();
$('archiveName').textContent = data.title + ' (' + data.files.length + ')';
$('archiveResults').classList.remove('hidden');
renderArchive();
} catch (e) { flash('Internet Archive: ' + e.message, true); }
finally { button.disabled = false; }
});
function renderArchive() {
const list = $('archiveFiles');
list.replaceChildren();
const text = $('archiveFilter').value.toLowerCase();
let displayed = 0;
for (const file of S.archiveFiles) {
if (!file.name.toLowerCase().includes(text)) continue;
if (displayed >= 400) break;
displayed++;
const row = el('label', 'file-row');
const checkbox = el('input');
checkbox.type = 'checkbox';
checkbox.checked = S.selected.has(file.name);
checkbox.addEventListener('change', () => {
if (checkbox.checked) S.selected.add(file.name);
else S.selected.delete(file.name);
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
});
row.append(checkbox, el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
list.append(row);
}
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
}
$('archiveFilter').addEventListener('input', renderArchive);
$('archiveAdd').addEventListener('click', async () => {
if (!S.selected.size) return flash('Zaznacz pliki.', true);
const button = $('archiveAdd');
button.disabled = true;
try {
await api('/archive/jobs', { method: 'POST', body: JSON.stringify({
identifier: S.archiveId, files: [...S.selected], title: S.archiveTitle, format: $('archiveFormat').value
}) });
S.selected.clear();
renderArchive();
flash('Dodano do kolejki.');
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
async function refreshImports() {
try {
const files = await api('/imports');
const list = $('importList');
list.replaceChildren();
if (!files.length) {
list.append(el('div', 'empty-state', 'Katalog data/imports/ jest pusty.'));
return;
}
for (const file of files) {
const row = el('div', 'file-row');
row.append(icon('file'), el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
const button = makeButton('Przetwórz', 'play');
button.addEventListener('click', async () => {
button.disabled = true;
try {
await api('/imports/jobs', { method: 'POST', body: JSON.stringify({
filename: file.name, format: $('importFormat').value
}) });
flash('Dodano: ' + file.name);
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
row.append(button);
list.append(row);
}
} catch (e) { flash('Import: ' + e.message, true); }
}
$('refreshImports').addEventListener('click', refreshImports);
$('refreshJobs').addEventListener('click', () => refreshJobs(true));
$('closeLog').addEventListener('click', () => $('logDialog').close());
refreshJobs();
refreshSystem();
refreshArchiveSession();
setInterval(() => refreshJobs(), 3000);
setInterval(refreshSystem, 30000);
+4
View File
@@ -0,0 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
<rect width="64" height="64" rx="10" fill="#1b1e21"/>
<path d="M17 16 32 32 47 16M17 48 32 32 47 48" fill="none" stroke="#91c99a" stroke-width="6" stroke-linecap="square"/>
</svg>

After

Width:  |  Height:  |  Size: 245 B

+151
View File
@@ -0,0 +1,151 @@
<!doctype html>
<html lang="pl">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="dark">
<meta name="theme-color" content="#141619">
<title>X360 Queue</title>
<link rel="icon" href="/static/icon.svg" type="image/svg+xml">
<link rel="stylesheet" href="/static/style.css">
<script src="/static/app.js" defer></script>
</head>
<body>
<svg class="svg-defs" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
<symbol id="i-download" viewBox="0 0 24 24"><path d="M12 3v12m-5-5 5 5 5-5M5 17v3h14v-3"/></symbol>
<symbol id="i-link" viewBox="0 0 24 24"><path d="M10 13a5 5 0 0 0 7.4.4l2-2a5 5 0 0 0-7.1-7.1l-1.1 1.1M14 11a5 5 0 0 0-7.4-.4l-2 2a5 5 0 0 0 7.1 7.1l1.1-1.1"/></symbol>
<symbol id="i-archive" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="4" rx="1"/><path d="M5 8v12h14V8M10 12h4"/></symbol>
<symbol id="i-folder" viewBox="0 0 24 24"><path d="M3 7a2 2 0 0 1 2-2h5l2 2h7a2 2 0 0 1 2 2v10H3z"/></symbol>
<symbol id="i-plus" viewBox="0 0 24 24"><path d="M12 5v14M5 12h14"/></symbol>
<symbol id="i-refresh" viewBox="0 0 24 24"><path d="M20 7v5h-5M4 17v-5h5"/><path d="M5.6 9A7 7 0 0 1 18 6l2 6M4 12l2 6a7 7 0 0 0 12.4-3"/></symbol>
<symbol id="i-pause" viewBox="0 0 24 24"><path d="M8 5v14M16 5v14"/></symbol>
<symbol id="i-play" viewBox="0 0 24 24"><path d="m8 5 11 7-11 7z"/></symbol>
<symbol id="i-x" viewBox="0 0 24 24"><path d="M6 6l12 12M18 6 6 18"/></symbol>
<symbol id="i-retry" viewBox="0 0 24 24"><path d="M3 12a9 9 0 1 0 3-6.7M3 4v5h5"/></symbol>
<symbol id="i-trash" viewBox="0 0 24 24"><path d="M4 7h16M10 4h4M6 7l1 13h10l1-13M10 11v5m4-5v5"/></symbol>
<symbol id="i-file" viewBox="0 0 24 24"><path d="M6 3h8l4 4v14H6zM14 3v5h5M9 12h6M9 16h6"/></symbol>
<symbol id="i-harddrive" viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="14" rx="2"/><path d="M3 13h18M7 16h.01M11 16h.01"/></symbol>
<symbol id="i-activity" viewBox="0 0 24 24"><path d="M3 12h4l3-7 4 14 3-7h4"/></symbol>
<symbol id="i-check" viewBox="0 0 24 24"><path d="m5 12 5 5L19 7"/></symbol>
<symbol id="i-clock" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6l4 2"/></symbol>
<symbol id="i-alert" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6m0 4v.1"/></symbol>
<symbol id="i-list" viewBox="0 0 24 24"><path d="M9 6h12M9 12h12M9 18h12M4 6h.01M4 12h.01M4 18h.01"/></symbol>
</svg>
<div class="shell">
<header class="topbar">
<div class="brand">
<div class="brand-mark" aria-hidden="true">X</div>
<div class="brand-text"><strong>X360 Queue</strong><span>Linux / RGH</span></div>
</div>
<div class="topbar-right">
<span class="connection" id="connectionState"><span class="connection-dot"></span><span id="connectionText">Łączenie…</span></span>
<span class="version">LOCAL</span>
</div>
</header>
<main>
<div class="page-title">
<div><h1>Pobieranie</h1><p>Dodawanie plików i konwersja XEX / GOD.</p></div>
</div>
<section class="metrics" aria-label="Stan aplikacji">
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-clock"/></svg> W kolejce</span><strong id="countQueued">0</strong></div>
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-activity"/></svg> W toku</span><strong id="countActive">0</strong></div>
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-check"/></svg> Gotowe</span><strong id="countDone">0</strong></div>
<div class="metric disk-metric">
<span class="metric-label"><svg class="icon"><use href="#i-harddrive"/></svg> Wolne miejsce</span>
<strong id="storageNum">—</strong>
<div class="meter"><span id="storageFill" style="width:0%"></span></div>
<small id="storageText">—</small>
</div>
</section>
<section class="panel" aria-labelledby="addHeading">
<div class="panel-heading"><h2 id="addHeading">Dodaj zadanie</h2><span class="muted-info">Zadania działają po zamknięciu przeglądarki</span></div>
<nav class="tabs" aria-label="Wybierz źródło">
<button class="tab active" type="button" data-tab="links" aria-selected="true"><svg class="icon"><use href="#i-link"/></svg>Link / magnet</button>
<button class="tab" type="button" data-tab="archive" aria-selected="false"><svg class="icon"><use href="#i-archive"/></svg>Internet Archive</button>
<button class="tab" type="button" data-tab="import" aria-selected="false"><svg class="icon"><use href="#i-folder"/></svg>Import</button>
</nav>
<div class="tab-body" id="pane-links">
<form id="linksForm">
<div class="field">
<label for="urls">Linki <span>— jeden adres w wierszu</span></label>
<textarea id="urls" rows="3" spellcheck="false" placeholder="https://example.org/obraz.iso&#10;magnet:?xt=urn:btih:..."></textarea>
</div>
<div class="input-row">
<div class="field title-field"><label for="linkTitle">Nazwa <span>(opcjonalnie)</span></label><input id="linkTitle" placeholder="Automatyczna" autocomplete="off"></div>
<div class="field format-field"><label for="linkFormat">Format wyjściowy</label><select id="linkFormat"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select></div>
<button class="primary submit-button" type="submit"><svg class="icon"><use href="#i-plus"/></svg>Dodaj do kolejki</button>
</div>
</form>
<div class="torrent-upload-inline"><label for="torrentFileLinks" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileLinks" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Alternatywa, gdy magnet nie zwraca metadanych.</span></div>
<p class="note">Magnet: najpierw lista plików, potem wybór i pobieranie. Vimm: tylko dostępny przycisk pobierania. Limity i zabezpieczenia serwisu pozostają aktywne.</p>
</div>
<div class="tab-body hidden" id="pane-archive">
<section class="archive-auth" aria-label="Konto Internet Archive">
<div class="auth-heading"><strong>Konto Archive.org</strong><span id="archiveSessionText">Sprawdzanie sesji…</span></div>
<form id="archiveLoginForm" class="auth-login hidden" autocomplete="on">
<div class="field"><label for="archiveEmail">E-mail</label><input id="archiveEmail" type="email" autocomplete="username" required placeholder="konto@example.org"></div>
<div class="field"><label for="archivePassword">Hasło</label><input id="archivePassword" type="password" autocomplete="current-password" required></div>
<button type="submit" class="secondary"><svg class="icon"><use href="#i-link"/></svg>Zaloguj</button>
</form>
<button id="archiveLogout" class="secondary hidden" type="button">Wyloguj</button>
<p class="note">Hasło nie jest zapisywane. Pobieranie używa sesji Archive.org; konto nie odblokowuje plików niedostępnych dla użytkownika. Panel udostępniaj przez HTTPS lub VPN.</p>
</section>
<form id="archiveForm">
<div class="field"><label for="archiveId">Adres lub identyfikator</label>
<div class="inline-input"><input id="archiveId" placeholder="archive.org/details/identyfikator" spellcheck="false"><button class="secondary" type="submit"><svg class="icon"><use href="#i-list"/></svg>Pokaż pliki</button></div>
</div>
</form>
<div class="archive-results hidden" id="archiveResults">
<div class="file-heading"><h3 id="archiveName"></h3><input id="archiveFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
<div class="file-table" id="archiveFiles"></div>
<div class="file-bottom"><span id="selectedCount">Wybrano: 0</span><div class="selection-controls"><select id="archiveFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="archiveAdd" class="primary" type="button"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
<p class="note">Części jednego archiwum zaznacz razem.</p>
</div>
</div>
<div class="tab-body hidden" id="pane-import">
<div class="import-toolbar"><div><p class="import-hint">Skopiuj pliki do <code id="importHostPath">/xbox/imports/</code>.</p></div><div class="selection-controls"><select id="importFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="refreshImports" class="secondary" type="button"><svg class="icon"><use href="#i-refresh"/></svg>Odśwież</button></div></div>
<div id="importList" class="file-table"></div>
</div>
</section>
<section class="queue-section" aria-labelledby="queueHeading">
<div class="queue-header">
<div class="queue-title"><h2 id="queueHeading">Kolejka <span id="jobsCount" class="count">0</span></h2><span class="muted-info">Odświeżanie co 3 s</span></div>
<div class="queue-controls">
<div class="filters" aria-label="Filtruj zadania">
<button type="button" class="filter active" data-filter="all" aria-pressed="true">Wszystkie</button>
<button type="button" class="filter" data-filter="active" aria-pressed="false">W toku</button>
<button type="button" class="filter" data-filter="done" aria-pressed="false">Gotowe</button>
<button type="button" class="filter" data-filter="error" aria-pressed="false">Błędy</button>
</div>
<button id="refreshJobs" class="secondary icon-button" type="button" title="Odśwież kolejkę" aria-label="Odśwież kolejkę"><svg class="icon"><use href="#i-refresh"/></svg></button>
</div>
</div>
<div id="jobsList" class="jobs"><div class="empty-state">Brak zadań.</div></div>
</section>
<footer class="footer"><span>Wyniki: <code id="readyXexPath">/xbox/ready/xex/</code> lub <code id="readyGodPath">/xbox/ready/god/</code></span><span>Pobieranie: <code id="downloadHostPath">/xbox/downloads/</code> · Robocze: <code id="workHostPath">/xbox/work/</code></span></footer>
</main>
</div>
<div id="toast" role="status" aria-live="polite"></div>
<dialog id="magnetDialog" aria-labelledby="magnetHeading">
<div class="dialog-heading"><h2 id="magnetHeading">Pliki w torrencie</h2><button type="button" class="secondary" id="closeMagnet"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div>
<p id="magnetStatus" class="preview-status">Pobieranie metadanych torrenta…</p>
<div class="torrent-upload-inline"><label for="torrentFileDialog" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileDialog" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Nie wymaga peerów ani oczekiwania na DHT.</span></div>
<div id="magnetResults" class="hidden">
<div class="file-heading"><h3 id="magnetName"></h3><input id="magnetFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
<div class="file-table" id="magnetFiles"></div>
<div class="file-bottom"><span id="magnetSelectedCount">Wybrano: 0</span><div class="selection-controls"><button type="button" id="magnetToggleAll" class="secondary">Zaznacz widoczne</button><button type="button" id="magnetAdd" class="primary"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
<p class="note">Pobieranie rozpocznie się dopiero po wybraniu plików. Przy archiwach wieloczęściowych zaznacz wszystkie części.</p>
</div>
</dialog>
<dialog id="logDialog" aria-labelledby="logHeading"><div class="dialog-heading"><h2 id="logHeading">Log zadania</h2><button type="button" class="secondary" id="closeLog"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div><pre id="logContent"></pre></dialog>
</body>
</html>
+214
View File
@@ -0,0 +1,214 @@
:root {
color-scheme: dark;
font-family: system-ui, -apple-system, "Segoe UI", Arial, sans-serif;
font-size: 14px;
font-synthesis: none;
--bg: #121416;
--panel: #1b1e21;
--panel-hover: #21252a;
--input: #151719;
--line: #34383d;
--line-soft: #2a2e32;
--text: #e4e7e9;
--muted: #a2a9af;
--faint: #788189;
--accent: #91c99a;
--accent-hover: #afe0b5;
--success-bg: #203427;
--danger: #e8a09b;
--danger-bg: #362324;
}
*, *::before, *::after { box-sizing: border-box; }
body { margin: 0; background: var(--bg); color: var(--text); }
button, input, textarea, select { font: inherit; }
button { cursor: pointer; }
button:disabled { opacity: .45; cursor: not-allowed; }
button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
.hidden, .svg-defs { display: none !important; }
.icon { width: 16px; height: 16px; flex: 0 0 auto; fill: none; stroke: currentColor; stroke-width: 1.7; stroke-linecap: round; stroke-linejoin: round; }
.shell { max-width: 1120px; margin: auto; padding: 0 24px; }
.topbar { height: 66px; display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid var(--line); }
.brand { display: flex; align-items: center; gap: 11px; }
.brand-mark { display: grid; place-items: center; width: 33px; height: 33px; border: 1px solid #72977c; color: #a9d6ad; background: #1d2a21; border-radius: 5px; font-weight: 800; font-size: 19px; line-height: 1; }
.brand-text { display: flex; flex-direction: column; gap: 2px; }
.brand-text strong { font-size: 14px; font-weight: 700; letter-spacing: .1px; }
.brand-text span { color: var(--faint); font-size: 11px; }
.topbar-right { display: flex; align-items: center; gap: 17px; }
.connection { display: inline-flex; align-items: center; gap: 7px; font-size: 12px; color: var(--muted); }
.connection-dot { width: 7px; height: 7px; background: var(--faint); border-radius: 50%; }
.connection.online .connection-dot { background: var(--accent); }
.connection.offline .connection-dot { background: var(--danger); }
.version { border: 1px solid var(--line); color: var(--faint); font: 600 10px ui-monospace, SFMono-Regular, Consolas, monospace; padding: 4px 7px; letter-spacing: .5px; border-radius: 3px; }
.page-title { padding: 31px 0 22px; }
h1, h2, h3, p { margin-top: 0; }
h1 { margin-bottom: 5px; font-size: 25px; font-weight: 680; letter-spacing: -.65px; line-height: 1.3; }
.page-title p { margin-bottom: 0; color: var(--muted); font-size: 13px; }
.metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); border: 1px solid var(--line); border-radius: 5px; background: var(--panel); overflow: hidden; }
.metric { display: flex; flex-direction: column; gap: 9px; padding: 16px 19px; min-height: 112px; }
.metric + .metric { border-left: 1px solid var(--line); }
.metric-label { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 12px; }
.metric-label .icon { width: 14px; height: 14px; color: var(--faint); }
.metric strong { color: var(--text); font-weight: 650; font-size: 25px; font-variant-numeric: tabular-nums; line-height: 1; }
.disk-metric { gap: 7px; }
.disk-metric strong { font-size: 20px; }
.disk-metric small { font-size: 10px; color: var(--faint); }
.meter { height: 4px; background: #353a3e; overflow: hidden; width: 100%; }
.meter span { display: block; height: 100%; background: var(--accent); transition: width .2s linear; }
.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; margin-top: 26px; }
.panel-heading { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 18px 20px 15px; }
h2 { margin: 0; font-size: 16px; font-weight: 670; letter-spacing: -.2px; }
.muted-info { font-size: 11px; color: var(--faint); }
.tabs { display: flex; gap: 3px; padding: 0 15px; border-bottom: 1px solid var(--line); }
.tab { display: inline-flex; align-items: center; gap: 8px; background: transparent; border: none; border-bottom: 2px solid transparent; padding: 11px 13px 12px; color: var(--muted); font-size: 12px; font-weight: 550; }
.tab:hover { color: var(--text); background: #25292d; }
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
.tab .icon { width: 15px; height: 15px; }
.tab-body { padding: 21px 20px 20px; }
.field { min-width: 0; }
label { display: block; margin-bottom: 7px; font-size: 12px; color: #cbd0d4; font-weight: 600; }
label span { color: var(--faint); font-weight: 400; }
input, select, textarea { display: block; width: 100%; min-width: 0; color: var(--text); background: var(--input); border: 1px solid #42474c; border-radius: 4px; padding: 9px 11px; outline: none; font-size: 12px; }
input, select { height: 38px; }
textarea { min-height: 89px; resize: vertical; line-height: 1.6; }
input::placeholder, textarea::placeholder { color: #737c84; }
select { appearance: auto; }
.input-row { display: flex; gap: 12px; align-items: flex-end; margin-top: 13px; }
.input-row .title-field { flex: 1 1 auto; }
.input-row .format-field { flex: 0 0 172px; }
button.primary, button.secondary, button.action { border-radius: 4px; display: inline-flex; align-items: center; justify-content: center; gap: 8px; font-size: 12px; font-weight: 620; white-space: nowrap; min-height: 38px; padding: 9px 12px; }
button.primary { border: 1px solid #9ccc9e; color: #18231b; background: var(--accent); }
button.primary:hover:not(:disabled) { background: var(--accent-hover); }
button.secondary, button.action { border: 1px solid #444a50; background: #292d31; color: #d9dee1; }
button.secondary:hover:not(:disabled), button.action:hover:not(:disabled) { background: #34393f; }
button.action { min-height: 29px; padding: 5px 9px; font-size: 11px; }
button.action.danger { color: #e3a6a4; }
button.action.danger:hover { background: #3a2829; }
button.icon-button { min-width: 36px; width: 36px; padding: 8px; }
.note { color: var(--faint); font-size: 11px; line-height: 1.55; margin: 13px 0 0; }
.inline-input { display: flex; align-items: center; gap: 9px; }
.inline-input input { flex: 1 1 auto; }
.file-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 24px; }
.file-heading h3 { font-size: 13px; font-weight: 600; margin: 0; max-width: 62%; overflow-wrap: anywhere; }
.file-heading input { max-width: 250px; }
.file-table { max-height: 320px; overflow: auto; margin-top: 12px; border: 1px solid var(--line); border-radius: 4px; }
.file-row { display: flex; gap: 12px; align-items: center; min-height: 44px; padding: 9px 11px; margin: 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; color: #d2d7da; font-weight: 400; }
.file-row:last-child { border-bottom: 0; }
.file-row:hover { background: var(--panel-hover); }
.file-row input[type=checkbox] { margin: 0; min-height: 0; width: 15px; height: 15px; accent-color: var(--accent); flex-shrink: 0; }
.file-name { flex: 1; min-width: 0; overflow-wrap: anywhere; }
.size { color: var(--faint); font-size: 11px; white-space: nowrap; }
.file-bottom { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 13px; color: var(--muted); font-size: 12px; }
.selection-controls { display: flex; align-items: center; gap: 9px; }
.selection-controls select { width: 145px; }
.import-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 15px; }
.import-hint { font-size: 12px; color: var(--muted); margin: 0; }
code { color: #b7d1bd; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 11px; }
.queue-section { padding-top: 29px; }
.queue-header { display: flex; justify-content: space-between; align-items: center; gap: 14px; margin-bottom: 14px; }
.queue-title { display: flex; align-items: baseline; gap: 13px; }
.queue-title h2 { display: flex; align-items: center; gap: 9px; }
.count { font-size: 11px; color: #bdc4c9; background: #2c3034; border: 1px solid #3d4247; min-width: 21px; padding: 2px 6px; text-align: center; border-radius: 3px; font-variant-numeric: tabular-nums; }
.queue-controls { display: flex; gap: 9px; align-items: center; }
.filters { display: flex; border: 1px solid var(--line); border-radius: 4px; overflow: hidden; }
.filter { background: #1c1f22; color: var(--muted); border: 0; border-right: 1px solid var(--line); padding: 9px 13px; font-size: 11px; min-height: 36px; }
.filter:last-child { border-right: 0; }
.filter:hover { color: var(--text); background: #272b30; }
.filter.active { color: var(--accent); background: #29342c; font-weight: 650; }
.jobs { display: flex; flex-direction: column; gap: 8px; }
.job { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; padding: 14px 17px; }
.job-top { display: flex; justify-content: space-between; gap: 20px; align-items: flex-start; }
.job-heading { min-width: 0; flex: 1; }
.job-name { font-weight: 650; font-size: 13px; line-height: 1.45; overflow-wrap: anywhere; }
.job-sub { color: var(--faint); font-size: 11px; margin-top: 3px; overflow-wrap: anywhere; max-height: 2.8em; overflow: hidden; }
.job-status { flex-shrink: 0; display: inline-flex; align-items: center; gap: 6px; border: 1px solid #4a5156; color: #bbc4ca; background: #272c30; border-radius: 3px; font-size: 10px; line-height: 1; padding: 6px 8px; white-space: nowrap; }
.job-status::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: #9ca5ac; }
.job-status.running { background: #233126; color: #add7af; border-color: #39563e; }
.job-status.running::before { background: var(--accent); }
.job-status.done { background: var(--success-bg); color: #a8d9ac; border-color: #3b5942; }
.job-status.done::before { background: #8fcd95; }
.job-status.error { background: var(--danger-bg); color: #f1aaa3; border-color: #704446; }
.job-status.error::before { background: var(--danger); }
.job-progress { display: flex; align-items: center; gap: 12px; margin-top: 12px; }
.progressbar { width: 100%; height: 5px; background: #373d41; overflow: hidden; }
.progressbar span { display: block; height: 100%; background: #8aba94; transition: width .25s linear; }
.progressbar.is-error span { background: #c47e78; }
.progress-text { color: #c5cbd0; text-align: right; font-variant-numeric: tabular-nums; font-size: 11px; flex: 0 0 50px; }
.job-meta { display: flex; align-items: center; justify-content: space-between; gap: 20px; margin-top: 8px; font-size: 11px; color: var(--faint); }
.job-meta .note-text { overflow-wrap: anywhere; }
.job-meta .speed { flex-shrink: 0; font-variant-numeric: tabular-nums; }
.job-bottom { display: flex; justify-content: space-between; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 11px; }
.job-chips { display: inline-flex; gap: 7px; align-items: center; }
.job-chip { color: #aab3ba; font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; border: 1px solid #3c4449; padding: 3px 7px; border-radius: 3px; }
.job-actions { display: flex; flex-wrap: wrap; gap: 6px; margin-left: auto; }
.job-error { margin-top: 11px; padding: 9px 11px; border-left: 2px solid #c6857e; color: #ebaba6; background: #2b2122; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.5; }
.job-output { color: #accdb5; overflow-wrap: anywhere; margin-top: 9px; font-size: 11px; }
.empty-state { color: var(--faint); text-align: center; border: 1px dashed #42484e; background: var(--panel); padding: 34px 12px; border-radius: 4px; font-size: 12px; }
.footer { display: flex; justify-content: space-between; gap: 20px; padding: 23px 0 36px; color: var(--faint); font-size: 11px; }
#toast { display: none; position: fixed; right: 20px; bottom: 20px; max-width: min(420px, calc(100vw - 40px)); z-index: 20; padding: 13px 16px; border: 1px solid #6e9e75; border-radius: 4px; background: #24402a; color: #d3f1d6; font-size: 12px; line-height: 1.45; white-space: pre-line; }
#toast.bad { background: #48282a; color: #ffd7d3; border-color: #955b60; }
dialog { width: min(760px, calc(100vw - 30px)); max-height: 85vh; background: var(--panel); color: var(--text); border: 1px solid #545a60; border-radius: 5px; padding: 18px; }
dialog::backdrop { background: rgba(0,0,0,.75); }
.dialog-heading { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--line); }
.dialog-heading h2 { font-size: 15px; }
pre { overflow: auto; max-height: 66vh; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.65; color: #c8cdd0; }
@media (max-width: 760px) {
.shell { padding: 0 15px; }
.topbar { height: 59px; }
.version { display: none; }
.page-title { padding: 22px 0 16px; }
.metrics { grid-template-columns: repeat(2, minmax(0,1fr)); }
.metric:nth-child(3) { border-left: 0; border-top: 1px solid var(--line); }
.metric:nth-child(4) { border-top: 1px solid var(--line); }
.metric { min-height: 99px; padding: 15px; }
.panel { margin-top: 19px; }
.panel-heading { padding: 16px 15px 13px; }
.panel-heading .muted-info { display: none; }
.tabs { padding: 0 6px; }
.tab { padding: 11px 9px; font-size: 11px; gap: 6px; }
.tab-body { padding: 16px 15px; }
.input-row { flex-wrap: wrap; }
.input-row .title-field { flex: 1 1 100%; }
.input-row .format-field { flex: 1 1 110px; }
.input-row .submit-button { flex: 1 1 auto; }
.file-heading { align-items: stretch; flex-direction: column; }
.file-heading h3 { max-width: 100%; }
.file-heading input { max-width: none; }
.queue-header { align-items: stretch; flex-direction: column; gap: 13px; }
.queue-controls { justify-content: space-between; }
.queue-title { justify-content: space-between; }
.filter { padding: 9px; }
.job { padding: 13px; }
.footer { flex-direction: column; gap: 7px; }
}
@media (max-width: 440px) {
.inline-input, .import-toolbar { align-items: stretch; flex-direction: column; }
.selection-controls { width: 100%; }
.selection-controls select { flex: 1; min-width: 0; }
.file-bottom { align-items: stretch; flex-direction: column; }
.job-top { gap: 9px; }
.job-status { max-width: 130px; white-space: normal; }
.queue-title .muted-info { display: none; }
}
@media (prefers-reduced-motion: reduce) {
*, *::before, *::after { transition-duration: 0ms !important; animation-duration: 0ms !important; }
}
/* Archive login and torrent metadata selection: compact, flat controls. */
.archive-auth { border: 1px solid var(--line); padding: 13px; margin-bottom: 20px; background: var(--input); border-radius: 4px; }
.auth-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; font-size: 12px; }
.auth-heading span { color: var(--muted); overflow-wrap: anywhere; font-size: 11px; }
.auth-login { display: flex; align-items: end; gap: 10px; flex-wrap: wrap; margin-top: 13px; }
.auth-login .field { flex: 1 1 190px; }
.archive-auth .note { margin-top: 10px; }
#archiveLogout { margin-top: 12px; }
#magnetDialog { width: min(840px, calc(100vw - 24px)); }
.preview-status { color: var(--muted); font-size: 12px; margin: 14px 0; }
#magnetDialog .file-table { max-height: 45vh; }
#magnetDialog .file-row input:disabled { opacity: .3; }
@media(max-width: 540px) { .auth-login .field, .auth-login button { flex: 1 1 100%; } }
/* Flat torrent metadata fallback controls. */
.torrent-upload-inline {display:flex; align-items:center; flex-wrap:wrap; gap:10px; margin:10px 0; color:var(--muted, #8d939d); font-size:12px;}
.file-label {display:inline-flex; align-items:center; justify-content:center; cursor:pointer; border:1px solid #434a51; background:#292e33; color:#d7dce1; padding:7px 10px; border-radius:3px; font-size:12px; font-weight:600;}
.file-label:hover {background:#32383e;}
#torrentFileLinks, #torrentFileDialog {display:none !important;}
+95
View File
@@ -0,0 +1,95 @@
"""SQLite-backed durable job queue. Single-worker, multiple API connections."""
import sqlite3
import time
from pathlib import Path
DATA = Path('/data')
DB_PATH = DATA / 'queue.sqlite3'
FIELDS = {
'title', 'source_type', 'source', 'format', 'state', 'progress', 'speed',
'total', 'completed', 'part_index', 'gid', 'error', 'output', 'note', 'updated',
}
def connection():
DATA.mkdir(parents=True, exist_ok=True)
con = sqlite3.connect(DB_PATH, timeout=30)
con.row_factory = sqlite3.Row
con.execute('PRAGMA busy_timeout=30000')
con.execute('PRAGMA journal_mode=WAL')
return con
def init_db():
with connection() as con:
con.execute('''CREATE TABLE IF NOT EXISTS jobs (
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
source_type TEXT NOT NULL,
source TEXT NOT NULL,
format TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'queued',
progress REAL NOT NULL DEFAULT 0,
speed INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
completed INTEGER NOT NULL DEFAULT 0,
part_index INTEGER NOT NULL DEFAULT 0,
gid TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
output TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
created REAL NOT NULL,
updated REAL NOT NULL
)''')
con.execute('CREATE INDEX IF NOT EXISTS jobs_created_idx ON jobs(created)')
# Zadania przerwane w trakcie rozpakowania/konwersji moga byc ponowione.
con.execute("UPDATE jobs SET state='preparing', note='Wznowiono konwersje po restarcie' "
"WHERE state IN ('extracting','converting')")
con.execute("UPDATE jobs SET state='queued', note='Przywrocono zadanie po restarcie' "
"WHERE state='vimm_browser'")
def create(job):
now = time.time()
with connection() as con:
con.execute('''INSERT INTO jobs(id,title,source_type,source,format,state,created,updated)
VALUES(?,?,?,?,?,'queued',?,?)''',
(job['id'], job['title'], job['source_type'],
job['source'], job['format'], now, now))
def get(job_id):
with connection() as con:
row = con.execute('SELECT * FROM jobs WHERE id=?', (job_id,)).fetchone()
return dict(row) if row else None
def list_jobs():
with connection() as con:
rows = con.execute('SELECT * FROM jobs ORDER BY created DESC LIMIT 300').fetchall()
return [dict(row) for row in rows]
def next_job():
with connection() as con:
row = con.execute('''SELECT * FROM jobs
WHERE state IN ('queued','downloading','preparing')
ORDER BY created ASC LIMIT 1''').fetchone()
return dict(row) if row else None
def update(job_id, **values):
if not values:
return
if not set(values).issubset(FIELDS):
raise ValueError('Niepoprawne pola aktualizacji bazy')
values['updated'] = time.time()
with connection() as con:
con.execute('UPDATE jobs SET ' + ','.join(f'{k}=?' for k in values)
+ ' WHERE id=?', (*values.values(), job_id))
def erase(job_id):
with connection() as con:
con.execute('DELETE FROM jobs WHERE id=?', (job_id,))
+181
View File
@@ -0,0 +1,181 @@
"""Safe, small bencode parser and ephemeral magnet metadata inspection state."""
import json
import os
import re
import shutil
import time
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from . import store
ID = re.compile(r'^[a-f0-9]{32}$')
MAX_METADATA = 32 * 1024 * 1024
class MetadataError(ValueError):
pass
def decode_bencode(raw):
if len(raw) > MAX_METADATA:
raise MetadataError('Metadane torrenta są zbyt duże')
position = 0
count = 0
def decode(depth=0):
nonlocal position, count
if depth > 24:
raise MetadataError('Metadane mają zbyt duże zagnieżdżenie')
count += 1
if count > 600000 or position >= len(raw):
raise MetadataError('Uszkodzone metadane torrenta')
start = raw[position]
if start == 105: # i<number>e
end = raw.find(b'e', position + 1)
if end < 0 or end - position > 25:
raise MetadataError('Błędna liczba bencode')
try:
number = int(raw[position + 1:end])
except ValueError as exc:
raise MetadataError('Błędna liczba bencode') from exc
position = end + 1
return number
if start in (108, 100): # l, d
position += 1
output = [] if start == 108 else {}
while position < len(raw) and raw[position] != 101:
if start == 108:
output.append(decode(depth + 1))
else:
key = decode(depth + 1)
if not isinstance(key, bytes):
raise MetadataError('Niepoprawny klucz bencode')
output[key] = decode(depth + 1)
if position >= len(raw):
raise MetadataError('Nieukończone metadane')
position += 1
return output
if 48 <= start <= 57: # <len>:<bytes>
colon = raw.find(b':', position, min(len(raw), position + 22))
if colon < 0:
raise MetadataError('Błędny ciąg bencode')
try:
length = int(raw[position:colon])
except ValueError as exc:
raise MetadataError('Błędna długość bencode') from exc
position = colon + 1
if length < 0 or length > len(raw) - position:
raise MetadataError('Niepoprawna długość pola')
result = raw[position:position + length]
position += length
return result
raise MetadataError('Uszkodzone metadane torrenta')
result = decode()
if position != len(raw) or not isinstance(result, dict):
raise MetadataError('Niepoprawny format torrenta')
return result
def _segment(raw):
if not isinstance(raw, bytes) or not raw or len(raw) > 255 or raw in (b'.', b'..'):
raise MetadataError('Niedozwolona nazwa w torrencie')
if any(char in raw for char in (b'/', b'\\', b'\0')):
raise MetadataError('Niebezpieczna ścieżka w torrencie')
return raw.decode('utf-8', errors='replace')
def read_torrent(path):
try:
data = decode_bencode(Path(path).read_bytes())
info = data[b'info']
name = _segment(info.get(b'name.utf-8') or info[b'name'])
result = []
if b'files' in info:
files = info[b'files']
if not isinstance(files, list) or len(files) > 50000:
raise MetadataError('Nieprawidłowa lista plików torrenta')
for index, file in enumerate(files, start=1):
parts = file.get(b'path.utf-8') or file[b'path']
if not isinstance(parts, list) or not parts:
raise MetadataError('Błędna ścieżka pliku')
length = file[b'length']
if not isinstance(length, int) or length < 0:
raise MetadataError('Nieprawidłowy rozmiar pliku')
result.append({'index': index, 'name': '/'.join([name] + [_segment(x) for x in parts]),
'size': length, 'padding': b'p' in file.get(b'attr', b'')})
else:
length = info[b'length']
if not isinstance(length, int) or length < 0:
raise MetadataError('Nieprawidłowy rozmiar pliku')
result.append({'index': 1, 'name': name, 'size': length, 'padding': False})
if not result:
raise MetadataError('Torrent nie zawiera plików')
return {'name': name, 'files': result}
except (KeyError, TypeError, OSError, ValueError) as exc:
if isinstance(exc, MetadataError):
raise
raise MetadataError('Nie można odczytać plików z torrenta') from exc
def suggested_indices(magnet, max_index):
"""magnet so indexes are zero-based; aria2 select-file indexes are one-based."""
so = parse_qs(urlsplit(magnet).query).get('so', [''])[0]
suggested = set()
for chunk in so.split(',')[:1000]:
try:
if '-' in chunk:
a, b = (int(x) for x in chunk.split('-', 1))
if b < a or b - a > 1000:
continue
values = range(a, b + 1)
else:
values = [int(chunk)]
suggested.update(n + 1 for n in values if 0 <= n < max_index)
except ValueError:
continue
return sorted(suggested)
def inspect_dir(preview_id):
if not ID.fullmatch(preview_id):
raise MetadataError('Nieprawidłowy identyfikator metadanych')
return store.DATA / 'metadata' / preview_id
def write_request(preview_id, data):
path = inspect_dir(preview_id)
path.mkdir(mode=0o700, parents=True, exist_ok=True)
filename = path / 'request.json'
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
with os.fdopen(os.open(filename, flags, 0o600), 'w', encoding='utf-8') as f:
json.dump({'created': time.time(), **data}, f)
def load_request(preview_id):
try:
info = json.loads((inspect_dir(preview_id) / 'request.json').read_text(encoding='utf-8'))
if not isinstance(info['gid'], str) or not isinstance(info['magnet'], str):
raise MetadataError('Błędne dane podglądu')
return info
except FileNotFoundError as exc:
raise MetadataError('Nie znaleziono podglądu magneta') from exc
except (ValueError, KeyError, TypeError) as exc:
raise MetadataError('Uszkodzone dane podglądu') from exc
def torrent_file(preview_id):
matches = list(inspect_dir(preview_id).glob('*.torrent'))
return matches[0] if len(matches) == 1 and matches[0].stat().st_size <= MAX_METADATA else None
def cleanup_old(max_age=86400):
root = store.DATA / 'metadata'
if not root.exists():
return
now = time.time()
for child in root.iterdir():
if (ID.fullmatch(child.name) and child.is_dir() and not child.is_symlink()
and now - child.stat().st_mtime > max_age):
shutil.rmtree(child)
+76
View File
@@ -0,0 +1,76 @@
"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
import os
import re
import asyncio
from pathlib import Path
from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser
import httpx
from . import store
async def browser_download(job):
if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
url = job['source']
u = urlparse(url)
if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
try:
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
r = await c.get('https://vimm.net/robots.txt')
if r.status_code == 200:
parser = RobotFileParser()
parser.parse(r.text.splitlines())
if not parser.can_fetch('x360-web-queue', url):
raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
elif r.status_code not in (404, 410):
raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
except httpx.HTTPError as exc:
raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
out = store.DATA / 'raw' / job['id']
out.mkdir(parents=True, exist_ok=True)
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True, executable_path='/usr/bin/chromium',
args=['--no-sandbox', '--disable-dev-shm-usage'])
try:
context = await browser.new_context(accept_downloads=True)
page = await context.new_page()
await page.goto(url, wait_until='domcontentloaded', timeout=60000)
# Only the original, visible button is actionable. Missing/taken-down
# buttons and challenges are not bypassed.
controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
usable = None
for i in range(min(await controls.count(), 10)):
c = controls.nth(i)
if await c.is_visible() and await c.is_enabled():
usable = c
break
if usable is None:
raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
try:
async with page.expect_download(timeout=timeout) as d:
await usable.click(timeout=15000)
download = await d.value
except PlaywrightTimeout as exc:
raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
if not name:
name = 'vimm-download.bin'
store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
failure = await download.failure()
if failure:
raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
finally:
await browser.close()