first commit

This commit is contained in:
Mateusz Gruszczyński committed 2026-10-15 14:15:10 +02:00
commit 58cfd95600
35 files changed
+5357

No files matched your search

+6
View File
@@ -0,0 +1,6 @@
.env
data/
__pycache__/
.pytest_cache/
*.pyc
.git/
+9
View File
@@ -0,0 +1,9 @@
WEB_USER=admin
WEB_PASSWORD=CHANGE_ME_TO_A_LONG_RANDOM_PASSWORD
ARIA2_SECRET=CHANGE_ME_TO_ANOTHER_LONG_RANDOM_TOKEN
WEB_PORT=8088
# Domyslnie panel dostepny w LAN. Dla localhost ustaw 127.0.0.1
WEB_BIND=0.0.0.0
# Automatyczna obsluga Vimm przez prawdziwa widoczna kontrolke pobierania
ENABLE_VIMM_BROWSER=true
VIMM_DOWNLOAD_TIMEOUT=7200
+4
View File
@@ -0,0 +1,4 @@
.env
.venv
venv
+29
View File
@@ -0,0 +1,29 @@
# Natywne linuksowe narzedzia do obslugi obrazow konsoli.
FROM rust:1.88-bookworm AS converters
ENV CARGO_BUILD_JOBS=2
RUN git clone --depth 1 --branch v1.8.1 https://github.com/iliazeus/iso2god-rs.git /src/iso2god \
&& cd /src/iso2god && cargo build --release --locked --bin iso2god
RUN git clone --depth 1 --branch v0.8.3 https://github.com/antangelo/xdvdfs.git /src/xdvdfs \
&& cd /src/xdvdfs && cargo build --release --locked -p xdvdfs-cli
FROM golang:1.24-bookworm AS bt-helper
WORKDIR /src
COPY bt/ /src/
RUN GOMAXPROCS=2 go mod tidy && GOMAXPROCS=2 CGO_ENABLED=0 go build -p 2 -trimpath -o /out/x360-bt .
FROM python:3.12-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends aria2 p7zip-full unar chromium ca-certificates \
&& rm -rf /var/lib/apt/lists/*
COPY --from=bt-helper /out/x360-bt /usr/local/bin/x360-bt
COPY --from=converters /src/iso2god/target/release/iso2god /usr/local/bin/iso2god
COPY --from=converters /src/xdvdfs/target/release/xdvdfs /usr/local/bin/xdvdfs
WORKDIR /opt/x360
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ app/
COPY entrypoint.sh ./
RUN chmod +x entrypoint.sh && mkdir -p /data
EXPOSE 8000
CMD ["/opt/x360/entrypoint.sh"]
+202
View File
@@ -0,0 +1,202 @@
# X360 Web Queue — Linux / Docker / Xbox 360 RGH
Samodzielna aplikacja WWW do pobierania i przygotowywania posiadanych legalnie obrazow gier Xbox 360. Wszystkie zadania wykonywane sa na serwerze/VM, niezaleznie od otwartej karty przegladarki.
## Start
Wymagania: **Linux x86_64**, Docker Engine z wtyczka Docker Compose, minimum ok. 2 GB RAM (zalecane 4 GB podczas budowania), duzo miejsca na dane.
```bash
unzip x360-web-queue-xbox.zip
cd x360-web-queue
# /xbox musi istniec i byc zapisywalny na hoscie Linux
./setup.sh
```
`setup.sh` przygotuje katalogi w `/xbox`, sam utworzy `.env` z mocnymi haslami i wykona `docker compose up -d --build`. Przy pierwszym starcie budowane sa `xdvdfs`, `iso2god-rs` i pomocniczy klient BitTorrent Go. Wirtualka potrzebuje dostepu do Docker Hub, GitHub, crates.io i proxy.golang.org. Na gospodarzu nie musisz instalowac Rusta ani Go.
Otworz `http://IP_TWOJEJ_WIRTUALKI:8088` i zaloguj sie jako `admin` za pomoca hasla wypisanego przez `setup.sh` (takze w `.env`).
Jesli Docker nie byl jeszcze zainstalowany, zainstaluj go, a pozniej:
```bash
./setup.sh
# lub, po utworzeniu .env:
docker compose up -d --build
```
Aktualny stan i logi:
```bash
docker compose ps
docker compose logs -f --tail=100
```
## Interfejs
Plaski panel w ciemnych barwach, lokalne ikony SVG (bez CDN), filtrowanie kolejki i widok mobilny. Interfejs nie wymaga dodatkowych zaleznosci JS i zachowuje API dotychczasowej aplikacji.
## Zrodla
- **HTTPS/HTTP**: wklej bezposredni adres pliku ISO/ZIP/7z/RAR, rowniez z serwisow umozliwiajacych normalny bezposredni transfer.
- **Magnet**: panel automatycznie usuwa z wklejonego adresu HTML `&`, przypadkowe znaki `\` i lamanie linii, sprawdza hash, a nastepnie pobiera wylacznie metadane torrenta. Metadane pobiera klient Go (`anacrolix/torrent`, taki sam stos jak `minerva-pick`), bez pobierania plikow gry. Okno pokazuje czas, liczbe polaczen i znanych peerow. Po 5 minutach bez metadanych pojawia sie blad zamiast bezterminowego oczekiwania. Parametr `so=` ustawia wstepne zaznaczenie (od 0). Transfer danych rusza dopiero po wyborze plikow.
- **Plik .torrent**: opcja `Wczytaj .torrent` w zakladce Link / magnet oraz w oknie podgladu. Pozwala natychmiast uzyskac liste plikow bez czekania na peerow. Maks. 32 MiB metadanych. Wczytanie .torrent *nie rozpoczyna* pobierania zawartosci, dopiero przycisk `Pobierz wybrane`. Dziala z duzymi katalogami Minerva (widok pokazuje maks. 500 plikow naraz, filtr pozwala wyszukac reszte).
- **Internet Archive**: zakladka umozliwia zalogowanie e-mailem i haslem Archive.org (mechanizm xauthn stosowany w oficjalnych narzedziach), nastepnie wklej `https://archive.org/details/IDENTYFIKATOR` i wybierz pliki. Aplikacja przechowuje **tylko ciasteczka sesji**, a nie haslo. Pliki sa pobierane bezposrednio przez backend z sesja Archive.org, bez starego przekaznika aria2. **Wszystkie pliki jednego zadania sa pobierane rownolegle, maksymalnie 50 jednoczesnie**. Duze pliki sa dodatkowo dzielone na segmenty HTTP Range. Limit `ARCHIVE_CONNECTIONS=50` dotyczy **sumy polaczen HTTP wszystkich plikow zadania**, nie kazdego pliku osobno. Postep segmentow jest zachowywany po restarcie; pobieranie wznawia tylko brakujace bajty. Gdy serwer nie obsluguje Range albo zwraca niezgodne zakresy, aplikacja automatycznie przechodzi na jedno polaczenie. Dzialaja ponowienia HTTP 500/502/503/504/429 i kontrola wersji pliku za pomoca ETag / Last-Modified, jesli serwer je podaje. Ciasteczka sesji nie trafiaja do aria2 ani do logow. Logowanie nie zapewnia dostepu do plikow niedostepnych dla Twojego konta. Nie obchodzimy ograniczen serwisu.
- **Vimm**: adres `https://vimm.net/vault/NUMER` przechodzi do **eksperymentalnego** adaptera uruchamiajacego Chromium. Wykorzystywany jest tylko faktycznie widoczny i dostepny przycisk `Download`. Brak omijania CAPTCHA, DRM, niedostepnych gier, antybota, limitow, kolejek i robots.txt. Gdy Vimm zmieni strone lub odrzuci automatyczny dostep, pobieranie nie bedzie dzialalo. Postep pobierania w trybie Vimm nie jest dostepny w % az do konca. Vimm uzywa jednej sesji naraz. Mozesz wylaczyc adapter w `.env`: `ENABLE_VIMM_BROWSER=false`.
- **Import lokalny**: skopiuj plik do `/xbox/imports/`, odswiez karte Import i dodaj zadanie.
Nie obslugujemy bypassowania zabezpieczen ani pobierania tresci wycofanych z serwisu. Uzywaj tylko plikow, do ktorych masz prawo.
## Przetwarzanie
| Wejscie | Format XEX | Format GOD |
|---|---|---|
| `.iso` Xbox 360 | `xdvdfs unpack` | `iso2god` |
| `.zip`, `.7z`, `.rar`, `.tar` zawierajace ISO | Rozpakuj, nastepnie XEX | Rozpakuj, nastepnie GOD |
| Folder z `default.xex` spakowany w ZIP/7z | Kopia folderu | `xdvdfs pack`, potem `iso2god` |
| Link magnet zawierajacy ISO | j.w. | j.w. |
Archiwa wieloczesciowe: wybierz wszystkie czesci w **jednym** zadaniu (zakladka Archive.org), a nie osobno. Obsługiwane sa m.in. `.part1.rar`, `.7z.001`. Nie kazdy wariant kompresji RAR dziala w p7zip — wtedy zadanie zakonczy sie czytelnym bledem. Hasla do zaszyfrowanych archiwow nie sa obslugiwane.
Dla XEX program sprawdza obecnosc `default.xex`. Dla GOD konwerter generuje katalogi TitleID/content type i pliki Data. Na dysku masz:
```text
/xbox/ # katalog na hoscie VM
downloads/<JOB_ID>/ # pobrane ISO, archiwa i torrenty (kontener: /data/raw)
downloads/<JOB_ID>/*.part # nieukonczone transfery HTTP
downloads/<JOB_ID>/*.part.segments/ # segmenty Archive.org do wznowienia
work/<JOB_ID>/ # rozpakowywanie i ISO tymczasowe (kontener: /data/work)
work/tmp/ # systemowe pliki tymczasowe (TMPDIR)
work/cache/ # cache Chromium/narzedzi (XDG_CACHE_HOME)
ready/xex/<GRA>/ # wynik XEX
ready/god/<GRA>/ # wynik GOD
imports/ # reczny import ISO lub archiwum
torrents/<JOB_ID>.torrent # metadane do wznowienia
metadata/<PODGLAD_ID>/ # metadane magneta i engine.log
private/archive-session.json # sesja Archive.org (0600)
logs/<JOB_ID>.log # log konwersji
logs/<JOB_ID>.bt.log # log BT
logs/<JOB_ID>.download.log # log transferu Archive
bt/<JOB_ID>.json # status klienta Go
aria2/ # stan i sesje aria2
queue.sqlite3 # kolejka i historia zadan
```
Na Xbox przez FTP skopiuj **zawartosc** folderu `Gra` do `Hdd1:/Games/Nazwa/` i uruchom `default.xex` (albo wykonaj skan biblioteki Aurora). Dla GOD skopiuj `Content/0000000000000000/...` do `Hdd1:/Content/0000000000000000/...` z zachowaniem struktury. Gry wielodyskowe moga wymagac osobnej konfiguracji w Aurora.
## Przyspieszenie Internet Archive (HTTP Range)
Nowe instalacje maja `ARCHIVE_CONNECTIONS=50` w `.env`. Przy aktualizacji istniejacego projektu dopisz lub zmien te ustawienie w `.env`, np.:
```dotenv
ARCHIVE_CONNECTIONS=50
```
Dozwolone jest **1-50**. `1` wylacza segmentowanie i ogranicza pobieranie do jednego pliku na raz. W zadaniu z 3 plikami przy limicie 50 przydzial wyniesie np. **17 + 17 + 16 polaczen**. Kazdy plik zaczyna sie pobierac od razu; kolejne pliki (pow. 50) uruchamiaja sie w zwolnionych slotach. Przydzial na plik jest ustalany na poczatku danej grupy plikow; gdy plik sie skonczy, pozostale zachowuja swoje segmenty i przydzielone sloty, a wolny slot moze rozpoczac nastepny plik. Mniejsze pliki oraz wczesniej podzielone na 8 segmentow moga wykorzystac mniej polaczen. Jesli Archive.org odrzuca zadania (np. HTTP 429) albo transfer zwalnia, ogranicz liczbe np. do 8, 16 lub 24. Po zmianie wykonaj `docker compose up -d --build`. Panel pokazuje liczbe plikow aktywnych, zbiorczy postep i sumaryczna predkosc, a w `/xbox/logs/<JOB_ID>.download.log` zapisany jest przydzial polaczen i bledy kazdego pliku.
Segmenty uruchamiaja sie tylko dla plikow dostatecznie duzych (co najmniej dwa zakresy po 8 MiB; 50 segmentow wymaga minimum 400 MiB pozostalych do pobrania), po potwierdzeniu obslugi `HTTP 206 Content-Range` przez serwer. Program nie omija ograniczen konta ani zabezpieczen serwisu. Gdy Archive.org ogranicza polaczenia (np. `HTTP 429`) aplikacja uwzglednia `Retry-After` w sekundach; obniz `ARCHIVE_CONNECTIONS`, jezeli problem sie powtarza.
Zadania, ktore maja juz zapisane segmenty z poprzedniej wersji, zachowuja pierwotna liczbe segmentow, aby mozna je bylo bezpiecznie wznowic. Po aktualizacji rownolegle rusza takze pobieranie pozostalych plikow z tego samego zadania. Nie trzeba kasowac pobranych czesci. Aby zastosowac nowy podzial 50 do juz podzielonego pliku, trzeba utworzyc nowe zadanie. Aktualizacja nie nadpisuje istniejacego `.env`: po podmianie projektu ustaw tam `ARCHIVE_CONNECTIONS=50` i zrestartuj kontener.
**Dysk:** przed konwersja potrzebne jest miejsce na segmenty i kolejna pelna kopie skladanego pliku. Sprawdz wolne miejsce w VM, zwlaszcza przy obrazach ISO po kilka GB.
## Eksploatacja
Aplikacja przechowuje kolejke SQLite, ma wznowienia HTTP i BitTorrent, jednego workera kolejki, do trzech podgladow magnet i przyciski stop/wznow/ponow. Wszystkie dane sa na hoscie w `/xbox` (domyslnie, bez wolumenow anonimowych), wiec pozostaja po restarcie kontenera. Zmiana lokalizacji jest mozliwa przez `XBOX_HOST_DIR` w `.env`. aria2 obsluguje zwykle linki HTTP/HTTPS, natomiast Go obsluguje magnety i pliki torrent. Czesciowo pobrane bloki torrenta sa weryfikowane po restarcie; Archive.org wznawia przez `.part` oraz katalog `.part.segments/` z oddzielnymi zakresami. Podczas koncowego laczenia czesci moze byc potrzebne dodatkowe miejsce na **jedna pelna kopie pliku**. Po poprawnym scaleniu czesci tymczasowe sa usuwane.
**Bezpieczenstwo:** panel uzywa HTTP Basic Auth (haslo przesylane przy kazdym zadaniu HTTP). Formularz Archive.org rowniez przesyla haslo do lokalnego serwera. Zabezpiecz panel przed logowaniem poprzez HTTPS lub VPN; w lokalnym kontenerze haslo IA jest tylko przekazywane do `https://archive.org/services/xauthn/?op=login` i nie jest zapisywane. Sesja jest przechowywana w `/xbox/private` z uprawnieniami 0700/0600. Nie umieszczaj `/xbox/private` w publicznych kopiach projektu. Nie wystawiaj `8088` bezposrednio do internetu. Do dostepu z zewnatrz uzyj VPN (np. WireGuard/Tailscale) albo reverse proxy z TLS. Domyslnie port nasluchuje na LAN (`WEB_BIND=0.0.0.0`); aby ograniczyc dostep do maszyny lokalnej, ustaw `WEB_BIND=127.0.0.1` w `.env` i wykonaj `docker compose up -d`.
## Ograniczenia obecnej wersji
- Brak bezposredniego FTP do konsoli i automatycznego rozpoznawania wszystkich wielodyskowych instalacji.
- Brak wyszukiwarki gier — podajesz wlasne URL/magnet/item Archive.
- Nie kazde konto Archive.org ma dostep do kazdego elementu. Nie ma gwarancji logowania, gdy serwis zmieni xauthn lub wymaga dodatkowej weryfikacji. Duzy transfer z Archive.org jest wielopolaczeniowy, jesli serwer akceptuje Range. Wieksza liczba polaczen nie gwarantuje wzrostu predkosci; zalezy to od serwera, trasy sieciowej i limitow konta. Jesli pojawiaja sie HTTP 429 albo 503, zmniejsz liczbe polaczen. Powtarzajacy sie HTTP 500 po 5 probach nadal oznacza blad po stronie serwera lub dostepu do pliku.
- Odczyt metadanych magneta wymaga dostepnych peerow. Jezeli liczba polaczen i znanych peerow pozostaje 0, sprawdz DHT/UDP lub uzyj pliku .torrent. Plik .torrent eliminuje czekanie na metadane, ale nie gwarantuje dostepnosci zawartosci. Brak dostepnych peerow oznacza, ze zadne ustawienie interfejsu nie gwarantuje pobrania metadanych. Limit podgladu: 5 min.
- W Docker Compose sa mapowane porty BitTorrent 6881/tcp i 6881/udp. Jezeli inna aplikacja uzywa 6881 na hoście, ustaw `BT_PORT=16881` w `.env` i przebuduj kontener. Przy NAT wirtualki moze byc potrzebne przekierowanie portow w hypervisorze/routerze. DHT wymaga dzialajacego ruchu UDP.
- Brak gwarancji dla stron wymagajacych JS/autoryzacji poza obslugiwanymi integracjami; sam aria2 nie imituje przegladarki.
- Uwaga na miejsce: w najgorszym przypadku podczas rozpakowania i konwersji potrzeba **2–4x** rozmiaru ISO.
- Nie testowano pobierania rzeczywistych plikow z Archive.org na prywatnym koncie, odzyskiwania wskazanego magneta z DHT ani konwersji realnego ISO w tym srodowisku. Testy API, sesji, HTTP Range, ponawiania HTTP 500, nadzorcy klienta Go, parsera torrenta i kolejki mozna uruchomic `PYTHONPATH=. pytest -q` z zaleznosciami z requirements i pytest. Nie przetestowano obrazu Dockera ani transferu na zywej sieci; wymagaja testu na VM.
## Licencje i projekty
Backend projektu: kod dostarczony tutaj. Zaleznosci i narzedzia sa instalowane niezaleznie na zasadach ich licencji: [aria2](https://aria2.github.io/), [anacrolix/torrent](https://github.com/anacrolix/torrent), [xdvdfs](https://github.com/antangelo/xdvdfs), [iso2god-rs](https://github.com/iliazeus/iso2god-rs), [Playwright](https://playwright.dev/), [FastAPI](https://fastapi.tiangolo.com/).
## Diagnostyka: magnet / torrent / Archive.org
Po aktualizacji najpierw uruchom nowe zadanie `.torrent` lub magnet. Stare zadania moga miec zapisane `gid` aria2; nowszy backend przy pierwszym uruchomieniu zatrzymuje ich transfery. W razie bledu nacisnij **Log** w panelu, a dodatkowo sprawdz:
```bash
docker compose ps
docker compose logs --tail=120 x360-web
# podstaw odpowiedni ID zadania z panelu lub katalogu /xbox/logs:
ls -lt /xbox/logs/*.bt.log /xbox/logs/*.download.log 2>/dev/null | head
# dla metadanych magnet: /xbox/metadata/<ID>/engine.log
```
Oczekiwanie na metadane z 0 peerow oznacza brak osiagalnego zrodla w danym momencie lub problem z siecia BT w VM. Wczytanie `.torrent` omija wyłącznie etap metadanych, a nie potrzebe aktywnych peerow. Go używa TCP/UDP; sprawdz DNS i wyjscie UDP w wirtualce. Przy NAT czasem przydaje sie przekierowanie `BT_PORT` przez hypervisor i router.
Archive.org **nie uzywa juz aria2** do plikow z `/download/`. Pliki jednego zadania ida rownolegle z lacznym limitem 1-50 polaczen HTTP (domyslnie 50), a HTTP 500 jest ponawiany maksymalnie 5 razy na probe/segment. Jesli nadal dostaniesz HTTP 500, oznacza to odpowiedz Archive.org lub posredniego serwera, nie samego panelu. Log pokazuje kazda probe, bez sesyjnych ciasteczek.
## Dane na dysku hosta: `/xbox`
Compose montuje fizyczny katalog `/xbox` do `/data` w kontenerze. Dodatkowo
`/xbox/downloads` jest montowany do `/data/raw`, dzieki czemu wszystkie stare
sciezki w kolejce i sesjach aria2 pozostaja zgodne. Rowniez rozpakowywanie,
konwersja i katalog tymczasowy (`TMPDIR`) korzystaja z tego samego katalogu
hosta, a wynik trafia do `/xbox/ready/`. Nic z pobranych danych nie zalezy
od dysku zapisywalnego kontenera.
**Przed uruchomieniem musi istniec `/xbox`**. Jesli to podmontowany dysk
lub udzial hosta, najpierw upewnij sie, ze jest dostepny i zapisywalny.
Compose uzywa `bind.create_host_path: false`, zeby nie tworzyc po cichu
brakujacej sciezki. **Jesli `/xbox` istnieje jako zwykly katalog pod
niezamontowanym dyskiem, Compose nie rozpozna tego bledu** — w takim
wypadku sprawdz `mountpoint /xbox` przed uruchomieniem.
Przy nowej instalacji (jezeli katalog nie istnieje, utworz go najpierw):
```bash
sudo mkdir -p /xbox
# Tylko dla nowego katalogu na lokalnym dysku (nie zmieniaj wlasciciela udzialu SMB/NFS):
sudo chown "$(id -un):$(id -gn)" /xbox
# Dla udzialu/montowanego dysku zapewnij uprawnienia zapisu innym sposobem.
cd x360-web-queue
./setup.sh
```
Jesli wolisz inna sciezke, dopisz do `.env` `XBOX_HOST_DIR=/sciezka/bezwzgledna`.
Program nie zapisuje wtedy nic do `/xbox`, tylko do wybranej sciezki.
Zmiana katalogu **nie przenosi** danych automatycznie.
### Aktualizacja istniejacej instalacji (migracja `./data`)
Zachowaj `.env` i `./data` przed wypakowaniem tej wersji. Rozpakuj ZIP w
katalogu nadrzednym starego `x360-web-queue/` i wykonaj:
```bash
cd x360-web-queue
# Upewnij sie, ze /xbox jest zamontowany i ma miejsce na dotychczasowe dane.
./migrate-to-xbox.sh
docker compose up -d --build
```
Skrypt zatrzymuje stary kontener **przed kopiowaniem**, przenosi przez kopiowanie
katalog `./data/raw/` do `/xbox/downloads/` oraz reszte `./data/` do `/xbox/`.
**Nie usuwa starego `./data` ani nie nadpisuje istniejacej bazy / plikow**.
Przerwana migracje mozna ponowic, o ile nie zostala uruchomiona nowa instancja.
Jezeli brakuje `rsync`, zainstaluj go (`sudo apt install rsync`).
Po sprawdzeniu dzialania i kompletności pobranych plikow mozesz samodzielnie
usunac stary `./data`, zwalniajac miejsce.
Weryfikacja:
```bash
docker compose ps
docker compose exec x360-web sh -c 'df -h /data/raw /data/work /data/ready; ls -ld /data/raw /data/work /data/ready'
ls -lah /xbox/downloads /xbox/work /xbox/ready
```
**Uwaga:** dla SQLite i wznawiania pobierania najlepiej uzywac lokalnego
systemu plikow z poprawnym blokowaniem (np. ext4 lub XFS). Przy montowaniu
SMB/NFS sprawdz blokowanie plikow, uprawnienia i stabilnosc udzialu.
View File
Whitespace-only changes.
+204
View File
@@ -0,0 +1,204 @@
"""Internet Archive user session and restricted download relay.
The relay keeps IA cookies inside this process. aria2 only receives a signed,
local endpoint, and never sees an IA session cookie or the account password.
"""
import hashlib
import hmac
import json
import os
import secrets
import tempfile
from pathlib import Path
from urllib.parse import urljoin, urlsplit
import httpx
from fastapi.responses import PlainTextResponse, Response, StreamingResponse
from . import store
class ArchiveAuthError(ValueError):
pass
def session_path():
return store.DATA / 'private' / 'archive-session.json'
def _read_session():
try:
value = json.loads(session_path().read_text(encoding='utf-8'))
cookies = value['cookies']
if not all(isinstance(cookies.get(k), str) and cookies[k] for k in ('logged-in-user', 'logged-in-sig')):
return None
return value
except (FileNotFoundError, KeyError, TypeError, ValueError, OSError):
return None
def session_status():
value = _read_session()
return {'authenticated': bool(value), 'email': value.get('email', '') if value else ''}
def session_cookies():
cookies = httpx.Cookies()
value = _read_session()
if value:
for key in ('logged-in-user', 'logged-in-sig'):
cookies.set(key, value['cookies'][key], domain='.archive.org', path='/')
return cookies
def _cookie_value(value):
# The official xauthn API includes cookie attributes after the value.
if not isinstance(value, str):
raise ArchiveAuthError('Brak danych sesji w odpowiedzi Archive.org')
value = value.split(';', 1)[0].strip()
if not value or len(value) > 4096 or any(x in value for x in ('\n', '\r', '\0')):
raise ArchiveAuthError('Niepoprawne dane sesji Archive.org')
return value
def _save_session(email, cookies):
path = session_path()
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
os.chmod(path.parent, 0o700)
fd, name = tempfile.mkstemp(dir=path.parent, prefix='.archive-session-', suffix='.tmp')
try:
with os.fdopen(fd, 'w', encoding='utf-8') as f:
json.dump({'email': email, 'cookies': cookies}, f)
f.flush()
os.fsync(f.fileno())
os.chmod(name, 0o600)
os.replace(name, path)
finally:
if os.path.exists(name):
os.unlink(name)
async def login(email, password):
"""Authenticate using IA's documented xauthn endpoint (no password stored)."""
try:
async with httpx.AsyncClient(timeout=25, follow_redirects=False) as client:
result = await client.post('https://archive.org/services/xauthn/', params={'op': 'login'},
data={'email': email, 'password': password})
result.raise_for_status()
data = result.json()
except (httpx.HTTPError, ValueError) as exc:
raise ArchiveAuthError('Błąd połączenia z logowaniem Archive.org') from exc
if not data.get('success'):
reason = str((data.get('values') or {}).get('reason') or data.get('error') or '')
if reason in ('account_bad_password', 'account_not_found'):
raise ArchiveAuthError('Nieprawidłowy adres e-mail lub hasło')
raise ArchiveAuthError('Archive.org odrzuciło logowanie: ' + (reason[:120] or 'brak szczegółów'))
try:
raw = data['values']['cookies']
cookies = {key: _cookie_value(raw[key]) for key in ('logged-in-user', 'logged-in-sig')}
except (KeyError, TypeError) as exc:
raise ArchiveAuthError('Logowanie udane, ale brak ciasteczek sesji w odpowiedzi') from exc
_save_session(email, cookies)
return session_status()
def logout():
session_path().unlink(missing_ok=True)
def is_archive_download(url):
try:
parsed = urlsplit(url)
return parsed.scheme == 'https' and parsed.hostname in ('archive.org', 'www.archive.org') and parsed.path.startswith('/download/')
except ValueError:
return False
def internal_url(job_id, index):
return f'http://127.0.0.1:8000/internal/archive/{job_id}/{index}'
def signed_header(job_id, index):
secret = os.environ['ARIA2_SECRET'].encode('utf-8')
message = f'archive:{job_id}:{index}'.encode('ascii')
digest = hmac.new(secret, message, hashlib.sha256).hexdigest()
return 'X-X360-Proxy-Token: ' + digest
def verify_token(job_id, index, token):
try:
expected = signed_header(job_id, index).split(': ', 1)[1]
except UnicodeEncodeError:
return False
return secrets.compare_digest(expected, token or '')
def allowed_redirect(url):
try:
parts = urlsplit(url)
name = (parts.hostname or '').lower()
return parts.scheme == 'https' and parts.port in (None, 443) and (name == 'archive.org' or name.endswith('.archive.org'))
except ValueError:
return False
async def stream_archive(url, method, headers):
"""Stream authorized bytes and preserve HTTP Range for aria2 resumes.
Only HTTPS Archive.org redirects are followed. Cookie domain scoping is
handled by httpx, unlike forwarding a raw Cookie header through redirects.
"""
outbound_headers = {'Accept-Encoding': 'identity', 'User-Agent': 'X360-Queue/1.1'}
for key in ('range', 'if-range', 'if-modified-since', 'if-none-match'):
if key in headers:
outbound_headers[key] = headers[key]
client = httpx.AsyncClient(timeout=httpx.Timeout(120, connect=25), cookies=session_cookies(),
follow_redirects=False, trust_env=False)
current = url
try:
for _ in range(7):
if not allowed_redirect(current):
return PlainTextResponse('Nieprawidłowe przekierowanie Archive.org', status_code=502)
request = client.build_request(method, current, headers=outbound_headers)
remote = await client.send(request, stream=True)
if remote.status_code in (301, 302, 303, 307, 308) and remote.headers.get('location'):
new_url = urljoin(current, remote.headers['location'])
await remote.aclose()
current = new_url
continue
break
else:
return PlainTextResponse('Zbyt wiele przekierowań Archive.org', status_code=502)
if remote.status_code not in (200, 206, 304, 416):
status = remote.status_code
await remote.aclose()
if status in (401, 403):
return PlainTextResponse('Archive.org odmówiło dostępu. Zaloguj się i sprawdź uprawnienia do pliku.', status_code=403)
return PlainTextResponse(f'Archive.org HTTP {status}', status_code=status if 400 <= status < 600 else 502)
if remote.status_code == 200 and 'text/html' in remote.headers.get('content-type', '').lower():
await remote.aclose()
return PlainTextResponse('Archive.org zwróciło stronę HTML zamiast pliku. Sprawdź logowanie.', status_code=403)
safe_headers = {k: v for k, v in remote.headers.items() if k.lower() in (
'content-length', 'content-range', 'accept-ranges', 'content-disposition',
'content-type', 'etag', 'last-modified')}
if method == 'HEAD' or remote.status_code in (304, 416):
await remote.aclose()
return Response(status_code=remote.status_code, headers=safe_headers)
async def chunks():
try:
async for chunk in remote.aiter_raw(chunk_size=131072):
yield chunk
finally:
await remote.aclose()
await client.aclose()
return StreamingResponse(chunks(), status_code=remote.status_code, headers=safe_headers,
media_type=safe_headers.get('content-type'))
except httpx.HTTPError:
return PlainTextResponse('Nie można połączyć się z Archive.org', status_code=502)
finally:
# A streamed response owns client until the last chunk is sent.
if 'remote' not in locals() or remote.is_closed:
await client.aclose()
+760
View File
@@ -0,0 +1,760 @@
"""Direct, authenticated Internet Archive downloader.
Avoids the old aria2 -> localhost HTTP relay (which hid origin errors behind
HTTP 500). Supports Range resumes, transient 5xx/429 retries and pause/cancel.
Only forwards IA cookies via httpx's domain-scoped cookie jar.
"""
import hashlib
import json
import os
import re
import shutil
import time
from collections import deque
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from contextlib import closing
from threading import Event, Lock
from pathlib import Path
from urllib.parse import urljoin
import httpx
from . import archive_auth, store
from .download import source_urls, target_filename
RETRY_CODES = {408, 425, 429, 500, 502, 503, 504}
MAX_ATTEMPTS = 5
MAX_CONNECTIONS = 50
MIN_SEGMENT_BYTES = 8 * 1024 * 1024
CHUNK_BYTES = 256 * 1024
RANGE = re.compile(r'^bytes (\d+)-(\d+)/(\d+|\*)$', re.I)
FINISH_RANGE = re.compile(r'^bytes \*/(\d+)$', re.I)
LOG_LOCK = Lock()
class ArchiveDownloadError(RuntimeError):
pass
class ArchiveStopped(Exception):
"""Pause or cancellation was requested, not a transfer error."""
def _client():
return httpx.Client(timeout=httpx.Timeout(90, connect=25), cookies=archive_auth.session_cookies(),
follow_redirects=False, trust_env=True)
def _check_state(jid, stop=None):
if stop is not None and stop.is_set():
raise ArchiveStopped()
current = store.get(jid)
if not current or current['state'] in ('paused', 'cancelled', 'error'):
raise ArchiveStopped()
def _response(client, url, headers):
current = url
for _ in range(8):
if not archive_auth.allowed_redirect(current):
raise ArchiveDownloadError('Archive.org: nieprawidłowe przekierowanie poza domenę Archive.org')
request = client.build_request('GET', current, headers=headers)
response = client.send(request, stream=True)
if response.status_code in (301, 302, 303, 307, 308) and 'location' in response.headers:
next_url = urljoin(current, response.headers['location'])
response.close()
current = next_url
continue
return response
raise ArchiveDownloadError('Archive.org: zbyt wiele przekierowań')
def _log(job_id, message):
target = store.DATA / 'logs' / (job_id + '.download.log')
target.parent.mkdir(parents=True, exist_ok=True)
with LOG_LOCK:
with target.open('a', encoding='utf-8') as fp:
fp.write(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + message + '\n')
def _transfer_one(job, *, batch=None, connections=None, stop=None):
jid = job['id']
ix = job['part_index']
urls = source_urls(job)
if ix >= len(urls):
store.update(jid, state='preparing')
return
uri = urls[ix]
if not archive_auth.is_archive_download(uri):
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
root = store.DATA / 'raw' / jid
root.mkdir(parents=True, exist_ok=True)
filename = target_filename(uri)
if not filename:
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
destination = root / filename
partial = root / (filename + '.part')
_check_state(jid, stop)
if batch is None:
store.update(jid, state='downloading', gid='', note=f'Archive.org: plik {ix+1}/{len(urls)}')
_log(jid, f'Rozpoczęto plik {ix+1}/{len(urls)}: {filename}')
if connections is None:
connections = _connection_limit()
# An existing manifest may contain many segments even if the new budget
# only allows one active connection. Keep those downloaded bytes.
if (connections > 1 or _segment_dir(partial).exists()) and _parallel_transfer(
job, ix, urls, uri, filename, partial, destination, connections, batch=batch, stop=stop):
return
# If the server cannot serve ranges (or parallel transfer was disabled),
# keep the verified single-stream prefix, not an unfinished assembly.
_discard_parallel_state(partial)
# Existing aria2 control files are not compatible with this downloader.
# Start from our own .part file rather than assuming an aria2 file is complete.
with _client() as client:
for attempt in range(MAX_ATTEMPTS):
_check_state(jid, stop)
start = partial.stat().st_size if partial.exists() else 0
headers = {'User-Agent': 'X360-Web-Queue/2.0', 'Accept-Encoding': 'identity'}
if start:
headers['Range'] = f'bytes={start}-'
try:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(jid, f'HTTP {code} z Archive.org, próba {attempt+1}/{MAX_ATTEMPTS}')
reason = f'Archive.org HTTP {code}'
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'{reason} po {MAX_ATTEMPTS} próbach; możliwy problem serwera Archive.org')
elif code in (401, 403):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia do pliku.')
elif code == 404:
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono wybranego pliku')
elif code == 416 and start:
m = FINISH_RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if m and int(m.group(1)) == start:
_check_state(jid, stop)
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename} (HTTP 416 potwierdziło rozmiar)')
return
partial.unlink(missing_ok=True)
continue
elif code not in (200, 206):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
else:
if 'text/html' in response.headers.get('Content-Type', '').lower():
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
if code == 206:
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or int(match.group(1)) != start:
raise ArchiveDownloadError('Archive.org zwróciło niezgodny zakres HTTP Range')
total = int(match.group(3)) if match.group(3) != '*' else 0
else:
# Server ignored Range: do not append duplicate bytes.
start = 0
total = int(response.headers.get('Content-Length') or 0)
written = start
last_update = 0.0
prev = written
prev_at = time.monotonic()
mode = 'ab' if start else 'wb'
with partial.open(mode) as fp:
for chunk in response.iter_raw(chunk_size=256 * 1024):
_check_state(jid, stop)
if not chunk:
continue
fp.write(chunk)
written += len(chunk)
now = time.monotonic()
if now - last_update >= 0.8:
elapsed = max(0.001, now - prev_at)
speed = int((written - prev) / elapsed)
prev, prev_at, last_update = written, now, now
portion = written / total if total else 0
if batch is not None:
batch.report(ix, written, total)
else:
store.update(jid, progress=round((ix + min(portion, 1)) / len(urls) * 100, 2),
speed=speed, total=total, completed=written,
note=f'Archive.org: plik {ix+1}/{len(urls)}, próba {attempt+1}')
if total and written != total:
raise httpx.ReadError(f'Przerwany transfer: pobrano {written} z {total} bajtów')
_check_state(jid, stop)
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename}: {written} bajtów')
return
except ArchiveStopped:
raise
except ArchiveDownloadError:
raise
except (httpx.HTTPError, OSError) as exc:
_log(jid, f'Błąd transferu {type(exc).__name__}: {str(exc)[:180]}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org: przerwane połączenie po {MAX_ATTEMPTS} próbach: {type(exc).__name__}: {exc}') from exc
# Applies to transient HTTP codes and connection exceptions.
if batch is None:
store.update(jid, note=f'Archive.org: ponawianie po HTTP/błędzie sieci ({attempt+2}/{MAX_ATTEMPTS})', speed=0)
for _ in range(min(2 ** attempt, 12) * 2):
_check_state(jid, stop)
time.sleep(0.5)
def _finish(job, ix, urls, *, batch=None):
jid = job['id']
if batch is not None:
batch.complete(ix)
return
if ix + 1 < len(urls):
store.update(jid, part_index=ix+1, state='queued', gid='', speed=0,
progress=round((ix+1)/len(urls) * 100, 2), note='Pobrano część archiwum')
else:
store.update(jid, state='preparing', progress=100, speed=0,
note='Pobieranie Archive.org ukończone')
class _RangeUnsupported(Exception):
"""A server stopped honoring byte ranges; caller can use the serial path."""
def __init__(self, message, reset_prefix=False):
super().__init__(message)
self.reset_prefix = reset_prefix
def _connection_limit():
try:
return max(1, min(MAX_CONNECTIONS, int(os.getenv('ARCHIVE_CONNECTIONS', '50'))))
except (ValueError, OverflowError):
return 50
def _segment_dir(partial):
return Path(str(partial) + '.segments')
def _read_manifest(folder):
try:
item = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
if not isinstance(item, dict):
return None
return item
except (OSError, ValueError):
return None
def _write_manifest(folder, manifest):
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
temp = folder / 'manifest.json.tmp'
with temp.open('w', encoding='utf-8') as f:
json.dump(manifest, f, sort_keys=True)
f.flush()
os.fsync(f.fileno())
temp.replace(folder / 'manifest.json')
def _discard_parallel_state(partial, reset_prefix=False):
"""Discard stale segment files without interpreting an assembled prefix as a resume."""
folder = _segment_dir(partial)
if not folder.exists():
return
manifest = _read_manifest(folder)
if reset_prefix:
partial.unlink(missing_ok=True)
elif manifest and manifest.get('assembling') and partial.exists():
prefix = manifest.get('prefix')
if isinstance(prefix, int) and 0 <= prefix <= partial.stat().st_size:
with partial.open('r+b') as f:
f.truncate(prefix)
else:
partial.unlink(missing_ok=True)
shutil.rmtree(folder)
def _validate_ranges(ranges, prefix, total):
if not isinstance(ranges, list) or len(ranges) < 2 or len(ranges) > MAX_CONNECTIONS:
return False
cursor = prefix
for item in ranges:
if (not isinstance(item, list) or len(item) != 2 or
not all(isinstance(x, int) and not isinstance(x, bool) for x in item)):
return False
start, end = item
if start != cursor or end < start or end >= total:
return False
cursor = end + 1
return cursor == total
def _prepare_segments(partial, uri, total, validator, connections):
folder = _segment_dir(partial)
source_hash = hashlib.sha256(uri.encode('utf-8')).hexdigest()
manifest = _read_manifest(folder)
valid = False
if manifest:
prefix = manifest.get('prefix')
numeric_prefix = isinstance(prefix, int) and not isinstance(prefix, bool)
prefix_present = numeric_prefix and (prefix == 0 or
(partial.exists() and partial.stat().st_size >= prefix))
valid = (manifest.get('version') == 1 and manifest.get('source_hash') == source_hash and
manifest.get('total') == total and manifest.get('validator') == validator and
isinstance(prefix, int) and not isinstance(prefix, bool) and 0 <= prefix < total and
_validate_ranges(manifest.get('ranges'), prefix, total) and prefix_present)
if valid and partial.exists() and partial.stat().st_size != prefix:
# A crash during assembly is safe to recover from: parts remain intact.
if manifest.get('assembling') and partial.stat().st_size > prefix:
with partial.open('r+b') as f:
f.truncate(prefix)
else:
valid = False
if valid:
return folder, manifest
if folder.exists():
# Invalid or mismatched metadata cannot establish which bytes are safe.
_discard_parallel_state(partial, reset_prefix=True)
prefix = partial.stat().st_size if partial.exists() else 0
if prefix > total:
partial.unlink(missing_ok=True)
prefix = 0
if prefix >= total:
return None, {'prefix': prefix, 'total': total}
remaining = total - prefix
n = min(connections, max(1, remaining // MIN_SEGMENT_BYTES))
if n < 2:
return None, None
quotient, extra = divmod(remaining, n)
cursor = prefix
ranges = []
for i in range(n):
end = cursor + quotient + (1 if i < extra else 0) - 1
ranges.append([cursor, end])
cursor = end + 1
manifest = {'version': 1, 'source_hash': source_hash, 'total': total,
'validator': validator, 'prefix': prefix, 'ranges': ranges,
'assembling': False}
_write_manifest(folder, manifest)
return folder, manifest
def _retry_wait(job_id, attempt, stop, retry_after=None, external_stop=None):
delay = min(2 ** attempt, 16)
if retry_after and retry_after.isdigit():
delay = min(max(1, int(retry_after)), 60)
for _ in range(delay * 2):
if stop.is_set():
raise ArchiveStopped()
_check_state(job_id, external_stop)
time.sleep(0.5)
def _check_content(response):
if 'text/html' in response.headers.get('Content-Type', '').lower():
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
if response.headers.get('Content-Encoding', 'identity').lower() not in ('', 'identity'):
raise ArchiveDownloadError('Archive.org zignorowało Accept-Encoding: identity; transfer w tym kodowaniu jest niebezpieczny')
def _probe_range(job_id, uri, stop=None):
"""Only a real 206 with a numeric total enables parallel mode."""
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity', 'Range': 'bytes=0-0'}
with _client() as client:
for attempt in range(MAX_ATTEMPTS):
_check_state(job_id, stop)
try:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(job_id, f'Archive.org: sprawdzenie Range HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org HTTP {code} podczas sprawdzania Range po {MAX_ATTEMPTS} próbach')
delay = response.headers.get('Retry-After')
elif code in (401, 403):
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia.')
elif code == 404:
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono pliku')
elif code == 200:
_check_content(response)
return None
elif code == 206:
_check_content(response)
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or (int(match.group(1)), int(match.group(2))) != (0, 0) or match.group(3) == '*':
return None
total = int(match.group(3))
if total <= 0:
return None
etag = response.headers.get('ETag', '').strip()
if etag.startswith('W/'):
etag = ''
validator = etag or response.headers.get('Last-Modified', '').strip()
if not validator or len(validator) > 256 or '\n' in validator:
validator = ''
return total, validator
else:
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
except ArchiveDownloadError:
raise
except (httpx.HTTPError, OSError) as exc:
_log(job_id, f'Archive.org: sprawdzenie Range - {type(exc).__name__}: {str(exc)[:150]}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Archive.org: brak odpowiedzi na żądanie Range po {MAX_ATTEMPTS} próbach') from exc
delay = None
_retry_wait(job_id, attempt, Event(), delay, stop)
return None
def _segment_worker(job_id, uri, folder, manifest, index, stop, external_stop=None):
begin, end = manifest['ranges'][index]
total = manifest['total']
limit = end - begin + 1
target = folder / f'{index:02d}.part'
if target.exists() and target.stat().st_size > limit:
target.unlink()
for attempt in range(MAX_ATTEMPTS):
if stop.is_set():
raise ArchiveStopped()
_check_state(job_id, external_stop)
present = target.stat().st_size if target.exists() else 0
if present == limit:
return
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity',
'Range': f'bytes={begin + present}-{end}'}
if manifest['validator']:
headers['If-Range'] = manifest['validator']
try:
with _client() as client:
with closing(_response(client, uri, headers)) as response:
code = response.status_code
if code in RETRY_CODES:
_log(job_id, f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code} po {MAX_ATTEMPTS} próbach')
delay = response.headers.get('Retry-After')
elif code in (401, 403):
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}: brak dostępu')
elif code == 200:
raise _RangeUnsupported('Archive.org zignorowało zakres HTTP Range / If-Range', reset_prefix=True)
elif code != 206:
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}')
else:
_check_content(response)
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
if not match or match.group(3) == '*' or (int(match.group(1)), int(match.group(2)), int(match.group(3))) != (begin + present, end, total):
raise _RangeUnsupported('Archive.org zwróciło niezgodny Content-Range', reset_prefix=True)
if manifest['validator']:
header = 'ETag' if manifest['validator'].startswith('"') else 'Last-Modified'
received_validator = response.headers.get(header, '').strip()
if received_validator and received_validator != manifest['validator']:
raise _RangeUnsupported('Archive.org zmieniło plik podczas pobierania', reset_prefix=True)
last_check = time.monotonic()
with target.open('ab') as f:
for chunk in response.iter_raw(chunk_size=CHUNK_BYTES):
if stop.is_set():
raise ArchiveStopped()
if time.monotonic() - last_check >= 1:
_check_state(job_id, external_stop)
last_check = time.monotonic()
if not chunk:
continue
if f.tell() + len(chunk) > limit:
raise _RangeUnsupported('Archive.org wysłało zbyt dużo danych w zakresie', reset_prefix=True)
f.write(chunk)
f.flush()
os.fsync(f.fileno())
if target.stat().st_size != limit:
raise httpx.ReadError('Przerwana transmisja segmentu')
return
except (ArchiveStopped, ArchiveDownloadError, _RangeUnsupported):
raise
except (httpx.HTTPError, OSError) as exc:
_log(job_id, f'Segment {index+1}: {type(exc).__name__}: {str(exc)[:150]}, próba {attempt+1}/{MAX_ATTEMPTS}')
if attempt == MAX_ATTEMPTS - 1:
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: przerwany transfer po {MAX_ATTEMPTS} próbach: {type(exc).__name__}') from exc
delay = None
_retry_wait(job_id, attempt, stop, delay, external_stop)
def _report_parallel(job_id, ix, url_count, partial, folder, manifest, last, batch=None):
prefix = manifest['prefix']
downloaded = prefix
for index, (start, end) in enumerate(manifest['ranges']):
path = folder / f'{index:02d}.part'
if path.exists():
downloaded += min(end - start + 1, path.stat().st_size)
now = time.monotonic()
prev_size, prev_at = last
speed = max(0, int((downloaded - prev_size) / max(0.001, now - prev_at)))
total = manifest['total']
if batch is None:
store.update(job_id, completed=downloaded, total=total, speed=speed,
progress=round((ix + min(downloaded / total, 1)) / url_count * 100, 2),
note=f'Archive.org: plik {ix+1}/{url_count}, {len(manifest["ranges"])} połączeń')
else:
batch.report(ix, downloaded, total)
return downloaded, now
def _assemble_segments(job_id, partial, destination, folder, manifest, stop=None):
prefix = manifest['prefix']
total = manifest['total']
for index, (start, end) in enumerate(manifest['ranges']):
part = folder / f'{index:02d}.part'
if not part.is_file() or part.stat().st_size != end - start + 1:
raise ArchiveDownloadError('Niekompletne segmenty; nie można złożyć pliku')
manifest['assembling'] = True
_write_manifest(folder, manifest)
with partial.open('r+b' if partial.exists() else 'w+b') as output:
output.truncate(prefix)
output.seek(prefix)
for index in range(len(manifest['ranges'])):
with (folder / f'{index:02d}.part').open('rb') as source:
while True:
_check_state(job_id, stop)
block = source.read(1024 * 1024)
if not block:
break
output.write(block)
output.flush()
os.fsync(output.fileno())
if partial.stat().st_size != total:
raise ArchiveDownloadError('Błąd scalania pliku: niezgodny rozmiar końcowy')
partial.replace(destination)
shutil.rmtree(folder)
def _parallel_transfer(job, ix, urls, uri, filename, partial, destination, connections, *, batch=None, stop=None):
jid = job['id']
batch_stop = stop
probe = _probe_range(jid, uri, stop)
if probe is None:
_log(jid, 'Archive.org: serwer nie obsługuje Range; pobieranie pojedynczym połączeniem')
return False
total, validator = probe
folder, manifest = _prepare_segments(partial, uri, total, validator, connections)
if folder is None and manifest is None:
_log(jid, 'Archive.org: mały plik lub końcówka; pobieranie pojedynczym połączeniem')
return False
if folder is None and manifest['prefix'] == total:
partial.replace(destination)
_finish(job, ix, urls, batch=batch)
return True
_log(jid, f'Archive.org: {len(manifest["ranges"])} połączeń HTTP Range, rozmiar {total} B, wznowienie od {manifest["prefix"]} B')
local_stop = Event()
current = (manifest['prefix'] + sum(min((folder / f'{idx:02d}.part').stat().st_size, end - start + 1)
for idx, (start, end) in enumerate(manifest['ranges'])
if (folder / f'{idx:02d}.part').exists()), time.monotonic())
try:
with ThreadPoolExecutor(max_workers=min(connections, len(manifest['ranges']))) as pool:
pending = {pool.submit(_segment_worker, jid, uri, folder, manifest, i, local_stop, batch_stop)
for i in range(len(manifest['ranges']))}
try:
while pending:
_check_state(jid, batch_stop)
done, pending = wait(pending, timeout=0.8, return_when=FIRST_COMPLETED)
current = _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
for task in done:
task.result()
finally:
local_stop.set()
for task in pending:
task.cancel()
except _RangeUnsupported as exc:
_log(jid, 'Archive.org: niezgodna obsługa Range, przełączam na jedno połączenie')
_discard_parallel_state(partial, reset_prefix=exc.reset_prefix)
return False
_check_state(jid, batch_stop)
_report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
_log(jid, 'Archive.org: scalanie segmentów')
if batch is None:
store.update(jid, note='Archive.org: scalanie segmentów', speed=0)
if batch_stop is None:
_assemble_segments(jid, partial, destination, folder, manifest)
else:
_assemble_segments(jid, partial, destination, folder, manifest, batch_stop)
_finish(job, ix, urls, batch=batch)
_log(jid, f'Ukończono {filename}: {total} bajtów, {len(manifest["ranges"])} połączeń')
return True
class _BatchProgress:
"""Collect progress from file threads; only the scheduler updates SQLite.
A single stream/segment never overwrites the total progress or total speed
from another file. No extra database table is needed for old installations.
"""
def __init__(self, job_id, filenames, limit):
self.job_id = job_id
self.filenames = filenames
self.limit = limit
self.lock = Lock()
self.sizes = [0] * len(filenames)
self.totals = [0] * len(filenames)
self.seen = [False] * len(filenames)
self.done = set()
self.active = set()
self.transferred = 0
self.last_publish = time.monotonic()
def start(self, index):
with self.lock:
self.active.add(index)
def report(self, index, downloaded, total):
with self.lock:
downloaded = max(0, int(downloaded))
if self.seen[index]:
self.transferred += max(0, downloaded - self.sizes[index])
else:
# Existing .part and .segments from an earlier run are not
# counted as freshly transferred bytes per second.
self.seen[index] = True
self.sizes[index] = downloaded
self.totals[index] = max(0, int(total))
def complete(self, index, size=None):
if size is None:
size = (store.DATA / 'raw' / self.job_id / self.filenames[index]).stat().st_size
with self.lock:
if self.seen[index]:
self.transferred += max(0, size - self.sizes[index])
self.seen[index] = True
self.sizes[index] = size
self.totals[index] = size
self.done.add(index)
self.active.discard(index)
def publish(self):
now = time.monotonic()
with self.lock:
elapsed = max(0.001, now - self.last_publish)
speed = max(0, int(self.transferred / elapsed))
self.transferred = 0
self.last_publish = now
total_files = len(self.filenames)
fractions = []
for index in range(total_files):
if index in self.done:
fractions.append(1.0)
elif self.totals[index]:
fractions.append(min(self.sizes[index] / self.totals[index], 1))
else:
fractions.append(0.0)
completed_bytes = sum(self.sizes)
total_bytes = sum(self.totals)
if total_bytes and all(self.totals):
# A 20 KiB NFO must not count the same as an 8 GiB ISO.
progress = round(100 * min(completed_bytes, total_bytes) / total_bytes, 2)
else:
# Until all content lengths are known, show the fraction of
# completed files/known segments as a provisional estimate.
progress = round(sum(fractions) / total_files * 100, 2)
finished = len(self.done)
active = len(self.active)
contiguous = next((i for i in range(total_files) if i not in self.done), total_files)
store.update(self.job_id, part_index=contiguous, progress=progress,
completed=completed_bytes, total=total_bytes, speed=speed,
note=f'Archive.org: {finished}/{total_files} gotowe, {active} równolegle, limit {self.limit} połączeń')
def _transfer_many(job):
"""Download multiple Archive.org files at once with ONE connection budget.
Each slot owns a fixed connection allowance. Reusing the slot for the next
file preserves the global cap even with 50-segment old resume manifests.
"""
jid = job['id']
urls = source_urls(job)
if not urls:
raise ArchiveDownloadError('Brak plików Archive.org do pobrania')
names = []
for uri in urls:
if not archive_auth.is_archive_download(uri):
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
name = target_filename(uri)
if not name:
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
names.append(name)
if len({name.casefold() for name in names}) != len(names):
raise ArchiveDownloadError('Wybrane pliki mają powtarzające się nazwy. Wybierz pliki o różnych nazwach.')
root = store.DATA / 'raw' / jid
root.mkdir(parents=True, exist_ok=True)
_check_state(jid)
limit = _connection_limit()
tracker = _BatchProgress(jid, names, limit)
waiting = deque()
for ix, name in enumerate(names):
dest = root / name
if dest.is_file() and not Path(str(dest) + '.aria2').exists():
# A completed destination was atomically renamed by the old or
# new downloader. A legacy aria2 control file means its sibling
# destination may only contain an incomplete preallocated file.
tracker.complete(ix, dest.stat().st_size)
_log(jid, f'Plik {ix+1}/{len(urls)} już gotowy: {name}')
else:
waiting.append(ix)
store.update(jid, state='downloading', gid='', note=f'Archive.org: start równoległego pobierania {len(waiting)} plików')
tracker.publish()
if not waiting:
_check_state(jid)
store.update(jid, state='preparing', progress=100, speed=0,
note='Pobieranie Archive.org ukończone')
return
slots = min(limit, len(waiting))
budgets = [limit // slots + (1 if i < limit % slots else 0) for i in range(slots)]
_log(jid, f'Archive.org: start {slots} plików równolegle; budżet {limit} połączeń HTTP '
f'(przydział na plik: {", ".join(map(str, budgets))})')
stopped = Event()
first_error = None
def launch(pool, slot):
ix = waiting.popleft()
item = dict(job, part_index=ix)
tracker.start(ix)
return pool.submit(_transfer_one, item, batch=tracker, connections=budgets[slot], stop=stopped)
with ThreadPoolExecutor(max_workers=slots, thread_name_prefix='archive-file') as pool:
running = {launch(pool, slot): slot for slot in range(slots)}
try:
while running:
_check_state(jid, stopped)
finished, _ = wait(running, timeout=0.8, return_when=FIRST_COMPLETED)
tracker.publish()
for future in finished:
slot = running.pop(future)
try:
future.result()
except Exception as exc:
first_error = first_error or exc
stopped.set()
else:
if waiting and not stopped.is_set():
running[launch(pool, slot)] = slot
if first_error is not None:
break
finally:
stopped.set()
for future in running:
future.cancel()
if first_error is not None:
tracker.publish()
raise first_error
_check_state(jid) # Especially important if a pause arrives at the last byte.
tracker.publish()
store.update(jid, state='preparing', progress=100, speed=0,
part_index=len(urls), note='Pobieranie Archive.org ukończone')
_log(jid, f'Archive.org: pobrano wszystkie {len(urls)} pliki')
async def run_one(job):
import asyncio
urls = source_urls(job)
if job['source_type'] == 'archive_many' and len(urls) > 1:
await asyncio.to_thread(_transfer_many, job)
else:
await asyncio.to_thread(_transfer_one, job)
+147
View File
@@ -0,0 +1,147 @@
"""Supervise anacrolix-based BitTorrent subprocesses (no aria2 for torrents).
Torrent payload data starts only after file selection. All subprocess state is
written as atomic JSON so a restart can detect interrupted work and resume.
"""
import asyncio
import json
import os
import time
from pathlib import Path
from . import store
BT_HELPER = '/usr/local/bin/x360-bt'
_INSPECTIONS = {}
_DOWNLOADS = {}
class BitTorrentError(RuntimeError):
pass
def exe():
candidate = os.getenv('BT_HELPER', BT_HELPER)
if not Path(candidate).is_file():
raise BitTorrentError(f'Brak silnika BitTorrent: {candidate} (przebuduj obraz Docker)')
return candidate
def _read(path):
try:
value = json.loads(Path(path).read_text(encoding='utf-8'))
return value if isinstance(value, dict) else {}
except (FileNotFoundError, ValueError, OSError):
return {}
def inspect_state(preview_id):
state = _read(store.DATA / 'metadata' / preview_id / 'state.json')
process = _INSPECTIONS.get(preview_id)
if not state and (not process or process.returncode is not None):
return {'status': 'error', 'error': 'Klient BitTorrent przerwał odczyt metadanych. Sprawdź engine.log podglądu lub ponów.'}
return state
def job_state(job_id):
return _read(store.DATA / 'bt' / (job_id + '.json'))
async def _spawn(args, log_path):
log_path.parent.mkdir(parents=True, exist_ok=True)
with log_path.open('ab') as log:
try:
return await asyncio.create_subprocess_exec(*args, stdin=asyncio.subprocess.DEVNULL,
stdout=log, stderr=asyncio.subprocess.STDOUT)
except (OSError, ValueError) as exc:
raise BitTorrentError('Nie można uruchomić silnika BT: ' + str(exc)) from exc
async def start_inspection(preview_id, magnet):
from . import torrent_meta
folder = torrent_meta.inspect_dir(preview_id)
args = [exe(), 'inspect', '-magnet', magnet, '-out', str(folder / 'metadata.torrent'),
'-status', str(folder / 'state.json'), '-timeout', '5m', '-port', '0']
process = await _spawn(args, folder / 'engine.log')
_INSPECTIONS[preview_id] = process
return process
async def _stop(process):
if process and process.returncode is None:
process.terminate()
try:
await asyncio.wait_for(process.wait(), 6)
except asyncio.TimeoutError:
process.kill()
await process.wait()
async def stop_inspection(preview_id):
await _stop(_INSPECTIONS.pop(preview_id, None))
async def start_download(job):
jid = job['id']
old = _DOWNLOADS.get(jid)
if old and old.returncode is None:
return
path = store.DATA / 'torrents' / (jid + '.torrent')
if not path.is_file():
raise BitTorrentError('Brak zapisanego pliku .torrent dla zadania')
try:
spec = json.loads(job['source'])
selection = spec['selection']
except (ValueError, KeyError, TypeError) as exc:
raise BitTorrentError('Uszkodzony zapis wyboru plików torrenta') from exc
output = store.DATA / 'raw' / jid
output.mkdir(parents=True, exist_ok=True)
status_path = store.DATA / 'bt' / (jid + '.json')
status_path.parent.mkdir(parents=True, exist_ok=True)
status_path.unlink(missing_ok=True)
args = [exe(), 'fetch', '-torrent', str(path), '-select', selection, '-out', str(output),
'-status', str(status_path), '-port', os.getenv('BT_PORT', '6881')]
_DOWNLOADS[jid] = await _spawn(args, store.DATA / 'logs' / (jid + '.bt.log'))
store.update(jid, gid='', state='downloading', note='BitTorrent: uruchomiono klienta')
async def poll_download(job):
jid = job['id']
process = _DOWNLOADS.get(jid)
if process is None:
# A container restart loses the subprocess; persistently downloaded
# pieces in data/raw are verified and resumed by the helper.
await start_download(job)
return
state = job_state(jid)
done, total = int(state.get('downloaded', 0) or 0), int(state.get('total', 0) or 0)
speed = int(state.get('speed', 0) or 0)
peers = int(state.get('connections', 0) or 0)
known = int(state.get('known_peers', 0) or 0)
if state.get('status') == 'done' and process.returncode == 0:
await process.wait()
_DOWNLOADS.pop(jid, None)
store.update(jid, state='preparing', progress=100, speed=0,
note='BitTorrent ukończony, przygotowanie plików')
return
if state.get('status') == 'error' or process.returncode not in (None, 0):
await process.wait()
_DOWNLOADS.pop(jid, None)
raise BitTorrentError(state.get('error') or 'Silnik BitTorrent zakończył się niepowodzeniem; sprawdź log BT')
if process.returncode == 0 and state.get('status') != 'done':
_DOWNLOADS.pop(jid, None)
raise BitTorrentError('Silnik BitTorrent zakończył pracę bez potwierdzenia ukończenia danych')
percent = round(done / total * 100, 2) if total else 0
store.update(jid, progress=percent, speed=speed, total=total, completed=done,
note=f'BitTorrent: {peers} połączeń, {known} znanych peerów')
async def stop_download(job_id):
await _stop(_DOWNLOADS.pop(job_id, None))
async def shutdown():
jobs = list(_INSPECTIONS.values()) + list(_DOWNLOADS.values())
_INSPECTIONS.clear()
_DOWNLOADS.clear()
await asyncio.gather(*(_stop(p) for p in jobs))
+109
View File
@@ -0,0 +1,109 @@
"""Validation, safe paths and Internet Archive integration."""
import html
import re
import socket
import ipaddress
from urllib.parse import urlparse, quote, unquote, parse_qs
from pathlib import Path, PurePosixPath
FORMATS = {'xex', 'god', 'raw'}
ARCHIVE_EXTS = ('.zip', '.7z', '.rar', '.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')
SAFE_ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,150}$')
VIMM_PAGE = re.compile(r'^/vault/[0-9]+/?$')
def safe_title(name):
value = re.sub(r'[^\w .()\[\]-]+', '_', name, flags=re.UNICODE).strip(' ._')[:78]
return value or 'Gra'
def normalize_magnet(value):
# Pasted magnet URLs often contain HTML entities, hard wraps or Markdown.
url = html.unescape(value).strip()
url = re.sub(r"(?i)^['\"`]*\*{0,2}magnet\*{0,2}:\?", 'magnet:?', url)
if not url.lower().startswith('magnet:?'):
return url
url = re.sub(r"\\\s*", '', url)
url = re.sub(r"\s+", '', url)
return html.unescape(url.strip("'\"`"))
def validate_url(value):
url = normalize_magnet(value)
if url.lower().startswith('magnet:?'):
if len(url) > 12000:
raise ValueError('Link magnet jest zbyt dlugi')
params = parse_qs(urlparse(url).query, keep_blank_values=True)
xt = params.get('xt', [])
if not any(re.fullmatch(r'urn:btih:(?:[0-9a-fA-F]{40}|[A-Za-z2-7]{32})', item, re.I)
or re.fullmatch(r'urn:btmh:1220[0-9a-fA-F]{64}', item, re.I)
for item in xt):
raise ValueError('Nieprawidlowy hash xt w magnet (BTIH: 40 znakow HEX lub 32 Base32)')
return 'magnet', url
parsed = urlparse(url)
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
raise ValueError('Podaj adres HTTP/HTTPS albo link magnet')
try:
port = parsed.port
except ValueError as exc:
raise ValueError('Niepoprawny numer portu w URL') from exc
if port not in (None, 80, 443):
raise ValueError('Niestandardowy port w URL jest niedozwolony')
host = parsed.hostname.lower().strip('.')
if host in ('localhost',) or host.endswith('.localhost') or host.endswith('.local'):
raise ValueError('Adres lokalny nie jest dozwolony')
try:
ip = ipaddress.ip_address(host)
except ValueError:
ip = None
if ip is not None and not ip.is_global:
raise ValueError('Adres prywatny lub zarezerwowany jest niedozwolony')
if host in ('vimm.net', 'www.vimm.net') and VIMM_PAGE.fullmatch(parsed.path):
return 'vimm', url
return 'url', url
def parse_archive_id(value):
value = value.strip()
u = urlparse(value)
if u.scheme in ('http', 'https') and u.hostname in ('archive.org', 'www.archive.org'):
pieces = [unquote(x) for x in u.path.split('/') if x]
if len(pieces) >= 2 and pieces[0] in ('details', 'download', 'metadata'):
value = pieces[1]
else:
raise ValueError('Link musi wskazywac element archive.org/details/<identyfikator>')
if not SAFE_ID.fullmatch(value):
raise ValueError('Bledny identyfikator elementu Archive.org')
return value
def archive_download_url(identifier, filename):
identifier = parse_archive_id(identifier)
if (not filename or filename.startswith('/') or '\\' in filename or '\x00' in filename
or any(p in ('..', '.') for p in PurePosixPath(filename).parts)):
raise ValueError('Niedozwolona sciezka pliku Archive.org')
return f'https://archive.org/download/{quote(identifier)}/{quote(filename, safe="/")}'
def safe_import_file(filename, root=Path('/data/imports')):
# User selects from server-side imports (including subdirectories).
if not filename or '\\' in filename or filename.startswith('/') or '\x00' in filename:
raise ValueError('Niepoprawna sciezka pliku')
item = root.joinpath(*PurePosixPath(filename).parts)
if not item.resolve().is_relative_to(root.resolve()):
raise ValueError('Plik spoza katalogu imports')
if not item.is_file() or item.is_symlink():
raise ValueError('Wskazany plik nie istnieje')
return item
def is_archive(filename):
name = filename.lower()
return name.endswith(ARCHIVE_EXTS) or bool(re.search(r'\.(?:7z|zip)\.001$', name)) or bool(re.search(r'\.part0*1\.rar$', name))
def is_additional_volume(filename):
name = filename.lower()
return bool(re.search(r'\.(?:7z|zip)\.0*0?[2-9][0-9]*$', name) or
re.search(r'\.part0*[2-9][0-9]*\.rar$', name) or
re.search(r'\.r\d\d$', name))
+156
View File
@@ -0,0 +1,156 @@
"""aria2 JSON-RPC. aria2 is started alongside the API server, not exposed publicly."""
import asyncio
import base64
import json
import os
import urllib.parse
from pathlib import Path
import httpx
from . import archive_auth, store
RPC = 'http://127.0.0.1:6800/jsonrpc'
async def rpc(method, *params):
payload = {
'jsonrpc': '2.0', 'id': 'x360', 'method': 'aria2.' + method,
'params': ['token:' + os.environ['ARIA2_SECRET'], *params],
}
# On a container restart the API may start slightly before aria2 RPC.
# Retry the connection instead of permanently failing restored jobs.
async with httpx.AsyncClient(timeout=15) as client:
for attempt in range(15):
try:
r = await client.post(RPC, json=payload)
r.raise_for_status()
answer = r.json()
if 'error' in answer:
raise RuntimeError(answer['error'].get('message', 'Bledna odpowiedz aria2'))
return answer['result']
except (httpx.ConnectError, httpx.ConnectTimeout):
if attempt == 14:
raise
await asyncio.sleep(1)
def source_urls(job):
if job['source_type'] == 'archive_many':
return json.loads(job['source'])
return [job['source']]
def target_filename(uri):
if uri.lower().startswith('magnet:'):
return None
name = urllib.parse.unquote(Path(urllib.parse.urlsplit(uri).path).name)
if not name or name in ('.', '..'):
return 'pobrany-plik.bin'
# Do not allow URI-provided paths to escape the download directory.
return ''.join(c if c not in '\\/:*?"<>|\x00\n\r' else '_' for c in name)[:200]
async def begin(job):
base = store.DATA / 'raw' / job['id']
base.mkdir(parents=True, exist_ok=True)
urls = source_urls(job)
ix = job['part_index']
if ix >= len(urls):
store.update(job['id'], state='preparing')
return
uri = urls[ix]
opts = {
'dir': str(base), 'continue': 'true', 'max-connection-per-server': '2',
'split': '2', 'seed-time': '0', 'auto-file-renaming': 'false',
'allow-overwrite': 'false', 'max-tries': '8', 'retry-wait': '30',
}
if job['source_type'] != 'torrent_selected':
name = target_filename(uri)
if name and '.' in name:
opts['out'] = name
elif name:
# Let the server's Content-Disposition supply the real filename.
opts['content-disposition'] = 'true'
if job['source_type'] == 'torrent_selected':
torrent = store.DATA / 'torrents' / (job['id'] + '.torrent')
encoded = base64.b64encode(torrent.read_bytes()).decode('ascii')
select = json.loads(job['source'])['selection']
opts['select-file'] = select
opts['bt-remove-unselected-file'] = 'true'
gid = await rpc('addTorrent', encoded, [], opts)
note = 'Pobieranie wybranych plików torrenta'
else:
if archive_auth.is_archive_download(uri):
# aria2 downloads from the local proxy. It never receives IA cookies.
opts['header'] = [archive_auth.signed_header(job['id'], ix)]
uri = archive_auth.internal_url(job['id'], ix)
opts['split'] = '1'
opts['max-connection-per-server'] = '1'
gid = await rpc('addUri', [uri], opts)
note = f'Pobieranie pliku {ix+1}/{len(urls)}'
store.update(job['id'], gid=gid, state='downloading', note=note)
async def poll(job):
urls = source_urls(job)
gid = job['gid']
if not gid:
await begin(job)
return
try:
response = await rpc('tellStatus', gid, ['gid','status','totalLength','completedLength',
'downloadSpeed','followedBy','errorMessage'])
except RuntimeError as ex:
# On restart aria2 may lose an old GID. Re-add to resume available .aria2 files.
if 'not found' in str(ex).lower():
store.update(job['id'], gid='', note='Odzyskiwanie transferu po restarcie')
return
raise
followed = response.get('followedBy', [])
if followed:
store.update(job['id'], gid=followed[0], note='Magnet: metadane gotowe')
return
state = response.get('status')
length = int(response.get('totalLength', 0) or 0)
completed = int(response.get('completedLength', 0) or 0)
speed = int(response.get('downloadSpeed', 0) or 0)
part_progress = (completed / length) if length else 0
ix = job['part_index']
pct = (ix + part_progress) / len(urls) * 100
store.update(job['id'], progress=round(pct, 2), speed=speed, total=length,
completed=completed, note=f'Pobieranie {ix+1}/{len(urls)}: {state}')
if state == 'complete':
if ix + 1 < len(urls):
store.update(job['id'], gid='', part_index=ix+1)
else:
store.update(job['id'], state='preparing', progress=100, speed=0,
note='Pobieranie ukonczone, przygotowuje pliki')
elif state in ('error','removed'):
message = response.get('errorMessage') or state
if job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(urls[ix])):
if any(x in message.lower() for x in ('authorization', '403', '401', 'forbidden')):
message += ' — Internet Archive: sprawdź logowanie i dostęp do pliku.'
raise RuntimeError('aria2: ' + message)
elif state == 'paused':
store.update(job['id'], state='paused', speed=0)
async def cancel_download(gid):
if not gid:
return
try:
await rpc('forceRemove', gid)
except (RuntimeError, httpx.HTTPError):
pass
async def pause_download(gid):
if gid:
await rpc('forcePause', gid)
async def resume_download(gid):
if gid:
await rpc('unpause', gid)
+515
View File
@@ -0,0 +1,515 @@
"""Self-hosted X360 Web Queue: authenticated FastAPI + one resilient worker."""
import asyncio
import contextlib
import json
import os
import re
import secrets
import shutil
import time
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from urllib.parse import urlparse, unquote
import httpx
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status
from fastapi.responses import FileResponse, PlainTextResponse
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel, Field
from . import archive_auth, archive_download, bt, download, pipeline, store, torrent_meta, vimm
from .common import (FORMATS, archive_download_url, parse_archive_id, safe_import_file,
safe_title, validate_url)
basic = HTTPBasic()
def authorize(credentials: HTTPBasicCredentials = Depends(basic)):
u = os.getenv('WEB_USER', 'admin').encode()
p = os.getenv('WEB_PASSWORD', '').encode()
if not p or not (secrets.compare_digest(credentials.username.encode(), u)
and secrets.compare_digest(credentials.password.encode(), p)):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
detail='Nieprawidlowy login lub haslo',
headers={'WWW-Authenticate': 'Basic'})
class AddJob(BaseModel):
url: str = Field(min_length=1, max_length=4000)
title: str = Field(default='', max_length=160)
format: str = 'xex'
class AddImports(BaseModel):
filename: str = Field(min_length=1, max_length=300)
format: str = 'xex'
class AddArchive(BaseModel):
identifier: str = Field(min_length=1, max_length=250)
files: list[str] = Field(min_length=1, max_length=150)
title: str = Field(default='', max_length=160)
format: str = 'xex'
class ItemLookup(BaseModel):
identifier: str = Field(min_length=1, max_length=250)
class ArchiveLogin(BaseModel):
email: str = Field(min_length=3, max_length=254)
password: str = Field(min_length=1, max_length=1024)
class MagnetInspect(BaseModel):
magnet: str = Field(min_length=20, max_length=12000)
class MagnetJob(BaseModel):
preview_id: str = Field(min_length=32, max_length=32)
files: list[int] = Field(min_length=1, max_length=20000)
title: str = Field(default='', max_length=160)
format: str = 'xex'
def new_job(source_type, source, title, fmt):
if fmt not in FORMATS:
raise HTTPException(status_code=400, detail='Format: xex, god albo raw')
jid = str(uuid.uuid4())
store.create({'id': jid, 'title': safe_title(title or 'Xbox 360'),
'source_type': source_type, 'source': source, 'format': fmt})
return store.get(jid)
async def worker():
while True:
job = store.next_job()
if not job:
await asyncio.sleep(2)
continue
jid = job['id']
try:
if job['state'] == 'preparing':
await pipeline.process_job(job)
elif job['source_type'] == 'import':
# Local import is copied into persistent raw dir, then converted.
src = safe_import_file(job['source'])
target = store.DATA / 'raw' / jid
target.mkdir(parents=True, exist_ok=True)
await asyncio.to_thread(shutil.copy2, src, target / src.name)
store.update(jid, state='preparing', progress=100)
elif job['source_type'] == 'vimm':
await vimm.browser_download(job)
elif job['source_type'] == 'torrent_selected':
if job['state'] == 'queued':
await bt.start_download(job)
else:
await bt.poll_download(job)
elif job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(job['source'])):
if job['gid']:
await download.cancel_download(job['gid'])
store.update(jid, gid='')
await archive_download.run_one(job)
elif job['state'] == 'queued':
# Resume if there is an aria2 GID instead of accidentally
# starting a duplicate download.
if job['gid']:
await download.resume_download(job['gid'])
store.update(jid, state='downloading')
else:
await download.begin(job)
else:
await download.poll(job)
except asyncio.CancelledError:
raise
except archive_download.ArchiveStopped:
pass
except Exception as ex:
store.update(jid, state='error', speed=0, error=str(ex)[:1800],
note='Błąd zadania: otwórz Log')
await asyncio.sleep(1.5)
@asynccontextmanager
async def lifespan(app):
for subdir in ('raw', 'ready', 'work', 'imports', 'logs', 'metadata', 'torrents'):
(store.DATA / subdir).mkdir(parents=True, exist_ok=True)
store.init_db()
task = asyncio.create_task(worker())
try:
yield
finally:
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
await bt.shutdown()
app = FastAPI(title='X360 Web Queue', version='0.1.0', lifespan=lifespan,
docs_url=None, redoc_url=None, openapi_url=None)
api = APIRouter(prefix='/api', dependencies=[Depends(authorize)])
MAX_INSPECT_SECONDS = 300
@app.middleware('http')
async def guard_cross_origin_requests(request: Request, call_next):
if request.method in ('POST', 'PUT', 'PATCH', 'DELETE'):
origin = request.headers.get('origin')
if origin and urlparse(origin).netloc != request.headers.get('host'):
return PlainTextResponse('Nieprawidlowa domena zrodla zadania', status_code=403)
return await call_next(request)
@app.get('/')
async def index(_: None = Depends(authorize)):
return FileResponse(Path(__file__).parent / 'static' / 'index.html')
@app.get('/healthz')
async def healthz():
return {'ok': True}
@api.get('/jobs')
async def jobs():
return store.list_jobs()
@api.post('/jobs')
async def add_download(body: AddJob):
try:
kind, url = validate_url(body.url)
except ValueError as ex:
raise HTTPException(status_code=400, detail=str(ex)) from ex
parsed = urlparse(url)
guessed = unquote(Path(parsed.path).name) if kind != 'magnet' else 'Torrent'
if kind == 'vimm':
guessed = 'Vimm-' + parsed.path.rstrip('/').split('/')[-1]
if kind == 'magnet':
raise HTTPException(400, 'Magnet wymaga najpierw wyboru plików (/api/magnet/inspect)')
return new_job(kind, url, body.title or guessed, body.format)
@api.get('/archive/session')
async def archive_session():
return archive_auth.session_status()
@api.post('/archive/session')
async def archive_login(body: ArchiveLogin):
if '@' not in body.email:
raise HTTPException(400, 'Niepoprawny adres e-mail')
try:
return await archive_auth.login(body.email.strip(), body.password)
except archive_auth.ArchiveAuthError as exc:
raise HTTPException(400, str(exc)) from exc
@api.delete('/archive/session')
async def archive_logout():
archive_auth.logout()
return archive_auth.session_status()
@app.api_route('/internal/archive/{job_id}/{part_index}', methods=['GET', 'HEAD'])
async def archive_proxy(job_id: str, part_index: int, request: Request):
if (not 0 <= part_index <= 149 or
not archive_auth.verify_token(job_id, part_index, request.headers.get('x-x360-proxy-token'))):
raise HTTPException(403, 'Odmowa dostępu')
job = store.get(job_id)
if not job or job['source_type'] not in ('url', 'archive_many'):
raise HTTPException(404, 'Nie znaleziono zadania')
urls = download.source_urls(job)
if part_index >= len(urls) or not archive_auth.is_archive_download(urls[part_index]):
raise HTTPException(404, 'Nie znaleziono pliku')
return await archive_auth.stream_archive(urls[part_index], request.method, request.headers)
@api.post('/magnet/inspect')
async def magnet_inspect(body: MagnetInspect):
try:
kind, magnet = validate_url(body.magnet)
if kind != 'magnet':
raise ValueError('Wklej link magnet')
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
await asyncio.to_thread(torrent_meta.cleanup_old)
root = store.DATA / 'metadata'
active_previews = sum(1 for item in root.iterdir() if item.is_dir() and not item.is_symlink() and
(item / 'request.json').is_file() and
not list(item.glob('*.torrent')) and
time.time() - item.stat().st_mtime < MAX_INSPECT_SECONDS)
if active_previews >= 3:
raise HTTPException(429, 'Trzy podglądy już pobierają metadane. Poczekaj na zakończenie.')
preview_id = uuid.uuid4().hex
directory = torrent_meta.inspect_dir(preview_id)
directory.mkdir(parents=True, mode=0o700)
try:
await bt.start_inspection(preview_id, magnet)
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': magnet, 'engine': 'anacrolix'})
except (RuntimeError, httpx.HTTPError, OSError) as exc:
await bt.stop_inspection(preview_id)
await asyncio.to_thread(shutil.rmtree, directory, True)
raise HTTPException(502, f'Nie można rozpocząć odczytu metadanych: {exc}') from exc
return {'preview_id': preview_id, 'status': 'loading'}
@api.post('/magnet/upload')
async def torrent_upload(request: Request):
# No multipart dependency. Transfer raw .torrent bytes with a fixed size limit.
if request.headers.get('content-type', '').split(';', 1)[0].strip() != 'application/x-bittorrent':
raise HTTPException(415, 'Wymagany plik .torrent')
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > torrent_meta.MAX_METADATA:
raise HTTPException(413, 'Plik .torrent przekracza limit 32 MiB')
raw.extend(chunk)
if not raw:
raise HTTPException(400, 'Pusty plik .torrent')
preview_id = uuid.uuid4().hex
directory = torrent_meta.inspect_dir(preview_id)
directory.mkdir(parents=True, exist_ok=False, mode=0o700)
target = directory / 'uploaded.torrent'
try:
# Validate before storing the metadata. No game payload is transferred.
decoded = torrent_meta.decode_bencode(bytes(raw))
if b'info' not in decoded:
raise torrent_meta.MetadataError('Plik nie zawiera pola info')
target.write_bytes(raw)
torrent_meta.read_torrent(target)
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': ''})
except (torrent_meta.MetadataError, OSError) as exc:
await asyncio.to_thread(shutil.rmtree, directory, True)
raise HTTPException(400, str(exc)) from exc
return {'preview_id': preview_id, 'status': 'ready'}
@api.get('/magnet/inspect/{preview_id}')
async def magnet_inspect_status(preview_id: str):
try:
req = torrent_meta.load_request(preview_id)
path = torrent_meta.torrent_file(preview_id)
if path:
info = torrent_meta.read_torrent(path)
await bt.stop_inspection(preview_id)
info.update({'status': 'ready', 'suggested': torrent_meta.suggested_indices(req['magnet'], len(info['files']))})
return info
elapsed = max(0, int(time.time() - req['created']))
status = bt.inspect_state(preview_id)
if status.get('status') == 'error':
return {'status': 'error', 'error': status.get('error', 'Silnik BT zgłosił błąd')[:500]}
if elapsed > MAX_INSPECT_SECONDS:
await bt.stop_inspection(preview_id)
return {'status': 'error', 'error': 'Brak metadanych po 5 minutach. Sprawdź dostępność peerów i DHT/UDP.'}
return {'status': 'loading', 'elapsed': elapsed,
'connections': int(status.get('connections') or 0),
'seeders': 0, 'known_peers': int(status.get('known_peers') or 0),
'aria_status': 'anacrolix'}
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
except (RuntimeError, httpx.HTTPError) as exc:
raise HTTPException(502, 'aria2: ' + str(exc)[:200]) from exc
@api.post('/magnet/inspect/{preview_id}/cancel')
async def magnet_inspect_cancel(preview_id: str):
try:
req = torrent_meta.load_request(preview_id)
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
if req.get('engine') == 'anacrolix':
await bt.stop_inspection(preview_id)
else:
await download.cancel_download(req['gid'])
directory = torrent_meta.inspect_dir(preview_id)
await asyncio.to_thread(shutil.rmtree, directory, True)
return {'ok': True}
@api.post('/magnet/jobs')
async def magnet_selected_job(body: MagnetJob):
if body.format not in FORMATS:
raise HTTPException(400, 'Format: xex, god albo raw')
try:
torrent_meta.load_request(body.preview_id)
path = torrent_meta.torrent_file(body.preview_id)
if path is None:
raise torrent_meta.MetadataError('Metadane jeszcze nie są gotowe')
info = torrent_meta.read_torrent(path)
await bt.stop_inspection(body.preview_id)
valid = {item['index'] for item in info['files'] if not item['padding']}
if len(body.files) != len(set(body.files)) or not set(body.files).issubset(valid):
raise torrent_meta.MetadataError('Wybierz poprawne indeksy plików')
except torrent_meta.MetadataError as exc:
raise HTTPException(400, str(exc)) from exc
jid = str(uuid.uuid4())
dest = store.DATA / 'torrents' / (jid + '.torrent')
dest.parent.mkdir(parents=True, exist_ok=True)
await asyncio.to_thread(shutil.copyfile, path, dest)
selection = ','.join(str(i) for i in sorted(body.files))
data = json.dumps({'selection': selection, 'count': len(body.files), 'name': info['name']})
try:
store.create({'id': jid, 'title': safe_title(body.title or info['name']),
'source_type': 'torrent_selected', 'source': data, 'format': body.format})
except Exception:
dest.unlink(missing_ok=True)
raise
return store.get(jid)
@api.post('/archive/lookup')
async def archive_lookup(body: ItemLookup):
try:
identifier = parse_archive_id(body.identifier)
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
try:
async with httpx.AsyncClient(timeout=25, follow_redirects=True, cookies=archive_auth.session_cookies()) as client:
res = await client.get('https://archive.org/metadata/' + identifier)
res.raise_for_status()
data = res.json()
except (httpx.HTTPError, ValueError) as ex:
raise HTTPException(502, 'Archive.org metadata: ' + str(ex)) from ex
result = []
for entry in data.get('files', []):
name = entry.get('name', '')
if not name or entry.get('source') == 'metadata' or name.startswith('__'):
continue
# Show all file types with an extension; user chooses exact archive parts.
if '.' not in Path(name).name:
continue
try:
archive_download_url(identifier, name)
except ValueError:
continue
result.append({'name': name, 'size': int(entry.get('size') or 0),
'type': entry.get('format', '')})
return {'identifier': identifier, 'title': data.get('metadata', {}).get('title', identifier),
'files': result[:1000]}
@api.post('/archive/jobs')
async def add_archive(body: AddArchive):
try:
identifier = parse_archive_id(body.identifier)
if len(set(body.files)) != len(body.files):
raise ValueError('Pliki nie moga sie powtarzac')
urls = [archive_download_url(identifier, file) for file in body.files]
if len({Path(unquote(urlparse(u).path)).name for u in urls}) != len(urls):
raise ValueError('Pliki maja identyczne nazwy; wybierz jeden folder na zadanie')
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
# Preserving filename order is important for multipart archives.
return new_job('archive_many', json.dumps(urls), body.title or identifier, body.format)
@api.get('/imports')
async def imports():
root = store.DATA / 'imports'
result = []
for item in root.rglob('*'):
if len(result) >= 200:
break
if item.is_file() and not item.is_symlink() and item.resolve().is_relative_to(root.resolve()):
result.append({'name': str(item.relative_to(root)), 'size': item.stat().st_size})
return result
@api.post('/imports/jobs')
async def import_job(body: AddImports):
try:
path = safe_import_file(body.filename)
except ValueError as ex:
raise HTTPException(400, str(ex)) from ex
return new_job('import', body.filename, path.stem, body.format)
@api.post('/jobs/{job_id}/{action}')
async def manage(job_id: str, action: str):
job = store.get(job_id)
if not job:
raise HTTPException(404, 'Nie znaleziono zadania')
state = job['state']
if action == 'pause' and state in ('queued', 'downloading'):
if state == 'downloading':
try:
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
else:
await download.pause_download(job['gid'])
except (RuntimeError, httpx.HTTPError) as ex:
raise HTTPException(502, str(ex)) from ex
store.update(job_id, state='paused', speed=0, note='Wstrzymano')
elif action == 'resume' and state == 'paused':
store.update(job_id, state='queued', note='Wznowienie zadania')
elif action == 'cancel' and state in ('queued', 'paused', 'downloading'):
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
elif job['gid']:
await download.cancel_download(job['gid'])
store.update(job_id, state='cancelled', speed=0, note='Anulowano')
elif action == 'retry' and state in ('error', 'cancelled'):
# For already-downloaded input, retry conversion without redownloading.
root = store.DATA / 'raw' / job_id
good_files = ([p for p in root.rglob('*') if p.is_file() and not p.name.endswith('.aria2')]
if root.exists() else [])
complete = job['progress'] >= 99.9 and bool(good_files)
if job['source_type'] == 'archive_many':
# One finished file (or a .part segment) never means the whole
# multi-file archive is ready to convert.
complete = all((root / download.target_filename(url)).is_file()
for url in download.source_urls(job))
store.update(job_id, state='preparing' if complete else 'queued',
gid='' if not complete else job['gid'], error='',
note='Ponowienie zadania')
elif action == 'remove' and state in ('error','cancelled','done','paused'):
if job['source_type'] == 'torrent_selected':
await bt.stop_download(job_id)
elif state == 'paused' and job['gid']:
await download.cancel_download(job['gid'])
store.erase(job_id)
return {'ok': True, 'note': 'Usunieto wpis z kolejki, pliki pozostaly na dysku'}
else:
raise HTTPException(409, 'Ta akcja nie jest dostepna dla obecnego stanu zadania')
return store.get(job_id)
@api.get('/jobs/{job_id}/log')
async def job_log(job_id: str):
if not store.get(job_id):
raise HTTPException(404, 'Nie znaleziono zadania')
path = store.DATA / 'logs' / f'{job_id}.log'
job = store.get(job_id)
lines = [f"Stan: {job['state']}", f"Etap: {job['note'] or 'brak'}"]
if job['error']:
lines.append('Błąd pobierania / zadania: ' + job['error'])
bt_log = store.DATA / 'logs' / f'{job_id}.bt.log'
if bt_log.is_file():
lines.append('BitTorrent (ostatnie 12 KiB):')
lines.append(bt_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
download_log = store.DATA / 'logs' / f'{job_id}.download.log'
if download_log.is_file():
lines.append('Pobieranie (ostatnie 12 KiB):')
lines.append(download_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
lines.append('')
lines.append(path.read_bytes()[-24000:].decode('utf-8', errors='replace')
if path.is_file() else 'Konwersja jeszcze nie została uruchomiona.')
return PlainTextResponse('\n'.join(lines))
@api.get('/system')
async def system():
usage = shutil.disk_usage(store.DATA / 'raw')
host_dir = os.getenv('XBOX_HOST_DIR', '/xbox').rstrip('/') or '/xbox'
return {'free': usage.free, 'total': usage.total,
'host_data_dir': host_dir,
'formats': ['xex','god','raw'],
'vimm_enabled': os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() in ('1','yes','true'),
'bt_engine': 'anacrolix (Go)', 'bt_port': os.getenv('BT_PORT', '6881')}
app.include_router(api)
app.mount('/static', StaticFiles(directory=str(Path(__file__).parent / 'static')), name='static')
+211
View File
@@ -0,0 +1,211 @@
"""Automatic ISO -> XEX/GOD pipeline; no console-side patching of executables."""
import asyncio
import json
import os
import re
import shutil
import tarfile
import zipfile
from pathlib import Path, PurePosixPath
from . import store
from .common import is_archive, is_additional_volume, safe_title
READY = store.DATA / 'ready'
WORK = store.DATA / 'work'
RAW = store.DATA / 'raw'
LOGS = store.DATA / 'logs'
class ProcessingError(RuntimeError):
pass
def safe_archive_path(name):
# Archive members are always relative, independent of platform.
norm = name.replace('\\', '/')
if (not norm or norm.startswith('/') or '\x00' in norm or
re.match(r'^[A-Za-z]:', norm) or
any(x == '..' for x in PurePosixPath(norm).parts)):
raise ProcessingError(f'Archiwum zawiera niebezpieczna sciezke: {name[:150]}')
def verify_no_symlinks(directory):
for root, dirs, files in os.walk(directory, followlinks=False):
for name in dirs + files:
path = Path(root) / name
if path.is_symlink():
raise ProcessingError('Pliki-symboliczne linki nie sa dopuszczalne: ' + name)
async def command(job_id, *args):
LOGS.mkdir(parents=True, exist_ok=True)
log_path = LOGS / f'{job_id}.log'
with log_path.open('ab') as log:
log.write(('\n>>> ' + ' '.join(str(x) for x in args) + '\n').encode('utf-8'))
process = await asyncio.create_subprocess_exec(*map(str, args), stdout=log, stderr=log,
stdin=asyncio.subprocess.DEVNULL)
try:
code = await process.wait()
except asyncio.CancelledError:
process.terminate()
await process.wait()
raise
if code:
data = log_path.read_bytes()[-2500:].decode('utf-8', errors='replace')
raise ProcessingError(f'Narzedzie {args[0]} zakonczone bledem {code}. Ostatni log:\n{data[-1400:]}')
def list_primary_archives(source):
files = [p for p in source.rglob('*') if p.is_file() and not p.is_symlink()]
archives = [p for p in files if is_archive(p.name) and not is_additional_volume(p.name)]
# Multipart .rar is detected by the first volume, subsequent parts are ignored.
# A single .zip with .z01 volumes is also supported by 7z.
return archives
async def unpack_archive(job_id, archive, destination):
destination.mkdir(parents=True, exist_ok=True)
lower = archive.name.lower()
store.update(job_id, state='extracting', note='Rozpakowywanie ' + archive.name[:100])
if lower.endswith('.zip') and not any(archive.parent.glob(archive.stem + '.z[0-9][0-9]')):
# Standard ZIP: validate entries to prevent writes outside destination.
with zipfile.ZipFile(archive) as z:
for member in z.infolist():
safe_archive_path(member.filename)
if (member.external_attr >> 16) & 0o170000 == 0o120000:
raise ProcessingError('Archiwum ZIP zawiera link symboliczny')
await asyncio.to_thread(z.extractall, destination)
elif lower.endswith(('.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')):
# tarfile's filters are version-dependent; reject links/special entries ourselves.
with tarfile.open(archive) as tf:
for member in tf.getmembers():
safe_archive_path(member.name)
if not (member.isfile() or member.isdir()):
raise ProcessingError('Archiwum TAR zawiera link lub plik specjalny')
await asyncio.to_thread(tf.extractall, destination, filter='data')
else:
# 7z/RAR/multipart. Validate file paths *before* invoking extraction.
listing = await asyncio.create_subprocess_exec('7z','l','-slt',str(archive),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
listing_out, listing_err = await listing.communicate()
if listing.returncode:
if not lower.endswith('.rar'):
raise ProcessingError('7z nie rozpoznaje archiwum: ' + listing_err[-300:].decode(errors='replace'))
# RAR5 fallback: libunarr (unar) understands many archives missing
# from Debian's p7zip package. Validate lsar entries first.
lsar = await asyncio.create_subprocess_exec('lsar', '-json', '-no-recursion', str(archive),
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await lsar.communicate()
if lsar.returncode:
raise ProcessingError('7z/unar nie rozpoznaje RAR: ' + stderr[-400:].decode(errors='replace'))
try:
items = json.loads(stdout).get('lsarContents', [])
if not items:
raise ProcessingError('Nie mozna odczytac zawartosci RAR')
for item in items:
safe_archive_path(item['XADFileName'])
if any(item.get(x) for x in ('XADIsLink', 'XADIsSymbolicLink', 'XADLinkDestination')):
raise ProcessingError('RAR zawiera link symboliczny')
except (ValueError, KeyError, TypeError) as exc:
raise ProcessingError('Niepoprawna lista plikow RAR') from exc
await command(job_id, 'unar', '-no-directory', '-force-overwrite',
'-output-directory', str(destination), str(archive))
verify_no_symlinks(destination)
return
data = listing_out.decode('utf-8', errors='replace')
entries = data.split('----------', 1)
if len(entries) != 2:
raise ProcessingError('Nie mozna zweryfikowac listy plikow archiwum')
for line in entries[1].splitlines():
if line.startswith('Path = '):
safe_archive_path(line[7:])
if line.startswith(('Symbolic Link = ', 'Hard Link = ')):
raise ProcessingError('Archiwum zawiera link symboliczny lub twardy')
await command(job_id, '7z', 'x', '-y', '-bd', str(archive), f'-o{destination}')
verify_no_symlinks(destination)
async def locate_payload(job_id, root, tmp):
"""Find X360 ISOs or default.xex, recursively extracting up to 2 archive layers."""
current = root
for depth in range(3):
verify_no_symlinks(current)
isos = sorted(p for p in current.rglob('*') if p.is_file() and p.suffix.lower() == '.iso')
xex_files = sorted(p for p in current.rglob('*') if p.is_file() and p.name.lower() == 'default.xex')
if isos or xex_files:
return isos, xex_files
archives = list_primary_archives(current)
if not archives:
break
if len(archives) != 1:
raise ProcessingError('Znaleziono kilka niezaleznych archiwow. Dodaj jedno na zadanie lub polacz czesci gry w jedno archiwum.')
if depth >= 2:
raise ProcessingError('Archiwa zagniezdzone glebiej niz 3 poziomy nie sa obsługiwane')
out = tmp / f'unpacked-{depth}'
await unpack_archive(job_id, archives[0], out)
current = out
raise ProcessingError('Brak obrazu .iso ani default.xex. Sprawdz czy pobrano wszystkie czesci archiwum.')
async def process_job(job):
jid = job['id']
fmt = job['format']
source = RAW / jid
temp = WORK / jid
slug = safe_title(job['title'])[:55] + '-' + jid[:8]
dest = READY / fmt / slug
atomic_dest = READY / fmt / ('.' + slug + '.working')
if not source.exists():
raise ProcessingError('Brak pobranych plikow w katalogu raw')
temp.mkdir(parents=True, exist_ok=True)
if atomic_dest.exists():
shutil.rmtree(atomic_dest)
atomic_dest.mkdir(parents=True, exist_ok=True)
store.update(jid, state='preparing', note='Analiza pobranych plikow', speed=0)
try:
verify_no_symlinks(source)
if fmt == 'raw':
await asyncio.to_thread(shutil.copytree, source, atomic_dest / 'files')
else:
isos, xex_files = await locate_payload(jid, source, temp)
if fmt == 'xex':
store.update(jid, state='converting', note='Tworzenie folderu XEX')
if xex_files:
if len(xex_files) > 1:
raise ProcessingError('Kilka plikow default.xex; wybierz jeden obraz ISO na zadanie')
await asyncio.to_thread(shutil.copytree, xex_files[0].parent, atomic_dest / 'Gra')
else:
for n, iso in enumerate(isos, 1):
dest_dir = atomic_dest / ('Gra' if len(isos) == 1 else f'Disc_{n}')
await command(jid, 'xdvdfs', 'unpack', str(iso), str(dest_dir))
if not any(p.name.lower() == 'default.xex' for p in dest_dir.rglob('*') if p.is_file()):
raise ProcessingError(f'Obraz ISO nie zawiera default.xex: {iso.name}')
else:
store.update(jid, state='converting', note='Konwersja ISO do GOD')
if not isos and xex_files:
if len(xex_files) > 1:
raise ProcessingError('Kilka folderow default.xex; GOD wymaga jednego obrazu')
packed = temp / 'packed.xiso.iso'
await command(jid, 'xdvdfs', 'pack', str(xex_files[0].parent), str(packed))
isos = [packed]
for n, iso in enumerate(isos, 1):
# iso2god creates <TitleID>/<ContentType>/<MediaID> inside provided root.
# Prefix these folders with Content/0000000000000000 for Xbox internal HDD.
disc_dir = atomic_dest if len(isos) == 1 else atomic_dest / f'Disc_{n}'
content = disc_dir / 'Content' / '0000000000000000'
content.mkdir(parents=True, exist_ok=True)
await command(jid, 'iso2god', str(iso), str(content))
if not any(p.is_dir() and re.fullmatch(r'[0-9A-Fa-f]{8}', p.name)
for p in content.iterdir()):
raise ProcessingError(f'Brak katalogu TitleID w wyniku ISO2GOD: {iso.name}')
if dest.exists():
raise ProcessingError('Folder wynikowy juz istnieje: ' + str(dest))
atomic_dest.rename(dest)
store.update(jid, state='done', progress=100, output=str(dest),
note='Gotowe do skopiowania na Xbox 360')
finally:
# Full download is preserved; only temporary extraction is deleted.
shutil.rmtree(temp, ignore_errors=True)
if atomic_dest.exists():
shutil.rmtree(atomic_dest, ignore_errors=True)
+577
View File
@@ -0,0 +1,577 @@
'use strict';
const $ = (id) => document.getElementById(id);
const NS = 'http://www.w3.org/2000/svg';
const UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
const ACTIVE = new Set(['downloading', 'preparing', 'extracting', 'converting', 'vimm_browser']);
const LABELS = {
queued: 'W kolejce', downloading: 'Pobieranie', paused: 'Wstrzymano',
preparing: 'Przygotowanie', extracting: 'Rozpakowanie', converting: 'Konwersja',
done: 'Gotowe', error: 'Błąd', cancelled: 'Anulowano', vimm_browser: 'Vimm'
};
const S = { archiveId: '', archiveTitle: '', archiveFiles: [], selected: new Set(),
jobs: [], filter: 'all', connected: false, hostRoot: '/xbox', magnetPreview: null,
magnetFiles: [], magnetSelected: new Set(), magnetName: '', magnetTitle: '', magnetFormat: 'xex' };
function normalizeMagnet(raw) {
return raw.trim()
.replace(/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i, 'magnet:?')
.replace(/&amp;|&#38;|&#x26;/gi, '&')
.replace(/\\\s*/g, '')
.replace(/\s+/g, '')
.replace(/[\'"`]+$/g, '');
}
function bytes(n) {
n = Math.max(0, Number(n) || 0);
let i = 0;
while (n >= 1024 && i < UNITS.length - 1) { n /= 1024; i++; }
return (i ? n.toFixed(1) : n.toFixed(0)) + ' ' + UNITS[i];
}
function el(tag, cls, text) {
const node = document.createElement(tag);
if (cls) node.className = cls;
if (text !== undefined && text !== null) node.textContent = text;
return node;
}
function icon(name) {
const svg = document.createElementNS(NS, 'svg');
svg.setAttribute('class', 'icon');
svg.setAttribute('aria-hidden', 'true');
const use = document.createElementNS(NS, 'use');
use.setAttribute('href', '#i-' + name);
svg.append(use);
return svg;
}
function makeButton(label, glyph, cls = 'secondary') {
const button = el('button', cls);
button.type = 'button';
button.append(icon(glyph), document.createTextNode(label));
return button;
}
async function api(path, options = {}) {
const response = await fetch('/api' + path, {
...options,
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
});
let result;
try { result = await response.json(); }
catch { result = { detail: 'Nieprawidłowa odpowiedź serwera' }; }
if (!response.ok) {
throw new Error(typeof result.detail === 'string' ? result.detail : JSON.stringify(result.detail || result));
}
return result;
}
function connection(online) {
S.connected = online;
$('connectionState').classList.toggle('online', online);
$('connectionState').classList.toggle('offline', !online);
$('connectionText').textContent = online ? 'Połączono' : 'Brak połączenia';
}
function flash(message, bad = false) {
const t = $('toast');
t.textContent = message;
t.classList.toggle('bad', bad);
t.style.display = 'block';
clearTimeout(window._toastTimeout);
window._toastTimeout = setTimeout(() => { t.style.display = 'none'; }, 5000);
}
function filterJobs(jobs) {
if (S.filter === 'active') return jobs.filter(j => !['done', 'error', 'cancelled'].includes(j.state));
if (S.filter === 'done') return jobs.filter(j => j.state === 'done');
if (S.filter === 'error') return jobs.filter(j => j.state === 'error');
return jobs;
}
function sourceDescription(job) {
if (job.source_type === 'archive_many') {
try { return 'Internet Archive · ' + JSON.parse(job.source).length + ' plików'; }
catch { return 'Internet Archive'; }
}
if (job.source_type === 'import') return 'Import · ' + job.source;
if (job.source_type === 'magnet') return 'Magnet';
if (job.source_type === 'torrent_selected') {
try { return 'Torrent · ' + JSON.parse(job.source).count + ' plików'; }
catch { return 'Torrent'; }
}
if (job.source_type === 'vimm') return 'Vimm · ' + job.source;
return job.source;
}
function jobAction(label, glyph, action, job, danger = false) {
const button = makeButton(label, glyph, 'action' + (danger ? ' danger' : ''));
button.addEventListener('click', async () => {
if (action === 'remove' && !confirm('Usunąć wpis? Pobrane pliki zostaną na dysku.')) return;
if (action === 'cancel' && !confirm('Anulować to zadanie?')) return;
button.disabled = true;
try {
await api('/jobs/' + encodeURIComponent(job.id) + '/' + action, { method: 'POST' });
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
return button;
}
async function showLog(job) {
try {
const response = await fetch('/api/jobs/' + encodeURIComponent(job.id) + '/log');
if (!response.ok) throw new Error('HTTP ' + response.status);
$('logContent').textContent = await response.text();
$('logDialog').showModal();
} catch (e) { flash(e.message, true); }
}
function hostPath(path) {
if (typeof path !== 'string') return '';
if (path === '/data/raw') return S.hostRoot + '/downloads';
if (path.startsWith('/data/raw/')) return S.hostRoot + '/downloads/' + path.slice('/data/raw/'.length);
if (path === '/data') return S.hostRoot;
if (path.startsWith('/data/')) return S.hostRoot + '/' + path.slice('/data/'.length);
return path;
}
function jobRow(job) {
const card = el('article', 'job');
const header = el('div', 'job-top');
const heading = el('div', 'job-heading');
heading.append(el('div', 'job-name', job.title), el('div', 'job-sub', sourceDescription(job)));
header.append(heading);
const statusClass = job.state === 'done' ? 'done' : job.state === 'error' ? 'error' : ACTIVE.has(job.state) ? 'running' : '';
header.append(el('span', 'job-status ' + statusClass, LABELS[job.state] || job.state));
card.append(header);
const pct = Math.max(0, Math.min(100, Number(job.progress) || 0));
const unknown = job.state === 'vimm_browser';
const progress = el('div', 'job-progress');
const track = el('div', 'progressbar' + (job.state === 'error' ? ' is-error' : ''));
const fill = el('span');
fill.style.width = pct + '%';
track.append(fill);
progress.append(track, el('span', 'progress-text', unknown ? '—' : pct.toFixed(1) + '%'));
card.append(progress);
const meta = el('div', 'job-meta');
meta.append(el('span', 'note-text', job.note || 'Oczekiwanie'));
meta.append(el('span', 'speed', Number(job.speed) > 0 ? bytes(job.speed) + '/s' : ''));
card.append(meta);
if (job.state === 'error' && job.error) card.append(el('div', 'job-error', job.error));
if (job.output) card.append(el('div', 'job-output', 'Zapisano: ' + hostPath(job.output)));
const bottom = el('div', 'job-bottom');
const chips = el('div', 'job-chips');
chips.append(el('span', 'job-chip', (job.format || 'raw').toUpperCase()));
bottom.append(chips);
const actions = el('div', 'job-actions');
if (['queued', 'downloading'].includes(job.state)) {
actions.append(jobAction('Pauza', 'pause', 'pause', job), jobAction('Anuluj', 'x', 'cancel', job, true));
}
if (job.state === 'paused') {
actions.append(jobAction('Wznów', 'play', 'resume', job), jobAction('Anuluj', 'x', 'cancel', job, true));
}
if (['error', 'cancelled'].includes(job.state)) actions.append(jobAction('Ponów', 'retry', 'retry', job));
if (['error', 'done', 'cancelled', 'paused'].includes(job.state)) actions.append(jobAction('Usuń', 'trash', 'remove', job, true));
const log = makeButton('Log', 'file', 'action');
log.addEventListener('click', () => showLog(job));
actions.append(log);
bottom.append(actions);
card.append(bottom);
return card;
}
function renderJobs() {
$('jobsCount').textContent = String(S.jobs.length);
$('countQueued').textContent = String(S.jobs.filter(j => j.state === 'queued' || j.state === 'paused').length);
$('countActive').textContent = String(S.jobs.filter(j => ACTIVE.has(j.state)).length);
$('countDone').textContent = String(S.jobs.filter(j => j.state === 'done').length);
const list = $('jobsList');
list.replaceChildren();
const filtered = filterJobs(S.jobs);
if (!filtered.length) {
list.append(el('div', 'empty-state', S.filter === 'all' ? 'Brak zadań. Dodaj link powyżej.' : 'Brak zadań w tym widoku.'));
return;
}
for (const job of filtered) list.append(jobRow(job));
}
async function refreshJobs(showError = false) {
try {
S.jobs = await api('/jobs');
renderJobs();
connection(true);
} catch (e) {
connection(false);
if (showError) flash('Nie można odczytać kolejki: ' + e.message, true);
}
}
async function refreshSystem() {
try {
const info = await api('/system');
$('storageNum').textContent = bytes(info.free);
const percent = info.total > 0 ? (100 * (1 - info.free / info.total)) : 0;
$('storageFill').style.width = Math.max(0, Math.min(100, percent)) + '%';
$('storageText').textContent = bytes(info.total) + ' łącznie';
const hostRoot = String(info.host_data_dir || '/xbox').replace(/\/+$/, '');
S.hostRoot = hostRoot;
$('importHostPath').textContent = hostRoot + '/imports/';
$('readyXexPath').textContent = hostRoot + '/ready/xex/';
$('readyGodPath').textContent = hostRoot + '/ready/god/';
$('downloadHostPath').textContent = hostRoot + '/downloads/';
$('workHostPath').textContent = hostRoot + '/work/';
connection(true);
} catch {
$('storageText').textContent = 'Nie udało się odczytać dysku';
connection(false);
}
}
for (const tab of document.querySelectorAll('.tab')) {
tab.addEventListener('click', () => {
for (const other of document.querySelectorAll('.tab')) {
const selected = other === tab;
other.classList.toggle('active', selected);
other.setAttribute('aria-selected', String(selected));
}
for (const name of ['links', 'archive', 'import']) {
$('pane-' + name).classList.toggle('hidden', name !== tab.dataset.tab);
}
if (tab.dataset.tab === 'import') refreshImports();
});
}
for (const filter of document.querySelectorAll('.filter')) {
filter.addEventListener('click', () => {
S.filter = filter.dataset.filter;
for (const other of document.querySelectorAll('.filter')) {
const selected = other === filter;
other.classList.toggle('active', selected);
other.setAttribute('aria-pressed', String(selected));
}
renderJobs();
});
}
async function refreshArchiveSession() {
try {
const result = await api('/archive/session');
$('archiveSessionText').textContent = result.authenticated ? ('Sesja zapisana · ' + result.email) : 'Niezalogowano';
$('archiveLoginForm').classList.toggle('hidden', result.authenticated);
$('archiveLogout').classList.toggle('hidden', !result.authenticated);
} catch (e) { $('archiveSessionText').textContent = 'Błąd sprawdzania sesji'; }
}
$('archiveLoginForm').addEventListener('submit', async (event) => {
event.preventDefault();
const button = $('archiveLoginForm').querySelector('[type=submit]');
button.disabled = true;
try {
const result = await api('/archive/session', { method: 'POST', body: JSON.stringify({
email: $('archiveEmail').value.trim(), password: $('archivePassword').value
}) });
$('archivePassword').value = '';
flash('Zapisano sesję Archive.org: ' + result.email);
await refreshArchiveSession();
} catch (e) { $('archivePassword').value = ''; flash('Archive.org: ' + e.message, true); }
finally { button.disabled = false; }
});
$('archiveLogout').addEventListener('click', async () => {
try {
await api('/archive/session', { method: 'DELETE' });
await refreshArchiveSession();
flash('Usunięto zapisaną sesję Archive.org.');
} catch (e) { flash(e.message, true); }
});
function renderMagnet() {
const list = $('magnetFiles');
list.replaceChildren();
const text = $('magnetFilter').value.toLowerCase();
let displayed = 0;
for (const file of S.magnetFiles) {
if (!file.name.toLowerCase().includes(text)) continue;
if (displayed >= 500) break;
displayed++;
const row = el('label', 'file-row');
const box = el('input');
box.type = 'checkbox';
box.checked = S.magnetSelected.has(file.index);
box.disabled = Boolean(file.padding);
box.addEventListener('change', () => {
if (box.checked) S.magnetSelected.add(file.index);
else S.magnetSelected.delete(file.index);
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
});
row.append(box, el('span', 'file-name', file.name),
el('span', 'size', file.padding ? 'Systemowy' : bytes(file.size)));
list.append(row);
}
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
if (displayed >= 500) list.append(el('div', 'empty-state', 'Pierwsze 500 plików. Zawęź filtr, aby wyszukać kolejne.'));
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
}
async function pollMagnet(previewId) {
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
try {
const data = await api('/magnet/inspect/' + encodeURIComponent(previewId));
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
if (data.status === 'ready') {
S.magnetName = data.name;
S.magnetFiles = data.files;
S.magnetSelected = new Set((data.suggested || []).filter(i => !data.files.find(f => f.index === i)?.padding));
$('magnetName').textContent = data.name + ' (' + data.files.length + ' plików)';
$('magnetStatus').textContent = 'Metadane pobrane. Wybierz pliki.';
$('magnetResults').classList.remove('hidden');
renderMagnet();
return;
}
if (data.status === 'loading') {
const seconds = Number(data.elapsed) || 0;
const connections = Number(data.connections) || 0;
const known = Number(data.known_peers) || 0;
let notice = 'Metadane: ' + seconds + ' s | połączenia: ' + connections + ' | znani peerzy: ' + known;
if (seconds >= 25 && connections === 0)
notice += ' | Brak połączeń. Sprawdź UDP/DHT lub wczytaj .torrent.';
$('magnetStatus').textContent = notice;
}
if (data.status === 'error') {
$('magnetStatus').textContent = 'Błąd: ' + data.error;
return;
}
} catch (e) {
$('magnetStatus').textContent = 'Błąd: ' + e.message;
return;
}
setTimeout(() => pollMagnet(previewId), 3000);
}
async function inspectMagnet(magnet, title, format) {
S.magnetPreview = null;
S.magnetTitle = title;
S.magnetFormat = format;
S.magnetFiles = [];
S.magnetSelected.clear();
$('magnetStatus').textContent = 'Oczekiwanie na metadane z sieci BitTorrent…';
$('magnetResults').classList.add('hidden');
$('magnetFilter').value = '';
$('magnetDialog').showModal();
try {
const data = await api('/magnet/inspect', { method: 'POST', body: JSON.stringify({ magnet }) });
if (!$('magnetDialog').open) return;
S.magnetPreview = data.preview_id;
pollMagnet(data.preview_id);
} catch (e) { $('magnetStatus').textContent = 'Błąd: ' + e.message; }
}
async function uploadTorrent(file, fromLinks) {
if (!file) return;
if (file.size > 32 * 1024 * 1024) return flash('Limit pliku .torrent: 32 MiB.', true);
if (fromLinks) {
S.magnetTitle = $('linkTitle').value.trim();
S.magnetFormat = $('linkFormat').value;
}
const previous = S.magnetPreview;
S.magnetPreview = null;
S.magnetFiles = [];
S.magnetSelected.clear();
$('magnetResults').classList.add('hidden');
$('magnetFilter').value = '';
$('magnetStatus').textContent = 'Wczytywanie pliku .torrent...';
if (!$('magnetDialog').open) $('magnetDialog').showModal();
if (previous) await api('/magnet/inspect/' + encodeURIComponent(previous) + '/cancel', { method: 'POST' }).catch(() => {});
try {
const response = await fetch('/api/magnet/upload', {
method: 'POST', headers: { 'Content-Type': 'application/x-bittorrent' }, body: file
});
const data = await response.json();
if (!response.ok) throw new Error(data.detail || 'HTTP ' + response.status);
if (!$('magnetDialog').open) return;
S.magnetPreview = data.preview_id;
await pollMagnet(data.preview_id);
} catch (error) {
$('magnetStatus').textContent = 'Błąd: ' + error.message;
}
}
for (const id of ['torrentFileLinks', 'torrentFileDialog']) {
$(id).addEventListener('change', async (event) => {
const file = event.target.files?.[0];
event.target.value = '';
await uploadTorrent(file, id === 'torrentFileLinks');
});
}
$('magnetFilter').addEventListener('input', renderMagnet);
$('magnetToggleAll').addEventListener('click', () => {
const visible = S.magnetFiles.filter(f => !f.padding && f.name.toLowerCase().includes($('magnetFilter').value.toLowerCase())).slice(0, 500);
const allSelected = visible.every(f => S.magnetSelected.has(f.index));
for (const file of visible) {
if (allSelected) S.magnetSelected.delete(file.index);
else S.magnetSelected.add(file.index);
}
renderMagnet();
});
$('closeMagnet').addEventListener('click', () => $('magnetDialog').close());
$('magnetDialog').addEventListener('close', () => {
const preview = S.magnetPreview;
S.magnetPreview = null;
if (preview) api('/magnet/inspect/' + encodeURIComponent(preview) + '/cancel', { method: 'POST' }).catch(() => {});
});
$('magnetAdd').addEventListener('click', async () => {
if (!S.magnetSelected.size) return flash('Zaznacz co najmniej jeden plik.', true);
const button = $('magnetAdd');
button.disabled = true;
try {
await api('/magnet/jobs', { method: 'POST', body: JSON.stringify({
preview_id: S.magnetPreview, files: [...S.magnetSelected],
title: S.magnetTitle, format: S.magnetFormat
}) });
$('urls').value = '';
S.magnetPreview = null;
$('magnetDialog').close();
flash('Wybrane pliki dodano do kolejki.');
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
$('linksForm').addEventListener('submit', async (event) => {
event.preventDefault();
const raw = $('urls').value.trim();
if (!raw) return flash('Podaj co najmniej jeden link.', true);
const button = $('linksForm').querySelector('[type=submit]');
const title = $('linkTitle').value.trim();
const format = $('linkFormat').value;
if (/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i.test(raw)) {
const magnet = normalizeMagnet(raw);
if ((magnet.match(/magnet:\?/gi) || []).length !== 1)
return flash('Magnet wklej pojedynczo, aby wybrać pliki.', true);
await inspectMagnet(magnet, title, format);
return;
}
const items = raw.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
let ok = 0;
const errors = [];
button.disabled = true;
try {
for (const url of items) {
try { await api('/jobs', { method: 'POST', body: JSON.stringify({ url, title, format }) }); ok++; }
catch (e) { errors.push(url.slice(0, 65) + ': ' + e.message); }
}
if (ok) {
$('urls').value = '';
flash('Dodano zadań: ' + ok);
await refreshJobs();
}
if (errors.length) flash(errors.join('\n'), true);
} finally { button.disabled = false; }
});
$('archiveForm').addEventListener('submit', async (event) => {
event.preventDefault();
const button = $('archiveForm').querySelector('[type=submit]');
button.disabled = true;
$('archiveResults').classList.add('hidden');
try {
const data = await api('/archive/lookup', { method: 'POST', body: JSON.stringify({ identifier: $('archiveId').value }) });
S.archiveId = data.identifier;
S.archiveTitle = data.title;
S.archiveFiles = data.files;
S.selected.clear();
$('archiveName').textContent = data.title + ' (' + data.files.length + ')';
$('archiveResults').classList.remove('hidden');
renderArchive();
} catch (e) { flash('Internet Archive: ' + e.message, true); }
finally { button.disabled = false; }
});
function renderArchive() {
const list = $('archiveFiles');
list.replaceChildren();
const text = $('archiveFilter').value.toLowerCase();
let displayed = 0;
for (const file of S.archiveFiles) {
if (!file.name.toLowerCase().includes(text)) continue;
if (displayed >= 400) break;
displayed++;
const row = el('label', 'file-row');
const checkbox = el('input');
checkbox.type = 'checkbox';
checkbox.checked = S.selected.has(file.name);
checkbox.addEventListener('change', () => {
if (checkbox.checked) S.selected.add(file.name);
else S.selected.delete(file.name);
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
});
row.append(checkbox, el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
list.append(row);
}
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
}
$('archiveFilter').addEventListener('input', renderArchive);
$('archiveAdd').addEventListener('click', async () => {
if (!S.selected.size) return flash('Zaznacz pliki.', true);
const button = $('archiveAdd');
button.disabled = true;
try {
await api('/archive/jobs', { method: 'POST', body: JSON.stringify({
identifier: S.archiveId, files: [...S.selected], title: S.archiveTitle, format: $('archiveFormat').value
}) });
S.selected.clear();
renderArchive();
flash('Dodano do kolejki.');
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
async function refreshImports() {
try {
const files = await api('/imports');
const list = $('importList');
list.replaceChildren();
if (!files.length) {
list.append(el('div', 'empty-state', 'Katalog data/imports/ jest pusty.'));
return;
}
for (const file of files) {
const row = el('div', 'file-row');
row.append(icon('file'), el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
const button = makeButton('Przetwórz', 'play');
button.addEventListener('click', async () => {
button.disabled = true;
try {
await api('/imports/jobs', { method: 'POST', body: JSON.stringify({
filename: file.name, format: $('importFormat').value
}) });
flash('Dodano: ' + file.name);
await refreshJobs();
} catch (e) { flash(e.message, true); }
finally { button.disabled = false; }
});
row.append(button);
list.append(row);
}
} catch (e) { flash('Import: ' + e.message, true); }
}
$('refreshImports').addEventListener('click', refreshImports);
$('refreshJobs').addEventListener('click', () => refreshJobs(true));
$('closeLog').addEventListener('click', () => $('logDialog').close());
refreshJobs();
refreshSystem();
refreshArchiveSession();
setInterval(() => refreshJobs(), 3000);
setInterval(refreshSystem, 30000);
+4
View File
@@ -0,0 +1,4 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
<rect width="64" height="64" rx="10" fill="#1b1e21"/>
<path d="M17 16 32 32 47 16M17 48 32 32 47 48" fill="none" stroke="#91c99a" stroke-width="6" stroke-linecap="square"/>
</svg>

After

Width:  |  Height:  |  Size: 245 B

+151
View File
@@ -0,0 +1,151 @@
<!doctype html>
<html lang="pl">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="color-scheme" content="dark">
<meta name="theme-color" content="#141619">
<title>X360 Queue</title>
<link rel="icon" href="/static/icon.svg" type="image/svg+xml">
<link rel="stylesheet" href="/static/style.css">
<script src="/static/app.js" defer></script>
</head>
<body>
<svg class="svg-defs" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
<symbol id="i-download" viewBox="0 0 24 24"><path d="M12 3v12m-5-5 5 5 5-5M5 17v3h14v-3"/></symbol>
<symbol id="i-link" viewBox="0 0 24 24"><path d="M10 13a5 5 0 0 0 7.4.4l2-2a5 5 0 0 0-7.1-7.1l-1.1 1.1M14 11a5 5 0 0 0-7.4-.4l-2 2a5 5 0 0 0 7.1 7.1l1.1-1.1"/></symbol>
<symbol id="i-archive" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="4" rx="1"/><path d="M5 8v12h14V8M10 12h4"/></symbol>
<symbol id="i-folder" viewBox="0 0 24 24"><path d="M3 7a2 2 0 0 1 2-2h5l2 2h7a2 2 0 0 1 2 2v10H3z"/></symbol>
<symbol id="i-plus" viewBox="0 0 24 24"><path d="M12 5v14M5 12h14"/></symbol>
<symbol id="i-refresh" viewBox="0 0 24 24"><path d="M20 7v5h-5M4 17v-5h5"/><path d="M5.6 9A7 7 0 0 1 18 6l2 6M4 12l2 6a7 7 0 0 0 12.4-3"/></symbol>
<symbol id="i-pause" viewBox="0 0 24 24"><path d="M8 5v14M16 5v14"/></symbol>
<symbol id="i-play" viewBox="0 0 24 24"><path d="m8 5 11 7-11 7z"/></symbol>
<symbol id="i-x" viewBox="0 0 24 24"><path d="M6 6l12 12M18 6 6 18"/></symbol>
<symbol id="i-retry" viewBox="0 0 24 24"><path d="M3 12a9 9 0 1 0 3-6.7M3 4v5h5"/></symbol>
<symbol id="i-trash" viewBox="0 0 24 24"><path d="M4 7h16M10 4h4M6 7l1 13h10l1-13M10 11v5m4-5v5"/></symbol>
<symbol id="i-file" viewBox="0 0 24 24"><path d="M6 3h8l4 4v14H6zM14 3v5h5M9 12h6M9 16h6"/></symbol>
<symbol id="i-harddrive" viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="14" rx="2"/><path d="M3 13h18M7 16h.01M11 16h.01"/></symbol>
<symbol id="i-activity" viewBox="0 0 24 24"><path d="M3 12h4l3-7 4 14 3-7h4"/></symbol>
<symbol id="i-check" viewBox="0 0 24 24"><path d="m5 12 5 5L19 7"/></symbol>
<symbol id="i-clock" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6l4 2"/></symbol>
<symbol id="i-alert" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6m0 4v.1"/></symbol>
<symbol id="i-list" viewBox="0 0 24 24"><path d="M9 6h12M9 12h12M9 18h12M4 6h.01M4 12h.01M4 18h.01"/></symbol>
</svg>
<div class="shell">
<header class="topbar">
<div class="brand">
<div class="brand-mark" aria-hidden="true">X</div>
<div class="brand-text"><strong>X360 Queue</strong><span>Linux / RGH</span></div>
</div>
<div class="topbar-right">
<span class="connection" id="connectionState"><span class="connection-dot"></span><span id="connectionText">Łączenie…</span></span>
<span class="version">LOCAL</span>
</div>
</header>
<main>
<div class="page-title">
<div><h1>Pobieranie</h1><p>Dodawanie plików i konwersja XEX / GOD.</p></div>
</div>
<section class="metrics" aria-label="Stan aplikacji">
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-clock"/></svg> W kolejce</span><strong id="countQueued">0</strong></div>
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-activity"/></svg> W toku</span><strong id="countActive">0</strong></div>
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-check"/></svg> Gotowe</span><strong id="countDone">0</strong></div>
<div class="metric disk-metric">
<span class="metric-label"><svg class="icon"><use href="#i-harddrive"/></svg> Wolne miejsce</span>
<strong id="storageNum">—</strong>
<div class="meter"><span id="storageFill" style="width:0%"></span></div>
<small id="storageText">—</small>
</div>
</section>
<section class="panel" aria-labelledby="addHeading">
<div class="panel-heading"><h2 id="addHeading">Dodaj zadanie</h2><span class="muted-info">Zadania działają po zamknięciu przeglądarki</span></div>
<nav class="tabs" aria-label="Wybierz źródło">
<button class="tab active" type="button" data-tab="links" aria-selected="true"><svg class="icon"><use href="#i-link"/></svg>Link / magnet</button>
<button class="tab" type="button" data-tab="archive" aria-selected="false"><svg class="icon"><use href="#i-archive"/></svg>Internet Archive</button>
<button class="tab" type="button" data-tab="import" aria-selected="false"><svg class="icon"><use href="#i-folder"/></svg>Import</button>
</nav>
<div class="tab-body" id="pane-links">
<form id="linksForm">
<div class="field">
<label for="urls">Linki <span>— jeden adres w wierszu</span></label>
<textarea id="urls" rows="3" spellcheck="false" placeholder="https://example.org/obraz.iso&#10;magnet:?xt=urn:btih:..."></textarea>
</div>
<div class="input-row">
<div class="field title-field"><label for="linkTitle">Nazwa <span>(opcjonalnie)</span></label><input id="linkTitle" placeholder="Automatyczna" autocomplete="off"></div>
<div class="field format-field"><label for="linkFormat">Format wyjściowy</label><select id="linkFormat"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select></div>
<button class="primary submit-button" type="submit"><svg class="icon"><use href="#i-plus"/></svg>Dodaj do kolejki</button>
</div>
</form>
<div class="torrent-upload-inline"><label for="torrentFileLinks" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileLinks" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Alternatywa, gdy magnet nie zwraca metadanych.</span></div>
<p class="note">Magnet: najpierw lista plików, potem wybór i pobieranie. Vimm: tylko dostępny przycisk pobierania. Limity i zabezpieczenia serwisu pozostają aktywne.</p>
</div>
<div class="tab-body hidden" id="pane-archive">
<section class="archive-auth" aria-label="Konto Internet Archive">
<div class="auth-heading"><strong>Konto Archive.org</strong><span id="archiveSessionText">Sprawdzanie sesji…</span></div>
<form id="archiveLoginForm" class="auth-login hidden" autocomplete="on">
<div class="field"><label for="archiveEmail">E-mail</label><input id="archiveEmail" type="email" autocomplete="username" required placeholder="konto@example.org"></div>
<div class="field"><label for="archivePassword">Hasło</label><input id="archivePassword" type="password" autocomplete="current-password" required></div>
<button type="submit" class="secondary"><svg class="icon"><use href="#i-link"/></svg>Zaloguj</button>
</form>
<button id="archiveLogout" class="secondary hidden" type="button">Wyloguj</button>
<p class="note">Hasło nie jest zapisywane. Pobieranie używa sesji Archive.org; konto nie odblokowuje plików niedostępnych dla użytkownika. Panel udostępniaj przez HTTPS lub VPN.</p>
</section>
<form id="archiveForm">
<div class="field"><label for="archiveId">Adres lub identyfikator</label>
<div class="inline-input"><input id="archiveId" placeholder="archive.org/details/identyfikator" spellcheck="false"><button class="secondary" type="submit"><svg class="icon"><use href="#i-list"/></svg>Pokaż pliki</button></div>
</div>
</form>
<div class="archive-results hidden" id="archiveResults">
<div class="file-heading"><h3 id="archiveName"></h3><input id="archiveFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
<div class="file-table" id="archiveFiles"></div>
<div class="file-bottom"><span id="selectedCount">Wybrano: 0</span><div class="selection-controls"><select id="archiveFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="archiveAdd" class="primary" type="button"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
<p class="note">Części jednego archiwum zaznacz razem.</p>
</div>
</div>
<div class="tab-body hidden" id="pane-import">
<div class="import-toolbar"><div><p class="import-hint">Skopiuj pliki do <code id="importHostPath">/xbox/imports/</code>.</p></div><div class="selection-controls"><select id="importFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="refreshImports" class="secondary" type="button"><svg class="icon"><use href="#i-refresh"/></svg>Odśwież</button></div></div>
<div id="importList" class="file-table"></div>
</div>
</section>
<section class="queue-section" aria-labelledby="queueHeading">
<div class="queue-header">
<div class="queue-title"><h2 id="queueHeading">Kolejka <span id="jobsCount" class="count">0</span></h2><span class="muted-info">Odświeżanie co 3 s</span></div>
<div class="queue-controls">
<div class="filters" aria-label="Filtruj zadania">
<button type="button" class="filter active" data-filter="all" aria-pressed="true">Wszystkie</button>
<button type="button" class="filter" data-filter="active" aria-pressed="false">W toku</button>
<button type="button" class="filter" data-filter="done" aria-pressed="false">Gotowe</button>
<button type="button" class="filter" data-filter="error" aria-pressed="false">Błędy</button>
</div>
<button id="refreshJobs" class="secondary icon-button" type="button" title="Odśwież kolejkę" aria-label="Odśwież kolejkę"><svg class="icon"><use href="#i-refresh"/></svg></button>
</div>
</div>
<div id="jobsList" class="jobs"><div class="empty-state">Brak zadań.</div></div>
</section>
<footer class="footer"><span>Wyniki: <code id="readyXexPath">/xbox/ready/xex/</code> lub <code id="readyGodPath">/xbox/ready/god/</code></span><span>Pobieranie: <code id="downloadHostPath">/xbox/downloads/</code> · Robocze: <code id="workHostPath">/xbox/work/</code></span></footer>
</main>
</div>
<div id="toast" role="status" aria-live="polite"></div>
<dialog id="magnetDialog" aria-labelledby="magnetHeading">
<div class="dialog-heading"><h2 id="magnetHeading">Pliki w torrencie</h2><button type="button" class="secondary" id="closeMagnet"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div>
<p id="magnetStatus" class="preview-status">Pobieranie metadanych torrenta…</p>
<div class="torrent-upload-inline"><label for="torrentFileDialog" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileDialog" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Nie wymaga peerów ani oczekiwania na DHT.</span></div>
<div id="magnetResults" class="hidden">
<div class="file-heading"><h3 id="magnetName"></h3><input id="magnetFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
<div class="file-table" id="magnetFiles"></div>
<div class="file-bottom"><span id="magnetSelectedCount">Wybrano: 0</span><div class="selection-controls"><button type="button" id="magnetToggleAll" class="secondary">Zaznacz widoczne</button><button type="button" id="magnetAdd" class="primary"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
<p class="note">Pobieranie rozpocznie się dopiero po wybraniu plików. Przy archiwach wieloczęściowych zaznacz wszystkie części.</p>
</div>
</dialog>
<dialog id="logDialog" aria-labelledby="logHeading"><div class="dialog-heading"><h2 id="logHeading">Log zadania</h2><button type="button" class="secondary" id="closeLog"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div><pre id="logContent"></pre></dialog>
</body>
</html>
+214
View File
@@ -0,0 +1,214 @@
:root {
color-scheme: dark;
font-family: system-ui, -apple-system, "Segoe UI", Arial, sans-serif;
font-size: 14px;
font-synthesis: none;
--bg: #121416;
--panel: #1b1e21;
--panel-hover: #21252a;
--input: #151719;
--line: #34383d;
--line-soft: #2a2e32;
--text: #e4e7e9;
--muted: #a2a9af;
--faint: #788189;
--accent: #91c99a;
--accent-hover: #afe0b5;
--success-bg: #203427;
--danger: #e8a09b;
--danger-bg: #362324;
}
*, *::before, *::after { box-sizing: border-box; }
body { margin: 0; background: var(--bg); color: var(--text); }
button, input, textarea, select { font: inherit; }
button { cursor: pointer; }
button:disabled { opacity: .45; cursor: not-allowed; }
button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
.hidden, .svg-defs { display: none !important; }
.icon { width: 16px; height: 16px; flex: 0 0 auto; fill: none; stroke: currentColor; stroke-width: 1.7; stroke-linecap: round; stroke-linejoin: round; }
.shell { max-width: 1120px; margin: auto; padding: 0 24px; }
.topbar { height: 66px; display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid var(--line); }
.brand { display: flex; align-items: center; gap: 11px; }
.brand-mark { display: grid; place-items: center; width: 33px; height: 33px; border: 1px solid #72977c; color: #a9d6ad; background: #1d2a21; border-radius: 5px; font-weight: 800; font-size: 19px; line-height: 1; }
.brand-text { display: flex; flex-direction: column; gap: 2px; }
.brand-text strong { font-size: 14px; font-weight: 700; letter-spacing: .1px; }
.brand-text span { color: var(--faint); font-size: 11px; }
.topbar-right { display: flex; align-items: center; gap: 17px; }
.connection { display: inline-flex; align-items: center; gap: 7px; font-size: 12px; color: var(--muted); }
.connection-dot { width: 7px; height: 7px; background: var(--faint); border-radius: 50%; }
.connection.online .connection-dot { background: var(--accent); }
.connection.offline .connection-dot { background: var(--danger); }
.version { border: 1px solid var(--line); color: var(--faint); font: 600 10px ui-monospace, SFMono-Regular, Consolas, monospace; padding: 4px 7px; letter-spacing: .5px; border-radius: 3px; }
.page-title { padding: 31px 0 22px; }
h1, h2, h3, p { margin-top: 0; }
h1 { margin-bottom: 5px; font-size: 25px; font-weight: 680; letter-spacing: -.65px; line-height: 1.3; }
.page-title p { margin-bottom: 0; color: var(--muted); font-size: 13px; }
.metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); border: 1px solid var(--line); border-radius: 5px; background: var(--panel); overflow: hidden; }
.metric { display: flex; flex-direction: column; gap: 9px; padding: 16px 19px; min-height: 112px; }
.metric + .metric { border-left: 1px solid var(--line); }
.metric-label { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 12px; }
.metric-label .icon { width: 14px; height: 14px; color: var(--faint); }
.metric strong { color: var(--text); font-weight: 650; font-size: 25px; font-variant-numeric: tabular-nums; line-height: 1; }
.disk-metric { gap: 7px; }
.disk-metric strong { font-size: 20px; }
.disk-metric small { font-size: 10px; color: var(--faint); }
.meter { height: 4px; background: #353a3e; overflow: hidden; width: 100%; }
.meter span { display: block; height: 100%; background: var(--accent); transition: width .2s linear; }
.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; margin-top: 26px; }
.panel-heading { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 18px 20px 15px; }
h2 { margin: 0; font-size: 16px; font-weight: 670; letter-spacing: -.2px; }
.muted-info { font-size: 11px; color: var(--faint); }
.tabs { display: flex; gap: 3px; padding: 0 15px; border-bottom: 1px solid var(--line); }
.tab { display: inline-flex; align-items: center; gap: 8px; background: transparent; border: none; border-bottom: 2px solid transparent; padding: 11px 13px 12px; color: var(--muted); font-size: 12px; font-weight: 550; }
.tab:hover { color: var(--text); background: #25292d; }
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
.tab .icon { width: 15px; height: 15px; }
.tab-body { padding: 21px 20px 20px; }
.field { min-width: 0; }
label { display: block; margin-bottom: 7px; font-size: 12px; color: #cbd0d4; font-weight: 600; }
label span { color: var(--faint); font-weight: 400; }
input, select, textarea { display: block; width: 100%; min-width: 0; color: var(--text); background: var(--input); border: 1px solid #42474c; border-radius: 4px; padding: 9px 11px; outline: none; font-size: 12px; }
input, select { height: 38px; }
textarea { min-height: 89px; resize: vertical; line-height: 1.6; }
input::placeholder, textarea::placeholder { color: #737c84; }
select { appearance: auto; }
.input-row { display: flex; gap: 12px; align-items: flex-end; margin-top: 13px; }
.input-row .title-field { flex: 1 1 auto; }
.input-row .format-field { flex: 0 0 172px; }
button.primary, button.secondary, button.action { border-radius: 4px; display: inline-flex; align-items: center; justify-content: center; gap: 8px; font-size: 12px; font-weight: 620; white-space: nowrap; min-height: 38px; padding: 9px 12px; }
button.primary { border: 1px solid #9ccc9e; color: #18231b; background: var(--accent); }
button.primary:hover:not(:disabled) { background: var(--accent-hover); }
button.secondary, button.action { border: 1px solid #444a50; background: #292d31; color: #d9dee1; }
button.secondary:hover:not(:disabled), button.action:hover:not(:disabled) { background: #34393f; }
button.action { min-height: 29px; padding: 5px 9px; font-size: 11px; }
button.action.danger { color: #e3a6a4; }
button.action.danger:hover { background: #3a2829; }
button.icon-button { min-width: 36px; width: 36px; padding: 8px; }
.note { color: var(--faint); font-size: 11px; line-height: 1.55; margin: 13px 0 0; }
.inline-input { display: flex; align-items: center; gap: 9px; }
.inline-input input { flex: 1 1 auto; }
.file-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 24px; }
.file-heading h3 { font-size: 13px; font-weight: 600; margin: 0; max-width: 62%; overflow-wrap: anywhere; }
.file-heading input { max-width: 250px; }
.file-table { max-height: 320px; overflow: auto; margin-top: 12px; border: 1px solid var(--line); border-radius: 4px; }
.file-row { display: flex; gap: 12px; align-items: center; min-height: 44px; padding: 9px 11px; margin: 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; color: #d2d7da; font-weight: 400; }
.file-row:last-child { border-bottom: 0; }
.file-row:hover { background: var(--panel-hover); }
.file-row input[type=checkbox] { margin: 0; min-height: 0; width: 15px; height: 15px; accent-color: var(--accent); flex-shrink: 0; }
.file-name { flex: 1; min-width: 0; overflow-wrap: anywhere; }
.size { color: var(--faint); font-size: 11px; white-space: nowrap; }
.file-bottom { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 13px; color: var(--muted); font-size: 12px; }
.selection-controls { display: flex; align-items: center; gap: 9px; }
.selection-controls select { width: 145px; }
.import-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 15px; }
.import-hint { font-size: 12px; color: var(--muted); margin: 0; }
code { color: #b7d1bd; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 11px; }
.queue-section { padding-top: 29px; }
.queue-header { display: flex; justify-content: space-between; align-items: center; gap: 14px; margin-bottom: 14px; }
.queue-title { display: flex; align-items: baseline; gap: 13px; }
.queue-title h2 { display: flex; align-items: center; gap: 9px; }
.count { font-size: 11px; color: #bdc4c9; background: #2c3034; border: 1px solid #3d4247; min-width: 21px; padding: 2px 6px; text-align: center; border-radius: 3px; font-variant-numeric: tabular-nums; }
.queue-controls { display: flex; gap: 9px; align-items: center; }
.filters { display: flex; border: 1px solid var(--line); border-radius: 4px; overflow: hidden; }
.filter { background: #1c1f22; color: var(--muted); border: 0; border-right: 1px solid var(--line); padding: 9px 13px; font-size: 11px; min-height: 36px; }
.filter:last-child { border-right: 0; }
.filter:hover { color: var(--text); background: #272b30; }
.filter.active { color: var(--accent); background: #29342c; font-weight: 650; }
.jobs { display: flex; flex-direction: column; gap: 8px; }
.job { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; padding: 14px 17px; }
.job-top { display: flex; justify-content: space-between; gap: 20px; align-items: flex-start; }
.job-heading { min-width: 0; flex: 1; }
.job-name { font-weight: 650; font-size: 13px; line-height: 1.45; overflow-wrap: anywhere; }
.job-sub { color: var(--faint); font-size: 11px; margin-top: 3px; overflow-wrap: anywhere; max-height: 2.8em; overflow: hidden; }
.job-status { flex-shrink: 0; display: inline-flex; align-items: center; gap: 6px; border: 1px solid #4a5156; color: #bbc4ca; background: #272c30; border-radius: 3px; font-size: 10px; line-height: 1; padding: 6px 8px; white-space: nowrap; }
.job-status::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: #9ca5ac; }
.job-status.running { background: #233126; color: #add7af; border-color: #39563e; }
.job-status.running::before { background: var(--accent); }
.job-status.done { background: var(--success-bg); color: #a8d9ac; border-color: #3b5942; }
.job-status.done::before { background: #8fcd95; }
.job-status.error { background: var(--danger-bg); color: #f1aaa3; border-color: #704446; }
.job-status.error::before { background: var(--danger); }
.job-progress { display: flex; align-items: center; gap: 12px; margin-top: 12px; }
.progressbar { width: 100%; height: 5px; background: #373d41; overflow: hidden; }
.progressbar span { display: block; height: 100%; background: #8aba94; transition: width .25s linear; }
.progressbar.is-error span { background: #c47e78; }
.progress-text { color: #c5cbd0; text-align: right; font-variant-numeric: tabular-nums; font-size: 11px; flex: 0 0 50px; }
.job-meta { display: flex; align-items: center; justify-content: space-between; gap: 20px; margin-top: 8px; font-size: 11px; color: var(--faint); }
.job-meta .note-text { overflow-wrap: anywhere; }
.job-meta .speed { flex-shrink: 0; font-variant-numeric: tabular-nums; }
.job-bottom { display: flex; justify-content: space-between; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 11px; }
.job-chips { display: inline-flex; gap: 7px; align-items: center; }
.job-chip { color: #aab3ba; font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; border: 1px solid #3c4449; padding: 3px 7px; border-radius: 3px; }
.job-actions { display: flex; flex-wrap: wrap; gap: 6px; margin-left: auto; }
.job-error { margin-top: 11px; padding: 9px 11px; border-left: 2px solid #c6857e; color: #ebaba6; background: #2b2122; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.5; }
.job-output { color: #accdb5; overflow-wrap: anywhere; margin-top: 9px; font-size: 11px; }
.empty-state { color: var(--faint); text-align: center; border: 1px dashed #42484e; background: var(--panel); padding: 34px 12px; border-radius: 4px; font-size: 12px; }
.footer { display: flex; justify-content: space-between; gap: 20px; padding: 23px 0 36px; color: var(--faint); font-size: 11px; }
#toast { display: none; position: fixed; right: 20px; bottom: 20px; max-width: min(420px, calc(100vw - 40px)); z-index: 20; padding: 13px 16px; border: 1px solid #6e9e75; border-radius: 4px; background: #24402a; color: #d3f1d6; font-size: 12px; line-height: 1.45; white-space: pre-line; }
#toast.bad { background: #48282a; color: #ffd7d3; border-color: #955b60; }
dialog { width: min(760px, calc(100vw - 30px)); max-height: 85vh; background: var(--panel); color: var(--text); border: 1px solid #545a60; border-radius: 5px; padding: 18px; }
dialog::backdrop { background: rgba(0,0,0,.75); }
.dialog-heading { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--line); }
.dialog-heading h2 { font-size: 15px; }
pre { overflow: auto; max-height: 66vh; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.65; color: #c8cdd0; }
@media (max-width: 760px) {
.shell { padding: 0 15px; }
.topbar { height: 59px; }
.version { display: none; }
.page-title { padding: 22px 0 16px; }
.metrics { grid-template-columns: repeat(2, minmax(0,1fr)); }
.metric:nth-child(3) { border-left: 0; border-top: 1px solid var(--line); }
.metric:nth-child(4) { border-top: 1px solid var(--line); }
.metric { min-height: 99px; padding: 15px; }
.panel { margin-top: 19px; }
.panel-heading { padding: 16px 15px 13px; }
.panel-heading .muted-info { display: none; }
.tabs { padding: 0 6px; }
.tab { padding: 11px 9px; font-size: 11px; gap: 6px; }
.tab-body { padding: 16px 15px; }
.input-row { flex-wrap: wrap; }
.input-row .title-field { flex: 1 1 100%; }
.input-row .format-field { flex: 1 1 110px; }
.input-row .submit-button { flex: 1 1 auto; }
.file-heading { align-items: stretch; flex-direction: column; }
.file-heading h3 { max-width: 100%; }
.file-heading input { max-width: none; }
.queue-header { align-items: stretch; flex-direction: column; gap: 13px; }
.queue-controls { justify-content: space-between; }
.queue-title { justify-content: space-between; }
.filter { padding: 9px; }
.job { padding: 13px; }
.footer { flex-direction: column; gap: 7px; }
}
@media (max-width: 440px) {
.inline-input, .import-toolbar { align-items: stretch; flex-direction: column; }
.selection-controls { width: 100%; }
.selection-controls select { flex: 1; min-width: 0; }
.file-bottom { align-items: stretch; flex-direction: column; }
.job-top { gap: 9px; }
.job-status { max-width: 130px; white-space: normal; }
.queue-title .muted-info { display: none; }
}
@media (prefers-reduced-motion: reduce) {
*, *::before, *::after { transition-duration: 0ms !important; animation-duration: 0ms !important; }
}
/* Archive login and torrent metadata selection: compact, flat controls. */
.archive-auth { border: 1px solid var(--line); padding: 13px; margin-bottom: 20px; background: var(--input); border-radius: 4px; }
.auth-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; font-size: 12px; }
.auth-heading span { color: var(--muted); overflow-wrap: anywhere; font-size: 11px; }
.auth-login { display: flex; align-items: end; gap: 10px; flex-wrap: wrap; margin-top: 13px; }
.auth-login .field { flex: 1 1 190px; }
.archive-auth .note { margin-top: 10px; }
#archiveLogout { margin-top: 12px; }
#magnetDialog { width: min(840px, calc(100vw - 24px)); }
.preview-status { color: var(--muted); font-size: 12px; margin: 14px 0; }
#magnetDialog .file-table { max-height: 45vh; }
#magnetDialog .file-row input:disabled { opacity: .3; }
@media(max-width: 540px) { .auth-login .field, .auth-login button { flex: 1 1 100%; } }
/* Flat torrent metadata fallback controls. */
.torrent-upload-inline {display:flex; align-items:center; flex-wrap:wrap; gap:10px; margin:10px 0; color:var(--muted, #8d939d); font-size:12px;}
.file-label {display:inline-flex; align-items:center; justify-content:center; cursor:pointer; border:1px solid #434a51; background:#292e33; color:#d7dce1; padding:7px 10px; border-radius:3px; font-size:12px; font-weight:600;}
.file-label:hover {background:#32383e;}
#torrentFileLinks, #torrentFileDialog {display:none !important;}
+95
View File
@@ -0,0 +1,95 @@
"""SQLite-backed durable job queue. Single-worker, multiple API connections."""
import sqlite3
import time
from pathlib import Path
DATA = Path('/data')
DB_PATH = DATA / 'queue.sqlite3'
FIELDS = {
'title', 'source_type', 'source', 'format', 'state', 'progress', 'speed',
'total', 'completed', 'part_index', 'gid', 'error', 'output', 'note', 'updated',
}
def connection():
DATA.mkdir(parents=True, exist_ok=True)
con = sqlite3.connect(DB_PATH, timeout=30)
con.row_factory = sqlite3.Row
con.execute('PRAGMA busy_timeout=30000')
con.execute('PRAGMA journal_mode=WAL')
return con
def init_db():
with connection() as con:
con.execute('''CREATE TABLE IF NOT EXISTS jobs (
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
source_type TEXT NOT NULL,
source TEXT NOT NULL,
format TEXT NOT NULL,
state TEXT NOT NULL DEFAULT 'queued',
progress REAL NOT NULL DEFAULT 0,
speed INTEGER NOT NULL DEFAULT 0,
total INTEGER NOT NULL DEFAULT 0,
completed INTEGER NOT NULL DEFAULT 0,
part_index INTEGER NOT NULL DEFAULT 0,
gid TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
output TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
created REAL NOT NULL,
updated REAL NOT NULL
)''')
con.execute('CREATE INDEX IF NOT EXISTS jobs_created_idx ON jobs(created)')
# Zadania przerwane w trakcie rozpakowania/konwersji moga byc ponowione.
con.execute("UPDATE jobs SET state='preparing', note='Wznowiono konwersje po restarcie' "
"WHERE state IN ('extracting','converting')")
con.execute("UPDATE jobs SET state='queued', note='Przywrocono zadanie po restarcie' "
"WHERE state='vimm_browser'")
def create(job):
now = time.time()
with connection() as con:
con.execute('''INSERT INTO jobs(id,title,source_type,source,format,state,created,updated)
VALUES(?,?,?,?,?,'queued',?,?)''',
(job['id'], job['title'], job['source_type'],
job['source'], job['format'], now, now))
def get(job_id):
with connection() as con:
row = con.execute('SELECT * FROM jobs WHERE id=?', (job_id,)).fetchone()
return dict(row) if row else None
def list_jobs():
with connection() as con:
rows = con.execute('SELECT * FROM jobs ORDER BY created DESC LIMIT 300').fetchall()
return [dict(row) for row in rows]
def next_job():
with connection() as con:
row = con.execute('''SELECT * FROM jobs
WHERE state IN ('queued','downloading','preparing')
ORDER BY created ASC LIMIT 1''').fetchone()
return dict(row) if row else None
def update(job_id, **values):
if not values:
return
if not set(values).issubset(FIELDS):
raise ValueError('Niepoprawne pola aktualizacji bazy')
values['updated'] = time.time()
with connection() as con:
con.execute('UPDATE jobs SET ' + ','.join(f'{k}=?' for k in values)
+ ' WHERE id=?', (*values.values(), job_id))
def erase(job_id):
with connection() as con:
con.execute('DELETE FROM jobs WHERE id=?', (job_id,))
+181
View File
@@ -0,0 +1,181 @@
"""Safe, small bencode parser and ephemeral magnet metadata inspection state."""
import json
import os
import re
import shutil
import time
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
from . import store
ID = re.compile(r'^[a-f0-9]{32}$')
MAX_METADATA = 32 * 1024 * 1024
class MetadataError(ValueError):
pass
def decode_bencode(raw):
if len(raw) > MAX_METADATA:
raise MetadataError('Metadane torrenta są zbyt duże')
position = 0
count = 0
def decode(depth=0):
nonlocal position, count
if depth > 24:
raise MetadataError('Metadane mają zbyt duże zagnieżdżenie')
count += 1
if count > 600000 or position >= len(raw):
raise MetadataError('Uszkodzone metadane torrenta')
start = raw[position]
if start == 105: # i<number>e
end = raw.find(b'e', position + 1)
if end < 0 or end - position > 25:
raise MetadataError('Błędna liczba bencode')
try:
number = int(raw[position + 1:end])
except ValueError as exc:
raise MetadataError('Błędna liczba bencode') from exc
position = end + 1
return number
if start in (108, 100): # l, d
position += 1
output = [] if start == 108 else {}
while position < len(raw) and raw[position] != 101:
if start == 108:
output.append(decode(depth + 1))
else:
key = decode(depth + 1)
if not isinstance(key, bytes):
raise MetadataError('Niepoprawny klucz bencode')
output[key] = decode(depth + 1)
if position >= len(raw):
raise MetadataError('Nieukończone metadane')
position += 1
return output
if 48 <= start <= 57: # <len>:<bytes>
colon = raw.find(b':', position, min(len(raw), position + 22))
if colon < 0:
raise MetadataError('Błędny ciąg bencode')
try:
length = int(raw[position:colon])
except ValueError as exc:
raise MetadataError('Błędna długość bencode') from exc
position = colon + 1
if length < 0 or length > len(raw) - position:
raise MetadataError('Niepoprawna długość pola')
result = raw[position:position + length]
position += length
return result
raise MetadataError('Uszkodzone metadane torrenta')
result = decode()
if position != len(raw) or not isinstance(result, dict):
raise MetadataError('Niepoprawny format torrenta')
return result
def _segment(raw):
if not isinstance(raw, bytes) or not raw or len(raw) > 255 or raw in (b'.', b'..'):
raise MetadataError('Niedozwolona nazwa w torrencie')
if any(char in raw for char in (b'/', b'\\', b'\0')):
raise MetadataError('Niebezpieczna ścieżka w torrencie')
return raw.decode('utf-8', errors='replace')
def read_torrent(path):
try:
data = decode_bencode(Path(path).read_bytes())
info = data[b'info']
name = _segment(info.get(b'name.utf-8') or info[b'name'])
result = []
if b'files' in info:
files = info[b'files']
if not isinstance(files, list) or len(files) > 50000:
raise MetadataError('Nieprawidłowa lista plików torrenta')
for index, file in enumerate(files, start=1):
parts = file.get(b'path.utf-8') or file[b'path']
if not isinstance(parts, list) or not parts:
raise MetadataError('Błędna ścieżka pliku')
length = file[b'length']
if not isinstance(length, int) or length < 0:
raise MetadataError('Nieprawidłowy rozmiar pliku')
result.append({'index': index, 'name': '/'.join([name] + [_segment(x) for x in parts]),
'size': length, 'padding': b'p' in file.get(b'attr', b'')})
else:
length = info[b'length']
if not isinstance(length, int) or length < 0:
raise MetadataError('Nieprawidłowy rozmiar pliku')
result.append({'index': 1, 'name': name, 'size': length, 'padding': False})
if not result:
raise MetadataError('Torrent nie zawiera plików')
return {'name': name, 'files': result}
except (KeyError, TypeError, OSError, ValueError) as exc:
if isinstance(exc, MetadataError):
raise
raise MetadataError('Nie można odczytać plików z torrenta') from exc
def suggested_indices(magnet, max_index):
"""magnet so indexes are zero-based; aria2 select-file indexes are one-based."""
so = parse_qs(urlsplit(magnet).query).get('so', [''])[0]
suggested = set()
for chunk in so.split(',')[:1000]:
try:
if '-' in chunk:
a, b = (int(x) for x in chunk.split('-', 1))
if b < a or b - a > 1000:
continue
values = range(a, b + 1)
else:
values = [int(chunk)]
suggested.update(n + 1 for n in values if 0 <= n < max_index)
except ValueError:
continue
return sorted(suggested)
def inspect_dir(preview_id):
if not ID.fullmatch(preview_id):
raise MetadataError('Nieprawidłowy identyfikator metadanych')
return store.DATA / 'metadata' / preview_id
def write_request(preview_id, data):
path = inspect_dir(preview_id)
path.mkdir(mode=0o700, parents=True, exist_ok=True)
filename = path / 'request.json'
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
with os.fdopen(os.open(filename, flags, 0o600), 'w', encoding='utf-8') as f:
json.dump({'created': time.time(), **data}, f)
def load_request(preview_id):
try:
info = json.loads((inspect_dir(preview_id) / 'request.json').read_text(encoding='utf-8'))
if not isinstance(info['gid'], str) or not isinstance(info['magnet'], str):
raise MetadataError('Błędne dane podglądu')
return info
except FileNotFoundError as exc:
raise MetadataError('Nie znaleziono podglądu magneta') from exc
except (ValueError, KeyError, TypeError) as exc:
raise MetadataError('Uszkodzone dane podglądu') from exc
def torrent_file(preview_id):
matches = list(inspect_dir(preview_id).glob('*.torrent'))
return matches[0] if len(matches) == 1 and matches[0].stat().st_size <= MAX_METADATA else None
def cleanup_old(max_age=86400):
root = store.DATA / 'metadata'
if not root.exists():
return
now = time.time()
for child in root.iterdir():
if (ID.fullmatch(child.name) and child.is_dir() and not child.is_symlink()
and now - child.stat().st_mtime > max_age):
shutil.rmtree(child)
+76
View File
@@ -0,0 +1,76 @@
"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
import os
import re
import asyncio
from pathlib import Path
from urllib.parse import urlparse
from urllib.robotparser import RobotFileParser
import httpx
from . import store
async def browser_download(job):
if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
url = job['source']
u = urlparse(url)
if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
try:
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
r = await c.get('https://vimm.net/robots.txt')
if r.status_code == 200:
parser = RobotFileParser()
parser.parse(r.text.splitlines())
if not parser.can_fetch('x360-web-queue', url):
raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
elif r.status_code not in (404, 410):
raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
except httpx.HTTPError as exc:
raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
out = store.DATA / 'raw' / job['id']
out.mkdir(parents=True, exist_ok=True)
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True, executable_path='/usr/bin/chromium',
args=['--no-sandbox', '--disable-dev-shm-usage'])
try:
context = await browser.new_context(accept_downloads=True)
page = await context.new_page()
await page.goto(url, wait_until='domcontentloaded', timeout=60000)
# Only the original, visible button is actionable. Missing/taken-down
# buttons and challenges are not bypassed.
controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
usable = None
for i in range(min(await controls.count(), 10)):
c = controls.nth(i)
if await c.is_visible() and await c.is_enabled():
usable = c
break
if usable is None:
raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
try:
async with page.expect_download(timeout=timeout) as d:
await usable.click(timeout=15000)
download = await d.value
except PlaywrightTimeout as exc:
raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
if not name:
name = 'vimm-download.bin'
store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
failure = await download.failure()
if failure:
raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
finally:
await browser.close()
+5
View File
@@ -0,0 +1,5 @@
module x360-bt-helper
go 1.24.0
require github.com/anacrolix/torrent v1.61.0
+260
View File
@@ -0,0 +1,260 @@
// x360-bt: non-interactive BitTorrent helper based on the anacrolix/torrent
// library, also used by the user's minerva-pick tool. It emits machine-readable
// status and never starts downloading payload data while inspecting a magnet.
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/anacrolix/torrent"
)
type state struct {
Status string `json:"status"`
Error string `json:"error,omitempty"`
Connections int `json:"connections"`
KnownPeers int `json:"known_peers"`
Downloaded int64 `json:"downloaded,omitempty"`
Total int64 `json:"total,omitempty"`
Speed int64 `json:"speed,omitempty"`
Elapsed int64 `json:"elapsed,omitempty"`
}
func writeState(path string, s state) {
if path == "" {
return
}
b, err := json.Marshal(s)
if err != nil {
return
}
if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return
}
f, err := os.CreateTemp(filepath.Dir(path), ".bt-state-*")
if err != nil {
return
}
defer os.Remove(f.Name())
if _, err = f.Write(b); err != nil {
f.Close()
return
}
if err = f.Close(); err != nil {
return
}
_ = os.Rename(f.Name(), path)
}
func newClient(dataDir string, port int) (*torrent.Client, error) {
cfg := torrent.NewDefaultClientConfig()
cfg.DisableIPv6 = true
cfg.DataDir = dataDir
cfg.Seed = false
cfg.NoDefaultPortForwarding = true
cfg.ListenPort = port
return torrent.NewClient(cfg)
}
func inspect(ctx context.Context, magnet, output, statusPath string, port int, timeout time.Duration) error {
if !strings.HasPrefix(strings.ToLower(magnet), "magnet:?") {
return errors.New("nieprawidlowy magnet")
}
if err := os.MkdirAll(filepath.Dir(output), 0700); err != nil {
return err
}
client, err := newClient(filepath.Dir(output), port)
if err != nil {
return fmt.Errorf("klient BT: %w", err)
}
defer client.Close()
t, err := client.AddMagnet(magnet)
if err != nil {
return fmt.Errorf("magnet: %w", err)
}
started := time.Now()
timer := time.NewTimer(timeout)
defer timer.Stop()
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
snapshot := state{Status: "loading", Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm()), Elapsed: int64(time.Since(started).Seconds())}
writeState(statusPath, snapshot)
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return fmt.Errorf("brak metadanych po %s (polaczenia=%d, znani peerzy=%d)", timeout, snapshot.Connections, snapshot.KnownPeers)
case <-t.GotInfo():
mi := t.Metainfo()
f, err := os.CreateTemp(filepath.Dir(output), ".bt-meta-*")
if err != nil {
return err
}
temp := f.Name()
defer os.Remove(temp)
if err = mi.Write(f); err != nil {
f.Close()
return err
}
if err = f.Close(); err != nil {
return err
}
if info, err := os.Stat(temp); err != nil || info.Size() > 32*1024*1024 {
return errors.New("metadane przekraczaja 32 MiB")
}
if err = os.Rename(temp, output); err != nil {
return err
}
writeState(statusPath, state{Status: "ready", Connections: snapshot.Connections, KnownPeers: snapshot.KnownPeers, Elapsed: snapshot.Elapsed})
return nil
case <-ticker.C:
}
}
}
func parseSelected(selection string, files []*torrent.File) ([]*torrent.File, error) {
if selection == "" {
return nil, errors.New("brak wybranych plikow")
}
used := make(map[int]bool)
out := make([]*torrent.File, 0)
for _, s := range strings.Split(selection, ",") {
n, err := strconv.Atoi(strings.TrimSpace(s))
if err != nil || n < 1 || n > len(files) || used[n] {
return nil, fmt.Errorf("niepoprawny indeks pliku %q", s)
}
used[n] = true
out = append(out, files[n-1])
}
return out, nil
}
func progress(files []*torrent.File) (done, total int64) {
for _, f := range files {
total += f.Length()
n := f.BytesCompleted()
if n > f.Length() {
n = f.Length()
}
done += n
}
return
}
func fetch(ctx context.Context, input, selection, outDir, statusPath string, port int) error {
if err := os.MkdirAll(outDir, 0755); err != nil {
return err
}
client, err := newClient(outDir, port)
if err != nil {
return fmt.Errorf("klient BT: %w", err)
}
defer client.Close()
t, err := client.AddTorrentFromFile(input)
if err != nil {
return fmt.Errorf("plik torrent: %w", err)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-t.GotInfo():
}
selected, err := parseSelected(selection, t.Files())
if err != nil {
return err
}
for _, f := range selected {
f.Download()
}
started := time.Now()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
lastDone := int64(0)
lastAt := time.Now()
for {
done, total := progress(selected)
now := time.Now()
seconds := now.Sub(lastAt).Seconds()
var speed int64
if seconds > 0 {
speed = int64(float64(done-lastDone) / seconds)
if speed < 0 {
speed = 0
}
}
lastDone = done
lastAt = now
if total == 0 || done >= total {
writeState(statusPath, state{Status: "done", Total: total, Downloaded: done, Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm())})
return nil
}
writeState(statusPath, state{Status: "downloading", Total: total, Downloaded: done, Speed: speed, Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm()), Elapsed: int64(time.Since(started).Seconds())})
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
}
}
}
func run(ctx context.Context) (string, error) {
if len(os.Args) < 2 {
return "", errors.New("tryb inspect albo fetch")
}
command := os.Args[1]
flags := flag.NewFlagSet(command, flag.ContinueOnError)
magnet := flags.String("magnet", "", "magnet link")
input := flags.String("torrent", "", "plik .torrent")
selection := flags.String("select", "", "indeksy 1-based oddzielone przecinkiem")
out := flags.String("out", "", "sciezka wyjsciowa")
statusPath := flags.String("status", "", "sciezka JSON statusu")
port := flags.Int("port", 0, "port TCP/UDP (0 = losowy)")
timeout := flags.Duration("timeout", 5*time.Minute, "timeout metadanych")
if err := flags.Parse(os.Args[2:]); err != nil {
return *statusPath, err
}
if *port < 0 || *port > 65535 {
return *statusPath, errors.New("niepoprawny port")
}
var err error
switch command {
case "inspect":
if *out == "" || *magnet == "" {
err = errors.New("podaj -magnet i -out")
} else {
err = inspect(ctx, *magnet, *out, *statusPath, *port, *timeout)
}
case "fetch":
if *input == "" || *out == "" {
err = errors.New("podaj -torrent i -out")
} else {
err = fetch(ctx, *input, *selection, *out, *statusPath, *port)
}
default:
err = errors.New("nieznany tryb: " + command)
}
return *statusPath, err
}
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
path, err := run(ctx)
if err != nil {
fmt.Fprintln(os.Stderr, "x360-bt:", err)
writeState(path, state{Status: "error", Error: err.Error()})
os.Exit(1)
}
}
+39
View File
@@ -0,0 +1,39 @@
services:
x360-web:
build: .
container_name: x360-web-queue
restart: unless-stopped
init: true
ports:
- "${WEB_BIND:-0.0.0.0}:${WEB_PORT:-8088}:8000"
- "${BT_PORT:-6881}:${BT_PORT:-6881}/tcp"
- "${BT_PORT:-6881}:${BT_PORT:-6881}/udp"
environment:
WEB_USER: ${WEB_USER:?Run ./setup.sh}
WEB_PASSWORD: ${WEB_PASSWORD:?Run ./setup.sh}
ARIA2_SECRET: ${ARIA2_SECRET:?Run ./setup.sh}
BT_PORT: ${BT_PORT:-6881}
ARCHIVE_CONNECTIONS: ${ARCHIVE_CONNECTIONS:-50}
ENABLE_VIMM_BROWSER: ${ENABLE_VIMM_BROWSER:-true}
VIMM_DOWNLOAD_TIMEOUT: ${VIMM_DOWNLOAD_TIMEOUT:-7200}
XBOX_HOST_DIR: ${XBOX_HOST_DIR:-/xbox}
# Python, Playwright and converter temporary files use host /xbox/work.
TMPDIR: /data/work/tmp
XDG_CACHE_HOME: /data/work/cache
volumes:
# Application state, metadata, work, ready files and logs on the host.
- type: bind
source: ${XBOX_HOST_DIR:-/xbox}
target: /data
bind:
create_host_path: false
# Preserve existing /data/raw paths for resumed jobs and aria2 sessions.
# Downloads are stored on the host as /xbox/downloads, not /xbox/raw.
- type: bind
source: ${XBOX_HOST_DIR:-/xbox}/downloads
target: /data/raw
bind:
create_host_path: false
shm_size: '512mb'
security_opt:
- no-new-privileges:true
Executable
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
: "${WEB_PASSWORD:?WEB_PASSWORD must be set}"
: "${ARIA2_SECRET:?ARIA2_SECRET must be set}"
: "${BT_PORT:=6881}"
case "$WEB_PASSWORD" in CHANGE_ME*|"") echo "Zmien WEB_PASSWORD w .env" >&2; exit 1;; esac
case "$ARIA2_SECRET" in CHANGE_ME*|"") echo "Zmien ARIA2_SECRET w .env" >&2; exit 1;; esac
mkdir -p /data/raw /data/ready /data/work /data/work/tmp /data/work/cache /data/imports /data/aria2
if [ ! -f /data/aria2/session.txt ]; then touch /data/aria2/session.txt; fi
aria2c \
--enable-rpc=true --rpc-listen-all=false --rpc-listen-port=6800 \
--rpc-secret="$ARIA2_SECRET" \
--dir=/data/raw \
--input-file=/data/aria2/session.txt \
--save-session=/data/aria2/session.txt --save-session-interval=30 \
--continue=true --max-concurrent-downloads=2 \
--max-connection-per-server=2 --split=2 --min-split-size=8M \
--seed-time=0 --bt-stop-timeout=900 \
--enable-dht=false --listen-port=6882 \
--file-allocation=none --auto-file-renaming=false \
--allow-overwrite=false --max-tries=8 --retry-wait=30 \
--console-log-level=warn \
> /data/aria2/aria2.log 2>&1 &
exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --no-access-log
+4
View File
@@ -0,0 +1,4 @@
fastapi==0.115.12
uvicorn==0.34.2
httpx==0.28.1
playwright==1.52.0
Executable
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")"
. ./xbox-paths.sh
XBOX_DIR=$(xbox_host_directory)
xbox_prepare_directories "$XBOX_DIR"
if [ -f ./data/queue.sqlite3 ] && [ ! -f "$XBOX_DIR/queue.sqlite3" ]; then
echo 'Wykryto poprzednia kolejke w ./data/queue.sqlite3.' >&2
echo 'Zanim uruchomisz nowa wersje, wykonaj: ./migrate-to-xbox.sh' >&2
exit 1
fi
if [ -f .env ]; then
echo '.env juz istnieje; nie nadpisuje hasel.'
else
umask 077
PASSWORD=$(head -c 48 /dev/urandom | od -An -tx1 | tr -d ' \n')
SECRET=$(head -c 48 /dev/urandom | od -An -tx1 | tr -d ' \n')
sed -e "s/^WEB_PASSWORD=.*/WEB_PASSWORD=$PASSWORD/" -e "s/^ARIA2_SECRET=.*/ARIA2_SECRET=$SECRET/" .env.example > .env
# Persist a customized host directory so future `docker compose up` uses it.
if [ "$XBOX_DIR" != /xbox ]; then
grep -v '^XBOX_HOST_DIR=' .env > .env.tmp
printf 'XBOX_HOST_DIR=%s\n' "$XBOX_DIR" >> .env.tmp
mv .env.tmp .env
fi
chmod 600 .env
echo 'Utworzono .env (login admin, losowe haslo).'
echo "Haslo panelu: $PASSWORD"
fi
echo "Dane hosta: $XBOX_DIR (pobrane: $XBOX_DIR/downloads, robocze: $XBOX_DIR/work)"
if command -v docker >/dev/null 2>&1 && [ "${XBOX_SKIP_DOCKER:-0}" != 1 ]; then
docker compose up -d --build
echo "Panel: http://ADRES_IP_WIRTUALKI:${WEB_PORT:-8088}"
else
echo 'Uruchom Docker Engine z Compose i wykonaj: docker compose up -d --build'
fi
+228
View File
@@ -0,0 +1,228 @@
"""Multi-file Archive.org scheduling, fair budgets and safe restarts."""
import asyncio
import json
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import httpx
import pytest
from fastapi.testclient import TestClient
from app import archive_download, main, store
class BytesStream(httpx.SyncByteStream):
def __init__(self, data):
self.data = data
def __iter__(self):
yield self.data
@pytest.fixture
def setup(tmp_path, monkeypatch):
monkeypatch.setattr(store, 'DATA', tmp_path)
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '50')
monkeypatch.setenv('WEB_USER', 'admin')
monkeypatch.setenv('WEB_PASSWORD', 'unit-test-password')
monkeypatch.setattr(archive_download, 'MIN_SEGMENT_BYTES', 16)
store.init_db()
return tmp_path
def multi_job(names=('A.iso', 'B.iso', 'C.iso')):
urls = [f'https://archive.org/download/test/{name}' for name in names]
return main.new_job('archive_many', json.dumps(urls), 'Multi', 'raw')
def mock_archive(monkeypatch, files, first_request_barrier=None, delay=0):
lock = threading.Lock()
active = 0
max_active = 0
seen = {}
requests = []
def handler(req):
nonlocal active, max_active
name = req.url.path.rsplit('/', 1)[-1]
data = files[name]
with lock:
active += 1
max_active = max(max_active, active)
first = name not in seen
seen[name] = seen.get(name, 0) + 1
requests.append((name, req.headers.get('range')))
try:
if first and first_request_barrier:
first_request_barrier.wait(timeout=5)
if delay:
time.sleep(delay)
span = req.headers.get('range')
if span:
match = re.fullmatch(r'bytes=(\d+)-(\d*)', span)
assert match, span
begin = int(match.group(1))
end = int(match.group(2)) if match.group(2) else len(data)-1
if begin >= len(data):
return httpx.Response(416, headers={'Content-Range': f'bytes */{len(data)}'})
assert 0 <= begin <= end < len(data)
return httpx.Response(206, stream=BytesStream(data[begin:end+1]), headers={
'Content-Range': f'bytes {begin}-{end}/{len(data)}',
'Content-Length': str(end - begin + 1), 'ETag': '"r1"'})
return httpx.Response(200, stream=BytesStream(data), headers={'Content-Length': str(len(data))})
finally:
with lock:
active -= 1
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
transport=httpx.MockTransport(handler), follow_redirects=False))
return requests, lambda: max_active
def test_three_files_start_together_and_share_fifty_connections(setup, monkeypatch):
names = ('A.iso', 'B.iso', 'C.iso')
contents = {name: bytes([i+10]) * 1024 for i, name in enumerate(names)}
barrier = threading.Barrier(3)
requests, max_connections = mock_archive(monkeypatch, contents, barrier, delay=0.005)
job = multi_job(names)
asyncio.run(archive_download.run_one(job))
for name in names:
assert (setup / 'raw' / job['id'] / name).read_bytes() == contents[name]
data = store.get(job['id'])
assert data['state'] == 'preparing'
assert data['progress'] == 100
assert data['part_index'] == 3
assert data['total'] == 3 * 1024
assert data['completed'] == 3 * 1024
assert max_connections() <= 50
assert max_connections() > 1
assert sum(1 for _, item in requests if item == 'bytes=0-0') == 3
log = (setup / 'logs' / f'{job["id"]}.download.log').read_text()
assert 'start 3 plików równolegle' in log
assert '17, 17, 16' in log
def test_existing_fifty_segment_manifests_do_not_exceed_global_limit(setup, monkeypatch):
files = {'A.iso': b'a' * 1024, 'B.iso': b'b' * 1024, 'C.iso': b'c' * 1024}
requests, max_connections = mock_archive(monkeypatch, files, delay=0.002)
job = multi_job()
root = setup / 'raw' / job['id']
root.mkdir(parents=True)
for url in json.loads(job['source']):
target = root / (url.rsplit('/', 1)[-1] + '.part')
_, manifest = archive_download._prepare_segments(target, url, 1024, '"r1"', 50)
assert len(manifest['ranges']) == 50
archive_download._transfer_many(job)
assert sum(bool(item and item != 'bytes=0-0') for _, item in requests) == 150
assert max_connections() <= 50
assert store.get(job['id'])['state'] == 'preparing'
for name, data in files.items():
assert (root / name).read_bytes() == data
def test_skip_completed_files_and_resume_incomplete_file(setup, monkeypatch):
files = {'A.iso': b'a' * 128, 'B.iso': b'b' * 128, 'C.iso': b'c' * 128}
requests, _ = mock_archive(monkeypatch, files)
job = multi_job()
raw = setup / 'raw' / job['id']
raw.mkdir(parents=True)
(raw / 'A.iso').write_bytes(files['A.iso'])
(raw / 'B.iso.part').write_bytes(files['B.iso'][:32])
archive_download._transfer_many(job)
assert store.get(job['id'])['state'] == 'preparing'
assert not any(name == 'A.iso' for name, _ in requests)
assert not any(name == 'B.iso' and value == 'bytes=0-31' for name, value in requests)
for name, data in files.items():
assert (raw / name).read_bytes() == data
def test_pause_stops_all_files_and_resume_keeps_segments(setup, monkeypatch):
files = {'A.iso': b'a' * 1024, 'B.iso': b'b' * 1024}
gate = threading.Event()
release = threading.Event()
original = httpx.MockTransport
def handler(req):
name = req.url.path.rsplit('/', 1)[-1]
data = files[name]
header = req.headers.get('range', '')
match = re.fullmatch(r'bytes=(\d+)-(\d*)', header)
assert match
start = int(match.group(1))
end = int(match.group(2)) if match.group(2) else len(data)-1
if (start, end) != (0, 0) and not gate.is_set():
gate.set()
assert release.wait(5), 'download thread was not released'
return httpx.Response(206, stream=BytesStream(data[start:end+1]), headers={
'ETag': '"r1"', 'Content-Range': f'bytes {start}-{end}/{len(data)}'})
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
transport=original(handler), follow_redirects=False))
job = multi_job(('A.iso', 'B.iso'))
with ThreadPoolExecutor(max_workers=1) as executor:
fut = executor.submit(archive_download._transfer_many, job)
assert gate.wait(5)
store.update(job['id'], state='paused')
release.set()
with pytest.raises(archive_download.ArchiveStopped):
fut.result(timeout=10)
assert store.get(job['id'])['state'] == 'paused'
store.update(job['id'], state='queued')
archive_download._transfer_many(store.get(job['id']))
assert store.get(job['id'])['state'] == 'preparing'
assert (setup / 'raw' / job['id'] / 'A.iso').read_bytes() == files['A.iso']
assert (setup / 'raw' / job['id'] / 'B.iso').read_bytes() == files['B.iso']
def test_duplicate_filenames_fail_without_writing(setup):
j = multi_job(('A/Game.iso', 'B/Game.iso'))
with pytest.raises(archive_download.ArchiveDownloadError, match='powtarzające'):
archive_download._transfer_many(j)
assert not (setup / 'raw' / j['id']).exists()
def test_retry_partial_archive_not_mistaken_for_ready_conversion(setup, monkeypatch):
j = multi_job()
raw = setup / 'raw' / j['id']
raw.mkdir(parents=True)
(raw / 'A.iso').write_bytes(b'a')
store.update(j['id'], state='error', progress=99.99)
async def idle_worker():
await asyncio.sleep(1000)
monkeypatch.setattr(main, 'worker', idle_worker)
with TestClient(main.app) as client:
response = client.post(f'/api/jobs/{j["id"]}/retry', auth=('admin', 'unit-test-password'))
assert response.status_code == 200
assert response.json()['state'] == 'queued'
def test_legacy_aria2_file_is_not_accepted_as_finished(setup, monkeypatch):
job = multi_job(('A.iso', 'B.iso'))
raw = setup / 'raw' / job['id']
raw.mkdir(parents=True)
(raw / 'A.iso').write_bytes(b'old incomplete file')
(raw / 'A.iso.aria2').write_bytes(b'aria2 control file')
files = {'A.iso': b'A' * 128, 'B.iso': b'B' * 128}
requests, _ = mock_archive(monkeypatch, files)
archive_download._transfer_many(job)
assert ('A.iso', 'bytes=0-0') in requests
assert (raw / 'A.iso').read_bytes() == files['A.iso']
assert (raw / 'B.iso').read_bytes() == files['B.iso']
def test_aggregate_progress_is_weighted_by_bytes_not_file_count(setup):
j = multi_job()
tracker = archive_download._BatchProgress(j['id'], ['A.iso', 'B.iso', 'C.iso'], 50)
tracker.report(0, 100, 100)
tracker.report(1, 10, 1000)
tracker.report(2, 20, 2000)
tracker.publish()
data = store.get(j['id'])
assert data['total'] == 3100
assert data['completed'] == 130
assert data['progress'] == 4.19
assert 'limit 50 połączeń' in data['note']
+255
View File
@@ -0,0 +1,255 @@
"""Deterministic tests for multi-range authenticated Archive.org downloads."""
import re
import httpx
import pytest
from app import archive_download, main, store
@pytest.fixture
def env(tmp_path, monkeypatch):
monkeypatch.setattr(store, 'DATA', tmp_path)
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '4')
monkeypatch.setattr(archive_download, 'MIN_SEGMENT_BYTES', 16)
monkeypatch.setattr(archive_download.time, 'sleep', lambda _: None)
store.init_db()
return tmp_path
def job():
return main.new_job('url', 'https://archive.org/download/demo/Game.iso', 'Demo', 'raw')
class BytesStream(httpx.SyncByteStream):
def __init__(self, data):
self.data = data
def __iter__(self):
yield self.data
def server(monkeypatch, data, custom=None, etag='"release-1"'):
requests = []
def handler(req):
requests.append(req)
if custom:
res = custom(req)
if res is not None:
return res
header = req.headers.get('range')
if not header:
return httpx.Response(200, stream=BytesStream(data), headers={'Content-Length': str(len(data))})
match = re.fullmatch(r'bytes=(\d+)-(\d*)', header)
assert match, header
start = int(match.group(1))
end = int(match.group(2)) if match.group(2) else len(data) - 1
assert end < len(data) and 0 <= start <= end
return httpx.Response(206, stream=BytesStream(data[start:end + 1]), headers={
'Content-Range': f'bytes {start}-{end}/{len(data)}',
'Content-Length': str(end - start + 1), 'ETag': etag})
transport = httpx.MockTransport(handler)
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
transport=transport, follow_redirects=False))
return requests
def final_path(env, j):
return env / 'raw' / j['id'] / 'Game.iso'
def test_segments_reconstruct_in_order_and_show_ranges(env, monkeypatch):
data = bytes(range(128))
requests = server(monkeypatch, data)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert store.get(j['id'])['state'] == 'preparing'
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
spans = [req.headers['range'] for req in requests]
assert spans[0] == 'bytes=0-0'
assert sorted(spans[1:]) == ['bytes=0-31', 'bytes=32-63', 'bytes=64-95', 'bytes=96-127']
assert all(req.headers.get('if-range') == '"release-1"' for req in requests[1:])
assert '4 połączeń HTTP Range' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
def test_existing_serial_prefix_is_reused(env, monkeypatch):
data = bytes(range(200))
requests = server(monkeypatch, data)
j = job()
folder = env / 'raw' / j['id']
folder.mkdir(parents=True)
(folder / 'Game.iso.part').write_bytes(data[:40])
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert len(requests) == 5
assert all(int(r.headers['range'].split('=')[1].split('-')[0]) >= 40 for r in requests[1:])
def test_interrupted_segment_resumes_only_missing_bytes(env, monkeypatch):
data = bytes(range(128))
failed = {'once': False}
monkeypatch.setattr(archive_download, 'CHUNK_BYTES', 4)
class BrokenStream(httpx.SyncByteStream):
def __iter__(self):
yield data[:5]
raise httpx.ReadError('connection reset')
def break_once(req):
if req.headers.get('range') == 'bytes=0-31' and not failed['once']:
failed['once'] = True
return httpx.Response(206, stream=BrokenStream(), headers={
'Content-Range': 'bytes 0-31/128', 'ETag': '"release-1"'})
return None
requests = server(monkeypatch, data, custom=break_once)
monkeypatch.setattr(archive_download, 'MAX_ATTEMPTS', 1)
j = job()
with pytest.raises(archive_download.ArchiveDownloadError, match='Segment'):
archive_download._transfer_one(j)
assert (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
assert failed['once']
requests.clear()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert any(r.headers.get('range') == 'bytes=4-31' for r in requests)
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
def test_server_ignores_segment_range_and_serial_fallback(env, monkeypatch):
data = bytes(range(128))
def ignore(req):
if req.headers.get('range') not in (None, 'bytes=0-0'):
return httpx.Response(200, stream=BytesStream(data))
return None
requests = server(monkeypatch, data, custom=ignore)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert any('range' not in r.headers for r in requests)
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
def test_http_503_one_segment_retries_without_losing_other_parts(env, monkeypatch):
data = bytes(range(128))
failed = {'once': False}
def temporary(req):
if req.headers.get('range') == 'bytes=32-63' and not failed['once']:
failed['once'] = True
return httpx.Response(503, headers={'Retry-After': '1'})
return None
requests = server(monkeypatch, data, custom=temporary)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert sum(req.headers.get('range') == 'bytes=32-63' for req in requests) == 2
assert 'HTTP 503' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
def test_interrupted_assembly_is_rebuilt_safely(env, monkeypatch):
data = bytes(range(128))
server(monkeypatch, data)
original = archive_download._assemble_segments
failed = {'once': False}
def broken_assembly(jid, partial, destination, folder, manifest):
if not failed['once']:
failed['once'] = True
manifest['assembling'] = True
archive_download._write_manifest(folder, manifest)
partial.write_bytes(b'partial-assembly')
raise OSError('simulated crash')
return original(jid, partial, destination, folder, manifest)
monkeypatch.setattr(archive_download, '_assemble_segments', broken_assembly)
j = job()
with pytest.raises(OSError, match='simulated crash'):
archive_download._transfer_one(j)
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
def test_small_file_uses_single_transfer(env, monkeypatch):
data = b'hello world'
requests = server(monkeypatch, data)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert len(requests) == 2
assert requests[0].headers['range'] == 'bytes=0-0'
assert 'range' not in requests[1].headers
def test_archive_connections_can_be_disabled(env, monkeypatch):
data = bytes(range(128))
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '1')
requests = server(monkeypatch, data)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert len(requests) == 1
assert 'range' not in requests[0].headers
def test_changed_etag_forces_clean_serial_restart(env, monkeypatch):
data = bytes(range(128))
def changed(req):
if req.headers.get('range') not in (None, 'bytes=0-0'):
value = req.headers['range'].split('=', 1)[1]
lo, hi = [int(x) for x in value.split('-')]
return httpx.Response(206, stream=BytesStream(data[lo:hi + 1]), headers={
'Content-Range': f'bytes {lo}-{hi}/128', 'ETag': '"changed"'})
return None
requests = server(monkeypatch, data, custom=changed)
j = job()
# This prefix must be discarded when the origin presents conflicting validators.
raw = env / 'raw' / j['id']
raw.mkdir(parents=True)
(raw / 'Game.iso.part').write_bytes(b'old-data')
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
assert any('range' not in req.headers for req in requests)
def test_archive_rejects_unexpected_compression(env, monkeypatch):
def compressed(_request):
return httpx.Response(206, headers={
'Content-Encoding': 'gzip', 'Content-Range': 'bytes 0-0/128'})
server(monkeypatch, bytes(range(128)), custom=compressed)
j = job()
with pytest.raises(archive_download.ArchiveDownloadError, match='Accept-Encoding'):
archive_download._transfer_one(j)
assert not final_path(env, j).exists()
def test_invalid_connection_settings_clamped(env, monkeypatch):
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '100')
assert archive_download._connection_limit() == 50
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '-1')
assert archive_download._connection_limit() == 1
monkeypatch.setenv('ARCHIVE_CONNECTIONS', 'invalid')
assert archive_download._connection_limit() == 50
def test_default_fifty_parallel_ranges_complete(env, monkeypatch):
"""The new default really runs 50 transfers, not only displaying that value."""
monkeypatch.delenv('ARCHIVE_CONNECTIONS', raising=False)
assert archive_download._connection_limit() == 50
data = bytes(range(256)) * 4 # Enough to split into 50 ranges in this test
requests = server(monkeypatch, data)
j = job()
archive_download._transfer_one(j)
assert final_path(env, j).read_bytes() == data
spans = [request.headers['range'] for request in requests]
assert spans[0] == 'bytes=0-0'
assert len(spans) == 51 # Range probe plus 50 segments
assert len(set(spans[1:])) == 50
assert store.get(j['id'])['state'] == 'preparing'
assert '50 połączeń HTTP Range' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
+124
View File
@@ -0,0 +1,124 @@
import asyncio
import io
import os
import uuid
import zipfile
from pathlib import Path
import pytest
from app import common, store, pipeline, download
@pytest.fixture
def isolated(tmp_path, monkeypatch):
monkeypatch.setattr(store, 'DATA', tmp_path)
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
monkeypatch.setattr(pipeline, 'RAW', tmp_path / 'raw')
monkeypatch.setattr(pipeline, 'READY', tmp_path / 'ready')
monkeypatch.setattr(pipeline, 'WORK', tmp_path / 'work')
monkeypatch.setattr(pipeline, 'LOGS', tmp_path / 'logs')
store.init_db()
return tmp_path
def test_uris():
assert common.validate_url('https://archive.org/download/demo/game.iso')[0] == 'url'
assert common.validate_url('https://vimm.net/vault/12345')[0] == 'vimm'
assert common.validate_url('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d')[0] == 'magnet'
for url in ('file:///etc/passwd', 'http://127.0.0.1/admin',
'https://user:password@example.org', 'https://localhost/file',
'ftp://example.org/file'):
with pytest.raises(ValueError):
common.validate_url(url)
def test_archive_id_and_escape():
assert common.parse_archive_id('https://archive.org/details/some-item') == 'some-item'
assert common.archive_download_url('some-item', 'Game (1).7z').endswith('Game%20%281%29.7z')
for name in ('../target.iso', '/etc/password', 'foo/../../abc', 'a\\b', ''):
with pytest.raises(ValueError):
common.archive_download_url('item', name)
for name in ('../../secret', '/absolute/file', 'C:\\Windows\\System32\\x', 'folder/../x'):
with pytest.raises(pipeline.ProcessingError):
pipeline.safe_archive_path(name)
def test_database_queue(isolated):
for t in ('Test1', 'Test2'):
store.create({'id': t, 'title': t, 'source_type': 'url',
'source': 'https://example.org/test.zip', 'format': 'xex'})
assert len(store.list_jobs()) == 2
assert store.next_job()['id'] == 'Test1'
store.update('Test1', state='done', progress=100)
assert store.next_job()['id'] == 'Test2'
def test_process_xex_zip(isolated):
jid = str(uuid.uuid4())
store.create({'id': jid, 'title': 'Demo Game', 'source_type': 'import',
'source': 'game.zip', 'format': 'xex'})
raw = isolated / 'raw' / jid
raw.mkdir(parents=True)
with zipfile.ZipFile(raw / 'game.zip', 'w') as archive:
archive.writestr('folder/default.xex', b'This is a fake XEX for queue testing')
archive.writestr('folder/data/readme.txt', b'hello')
asyncio.run(pipeline.process_job(store.get(jid)))
job = store.get(jid)
assert job['state'] == 'done'
assert (Path(job['output']) / 'Gra' / 'default.xex').exists()
assert (Path(job['output']) / 'Gra' / 'data' / 'readme.txt').read_bytes() == b'hello'
def test_zip_traversal_rejected(isolated):
jid = 'traversal'
store.create({'id': jid, 'title': 'Bad', 'source_type': 'import',
'source': 'bad.zip', 'format': 'xex'})
raw = isolated / 'raw' / jid
raw.mkdir(parents=True)
with zipfile.ZipFile(raw / 'bad.zip', 'w') as archive:
archive.writestr('../../pwned', b'nope')
with pytest.raises(pipeline.ProcessingError):
asyncio.run(pipeline.process_job(store.get(jid)))
assert not (isolated / 'pwned').exists()
def test_raw_copy(isolated):
jid = 'simple'
store.create({'id': jid, 'title': 'Keep File', 'source_type': 'import',
'source': 'anything.bin', 'format': 'raw'})
raw = isolated / 'raw' / jid
raw.mkdir(parents=True)
(raw / 'anything.bin').write_bytes(b'abc')
asyncio.run(pipeline.process_job(store.get(jid)))
assert (Path(store.get(jid)['output']) / 'files' / 'anything.bin').read_bytes() == b'abc'
def test_multipart_urls():
job={'source_type':'archive_many', 'source':'["https://host/A.part1.rar", "https://host/A.part2.rar"]'}
assert len(download.source_urls(job)) == 2
assert download.target_filename('https://example.com/Gra%20XYZ.iso') == 'Gra XYZ.iso'
def test_api_auth_and_add(isolated, monkeypatch):
from fastapi.testclient import TestClient
from app import main
monkeypatch.setenv('WEB_USER', 'admin')
monkeypatch.setenv('WEB_PASSWORD', 'secretsecret1234')
async def idle_worker():
await asyncio.sleep(10000)
monkeypatch.setattr(main, 'worker', idle_worker)
with TestClient(main.app) as client:
assert client.get('/').status_code == 401
assert client.get('/api/jobs').status_code == 401
auth = ('admin', 'secretsecret1234')
assert client.get('/', auth=auth).status_code == 200
r = client.post('/api/jobs', auth=auth,
json={'url': 'https://example.com/DEMO.iso','format': 'xex'})
assert r.status_code == 200, r.text
assert r.json()['title'] == 'DEMO.iso'
assert client.get('/api/jobs', auth=auth).json()[0]['id'] == r.json()['id']
assert client.post('/api/jobs', auth=auth,
json={'url':'file:///etc/passwd','format':'xex'}).status_code == 400
assert client.post('/api/jobs', auth=auth,
json={'url':'https://example.org/foo.iso','format':'oops'}).status_code == 400
+255
View File
@@ -0,0 +1,255 @@
import asyncio
import base64
import json
import os
from pathlib import Path
import httpx
import pytest
from fastapi.responses import PlainTextResponse
from fastapi.testclient import TestClient
from app import archive_auth, archive_download, bt, common, download, main, store, torrent_meta
def bencode(item):
if isinstance(item, int):
return b'i' + str(item).encode() + b'e'
if isinstance(item, str):
item = item.encode()
if isinstance(item, bytes):
return str(len(item)).encode() + b':' + item
if isinstance(item, (list, tuple)):
return b'l' + b''.join(bencode(x) for x in item) + b'e'
if isinstance(item, dict):
return b'd' + b''.join(bencode(k) + bencode(v) for k, v in sorted(item.items())) + b'e'
raise TypeError(item)
def sample_torrent():
return bencode({b'announce': b'https://tracker.example/announce', b'info': {
b'name': b'MyGame', b'files': [
{b'length': 100, b'path': [b'Game.iso']},
{b'length': 40, b'path': [b'readme.txt']},
{b'length': 12, b'path': [b'..padding', b'pad'], b'attr': b'p'},
]}})
@pytest.fixture
def isolated(tmp_path, monkeypatch):
monkeypatch.setattr(store, 'DATA', tmp_path)
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
monkeypatch.setenv('WEB_USER', 'admin')
monkeypatch.setenv('WEB_PASSWORD', 'mysecretpassword')
monkeypatch.setenv('ARIA2_SECRET', 'secret-rpc-key')
store.init_db()
async def idle_worker():
await asyncio.sleep(10000)
monkeypatch.setattr(main, 'worker', idle_worker)
return tmp_path
def test_torrent_metadata_parser_and_magnet_selection(tmp_path):
file = tmp_path / 'test.torrent'
file.write_bytes(sample_torrent())
meta = torrent_meta.read_torrent(file)
assert [row['name'] for row in meta['files']] == [
'MyGame/Game.iso', 'MyGame/readme.txt', 'MyGame/..padding/pad']
assert meta['files'][0]['index'] == 1
assert meta['files'][2]['padding'] is True
assert torrent_meta.suggested_indices('magnet:?xt=urn:btih:123&so=15,0-1', 20) == [1, 2, 16]
def test_torrent_path_traversal_rejected(tmp_path):
file = tmp_path / 'bad.torrent'
file.write_bytes(bencode({b'info': {b'name': b'Game', b'files': [
{b'length': 5, b'path': [b'..', b'secret']}]}}))
with pytest.raises(torrent_meta.MetadataError):
torrent_meta.read_torrent(file)
with pytest.raises(torrent_meta.MetadataError):
torrent_meta.decode_bencode(b'd3:fooi1eeTRAILING')
def test_archive_session_secrets_and_logout(isolated):
archive_auth._save_session('test@archive.org', {
'logged-in-user': 'member%40archive.org', 'logged-in-sig': 'signedsession'})
path = archive_auth.session_path()
assert oct(path.stat().st_mode & 0o777) == '0o600'
assert archive_auth.session_status() == {'authenticated': True, 'email': 'test@archive.org'}
assert archive_auth.session_cookies().get('logged-in-sig') == 'signedsession'
archive_auth.logout()
assert archive_auth.session_status()['authenticated'] is False
assert not path.exists()
def test_archive_xauthn_login_no_password_saved(isolated, monkeypatch):
original = httpx.AsyncClient
def handler(request):
assert request.method == 'POST'
assert request.url.host == 'archive.org'
assert b'password=not-saved' in request.content
return httpx.Response(200, json={'success': True, 'values': {'cookies': {
'logged-in-user': 'user%40example.org; Path=/',
'logged-in-sig': 'sig%3Dabc; Path=/'}}})
transport = httpx.MockTransport(handler)
monkeypatch.setattr(archive_auth.httpx, 'AsyncClient', lambda **kw: original(**kw, transport=transport))
status = asyncio.run(archive_auth.login('user@example.org', 'not-saved'))
assert status['authenticated'] is True
content = archive_auth.session_path().read_text()
assert 'not-saved' not in content
assert 'sig%3Dabc' in content
def test_archive_relay_redirect_cookie_scoping_and_range(isolated, monkeypatch):
archive_auth._save_session('user@example.org', {
'logged-in-user': 'member', 'logged-in-sig': 'secret-cookie'})
originals = httpx.AsyncClient
seen = []
def handler(request):
seen.append((request.url.host, request.headers.get('cookie'), request.headers.get('range')))
if request.url.host == 'archive.org':
return httpx.Response(302, headers={'Location': 'https://ia808.us.archive.org/download/asset.zip'})
class FakeStream(httpx.AsyncByteStream):
async def __aiter__(self):
yield b'bcd'
return httpx.Response(206, headers={'Content-Length': '3', 'Content-Range': 'bytes 1-3/4',
'Content-Type': 'application/octet-stream'}, stream=FakeStream())
transport = httpx.MockTransport(handler)
monkeypatch.setattr(archive_auth.httpx, 'AsyncClient', lambda **kw: originals(**kw, transport=transport))
async def call():
response = await archive_auth.stream_archive('https://archive.org/download/item/asset.zip', 'GET',
{'range': 'bytes=1-3'})
data = b''
async for part in response.body_iterator:
data += part
return response, data
response, data = asyncio.run(call())
assert response.status_code == 206
assert data == b'bcd'
assert len(seen) == 2
assert all('secret-cookie' in (item[1] or '') for item in seen)
assert all(item[2] == 'bytes=1-3' for item in seen)
assert archive_auth.allowed_redirect('https://attacker.example/') is False
assert archive_auth.allowed_redirect('http://archive.org/') is False
def test_magnet_inspection_selection_and_signed_archive_proxy(isolated, monkeypatch):
started = []
stopped = []
async def fake_start(preview_id, magnet):
started.append((preview_id, magnet))
async def fake_stop(preview_id):
stopped.append(preview_id)
monkeypatch.setattr(bt, 'start_inspection', fake_start)
monkeypatch.setattr(bt, 'stop_inspection', fake_stop)
monkeypatch.setattr(bt, 'inspect_state', lambda pid: {'connections': 2, 'known_peers': 5})
with TestClient(main.app) as client:
auth = ('admin', 'mysecretpassword')
magnet = 'magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&dn=Test&so=1'
r = client.post('/api/magnet/inspect', auth=auth, json={'magnet': magnet})
assert r.status_code == 200, r.text
pid = r.json()['preview_id']
status = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
assert status['status'] == 'loading'
assert status['connections'] == 2 and status['known_peers'] == 5
assert started == [(pid, magnet)]
folder = torrent_meta.inspect_dir(pid)
(folder / 'hash.torrent').write_bytes(sample_torrent())
info = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
assert info['status'] == 'ready'
assert info['suggested'] == [2]
assert info['files'][1]['name'] == 'MyGame/readme.txt'
assert client.post('/api/magnet/jobs', auth=auth,
json={'preview_id': pid, 'files': [4], 'format': 'raw'}).status_code == 400
assert client.post('/api/magnet/jobs', auth=auth,
json={'preview_id': pid, 'files': [3], 'format': 'raw'}).status_code == 400
r = client.post('/api/magnet/jobs', auth=auth,
json={'preview_id': pid, 'files': [2], 'format': 'raw'})
assert r.status_code == 200, r.text
job = r.json()
assert job['source_type'] == 'torrent_selected'
assert (isolated / 'torrents' / (job['id'] + '.torrent')).read_bytes() == sample_torrent()
assert json.loads(job['source'])['selection'] == '2'
assert pid in stopped
archive_job = main.new_job('archive_many', json.dumps(['https://archive.org/download/test/game.iso']), 'Demo', 'raw')
uri = '/internal/archive/' + archive_job['id'] + '/0'
assert client.get(uri).status_code == 403
assert client.get(uri, headers={'X-X360-Proxy-Token': 'invalid'}).status_code == 403
async def fake_stream(url, method, headers):
assert url == 'https://archive.org/download/test/game.iso'
return PlainTextResponse('proxy-ok')
monkeypatch.setattr(archive_auth, 'stream_archive', fake_stream)
token = archive_auth.signed_header(archive_job['id'], 0).split(': ', 1)[1]
assert client.get(uri, headers={'X-X360-Proxy-Token': token}).text == 'proxy-ok'
assert client.post('/api/jobs', auth=auth,
json={'url': magnet, 'format': 'raw'}).status_code == 400
def test_wrapped_and_html_escaped_magnet():
hash_value = 'c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d'
pasted = "'**magnet**:?xt=urn:btih:" + hash_value + "&amp;dn=Minerva_Myrient\\\n&amp;so=15&amp;tr=udp%3A%2F%2Ftracker.ope\\\nnbittorrent.com%3A6969%2Fannounce'"
kind, cleaned = common.validate_url(pasted)
assert kind == 'magnet'
assert cleaned.startswith('magnet:?xt=urn:btih:' + hash_value + '&dn=')
assert '&so=15&tr=' in cleaned
assert 'tracker.openbittorrent.com' in cleaned
assert '\\' not in cleaned and '&amp;' not in cleaned and '\n' not in cleaned
assert torrent_meta.suggested_indices(cleaned, 20) == [16]
with pytest.raises(ValueError):
common.validate_url('magnet:?xt=urn:btih:abcdef')
def test_upload_torrent_and_queue_selection(isolated, monkeypatch):
async def fake_rpc(method, *args):
if method == 'addTorrent':
return '123456'
if method == 'tellStatus':
return {'status': 'active', 'connections': '0'}
return 'ok'
monkeypatch.setattr(download, 'rpc', fake_rpc)
with TestClient(main.app) as client:
auth = ('admin', 'mysecretpassword')
uploaded = client.post('/api/magnet/upload', auth=auth,
content=sample_torrent(), headers={'Content-Type': 'application/x-bittorrent'})
assert uploaded.status_code == 200, uploaded.text
pid = uploaded.json()['preview_id']
info = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
assert info['status'] == 'ready'
assert info['files'][0]['name'] == 'MyGame/Game.iso'
assert info['suggested'] == []
job = client.post('/api/magnet/jobs', auth=auth,
json={'preview_id': pid, 'files': [1], 'format': 'xex'})
assert job.status_code == 200, job.text
assert (isolated / 'torrents' / (job.json()['id'] + '.torrent')).exists()
assert client.post('/api/magnet/upload', auth=auth,
content=b'garbage', headers={'Content-Type': 'application/x-bittorrent'}).status_code == 400
assert client.post('/api/magnet/upload', auth=auth,
content=sample_torrent(), headers={'Content-Type': 'application/json'}).status_code == 415
def test_metadata_status_includes_connections_and_timeout(isolated, monkeypatch):
stopped = []
async def fake_start(preview_id, magnet):
pass
async def fake_stop(preview_id):
stopped.append(preview_id)
monkeypatch.setattr(bt, 'start_inspection', fake_start)
monkeypatch.setattr(bt, 'stop_inspection', fake_stop)
monkeypatch.setattr(bt, 'inspect_state', lambda pid: {'connections': 0, 'known_peers': 0})
magnet='magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=15'
with TestClient(main.app) as client:
auth=('admin','mysecretpassword')
resp=client.post('/api/magnet/inspect',auth=auth,json={'magnet':magnet})
assert resp.status_code == 200
pid=resp.json()['preview_id']
status=client.get('/api/magnet/inspect/'+pid,auth=auth).json()
assert status['status']=='loading' and status['connections']==0
request_path=torrent_meta.inspect_dir(pid)/'request.json'
state=json.loads(request_path.read_text())
state['created'] -= 310
request_path.write_text(json.dumps(state))
expired=client.get('/api/magnet/inspect/'+pid,auth=auth).json()
assert expired['status']=='error'
assert pid in stopped
+138
View File
@@ -0,0 +1,138 @@
"""Regression checks for host-side /xbox storage and safe upgrade migration."""
import os
import shutil
import subprocess
from pathlib import Path
import pytest
from app import main
PROJECT = Path(__file__).resolve().parents[1]
def _copy_scripts(dst):
for name in ('xbox-paths.sh', 'migrate-to-xbox.sh', 'setup.sh', '.env.example'):
shutil.copy2(PROJECT / name, dst / name)
def _run_script(project, name, root, **extra):
env = os.environ.copy()
env.update(XBOX_HOST_DIR=str(root), XBOX_MIGRATION_OFFLINE='1', XBOX_SKIP_DOCKER='1', **extra)
return subprocess.run(['sh', name], cwd=project, env=env,
capture_output=True, text=True, timeout=30)
def test_config_uses_host_and_dedicated_downloads():
compose = (PROJECT / 'compose.yaml').read_text()
assert 'source: ${XBOX_HOST_DIR:-/xbox}' in compose
assert 'source: ${XBOX_HOST_DIR:-/xbox}/downloads' in compose
assert 'target: /data/raw' in compose
assert 'target: /data' in compose
assert 'create_host_path: false' in compose
assert 'TMPDIR: /data/work/tmp' in compose
assert './data:/data' not in compose
assert 'XBOX_HOST_DIR=/xbox' in (PROJECT / '.env.example').read_text()
assert '--dir=/data/raw' in (PROJECT / 'entrypoint.sh').read_text()
@pytest.mark.skipif(not shutil.which('rsync'), reason='rsync not installed on test host')
def test_migrate_preserves_original_data_and_queue(tmp_path):
project = tmp_path / 'old project'
project.mkdir()
_copy_scripts(project)
host = tmp_path / 'host volume'
host.mkdir()
data = project / 'data'
(data / 'raw' / 'task-1').mkdir(parents=True)
(data / 'raw' / 'task-1' / 'demo.iso.part').write_bytes(b'12345')
(data / 'work' / 'task-1').mkdir(parents=True)
(data / 'work' / 'task-1' / 'unpacked.iso').write_bytes(b'unpacked')
(data / 'ready' / 'xex' / 'demo').mkdir(parents=True)
(data / 'ready' / 'xex' / 'demo' / 'default.xex').write_bytes(b'xex')
(data / 'private').mkdir()
(data / 'private' / 'archive-session.json').write_text('{"session": "example"}')
(data / 'queue.sqlite3').write_bytes(b'queue')
(data / 'queue.sqlite3-wal').write_bytes(b'wal')
(data / 'aria2').mkdir()
(data / 'aria2' / 'session.txt').write_text('/data/raw/task-1/demo.iso')
result = _run_script(project, 'migrate-to-xbox.sh', host)
assert result.returncode == 0, result.stderr
assert (host / 'downloads' / 'task-1' / 'demo.iso.part').read_bytes() == b'12345'
assert (host / 'work' / 'task-1' / 'unpacked.iso').read_bytes() == b'unpacked'
assert (host / 'ready' / 'xex' / 'demo' / 'default.xex').read_bytes() == b'xex'
assert (host / 'queue.sqlite3').read_bytes() == b'queue'
assert (host / 'queue.sqlite3-wal').read_bytes() == b'wal'
assert (host / 'private' / 'archive-session.json').is_file()
assert (host / 'aria2' / 'session.txt').read_text() == '/data/raw/task-1/demo.iso'
assert not (host / 'raw').exists()
assert not (host / '.x360-migration-in-progress').exists()
assert (data / 'raw' / 'task-1' / 'demo.iso.part').read_bytes() == b'12345'
assert (data / 'queue.sqlite3').read_bytes() == b'queue'
@pytest.mark.skipif(not shutil.which('rsync'), reason='rsync not installed on test host')
def test_migration_does_not_replace_existing_queue(tmp_path):
project = tmp_path / 'project'
project.mkdir()
_copy_scripts(project)
host = tmp_path / 'xbox'
host.mkdir()
(project / 'data').mkdir()
(project / 'data' / 'queue.sqlite3').write_bytes(b'old')
(host / 'queue.sqlite3').write_bytes(b'new')
result = _run_script(project, 'migrate-to-xbox.sh', host)
assert result.returncode != 0
assert (host / 'queue.sqlite3').read_bytes() == b'new'
assert not (host / '.x360-migration-in-progress').exists()
def test_setup_does_not_silently_drop_existing_database(tmp_path):
project = tmp_path / 'project'
project.mkdir()
_copy_scripts(project)
host = tmp_path / 'xbox'
host.mkdir()
(project / 'data').mkdir()
(project / 'data' / 'queue.sqlite3').write_bytes(b'important')
result = _run_script(project, 'setup.sh', host)
assert result.returncode != 0
assert 'migrate-to-xbox.sh' in result.stderr
assert not (project / '.env').exists()
def test_setup_persists_custom_host_path(tmp_path):
project = tmp_path / 'project'
project.mkdir()
_copy_scripts(project)
host = tmp_path / 'external-host-share'
host.mkdir()
result = _run_script(project, 'setup.sh', host)
assert result.returncode == 0, result.stderr
assert f'XBOX_HOST_DIR={host}' in (project / '.env').read_text()
assert (host / 'downloads').is_dir()
assert (host / 'work' / 'tmp').is_dir()
assert (host / 'ready').is_dir()
def test_missing_host_directory_fails_without_creating(tmp_path):
project = tmp_path / 'project'
project.mkdir()
_copy_scripts(project)
host = tmp_path / 'not-mounted'
result = _run_script(project, 'setup.sh', host)
assert result.returncode != 0
assert not host.exists()
assert not (project / '.env').exists()
@pytest.mark.asyncio
async def test_system_shows_actual_host_path(tmp_path, monkeypatch):
from app import store
monkeypatch.setattr(store, 'DATA', tmp_path)
(tmp_path / 'raw').mkdir(parents=True, exist_ok=True)
monkeypatch.setenv('XBOX_HOST_DIR', '/xbox')
result = await main.system()
assert result['host_data_dir'] == '/xbox'
assert result['total'] > 0
+164
View File
@@ -0,0 +1,164 @@
import asyncio
import json
import os
import sys
from pathlib import Path
import httpx
import pytest
from app import archive_auth, archive_download, bt, main, store
@pytest.fixture
def isolated(tmp_path, monkeypatch):
monkeypatch.setattr(store, 'DATA', tmp_path)
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
monkeypatch.setenv('ARIA2_SECRET', 'test')
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '1') # Old single-stream behavior is covered separately.
store.init_db()
return tmp_path
def create_archive(url='https://archive.org/download/item/Game.iso'):
return main.new_job('url', url, 'Game', 'raw')
def fake_http_client(monkeypatch, handler):
transport = httpx.MockTransport(handler)
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
transport=transport, cookies=archive_auth.session_cookies(), follow_redirects=False))
monkeypatch.setattr(archive_download.time, 'sleep', lambda delay: None)
class BytesStream(httpx.SyncByteStream):
def __init__(self, data):
self.data = data
def __iter__(self):
yield self.data
def test_archive_transient_http_500_is_retried_and_logged(isolated, monkeypatch):
archive_auth._save_session('user@example.org', {
'logged-in-user': 'user', 'logged-in-sig': 'cookie-secret'})
requests = []
def handler(request):
requests.append(request)
if len(requests) < 3:
return httpx.Response(500, text='upstream temporarily broken')
return httpx.Response(200, stream=BytesStream(b'ISO-CONTENT'), headers={'Content-Length': '11'})
fake_http_client(monkeypatch, handler)
job = create_archive()
archive_download._transfer_one(job)
assert store.get(job['id'])['state'] == 'preparing'
assert (isolated / 'raw' / job['id'] / 'Game.iso').read_bytes() == b'ISO-CONTENT'
assert len(requests) == 3
assert all('cookie-secret' in req.headers.get('cookie', '') for req in requests)
text = (isolated / 'logs' / (job['id'] + '.download.log')).read_text()
assert 'HTTP 500' in text and 'Ukończono' in text
assert 'cookie-secret' not in text
def test_archive_range_resume_with_redirect(isolated, monkeypatch):
requests = []
def handler(request):
requests.append((request.url.host, request.headers.get('range')))
if request.url.host == 'archive.org':
return httpx.Response(302, headers={'Location': 'https://ia444.us.archive.org/download/item/Game.iso'})
return httpx.Response(206, stream=BytesStream(b'DEF'), headers={'Content-Range': 'bytes 3-5/6',
'Content-Length': '3'})
fake_http_client(monkeypatch, handler)
job = create_archive()
raw = isolated / 'raw' / job['id']
raw.mkdir(parents=True)
(raw / 'Game.iso.part').write_bytes(b'ABC')
archive_download._transfer_one(job)
assert (raw / 'Game.iso').read_bytes() == b'ABCDEF'
assert not (raw / 'Game.iso.part').exists()
assert requests == [('archive.org', 'bytes=3-'), ('ia444.us.archive.org', 'bytes=3-')]
def test_archive_403_is_explicit_and_not_retried(isolated, monkeypatch):
seen = []
def handler(request):
seen.append(1)
return httpx.Response(403, text='denied')
fake_http_client(monkeypatch, handler)
job = create_archive()
with pytest.raises(archive_download.ArchiveDownloadError, match='HTTP 403'):
archive_download._transfer_one(job)
assert len(seen) == 1
def test_archive_persistent_500_reports_origin_failure(isolated, monkeypatch):
seen = []
def handler(request):
seen.append(1)
return httpx.Response(500)
fake_http_client(monkeypatch, handler)
job = create_archive()
with pytest.raises(archive_download.ArchiveDownloadError, match='HTTP 500 po 5 próbach'):
archive_download._transfer_one(job)
assert len(seen) == 5
def test_archive_cancel_keeps_partial_files(isolated, monkeypatch):
job = create_archive()
store.update(job['id'], state='paused')
fake_http_client(monkeypatch, lambda req: pytest.fail('Do not download paused job'))
with pytest.raises(archive_download.ArchiveStopped):
archive_download._transfer_one(job)
def test_bt_helper_subprocess_inspection_and_fetch(isolated, monkeypatch):
"""Exercise process supervisor and status semantics without real peers."""
from tests.test_features import sample_torrent
helper = isolated / 'fake-helper'
# Tests the Python supervisor lifecycle, not the compiled Go / network engine.
helper.write_text('#!' + sys.executable + '\n' + '''
import json, os, sys
from pathlib import Path
args = sys.argv[1:]
mode = args[0]
def param(x): return args[args.index(x)+1]
state = Path(param('-status'))
state.parent.mkdir(parents=True, exist_ok=True)
if mode == 'inspect':
Path(param('-out')).write_bytes(bytes.fromhex(os.environ['FAKE_TORRENT_HEX']))
state.write_text(json.dumps({'status':'ready','connections':2}))
else:
output=Path(param('-out'))
output.mkdir(parents=True, exist_ok=True)
(output/'Game.iso').write_bytes(b'foo')
state.write_text(json.dumps({'status':'done','downloaded':3,'total':3}))
''')
helper.chmod(0o755)
monkeypatch.setenv('FAKE_TORRENT_HEX', sample_torrent().hex())
monkeypatch.setenv('BT_HELPER', str(helper))
monkeypatch.setattr(bt, '_DOWNLOADS', {})
monkeypatch.setattr(bt, '_INSPECTIONS', {})
async def check():
pid = 'a' * 32
folder = isolated / 'metadata' / pid
folder.mkdir(parents=True)
proc = await bt.start_inspection(pid, 'magnet:?xt=urn:btih:' + 'a'*40)
assert await proc.wait() == 0
assert bt.inspect_state(pid)['status'] == 'ready'
await bt.stop_inspection(pid)
job = main.new_job('torrent_selected', json.dumps({'selection':'1'}), 'Game', 'raw')
(isolated/'torrents').mkdir()
(isolated/'torrents'/(job['id']+'.torrent')).write_bytes(sample_torrent())
await bt.start_download(job)
proc = bt._DOWNLOADS[job['id']]
assert await proc.wait() == 0
await bt.poll_download(store.get(job['id']))
assert store.get(job['id'])['state'] == 'preparing'
await bt.shutdown()
asyncio.run(check())
+54
View File
@@ -0,0 +1,54 @@
"""Browser test: pasted magnet cleanup, live diagnostics and .torrent upload."""
from pathlib import Path
from playwright.sync_api import sync_playwright
root = Path(__file__).resolve().parents[1] / 'app' / 'static'
html = (root / 'index.html').read_text().replace('<link rel="stylesheet" href="/static/style.css">', '').replace('<script src="/static/app.js" defer></script>', '')
css = (root / 'style.css').read_text()
js = (root / 'app.js').read_text()
with sync_playwright() as playwright:
browser = playwright.chromium.launch(executable_path='/usr/bin/chromium', args=['--no-sandbox', '--disable-dev-shm-usage'])
page = browser.new_page(viewport={'width': 1280, 'height': 900})
errors = []
page.on('pageerror', lambda err: errors.append(str(err)))
page.set_content(html)
page.add_style_tag(content=css)
page.evaluate('''() => {
window.__seenMagnet = null;
window.fetch = async (url, options = {}) => {
const u = String(url), m = options.method || 'GET';
let reply = {};
if (u === '/api/system') reply = {free:1000000000,total:2000000000};
else if (u === '/api/jobs' && m === 'GET') reply = [];
else if (u === '/api/archive/session') reply = {authenticated:false};
else if (u === '/api/magnet/inspect' && m === 'POST') {
window.__seenMagnet = JSON.parse(options.body).magnet;
reply = {status:'loading',preview_id:'a'.repeat(32)};
}
else if (u === '/api/magnet/upload') reply = {status:'ready',preview_id:'b'.repeat(32)};
else if (u.includes('/magnet/inspect/' + 'a'.repeat(32)) && m === 'GET')
reply = {status:'loading',elapsed:35,connections:0,aria_status:'active'};
else if (u.includes('/magnet/inspect/' + 'b'.repeat(32)) && m === 'GET')
reply = {status:'ready',name:'Demo',suggested:[],files:[{index:1,name:'Demo/Game.iso',size:1024,padding:false}]};
else if (u.startsWith('/api/magnet/inspect/') && m === 'POST') reply = {ok:true};
return new Response(JSON.stringify(reply), {status:200,headers:{'Content-Type':'application/json'}});
};
}''')
page.add_script_tag(content=js)
raw = "'**magnet**:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&amp;so=15\\\n&amp;tr=udp%3A%2F%2Fexample.com%2Fannounce'"
page.locator('#urls').fill(raw)
page.locator('#linksForm button[type=submit]').click()
page.locator('#magnetDialog').wait_for(state='visible')
page.locator('#magnetStatus').get_by_text('Brak połączeń', exact=False).wait_for()
parsed = page.evaluate('window.__seenMagnet')
assert parsed.startswith('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=15&tr='), parsed
assert '\n' not in parsed and '&amp;' not in parsed
page.screenshot(path='/mnt/data/x360-magnet-diagnostics-preview.png', full_page=True)
page.locator('#torrentFileDialog').set_input_files({'name':'demo.torrent','mimeType':'application/x-bittorrent','buffer':b'd4:infod4:name4:Demoe e'})
page.locator('#magnetResults').wait_for(state='visible')
assert page.locator('#magnetFiles input[type=checkbox]').count() == 1
page.screenshot(path='/mnt/data/x360-magnet-upload-preview.png', full_page=True)
assert not errors, errors
browser.close()
print('UI magnet fallback PASS')
+63
View File
@@ -0,0 +1,63 @@
"""Browser UI smoke test with mocked fetch; Chromium policies block navigation."""
import json
from pathlib import Path
from playwright.sync_api import sync_playwright
ROOT = Path(__file__).resolve().parents[1] / 'app' / 'static'
html = (ROOT / 'index.html').read_text()
html = html.replace('<link rel="stylesheet" href="/static/style.css">', '')
html = html.replace('<script src="/static/app.js" defer></script>', '')
css = (ROOT / 'style.css').read_text()
js = (ROOT / 'app.js').read_text()
with sync_playwright() as p:
browser = p.chromium.launch(executable_path='/usr/bin/chromium',
args=['--no-sandbox', '--disable-dev-shm-usage'])
page = browser.new_page(viewport={'width': 1250, 'height': 950})
errors = []
page.on('pageerror', lambda error: errors.append(str(error)))
page.set_content(html)
page.add_style_tag(content=css)
page.evaluate(r'''() => {
window.__submitted = [];
window.fetch = async (url, opts = {}) => {
const u = String(url);
const m = opts.method || 'GET';
let value = {ok:true};
if (u === '/api/system') value = {free:100000000000, total:200000000000};
else if (u === '/api/jobs' && m === 'GET') value = [];
else if (u === '/api/archive/session' && m === 'GET') value = {authenticated:window.__loggedIn === true,email:window.__loggedIn ? 'demo@example.org' : ''};
else if (u === '/api/archive/session' && m === 'POST') {window.__loggedIn=true;value={authenticated:true,email:'demo@example.org'};}
else if (u === '/api/magnet/inspect' && m === 'POST') value = {preview_id:'a'.repeat(32),status:'loading'};
else if (u.startsWith('/api/magnet/inspect/') && m === 'GET') value = {status:'ready',name:'Sample',suggested:[2],files:[
{index:1,name:'Sample/Game.iso',size:7210000000,padding:false},
{index:2,name:'Sample/Manual.pdf',size:15313,padding:false},
{index:3,name:'Sample/.pad',size:700,padding:true}
]};
else if (u === '/api/magnet/jobs' && m === 'POST') {window.__submitted.push(JSON.parse(opts.body));value={ok:true};}
return new Response(JSON.stringify(value), {status:200,headers:{'Content-Type':'application/json'}});
};
}''')
page.add_script_tag(content=js)
page.locator('button[data-tab="archive"]').click()
assert page.locator('#archiveEmail').is_visible()
page.locator('#archiveEmail').fill('demo@example.org')
page.locator('#archivePassword').fill('dummy-password')
page.locator('#archiveLoginForm button[type=submit]').click()
page.locator('#archiveLogout').wait_for(state='visible')
page.screenshot(path='/mnt/data/x360-archive-auth-preview.png', full_page=True)
page.locator('button[data-tab="links"]').click()
page.locator('#urls').fill('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=1')
page.locator('#linksForm [type=submit]').click()
page.locator('#magnetResults').wait_for(state='visible')
assert page.locator('#magnetFiles input[type="checkbox"]').count() == 3
assert page.locator('#magnetFiles input[type="checkbox"]').nth(1).is_checked()
page.screenshot(path='/mnt/data/x360-magnet-select-preview.png', full_page=True)
page.locator('#magnetFiles input[type="checkbox"]').nth(0).check()
page.locator('#magnetAdd').click()
page.locator('#magnetDialog').wait_for(state='hidden')
assert page.evaluate('window.__submitted') == [
{'preview_id':'a'*32,'files':[2,1],'title':'','format':'xex'}]
assert errors == [], errors
browser.close()
print('UI smoke PASS: Archive login form, torrent modal, file selection, submit')
+55
View File
@@ -0,0 +1,55 @@
#!/bin/sh
# Shared POSIX shell helpers. Must be sourced from the project directory.
xbox_host_directory() {
if [ -n "${XBOX_HOST_DIR:-}" ]; then
root=$XBOX_HOST_DIR
elif [ -f .env ]; then
root=$(sed -n 's/^XBOX_HOST_DIR=//p' .env | tail -n 1)
# Handle normal quoted .env values without executing file contents.
case "$root" in
'"'*'"') root=${root#\"}; root=${root%\"} ;;
"'"*"'") root=${root#\'}; root=${root%\'} ;;
esac
root=${root:-/xbox}
else
root=/xbox
fi
case "$root" in
/*) ;;
*) echo "XBOX_HOST_DIR musi byc sciezka bezwzgledna: $root" >&2; return 1 ;;
esac
if [ ! -d "$root" ]; then
echo "Nie znaleziono $root na hoscie Linux." >&2
echo "Utworz go przed uruchomieniem, np.: sudo mkdir -p '$root'" >&2
echo "Jesli to montowany dysk, upewnij sie, ze jest zamontowany." >&2
return 1
fi
root=$(cd "$root" && pwd -P) || return 1
if [ "$root" = / ]; then
echo 'Odmowa uzycia / jako katalogu danych.' >&2
return 1
fi
printf '%s\n' "$root"
}
xbox_prepare_directories() {
root=$1
for rel in downloads work work/tmp work/cache ready imports logs metadata torrents bt aria2; do
if [ -L "$root/$rel" ]; then
echo "Odmowa uzycia dowiazania symbolicznego: $root/$rel" >&2
return 1
fi
if ! mkdir -p "$root/$rel"; then
echo "Brak uprawnien zapisu w $root. Ustaw prawa katalogu na hoscie." >&2
return 1
fi
done
if [ -L "$root/private" ]; then
echo "Odmowa uzycia dowiazania symbolicznego: $root/private" >&2
return 1
fi
if [ ! -d "$root/private" ]; then
mkdir -m 700 "$root/private" || return 1
fi
}