first commit
This commit is contained in:
commit
58cfd95600
35 files changed
+5357
No files matched your search
@@ -0,0 +1,6 @@
|
||||
.env
|
||||
data/
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
*.pyc
|
||||
.git/
|
||||
@@ -0,0 +1,9 @@
|
||||
WEB_USER=admin
|
||||
WEB_PASSWORD=CHANGE_ME_TO_A_LONG_RANDOM_PASSWORD
|
||||
ARIA2_SECRET=CHANGE_ME_TO_ANOTHER_LONG_RANDOM_TOKEN
|
||||
WEB_PORT=8088
|
||||
# Domyslnie panel dostepny w LAN. Dla localhost ustaw 127.0.0.1
|
||||
WEB_BIND=0.0.0.0
|
||||
# Automatyczna obsluga Vimm przez prawdziwa widoczna kontrolke pobierania
|
||||
ENABLE_VIMM_BROWSER=true
|
||||
VIMM_DOWNLOAD_TIMEOUT=7200
|
||||
@@ -0,0 +1,4 @@
|
||||
.env
|
||||
.venv
|
||||
venv
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
# Natywne linuksowe narzedzia do obslugi obrazow konsoli.
|
||||
FROM rust:1.88-bookworm AS converters
|
||||
ENV CARGO_BUILD_JOBS=2
|
||||
RUN git clone --depth 1 --branch v1.8.1 https://github.com/iliazeus/iso2god-rs.git /src/iso2god \
|
||||
&& cd /src/iso2god && cargo build --release --locked --bin iso2god
|
||||
RUN git clone --depth 1 --branch v0.8.3 https://github.com/antangelo/xdvdfs.git /src/xdvdfs \
|
||||
&& cd /src/xdvdfs && cargo build --release --locked -p xdvdfs-cli
|
||||
|
||||
FROM golang:1.24-bookworm AS bt-helper
|
||||
WORKDIR /src
|
||||
COPY bt/ /src/
|
||||
RUN GOMAXPROCS=2 go mod tidy && GOMAXPROCS=2 CGO_ENABLED=0 go build -p 2 -trimpath -o /out/x360-bt .
|
||||
|
||||
FROM python:3.12-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends aria2 p7zip-full unar chromium ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY --from=bt-helper /out/x360-bt /usr/local/bin/x360-bt
|
||||
COPY --from=converters /src/iso2god/target/release/iso2god /usr/local/bin/iso2god
|
||||
COPY --from=converters /src/xdvdfs/target/release/xdvdfs /usr/local/bin/xdvdfs
|
||||
WORKDIR /opt/x360
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app/ app/
|
||||
COPY entrypoint.sh ./
|
||||
RUN chmod +x entrypoint.sh && mkdir -p /data
|
||||
EXPOSE 8000
|
||||
CMD ["/opt/x360/entrypoint.sh"]
|
||||
@@ -0,0 +1,202 @@
|
||||
# X360 Web Queue — Linux / Docker / Xbox 360 RGH
|
||||
|
||||
Samodzielna aplikacja WWW do pobierania i przygotowywania posiadanych legalnie obrazow gier Xbox 360. Wszystkie zadania wykonywane sa na serwerze/VM, niezaleznie od otwartej karty przegladarki.
|
||||
|
||||
## Start
|
||||
|
||||
Wymagania: **Linux x86_64**, Docker Engine z wtyczka Docker Compose, minimum ok. 2 GB RAM (zalecane 4 GB podczas budowania), duzo miejsca na dane.
|
||||
|
||||
```bash
|
||||
unzip x360-web-queue-xbox.zip
|
||||
cd x360-web-queue
|
||||
# /xbox musi istniec i byc zapisywalny na hoscie Linux
|
||||
./setup.sh
|
||||
```
|
||||
|
||||
`setup.sh` przygotuje katalogi w `/xbox`, sam utworzy `.env` z mocnymi haslami i wykona `docker compose up -d --build`. Przy pierwszym starcie budowane sa `xdvdfs`, `iso2god-rs` i pomocniczy klient BitTorrent Go. Wirtualka potrzebuje dostepu do Docker Hub, GitHub, crates.io i proxy.golang.org. Na gospodarzu nie musisz instalowac Rusta ani Go.
|
||||
|
||||
Otworz `http://IP_TWOJEJ_WIRTUALKI:8088` i zaloguj sie jako `admin` za pomoca hasla wypisanego przez `setup.sh` (takze w `.env`).
|
||||
|
||||
Jesli Docker nie byl jeszcze zainstalowany, zainstaluj go, a pozniej:
|
||||
|
||||
```bash
|
||||
./setup.sh
|
||||
# lub, po utworzeniu .env:
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Aktualny stan i logi:
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
docker compose logs -f --tail=100
|
||||
```
|
||||
|
||||
## Interfejs
|
||||
|
||||
Plaski panel w ciemnych barwach, lokalne ikony SVG (bez CDN), filtrowanie kolejki i widok mobilny. Interfejs nie wymaga dodatkowych zaleznosci JS i zachowuje API dotychczasowej aplikacji.
|
||||
|
||||
## Zrodla
|
||||
|
||||
- **HTTPS/HTTP**: wklej bezposredni adres pliku ISO/ZIP/7z/RAR, rowniez z serwisow umozliwiajacych normalny bezposredni transfer.
|
||||
- **Magnet**: panel automatycznie usuwa z wklejonego adresu HTML `&`, przypadkowe znaki `\` i lamanie linii, sprawdza hash, a nastepnie pobiera wylacznie metadane torrenta. Metadane pobiera klient Go (`anacrolix/torrent`, taki sam stos jak `minerva-pick`), bez pobierania plikow gry. Okno pokazuje czas, liczbe polaczen i znanych peerow. Po 5 minutach bez metadanych pojawia sie blad zamiast bezterminowego oczekiwania. Parametr `so=` ustawia wstepne zaznaczenie (od 0). Transfer danych rusza dopiero po wyborze plikow.
|
||||
- **Plik .torrent**: opcja `Wczytaj .torrent` w zakladce Link / magnet oraz w oknie podgladu. Pozwala natychmiast uzyskac liste plikow bez czekania na peerow. Maks. 32 MiB metadanych. Wczytanie .torrent *nie rozpoczyna* pobierania zawartosci, dopiero przycisk `Pobierz wybrane`. Dziala z duzymi katalogami Minerva (widok pokazuje maks. 500 plikow naraz, filtr pozwala wyszukac reszte).
|
||||
- **Internet Archive**: zakladka umozliwia zalogowanie e-mailem i haslem Archive.org (mechanizm xauthn stosowany w oficjalnych narzedziach), nastepnie wklej `https://archive.org/details/IDENTYFIKATOR` i wybierz pliki. Aplikacja przechowuje **tylko ciasteczka sesji**, a nie haslo. Pliki sa pobierane bezposrednio przez backend z sesja Archive.org, bez starego przekaznika aria2. **Wszystkie pliki jednego zadania sa pobierane rownolegle, maksymalnie 50 jednoczesnie**. Duze pliki sa dodatkowo dzielone na segmenty HTTP Range. Limit `ARCHIVE_CONNECTIONS=50` dotyczy **sumy polaczen HTTP wszystkich plikow zadania**, nie kazdego pliku osobno. Postep segmentow jest zachowywany po restarcie; pobieranie wznawia tylko brakujace bajty. Gdy serwer nie obsluguje Range albo zwraca niezgodne zakresy, aplikacja automatycznie przechodzi na jedno polaczenie. Dzialaja ponowienia HTTP 500/502/503/504/429 i kontrola wersji pliku za pomoca ETag / Last-Modified, jesli serwer je podaje. Ciasteczka sesji nie trafiaja do aria2 ani do logow. Logowanie nie zapewnia dostepu do plikow niedostepnych dla Twojego konta. Nie obchodzimy ograniczen serwisu.
|
||||
- **Vimm**: adres `https://vimm.net/vault/NUMER` przechodzi do **eksperymentalnego** adaptera uruchamiajacego Chromium. Wykorzystywany jest tylko faktycznie widoczny i dostepny przycisk `Download`. Brak omijania CAPTCHA, DRM, niedostepnych gier, antybota, limitow, kolejek i robots.txt. Gdy Vimm zmieni strone lub odrzuci automatyczny dostep, pobieranie nie bedzie dzialalo. Postep pobierania w trybie Vimm nie jest dostepny w % az do konca. Vimm uzywa jednej sesji naraz. Mozesz wylaczyc adapter w `.env`: `ENABLE_VIMM_BROWSER=false`.
|
||||
- **Import lokalny**: skopiuj plik do `/xbox/imports/`, odswiez karte Import i dodaj zadanie.
|
||||
|
||||
Nie obslugujemy bypassowania zabezpieczen ani pobierania tresci wycofanych z serwisu. Uzywaj tylko plikow, do ktorych masz prawo.
|
||||
|
||||
## Przetwarzanie
|
||||
|
||||
| Wejscie | Format XEX | Format GOD |
|
||||
|---|---|---|
|
||||
| `.iso` Xbox 360 | `xdvdfs unpack` | `iso2god` |
|
||||
| `.zip`, `.7z`, `.rar`, `.tar` zawierajace ISO | Rozpakuj, nastepnie XEX | Rozpakuj, nastepnie GOD |
|
||||
| Folder z `default.xex` spakowany w ZIP/7z | Kopia folderu | `xdvdfs pack`, potem `iso2god` |
|
||||
| Link magnet zawierajacy ISO | j.w. | j.w. |
|
||||
|
||||
Archiwa wieloczesciowe: wybierz wszystkie czesci w **jednym** zadaniu (zakladka Archive.org), a nie osobno. Obsługiwane sa m.in. `.part1.rar`, `.7z.001`. Nie kazdy wariant kompresji RAR dziala w p7zip — wtedy zadanie zakonczy sie czytelnym bledem. Hasla do zaszyfrowanych archiwow nie sa obslugiwane.
|
||||
|
||||
Dla XEX program sprawdza obecnosc `default.xex`. Dla GOD konwerter generuje katalogi TitleID/content type i pliki Data. Na dysku masz:
|
||||
|
||||
```text
|
||||
/xbox/ # katalog na hoscie VM
|
||||
downloads/<JOB_ID>/ # pobrane ISO, archiwa i torrenty (kontener: /data/raw)
|
||||
downloads/<JOB_ID>/*.part # nieukonczone transfery HTTP
|
||||
downloads/<JOB_ID>/*.part.segments/ # segmenty Archive.org do wznowienia
|
||||
work/<JOB_ID>/ # rozpakowywanie i ISO tymczasowe (kontener: /data/work)
|
||||
work/tmp/ # systemowe pliki tymczasowe (TMPDIR)
|
||||
work/cache/ # cache Chromium/narzedzi (XDG_CACHE_HOME)
|
||||
ready/xex/<GRA>/ # wynik XEX
|
||||
ready/god/<GRA>/ # wynik GOD
|
||||
imports/ # reczny import ISO lub archiwum
|
||||
torrents/<JOB_ID>.torrent # metadane do wznowienia
|
||||
metadata/<PODGLAD_ID>/ # metadane magneta i engine.log
|
||||
private/archive-session.json # sesja Archive.org (0600)
|
||||
logs/<JOB_ID>.log # log konwersji
|
||||
logs/<JOB_ID>.bt.log # log BT
|
||||
logs/<JOB_ID>.download.log # log transferu Archive
|
||||
bt/<JOB_ID>.json # status klienta Go
|
||||
aria2/ # stan i sesje aria2
|
||||
queue.sqlite3 # kolejka i historia zadan
|
||||
```
|
||||
|
||||
|
||||
Na Xbox przez FTP skopiuj **zawartosc** folderu `Gra` do `Hdd1:/Games/Nazwa/` i uruchom `default.xex` (albo wykonaj skan biblioteki Aurora). Dla GOD skopiuj `Content/0000000000000000/...` do `Hdd1:/Content/0000000000000000/...` z zachowaniem struktury. Gry wielodyskowe moga wymagac osobnej konfiguracji w Aurora.
|
||||
|
||||
## Przyspieszenie Internet Archive (HTTP Range)
|
||||
|
||||
Nowe instalacje maja `ARCHIVE_CONNECTIONS=50` w `.env`. Przy aktualizacji istniejacego projektu dopisz lub zmien te ustawienie w `.env`, np.:
|
||||
|
||||
```dotenv
|
||||
ARCHIVE_CONNECTIONS=50
|
||||
```
|
||||
|
||||
Dozwolone jest **1-50**. `1` wylacza segmentowanie i ogranicza pobieranie do jednego pliku na raz. W zadaniu z 3 plikami przy limicie 50 przydzial wyniesie np. **17 + 17 + 16 polaczen**. Kazdy plik zaczyna sie pobierac od razu; kolejne pliki (pow. 50) uruchamiaja sie w zwolnionych slotach. Przydzial na plik jest ustalany na poczatku danej grupy plikow; gdy plik sie skonczy, pozostale zachowuja swoje segmenty i przydzielone sloty, a wolny slot moze rozpoczac nastepny plik. Mniejsze pliki oraz wczesniej podzielone na 8 segmentow moga wykorzystac mniej polaczen. Jesli Archive.org odrzuca zadania (np. HTTP 429) albo transfer zwalnia, ogranicz liczbe np. do 8, 16 lub 24. Po zmianie wykonaj `docker compose up -d --build`. Panel pokazuje liczbe plikow aktywnych, zbiorczy postep i sumaryczna predkosc, a w `/xbox/logs/<JOB_ID>.download.log` zapisany jest przydzial polaczen i bledy kazdego pliku.
|
||||
|
||||
Segmenty uruchamiaja sie tylko dla plikow dostatecznie duzych (co najmniej dwa zakresy po 8 MiB; 50 segmentow wymaga minimum 400 MiB pozostalych do pobrania), po potwierdzeniu obslugi `HTTP 206 Content-Range` przez serwer. Program nie omija ograniczen konta ani zabezpieczen serwisu. Gdy Archive.org ogranicza polaczenia (np. `HTTP 429`) aplikacja uwzglednia `Retry-After` w sekundach; obniz `ARCHIVE_CONNECTIONS`, jezeli problem sie powtarza.
|
||||
|
||||
Zadania, ktore maja juz zapisane segmenty z poprzedniej wersji, zachowuja pierwotna liczbe segmentow, aby mozna je bylo bezpiecznie wznowic. Po aktualizacji rownolegle rusza takze pobieranie pozostalych plikow z tego samego zadania. Nie trzeba kasowac pobranych czesci. Aby zastosowac nowy podzial 50 do juz podzielonego pliku, trzeba utworzyc nowe zadanie. Aktualizacja nie nadpisuje istniejacego `.env`: po podmianie projektu ustaw tam `ARCHIVE_CONNECTIONS=50` i zrestartuj kontener.
|
||||
|
||||
**Dysk:** przed konwersja potrzebne jest miejsce na segmenty i kolejna pelna kopie skladanego pliku. Sprawdz wolne miejsce w VM, zwlaszcza przy obrazach ISO po kilka GB.
|
||||
|
||||
## Eksploatacja
|
||||
|
||||
Aplikacja przechowuje kolejke SQLite, ma wznowienia HTTP i BitTorrent, jednego workera kolejki, do trzech podgladow magnet i przyciski stop/wznow/ponow. Wszystkie dane sa na hoscie w `/xbox` (domyslnie, bez wolumenow anonimowych), wiec pozostaja po restarcie kontenera. Zmiana lokalizacji jest mozliwa przez `XBOX_HOST_DIR` w `.env`. aria2 obsluguje zwykle linki HTTP/HTTPS, natomiast Go obsluguje magnety i pliki torrent. Czesciowo pobrane bloki torrenta sa weryfikowane po restarcie; Archive.org wznawia przez `.part` oraz katalog `.part.segments/` z oddzielnymi zakresami. Podczas koncowego laczenia czesci moze byc potrzebne dodatkowe miejsce na **jedna pelna kopie pliku**. Po poprawnym scaleniu czesci tymczasowe sa usuwane.
|
||||
|
||||
**Bezpieczenstwo:** panel uzywa HTTP Basic Auth (haslo przesylane przy kazdym zadaniu HTTP). Formularz Archive.org rowniez przesyla haslo do lokalnego serwera. Zabezpiecz panel przed logowaniem poprzez HTTPS lub VPN; w lokalnym kontenerze haslo IA jest tylko przekazywane do `https://archive.org/services/xauthn/?op=login` i nie jest zapisywane. Sesja jest przechowywana w `/xbox/private` z uprawnieniami 0700/0600. Nie umieszczaj `/xbox/private` w publicznych kopiach projektu. Nie wystawiaj `8088` bezposrednio do internetu. Do dostepu z zewnatrz uzyj VPN (np. WireGuard/Tailscale) albo reverse proxy z TLS. Domyslnie port nasluchuje na LAN (`WEB_BIND=0.0.0.0`); aby ograniczyc dostep do maszyny lokalnej, ustaw `WEB_BIND=127.0.0.1` w `.env` i wykonaj `docker compose up -d`.
|
||||
|
||||
## Ograniczenia obecnej wersji
|
||||
|
||||
- Brak bezposredniego FTP do konsoli i automatycznego rozpoznawania wszystkich wielodyskowych instalacji.
|
||||
- Brak wyszukiwarki gier — podajesz wlasne URL/magnet/item Archive.
|
||||
- Nie kazde konto Archive.org ma dostep do kazdego elementu. Nie ma gwarancji logowania, gdy serwis zmieni xauthn lub wymaga dodatkowej weryfikacji. Duzy transfer z Archive.org jest wielopolaczeniowy, jesli serwer akceptuje Range. Wieksza liczba polaczen nie gwarantuje wzrostu predkosci; zalezy to od serwera, trasy sieciowej i limitow konta. Jesli pojawiaja sie HTTP 429 albo 503, zmniejsz liczbe polaczen. Powtarzajacy sie HTTP 500 po 5 probach nadal oznacza blad po stronie serwera lub dostepu do pliku.
|
||||
- Odczyt metadanych magneta wymaga dostepnych peerow. Jezeli liczba polaczen i znanych peerow pozostaje 0, sprawdz DHT/UDP lub uzyj pliku .torrent. Plik .torrent eliminuje czekanie na metadane, ale nie gwarantuje dostepnosci zawartosci. Brak dostepnych peerow oznacza, ze zadne ustawienie interfejsu nie gwarantuje pobrania metadanych. Limit podgladu: 5 min.
|
||||
- W Docker Compose sa mapowane porty BitTorrent 6881/tcp i 6881/udp. Jezeli inna aplikacja uzywa 6881 na hoście, ustaw `BT_PORT=16881` w `.env` i przebuduj kontener. Przy NAT wirtualki moze byc potrzebne przekierowanie portow w hypervisorze/routerze. DHT wymaga dzialajacego ruchu UDP.
|
||||
- Brak gwarancji dla stron wymagajacych JS/autoryzacji poza obslugiwanymi integracjami; sam aria2 nie imituje przegladarki.
|
||||
- Uwaga na miejsce: w najgorszym przypadku podczas rozpakowania i konwersji potrzeba **2–4x** rozmiaru ISO.
|
||||
- Nie testowano pobierania rzeczywistych plikow z Archive.org na prywatnym koncie, odzyskiwania wskazanego magneta z DHT ani konwersji realnego ISO w tym srodowisku. Testy API, sesji, HTTP Range, ponawiania HTTP 500, nadzorcy klienta Go, parsera torrenta i kolejki mozna uruchomic `PYTHONPATH=. pytest -q` z zaleznosciami z requirements i pytest. Nie przetestowano obrazu Dockera ani transferu na zywej sieci; wymagaja testu na VM.
|
||||
|
||||
## Licencje i projekty
|
||||
|
||||
Backend projektu: kod dostarczony tutaj. Zaleznosci i narzedzia sa instalowane niezaleznie na zasadach ich licencji: [aria2](https://aria2.github.io/), [anacrolix/torrent](https://github.com/anacrolix/torrent), [xdvdfs](https://github.com/antangelo/xdvdfs), [iso2god-rs](https://github.com/iliazeus/iso2god-rs), [Playwright](https://playwright.dev/), [FastAPI](https://fastapi.tiangolo.com/).
|
||||
|
||||
## Diagnostyka: magnet / torrent / Archive.org
|
||||
|
||||
Po aktualizacji najpierw uruchom nowe zadanie `.torrent` lub magnet. Stare zadania moga miec zapisane `gid` aria2; nowszy backend przy pierwszym uruchomieniu zatrzymuje ich transfery. W razie bledu nacisnij **Log** w panelu, a dodatkowo sprawdz:
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
docker compose logs --tail=120 x360-web
|
||||
# podstaw odpowiedni ID zadania z panelu lub katalogu /xbox/logs:
|
||||
ls -lt /xbox/logs/*.bt.log /xbox/logs/*.download.log 2>/dev/null | head
|
||||
# dla metadanych magnet: /xbox/metadata/<ID>/engine.log
|
||||
```
|
||||
|
||||
Oczekiwanie na metadane z 0 peerow oznacza brak osiagalnego zrodla w danym momencie lub problem z siecia BT w VM. Wczytanie `.torrent` omija wyłącznie etap metadanych, a nie potrzebe aktywnych peerow. Go używa TCP/UDP; sprawdz DNS i wyjscie UDP w wirtualce. Przy NAT czasem przydaje sie przekierowanie `BT_PORT` przez hypervisor i router.
|
||||
|
||||
Archive.org **nie uzywa juz aria2** do plikow z `/download/`. Pliki jednego zadania ida rownolegle z lacznym limitem 1-50 polaczen HTTP (domyslnie 50), a HTTP 500 jest ponawiany maksymalnie 5 razy na probe/segment. Jesli nadal dostaniesz HTTP 500, oznacza to odpowiedz Archive.org lub posredniego serwera, nie samego panelu. Log pokazuje kazda probe, bez sesyjnych ciasteczek.
|
||||
|
||||
## Dane na dysku hosta: `/xbox`
|
||||
|
||||
Compose montuje fizyczny katalog `/xbox` do `/data` w kontenerze. Dodatkowo
|
||||
`/xbox/downloads` jest montowany do `/data/raw`, dzieki czemu wszystkie stare
|
||||
sciezki w kolejce i sesjach aria2 pozostaja zgodne. Rowniez rozpakowywanie,
|
||||
konwersja i katalog tymczasowy (`TMPDIR`) korzystaja z tego samego katalogu
|
||||
hosta, a wynik trafia do `/xbox/ready/`. Nic z pobranych danych nie zalezy
|
||||
od dysku zapisywalnego kontenera.
|
||||
|
||||
**Przed uruchomieniem musi istniec `/xbox`**. Jesli to podmontowany dysk
|
||||
lub udzial hosta, najpierw upewnij sie, ze jest dostepny i zapisywalny.
|
||||
Compose uzywa `bind.create_host_path: false`, zeby nie tworzyc po cichu
|
||||
brakujacej sciezki. **Jesli `/xbox` istnieje jako zwykly katalog pod
|
||||
niezamontowanym dyskiem, Compose nie rozpozna tego bledu** — w takim
|
||||
wypadku sprawdz `mountpoint /xbox` przed uruchomieniem.
|
||||
|
||||
Przy nowej instalacji (jezeli katalog nie istnieje, utworz go najpierw):
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /xbox
|
||||
# Tylko dla nowego katalogu na lokalnym dysku (nie zmieniaj wlasciciela udzialu SMB/NFS):
|
||||
sudo chown "$(id -un):$(id -gn)" /xbox
|
||||
# Dla udzialu/montowanego dysku zapewnij uprawnienia zapisu innym sposobem.
|
||||
cd x360-web-queue
|
||||
./setup.sh
|
||||
```
|
||||
|
||||
Jesli wolisz inna sciezke, dopisz do `.env` `XBOX_HOST_DIR=/sciezka/bezwzgledna`.
|
||||
Program nie zapisuje wtedy nic do `/xbox`, tylko do wybranej sciezki.
|
||||
Zmiana katalogu **nie przenosi** danych automatycznie.
|
||||
|
||||
### Aktualizacja istniejacej instalacji (migracja `./data`)
|
||||
|
||||
Zachowaj `.env` i `./data` przed wypakowaniem tej wersji. Rozpakuj ZIP w
|
||||
katalogu nadrzednym starego `x360-web-queue/` i wykonaj:
|
||||
|
||||
```bash
|
||||
cd x360-web-queue
|
||||
# Upewnij sie, ze /xbox jest zamontowany i ma miejsce na dotychczasowe dane.
|
||||
./migrate-to-xbox.sh
|
||||
docker compose up -d --build
|
||||
```
|
||||
|
||||
Skrypt zatrzymuje stary kontener **przed kopiowaniem**, przenosi przez kopiowanie
|
||||
katalog `./data/raw/` do `/xbox/downloads/` oraz reszte `./data/` do `/xbox/`.
|
||||
**Nie usuwa starego `./data` ani nie nadpisuje istniejacej bazy / plikow**.
|
||||
Przerwana migracje mozna ponowic, o ile nie zostala uruchomiona nowa instancja.
|
||||
Jezeli brakuje `rsync`, zainstaluj go (`sudo apt install rsync`).
|
||||
Po sprawdzeniu dzialania i kompletności pobranych plikow mozesz samodzielnie
|
||||
usunac stary `./data`, zwalniajac miejsce.
|
||||
|
||||
Weryfikacja:
|
||||
|
||||
```bash
|
||||
docker compose ps
|
||||
docker compose exec x360-web sh -c 'df -h /data/raw /data/work /data/ready; ls -ld /data/raw /data/work /data/ready'
|
||||
ls -lah /xbox/downloads /xbox/work /xbox/ready
|
||||
```
|
||||
|
||||
**Uwaga:** dla SQLite i wznawiania pobierania najlepiej uzywac lokalnego
|
||||
systemu plikow z poprawnym blokowaniem (np. ext4 lub XFS). Przy montowaniu
|
||||
SMB/NFS sprawdz blokowanie plikow, uprawnienia i stabilnosc udzialu.
|
||||
Whitespace-only changes.
@@ -0,0 +1,204 @@
|
||||
"""Internet Archive user session and restricted download relay.
|
||||
|
||||
The relay keeps IA cookies inside this process. aria2 only receives a signed,
|
||||
local endpoint, and never sees an IA session cookie or the account password.
|
||||
"""
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
import httpx
|
||||
from fastapi.responses import PlainTextResponse, Response, StreamingResponse
|
||||
|
||||
from . import store
|
||||
|
||||
|
||||
class ArchiveAuthError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def session_path():
|
||||
return store.DATA / 'private' / 'archive-session.json'
|
||||
|
||||
|
||||
def _read_session():
|
||||
try:
|
||||
value = json.loads(session_path().read_text(encoding='utf-8'))
|
||||
cookies = value['cookies']
|
||||
if not all(isinstance(cookies.get(k), str) and cookies[k] for k in ('logged-in-user', 'logged-in-sig')):
|
||||
return None
|
||||
return value
|
||||
except (FileNotFoundError, KeyError, TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def session_status():
|
||||
value = _read_session()
|
||||
return {'authenticated': bool(value), 'email': value.get('email', '') if value else ''}
|
||||
|
||||
|
||||
def session_cookies():
|
||||
cookies = httpx.Cookies()
|
||||
value = _read_session()
|
||||
if value:
|
||||
for key in ('logged-in-user', 'logged-in-sig'):
|
||||
cookies.set(key, value['cookies'][key], domain='.archive.org', path='/')
|
||||
return cookies
|
||||
|
||||
|
||||
def _cookie_value(value):
|
||||
# The official xauthn API includes cookie attributes after the value.
|
||||
if not isinstance(value, str):
|
||||
raise ArchiveAuthError('Brak danych sesji w odpowiedzi Archive.org')
|
||||
value = value.split(';', 1)[0].strip()
|
||||
if not value or len(value) > 4096 or any(x in value for x in ('\n', '\r', '\0')):
|
||||
raise ArchiveAuthError('Niepoprawne dane sesji Archive.org')
|
||||
return value
|
||||
|
||||
|
||||
def _save_session(email, cookies):
|
||||
path = session_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
os.chmod(path.parent, 0o700)
|
||||
fd, name = tempfile.mkstemp(dir=path.parent, prefix='.archive-session-', suffix='.tmp')
|
||||
try:
|
||||
with os.fdopen(fd, 'w', encoding='utf-8') as f:
|
||||
json.dump({'email': email, 'cookies': cookies}, f)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.chmod(name, 0o600)
|
||||
os.replace(name, path)
|
||||
finally:
|
||||
if os.path.exists(name):
|
||||
os.unlink(name)
|
||||
|
||||
|
||||
async def login(email, password):
|
||||
"""Authenticate using IA's documented xauthn endpoint (no password stored)."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=25, follow_redirects=False) as client:
|
||||
result = await client.post('https://archive.org/services/xauthn/', params={'op': 'login'},
|
||||
data={'email': email, 'password': password})
|
||||
result.raise_for_status()
|
||||
data = result.json()
|
||||
except (httpx.HTTPError, ValueError) as exc:
|
||||
raise ArchiveAuthError('Błąd połączenia z logowaniem Archive.org') from exc
|
||||
|
||||
if not data.get('success'):
|
||||
reason = str((data.get('values') or {}).get('reason') or data.get('error') or '')
|
||||
if reason in ('account_bad_password', 'account_not_found'):
|
||||
raise ArchiveAuthError('Nieprawidłowy adres e-mail lub hasło')
|
||||
raise ArchiveAuthError('Archive.org odrzuciło logowanie: ' + (reason[:120] or 'brak szczegółów'))
|
||||
try:
|
||||
raw = data['values']['cookies']
|
||||
cookies = {key: _cookie_value(raw[key]) for key in ('logged-in-user', 'logged-in-sig')}
|
||||
except (KeyError, TypeError) as exc:
|
||||
raise ArchiveAuthError('Logowanie udane, ale brak ciasteczek sesji w odpowiedzi') from exc
|
||||
_save_session(email, cookies)
|
||||
return session_status()
|
||||
|
||||
|
||||
def logout():
|
||||
session_path().unlink(missing_ok=True)
|
||||
|
||||
|
||||
def is_archive_download(url):
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
return parsed.scheme == 'https' and parsed.hostname in ('archive.org', 'www.archive.org') and parsed.path.startswith('/download/')
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def internal_url(job_id, index):
|
||||
return f'http://127.0.0.1:8000/internal/archive/{job_id}/{index}'
|
||||
|
||||
|
||||
def signed_header(job_id, index):
|
||||
secret = os.environ['ARIA2_SECRET'].encode('utf-8')
|
||||
message = f'archive:{job_id}:{index}'.encode('ascii')
|
||||
digest = hmac.new(secret, message, hashlib.sha256).hexdigest()
|
||||
return 'X-X360-Proxy-Token: ' + digest
|
||||
|
||||
|
||||
def verify_token(job_id, index, token):
|
||||
try:
|
||||
expected = signed_header(job_id, index).split(': ', 1)[1]
|
||||
except UnicodeEncodeError:
|
||||
return False
|
||||
return secrets.compare_digest(expected, token or '')
|
||||
|
||||
|
||||
def allowed_redirect(url):
|
||||
try:
|
||||
parts = urlsplit(url)
|
||||
name = (parts.hostname or '').lower()
|
||||
return parts.scheme == 'https' and parts.port in (None, 443) and (name == 'archive.org' or name.endswith('.archive.org'))
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
async def stream_archive(url, method, headers):
|
||||
"""Stream authorized bytes and preserve HTTP Range for aria2 resumes.
|
||||
|
||||
Only HTTPS Archive.org redirects are followed. Cookie domain scoping is
|
||||
handled by httpx, unlike forwarding a raw Cookie header through redirects.
|
||||
"""
|
||||
outbound_headers = {'Accept-Encoding': 'identity', 'User-Agent': 'X360-Queue/1.1'}
|
||||
for key in ('range', 'if-range', 'if-modified-since', 'if-none-match'):
|
||||
if key in headers:
|
||||
outbound_headers[key] = headers[key]
|
||||
client = httpx.AsyncClient(timeout=httpx.Timeout(120, connect=25), cookies=session_cookies(),
|
||||
follow_redirects=False, trust_env=False)
|
||||
current = url
|
||||
try:
|
||||
for _ in range(7):
|
||||
if not allowed_redirect(current):
|
||||
return PlainTextResponse('Nieprawidłowe przekierowanie Archive.org', status_code=502)
|
||||
request = client.build_request(method, current, headers=outbound_headers)
|
||||
remote = await client.send(request, stream=True)
|
||||
if remote.status_code in (301, 302, 303, 307, 308) and remote.headers.get('location'):
|
||||
new_url = urljoin(current, remote.headers['location'])
|
||||
await remote.aclose()
|
||||
current = new_url
|
||||
continue
|
||||
break
|
||||
else:
|
||||
return PlainTextResponse('Zbyt wiele przekierowań Archive.org', status_code=502)
|
||||
|
||||
if remote.status_code not in (200, 206, 304, 416):
|
||||
status = remote.status_code
|
||||
await remote.aclose()
|
||||
if status in (401, 403):
|
||||
return PlainTextResponse('Archive.org odmówiło dostępu. Zaloguj się i sprawdź uprawnienia do pliku.', status_code=403)
|
||||
return PlainTextResponse(f'Archive.org HTTP {status}', status_code=status if 400 <= status < 600 else 502)
|
||||
if remote.status_code == 200 and 'text/html' in remote.headers.get('content-type', '').lower():
|
||||
await remote.aclose()
|
||||
return PlainTextResponse('Archive.org zwróciło stronę HTML zamiast pliku. Sprawdź logowanie.', status_code=403)
|
||||
safe_headers = {k: v for k, v in remote.headers.items() if k.lower() in (
|
||||
'content-length', 'content-range', 'accept-ranges', 'content-disposition',
|
||||
'content-type', 'etag', 'last-modified')}
|
||||
if method == 'HEAD' or remote.status_code in (304, 416):
|
||||
await remote.aclose()
|
||||
return Response(status_code=remote.status_code, headers=safe_headers)
|
||||
|
||||
async def chunks():
|
||||
try:
|
||||
async for chunk in remote.aiter_raw(chunk_size=131072):
|
||||
yield chunk
|
||||
finally:
|
||||
await remote.aclose()
|
||||
await client.aclose()
|
||||
return StreamingResponse(chunks(), status_code=remote.status_code, headers=safe_headers,
|
||||
media_type=safe_headers.get('content-type'))
|
||||
except httpx.HTTPError:
|
||||
return PlainTextResponse('Nie można połączyć się z Archive.org', status_code=502)
|
||||
finally:
|
||||
# A streamed response owns client until the last chunk is sent.
|
||||
if 'remote' not in locals() or remote.is_closed:
|
||||
await client.aclose()
|
||||
@@ -0,0 +1,760 @@
|
||||
"""Direct, authenticated Internet Archive downloader.
|
||||
|
||||
Avoids the old aria2 -> localhost HTTP relay (which hid origin errors behind
|
||||
HTTP 500). Supports Range resumes, transient 5xx/429 retries and pause/cancel.
|
||||
Only forwards IA cookies via httpx's domain-scoped cookie jar.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from collections import deque
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
from contextlib import closing
|
||||
from threading import Event, Lock
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import httpx
|
||||
|
||||
from . import archive_auth, store
|
||||
from .download import source_urls, target_filename
|
||||
|
||||
RETRY_CODES = {408, 425, 429, 500, 502, 503, 504}
|
||||
MAX_ATTEMPTS = 5
|
||||
MAX_CONNECTIONS = 50
|
||||
MIN_SEGMENT_BYTES = 8 * 1024 * 1024
|
||||
CHUNK_BYTES = 256 * 1024
|
||||
RANGE = re.compile(r'^bytes (\d+)-(\d+)/(\d+|\*)$', re.I)
|
||||
FINISH_RANGE = re.compile(r'^bytes \*/(\d+)$', re.I)
|
||||
LOG_LOCK = Lock()
|
||||
|
||||
|
||||
class ArchiveDownloadError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ArchiveStopped(Exception):
|
||||
"""Pause or cancellation was requested, not a transfer error."""
|
||||
|
||||
|
||||
def _client():
|
||||
return httpx.Client(timeout=httpx.Timeout(90, connect=25), cookies=archive_auth.session_cookies(),
|
||||
follow_redirects=False, trust_env=True)
|
||||
|
||||
|
||||
def _check_state(jid, stop=None):
|
||||
if stop is not None and stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
current = store.get(jid)
|
||||
if not current or current['state'] in ('paused', 'cancelled', 'error'):
|
||||
raise ArchiveStopped()
|
||||
|
||||
|
||||
def _response(client, url, headers):
|
||||
current = url
|
||||
for _ in range(8):
|
||||
if not archive_auth.allowed_redirect(current):
|
||||
raise ArchiveDownloadError('Archive.org: nieprawidłowe przekierowanie poza domenę Archive.org')
|
||||
request = client.build_request('GET', current, headers=headers)
|
||||
response = client.send(request, stream=True)
|
||||
if response.status_code in (301, 302, 303, 307, 308) and 'location' in response.headers:
|
||||
next_url = urljoin(current, response.headers['location'])
|
||||
response.close()
|
||||
current = next_url
|
||||
continue
|
||||
return response
|
||||
raise ArchiveDownloadError('Archive.org: zbyt wiele przekierowań')
|
||||
|
||||
|
||||
def _log(job_id, message):
|
||||
target = store.DATA / 'logs' / (job_id + '.download.log')
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with LOG_LOCK:
|
||||
with target.open('a', encoding='utf-8') as fp:
|
||||
fp.write(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + message + '\n')
|
||||
|
||||
|
||||
def _transfer_one(job, *, batch=None, connections=None, stop=None):
|
||||
jid = job['id']
|
||||
ix = job['part_index']
|
||||
urls = source_urls(job)
|
||||
if ix >= len(urls):
|
||||
store.update(jid, state='preparing')
|
||||
return
|
||||
uri = urls[ix]
|
||||
if not archive_auth.is_archive_download(uri):
|
||||
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
|
||||
root = store.DATA / 'raw' / jid
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
filename = target_filename(uri)
|
||||
if not filename:
|
||||
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
|
||||
destination = root / filename
|
||||
partial = root / (filename + '.part')
|
||||
_check_state(jid, stop)
|
||||
if batch is None:
|
||||
store.update(jid, state='downloading', gid='', note=f'Archive.org: plik {ix+1}/{len(urls)}')
|
||||
_log(jid, f'Rozpoczęto plik {ix+1}/{len(urls)}: {filename}')
|
||||
|
||||
if connections is None:
|
||||
connections = _connection_limit()
|
||||
# An existing manifest may contain many segments even if the new budget
|
||||
# only allows one active connection. Keep those downloaded bytes.
|
||||
if (connections > 1 or _segment_dir(partial).exists()) and _parallel_transfer(
|
||||
job, ix, urls, uri, filename, partial, destination, connections, batch=batch, stop=stop):
|
||||
return
|
||||
# If the server cannot serve ranges (or parallel transfer was disabled),
|
||||
# keep the verified single-stream prefix, not an unfinished assembly.
|
||||
_discard_parallel_state(partial)
|
||||
# Existing aria2 control files are not compatible with this downloader.
|
||||
# Start from our own .part file rather than assuming an aria2 file is complete.
|
||||
with _client() as client:
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
_check_state(jid, stop)
|
||||
start = partial.stat().st_size if partial.exists() else 0
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.0', 'Accept-Encoding': 'identity'}
|
||||
if start:
|
||||
headers['Range'] = f'bytes={start}-'
|
||||
try:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(jid, f'HTTP {code} z Archive.org, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
reason = f'Archive.org HTTP {code}'
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'{reason} po {MAX_ATTEMPTS} próbach; możliwy problem serwera Archive.org')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia do pliku.')
|
||||
elif code == 404:
|
||||
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono wybranego pliku')
|
||||
elif code == 416 and start:
|
||||
m = FINISH_RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if m and int(m.group(1)) == start:
|
||||
_check_state(jid, stop)
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename} (HTTP 416 potwierdziło rozmiar)')
|
||||
return
|
||||
partial.unlink(missing_ok=True)
|
||||
continue
|
||||
elif code not in (200, 206):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
|
||||
else:
|
||||
if 'text/html' in response.headers.get('Content-Type', '').lower():
|
||||
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
|
||||
if code == 206:
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or int(match.group(1)) != start:
|
||||
raise ArchiveDownloadError('Archive.org zwróciło niezgodny zakres HTTP Range')
|
||||
total = int(match.group(3)) if match.group(3) != '*' else 0
|
||||
else:
|
||||
# Server ignored Range: do not append duplicate bytes.
|
||||
start = 0
|
||||
total = int(response.headers.get('Content-Length') or 0)
|
||||
written = start
|
||||
last_update = 0.0
|
||||
prev = written
|
||||
prev_at = time.monotonic()
|
||||
mode = 'ab' if start else 'wb'
|
||||
with partial.open(mode) as fp:
|
||||
for chunk in response.iter_raw(chunk_size=256 * 1024):
|
||||
_check_state(jid, stop)
|
||||
if not chunk:
|
||||
continue
|
||||
fp.write(chunk)
|
||||
written += len(chunk)
|
||||
now = time.monotonic()
|
||||
if now - last_update >= 0.8:
|
||||
elapsed = max(0.001, now - prev_at)
|
||||
speed = int((written - prev) / elapsed)
|
||||
prev, prev_at, last_update = written, now, now
|
||||
portion = written / total if total else 0
|
||||
if batch is not None:
|
||||
batch.report(ix, written, total)
|
||||
else:
|
||||
store.update(jid, progress=round((ix + min(portion, 1)) / len(urls) * 100, 2),
|
||||
speed=speed, total=total, completed=written,
|
||||
note=f'Archive.org: plik {ix+1}/{len(urls)}, próba {attempt+1}')
|
||||
if total and written != total:
|
||||
raise httpx.ReadError(f'Przerwany transfer: pobrano {written} z {total} bajtów')
|
||||
_check_state(jid, stop)
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename}: {written} bajtów')
|
||||
return
|
||||
except ArchiveStopped:
|
||||
raise
|
||||
except ArchiveDownloadError:
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(jid, f'Błąd transferu {type(exc).__name__}: {str(exc)[:180]}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org: przerwane połączenie po {MAX_ATTEMPTS} próbach: {type(exc).__name__}: {exc}') from exc
|
||||
# Applies to transient HTTP codes and connection exceptions.
|
||||
if batch is None:
|
||||
store.update(jid, note=f'Archive.org: ponawianie po HTTP/błędzie sieci ({attempt+2}/{MAX_ATTEMPTS})', speed=0)
|
||||
for _ in range(min(2 ** attempt, 12) * 2):
|
||||
_check_state(jid, stop)
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _finish(job, ix, urls, *, batch=None):
|
||||
jid = job['id']
|
||||
if batch is not None:
|
||||
batch.complete(ix)
|
||||
return
|
||||
if ix + 1 < len(urls):
|
||||
store.update(jid, part_index=ix+1, state='queued', gid='', speed=0,
|
||||
progress=round((ix+1)/len(urls) * 100, 2), note='Pobrano część archiwum')
|
||||
else:
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie Archive.org ukończone')
|
||||
|
||||
|
||||
|
||||
class _RangeUnsupported(Exception):
|
||||
"""A server stopped honoring byte ranges; caller can use the serial path."""
|
||||
def __init__(self, message, reset_prefix=False):
|
||||
super().__init__(message)
|
||||
self.reset_prefix = reset_prefix
|
||||
|
||||
|
||||
def _connection_limit():
|
||||
try:
|
||||
return max(1, min(MAX_CONNECTIONS, int(os.getenv('ARCHIVE_CONNECTIONS', '50'))))
|
||||
except (ValueError, OverflowError):
|
||||
return 50
|
||||
|
||||
|
||||
def _segment_dir(partial):
|
||||
return Path(str(partial) + '.segments')
|
||||
|
||||
|
||||
def _read_manifest(folder):
|
||||
try:
|
||||
item = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
return item
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _write_manifest(folder, manifest):
|
||||
folder.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
temp = folder / 'manifest.json.tmp'
|
||||
with temp.open('w', encoding='utf-8') as f:
|
||||
json.dump(manifest, f, sort_keys=True)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
temp.replace(folder / 'manifest.json')
|
||||
|
||||
|
||||
def _discard_parallel_state(partial, reset_prefix=False):
|
||||
"""Discard stale segment files without interpreting an assembled prefix as a resume."""
|
||||
folder = _segment_dir(partial)
|
||||
if not folder.exists():
|
||||
return
|
||||
manifest = _read_manifest(folder)
|
||||
if reset_prefix:
|
||||
partial.unlink(missing_ok=True)
|
||||
elif manifest and manifest.get('assembling') and partial.exists():
|
||||
prefix = manifest.get('prefix')
|
||||
if isinstance(prefix, int) and 0 <= prefix <= partial.stat().st_size:
|
||||
with partial.open('r+b') as f:
|
||||
f.truncate(prefix)
|
||||
else:
|
||||
partial.unlink(missing_ok=True)
|
||||
shutil.rmtree(folder)
|
||||
|
||||
|
||||
def _validate_ranges(ranges, prefix, total):
|
||||
if not isinstance(ranges, list) or len(ranges) < 2 or len(ranges) > MAX_CONNECTIONS:
|
||||
return False
|
||||
cursor = prefix
|
||||
for item in ranges:
|
||||
if (not isinstance(item, list) or len(item) != 2 or
|
||||
not all(isinstance(x, int) and not isinstance(x, bool) for x in item)):
|
||||
return False
|
||||
start, end = item
|
||||
if start != cursor or end < start or end >= total:
|
||||
return False
|
||||
cursor = end + 1
|
||||
return cursor == total
|
||||
|
||||
|
||||
def _prepare_segments(partial, uri, total, validator, connections):
|
||||
folder = _segment_dir(partial)
|
||||
source_hash = hashlib.sha256(uri.encode('utf-8')).hexdigest()
|
||||
manifest = _read_manifest(folder)
|
||||
valid = False
|
||||
if manifest:
|
||||
prefix = manifest.get('prefix')
|
||||
numeric_prefix = isinstance(prefix, int) and not isinstance(prefix, bool)
|
||||
prefix_present = numeric_prefix and (prefix == 0 or
|
||||
(partial.exists() and partial.stat().st_size >= prefix))
|
||||
valid = (manifest.get('version') == 1 and manifest.get('source_hash') == source_hash and
|
||||
manifest.get('total') == total and manifest.get('validator') == validator and
|
||||
isinstance(prefix, int) and not isinstance(prefix, bool) and 0 <= prefix < total and
|
||||
_validate_ranges(manifest.get('ranges'), prefix, total) and prefix_present)
|
||||
if valid and partial.exists() and partial.stat().st_size != prefix:
|
||||
# A crash during assembly is safe to recover from: parts remain intact.
|
||||
if manifest.get('assembling') and partial.stat().st_size > prefix:
|
||||
with partial.open('r+b') as f:
|
||||
f.truncate(prefix)
|
||||
else:
|
||||
valid = False
|
||||
if valid:
|
||||
return folder, manifest
|
||||
|
||||
if folder.exists():
|
||||
# Invalid or mismatched metadata cannot establish which bytes are safe.
|
||||
_discard_parallel_state(partial, reset_prefix=True)
|
||||
prefix = partial.stat().st_size if partial.exists() else 0
|
||||
if prefix > total:
|
||||
partial.unlink(missing_ok=True)
|
||||
prefix = 0
|
||||
if prefix >= total:
|
||||
return None, {'prefix': prefix, 'total': total}
|
||||
remaining = total - prefix
|
||||
n = min(connections, max(1, remaining // MIN_SEGMENT_BYTES))
|
||||
if n < 2:
|
||||
return None, None
|
||||
quotient, extra = divmod(remaining, n)
|
||||
cursor = prefix
|
||||
ranges = []
|
||||
for i in range(n):
|
||||
end = cursor + quotient + (1 if i < extra else 0) - 1
|
||||
ranges.append([cursor, end])
|
||||
cursor = end + 1
|
||||
manifest = {'version': 1, 'source_hash': source_hash, 'total': total,
|
||||
'validator': validator, 'prefix': prefix, 'ranges': ranges,
|
||||
'assembling': False}
|
||||
_write_manifest(folder, manifest)
|
||||
return folder, manifest
|
||||
|
||||
|
||||
def _retry_wait(job_id, attempt, stop, retry_after=None, external_stop=None):
|
||||
delay = min(2 ** attempt, 16)
|
||||
if retry_after and retry_after.isdigit():
|
||||
delay = min(max(1, int(retry_after)), 60)
|
||||
for _ in range(delay * 2):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
_check_state(job_id, external_stop)
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def _check_content(response):
|
||||
if 'text/html' in response.headers.get('Content-Type', '').lower():
|
||||
raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
|
||||
if response.headers.get('Content-Encoding', 'identity').lower() not in ('', 'identity'):
|
||||
raise ArchiveDownloadError('Archive.org zignorowało Accept-Encoding: identity; transfer w tym kodowaniu jest niebezpieczny')
|
||||
|
||||
|
||||
def _probe_range(job_id, uri, stop=None):
|
||||
"""Only a real 206 with a numeric total enables parallel mode."""
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity', 'Range': 'bytes=0-0'}
|
||||
with _client() as client:
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
_check_state(job_id, stop)
|
||||
try:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(job_id, f'Archive.org: sprawdzenie Range HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code} podczas sprawdzania Range po {MAX_ATTEMPTS} próbach')
|
||||
delay = response.headers.get('Retry-After')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia.')
|
||||
elif code == 404:
|
||||
raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono pliku')
|
||||
elif code == 200:
|
||||
_check_content(response)
|
||||
return None
|
||||
elif code == 206:
|
||||
_check_content(response)
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or (int(match.group(1)), int(match.group(2))) != (0, 0) or match.group(3) == '*':
|
||||
return None
|
||||
total = int(match.group(3))
|
||||
if total <= 0:
|
||||
return None
|
||||
etag = response.headers.get('ETag', '').strip()
|
||||
if etag.startswith('W/'):
|
||||
etag = ''
|
||||
validator = etag or response.headers.get('Last-Modified', '').strip()
|
||||
if not validator or len(validator) > 256 or '\n' in validator:
|
||||
validator = ''
|
||||
return total, validator
|
||||
else:
|
||||
raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
|
||||
except ArchiveDownloadError:
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(job_id, f'Archive.org: sprawdzenie Range - {type(exc).__name__}: {str(exc)[:150]}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Archive.org: brak odpowiedzi na żądanie Range po {MAX_ATTEMPTS} próbach') from exc
|
||||
delay = None
|
||||
_retry_wait(job_id, attempt, Event(), delay, stop)
|
||||
return None
|
||||
|
||||
|
||||
def _segment_worker(job_id, uri, folder, manifest, index, stop, external_stop=None):
|
||||
begin, end = manifest['ranges'][index]
|
||||
total = manifest['total']
|
||||
limit = end - begin + 1
|
||||
target = folder / f'{index:02d}.part'
|
||||
if target.exists() and target.stat().st_size > limit:
|
||||
target.unlink()
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
_check_state(job_id, external_stop)
|
||||
present = target.stat().st_size if target.exists() else 0
|
||||
if present == limit:
|
||||
return
|
||||
headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity',
|
||||
'Range': f'bytes={begin + present}-{end}'}
|
||||
if manifest['validator']:
|
||||
headers['If-Range'] = manifest['validator']
|
||||
try:
|
||||
with _client() as client:
|
||||
with closing(_response(client, uri, headers)) as response:
|
||||
code = response.status_code
|
||||
if code in RETRY_CODES:
|
||||
_log(job_id, f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code} po {MAX_ATTEMPTS} próbach')
|
||||
delay = response.headers.get('Retry-After')
|
||||
elif code in (401, 403):
|
||||
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}: brak dostępu')
|
||||
elif code == 200:
|
||||
raise _RangeUnsupported('Archive.org zignorowało zakres HTTP Range / If-Range', reset_prefix=True)
|
||||
elif code != 206:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}')
|
||||
else:
|
||||
_check_content(response)
|
||||
match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
|
||||
if not match or match.group(3) == '*' or (int(match.group(1)), int(match.group(2)), int(match.group(3))) != (begin + present, end, total):
|
||||
raise _RangeUnsupported('Archive.org zwróciło niezgodny Content-Range', reset_prefix=True)
|
||||
if manifest['validator']:
|
||||
header = 'ETag' if manifest['validator'].startswith('"') else 'Last-Modified'
|
||||
received_validator = response.headers.get(header, '').strip()
|
||||
if received_validator and received_validator != manifest['validator']:
|
||||
raise _RangeUnsupported('Archive.org zmieniło plik podczas pobierania', reset_prefix=True)
|
||||
last_check = time.monotonic()
|
||||
with target.open('ab') as f:
|
||||
for chunk in response.iter_raw(chunk_size=CHUNK_BYTES):
|
||||
if stop.is_set():
|
||||
raise ArchiveStopped()
|
||||
if time.monotonic() - last_check >= 1:
|
||||
_check_state(job_id, external_stop)
|
||||
last_check = time.monotonic()
|
||||
if not chunk:
|
||||
continue
|
||||
if f.tell() + len(chunk) > limit:
|
||||
raise _RangeUnsupported('Archive.org wysłało zbyt dużo danych w zakresie', reset_prefix=True)
|
||||
f.write(chunk)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
if target.stat().st_size != limit:
|
||||
raise httpx.ReadError('Przerwana transmisja segmentu')
|
||||
return
|
||||
except (ArchiveStopped, ArchiveDownloadError, _RangeUnsupported):
|
||||
raise
|
||||
except (httpx.HTTPError, OSError) as exc:
|
||||
_log(job_id, f'Segment {index+1}: {type(exc).__name__}: {str(exc)[:150]}, próba {attempt+1}/{MAX_ATTEMPTS}')
|
||||
if attempt == MAX_ATTEMPTS - 1:
|
||||
raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: przerwany transfer po {MAX_ATTEMPTS} próbach: {type(exc).__name__}') from exc
|
||||
delay = None
|
||||
_retry_wait(job_id, attempt, stop, delay, external_stop)
|
||||
|
||||
|
||||
def _report_parallel(job_id, ix, url_count, partial, folder, manifest, last, batch=None):
|
||||
prefix = manifest['prefix']
|
||||
downloaded = prefix
|
||||
for index, (start, end) in enumerate(manifest['ranges']):
|
||||
path = folder / f'{index:02d}.part'
|
||||
if path.exists():
|
||||
downloaded += min(end - start + 1, path.stat().st_size)
|
||||
now = time.monotonic()
|
||||
prev_size, prev_at = last
|
||||
speed = max(0, int((downloaded - prev_size) / max(0.001, now - prev_at)))
|
||||
total = manifest['total']
|
||||
if batch is None:
|
||||
store.update(job_id, completed=downloaded, total=total, speed=speed,
|
||||
progress=round((ix + min(downloaded / total, 1)) / url_count * 100, 2),
|
||||
note=f'Archive.org: plik {ix+1}/{url_count}, {len(manifest["ranges"])} połączeń')
|
||||
else:
|
||||
batch.report(ix, downloaded, total)
|
||||
return downloaded, now
|
||||
|
||||
|
||||
def _assemble_segments(job_id, partial, destination, folder, manifest, stop=None):
|
||||
prefix = manifest['prefix']
|
||||
total = manifest['total']
|
||||
for index, (start, end) in enumerate(manifest['ranges']):
|
||||
part = folder / f'{index:02d}.part'
|
||||
if not part.is_file() or part.stat().st_size != end - start + 1:
|
||||
raise ArchiveDownloadError('Niekompletne segmenty; nie można złożyć pliku')
|
||||
manifest['assembling'] = True
|
||||
_write_manifest(folder, manifest)
|
||||
with partial.open('r+b' if partial.exists() else 'w+b') as output:
|
||||
output.truncate(prefix)
|
||||
output.seek(prefix)
|
||||
for index in range(len(manifest['ranges'])):
|
||||
with (folder / f'{index:02d}.part').open('rb') as source:
|
||||
while True:
|
||||
_check_state(job_id, stop)
|
||||
block = source.read(1024 * 1024)
|
||||
if not block:
|
||||
break
|
||||
output.write(block)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
if partial.stat().st_size != total:
|
||||
raise ArchiveDownloadError('Błąd scalania pliku: niezgodny rozmiar końcowy')
|
||||
partial.replace(destination)
|
||||
shutil.rmtree(folder)
|
||||
|
||||
|
||||
def _parallel_transfer(job, ix, urls, uri, filename, partial, destination, connections, *, batch=None, stop=None):
|
||||
jid = job['id']
|
||||
batch_stop = stop
|
||||
probe = _probe_range(jid, uri, stop)
|
||||
if probe is None:
|
||||
_log(jid, 'Archive.org: serwer nie obsługuje Range; pobieranie pojedynczym połączeniem')
|
||||
return False
|
||||
total, validator = probe
|
||||
folder, manifest = _prepare_segments(partial, uri, total, validator, connections)
|
||||
if folder is None and manifest is None:
|
||||
_log(jid, 'Archive.org: mały plik lub końcówka; pobieranie pojedynczym połączeniem')
|
||||
return False
|
||||
if folder is None and manifest['prefix'] == total:
|
||||
partial.replace(destination)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
return True
|
||||
_log(jid, f'Archive.org: {len(manifest["ranges"])} połączeń HTTP Range, rozmiar {total} B, wznowienie od {manifest["prefix"]} B')
|
||||
local_stop = Event()
|
||||
current = (manifest['prefix'] + sum(min((folder / f'{idx:02d}.part').stat().st_size, end - start + 1)
|
||||
for idx, (start, end) in enumerate(manifest['ranges'])
|
||||
if (folder / f'{idx:02d}.part').exists()), time.monotonic())
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=min(connections, len(manifest['ranges']))) as pool:
|
||||
pending = {pool.submit(_segment_worker, jid, uri, folder, manifest, i, local_stop, batch_stop)
|
||||
for i in range(len(manifest['ranges']))}
|
||||
try:
|
||||
while pending:
|
||||
_check_state(jid, batch_stop)
|
||||
done, pending = wait(pending, timeout=0.8, return_when=FIRST_COMPLETED)
|
||||
current = _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
|
||||
for task in done:
|
||||
task.result()
|
||||
finally:
|
||||
local_stop.set()
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
except _RangeUnsupported as exc:
|
||||
_log(jid, 'Archive.org: niezgodna obsługa Range, przełączam na jedno połączenie')
|
||||
_discard_parallel_state(partial, reset_prefix=exc.reset_prefix)
|
||||
return False
|
||||
_check_state(jid, batch_stop)
|
||||
_report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
|
||||
_log(jid, 'Archive.org: scalanie segmentów')
|
||||
if batch is None:
|
||||
store.update(jid, note='Archive.org: scalanie segmentów', speed=0)
|
||||
if batch_stop is None:
|
||||
_assemble_segments(jid, partial, destination, folder, manifest)
|
||||
else:
|
||||
_assemble_segments(jid, partial, destination, folder, manifest, batch_stop)
|
||||
_finish(job, ix, urls, batch=batch)
|
||||
_log(jid, f'Ukończono {filename}: {total} bajtów, {len(manifest["ranges"])} połączeń')
|
||||
return True
|
||||
|
||||
|
||||
class _BatchProgress:
|
||||
"""Collect progress from file threads; only the scheduler updates SQLite.
|
||||
|
||||
A single stream/segment never overwrites the total progress or total speed
|
||||
from another file. No extra database table is needed for old installations.
|
||||
"""
|
||||
|
||||
def __init__(self, job_id, filenames, limit):
|
||||
self.job_id = job_id
|
||||
self.filenames = filenames
|
||||
self.limit = limit
|
||||
self.lock = Lock()
|
||||
self.sizes = [0] * len(filenames)
|
||||
self.totals = [0] * len(filenames)
|
||||
self.seen = [False] * len(filenames)
|
||||
self.done = set()
|
||||
self.active = set()
|
||||
self.transferred = 0
|
||||
self.last_publish = time.monotonic()
|
||||
|
||||
def start(self, index):
|
||||
with self.lock:
|
||||
self.active.add(index)
|
||||
|
||||
def report(self, index, downloaded, total):
|
||||
with self.lock:
|
||||
downloaded = max(0, int(downloaded))
|
||||
if self.seen[index]:
|
||||
self.transferred += max(0, downloaded - self.sizes[index])
|
||||
else:
|
||||
# Existing .part and .segments from an earlier run are not
|
||||
# counted as freshly transferred bytes per second.
|
||||
self.seen[index] = True
|
||||
self.sizes[index] = downloaded
|
||||
self.totals[index] = max(0, int(total))
|
||||
|
||||
def complete(self, index, size=None):
|
||||
if size is None:
|
||||
size = (store.DATA / 'raw' / self.job_id / self.filenames[index]).stat().st_size
|
||||
with self.lock:
|
||||
if self.seen[index]:
|
||||
self.transferred += max(0, size - self.sizes[index])
|
||||
self.seen[index] = True
|
||||
self.sizes[index] = size
|
||||
self.totals[index] = size
|
||||
self.done.add(index)
|
||||
self.active.discard(index)
|
||||
|
||||
def publish(self):
|
||||
now = time.monotonic()
|
||||
with self.lock:
|
||||
elapsed = max(0.001, now - self.last_publish)
|
||||
speed = max(0, int(self.transferred / elapsed))
|
||||
self.transferred = 0
|
||||
self.last_publish = now
|
||||
total_files = len(self.filenames)
|
||||
fractions = []
|
||||
for index in range(total_files):
|
||||
if index in self.done:
|
||||
fractions.append(1.0)
|
||||
elif self.totals[index]:
|
||||
fractions.append(min(self.sizes[index] / self.totals[index], 1))
|
||||
else:
|
||||
fractions.append(0.0)
|
||||
completed_bytes = sum(self.sizes)
|
||||
total_bytes = sum(self.totals)
|
||||
if total_bytes and all(self.totals):
|
||||
# A 20 KiB NFO must not count the same as an 8 GiB ISO.
|
||||
progress = round(100 * min(completed_bytes, total_bytes) / total_bytes, 2)
|
||||
else:
|
||||
# Until all content lengths are known, show the fraction of
|
||||
# completed files/known segments as a provisional estimate.
|
||||
progress = round(sum(fractions) / total_files * 100, 2)
|
||||
finished = len(self.done)
|
||||
active = len(self.active)
|
||||
contiguous = next((i for i in range(total_files) if i not in self.done), total_files)
|
||||
store.update(self.job_id, part_index=contiguous, progress=progress,
|
||||
completed=completed_bytes, total=total_bytes, speed=speed,
|
||||
note=f'Archive.org: {finished}/{total_files} gotowe, {active} równolegle, limit {self.limit} połączeń')
|
||||
|
||||
|
||||
def _transfer_many(job):
|
||||
"""Download multiple Archive.org files at once with ONE connection budget.
|
||||
|
||||
Each slot owns a fixed connection allowance. Reusing the slot for the next
|
||||
file preserves the global cap even with 50-segment old resume manifests.
|
||||
"""
|
||||
jid = job['id']
|
||||
urls = source_urls(job)
|
||||
if not urls:
|
||||
raise ArchiveDownloadError('Brak plików Archive.org do pobrania')
|
||||
names = []
|
||||
for uri in urls:
|
||||
if not archive_auth.is_archive_download(uri):
|
||||
raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
|
||||
name = target_filename(uri)
|
||||
if not name:
|
||||
raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
|
||||
names.append(name)
|
||||
if len({name.casefold() for name in names}) != len(names):
|
||||
raise ArchiveDownloadError('Wybrane pliki mają powtarzające się nazwy. Wybierz pliki o różnych nazwach.')
|
||||
|
||||
root = store.DATA / 'raw' / jid
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
_check_state(jid)
|
||||
limit = _connection_limit()
|
||||
tracker = _BatchProgress(jid, names, limit)
|
||||
waiting = deque()
|
||||
for ix, name in enumerate(names):
|
||||
dest = root / name
|
||||
if dest.is_file() and not Path(str(dest) + '.aria2').exists():
|
||||
# A completed destination was atomically renamed by the old or
|
||||
# new downloader. A legacy aria2 control file means its sibling
|
||||
# destination may only contain an incomplete preallocated file.
|
||||
tracker.complete(ix, dest.stat().st_size)
|
||||
_log(jid, f'Plik {ix+1}/{len(urls)} już gotowy: {name}')
|
||||
else:
|
||||
waiting.append(ix)
|
||||
|
||||
store.update(jid, state='downloading', gid='', note=f'Archive.org: start równoległego pobierania {len(waiting)} plików')
|
||||
tracker.publish()
|
||||
if not waiting:
|
||||
_check_state(jid)
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie Archive.org ukończone')
|
||||
return
|
||||
|
||||
slots = min(limit, len(waiting))
|
||||
budgets = [limit // slots + (1 if i < limit % slots else 0) for i in range(slots)]
|
||||
_log(jid, f'Archive.org: start {slots} plików równolegle; budżet {limit} połączeń HTTP '
|
||||
f'(przydział na plik: {", ".join(map(str, budgets))})')
|
||||
stopped = Event()
|
||||
first_error = None
|
||||
|
||||
def launch(pool, slot):
|
||||
ix = waiting.popleft()
|
||||
item = dict(job, part_index=ix)
|
||||
tracker.start(ix)
|
||||
return pool.submit(_transfer_one, item, batch=tracker, connections=budgets[slot], stop=stopped)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=slots, thread_name_prefix='archive-file') as pool:
|
||||
running = {launch(pool, slot): slot for slot in range(slots)}
|
||||
try:
|
||||
while running:
|
||||
_check_state(jid, stopped)
|
||||
finished, _ = wait(running, timeout=0.8, return_when=FIRST_COMPLETED)
|
||||
tracker.publish()
|
||||
for future in finished:
|
||||
slot = running.pop(future)
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
first_error = first_error or exc
|
||||
stopped.set()
|
||||
else:
|
||||
if waiting and not stopped.is_set():
|
||||
running[launch(pool, slot)] = slot
|
||||
if first_error is not None:
|
||||
break
|
||||
finally:
|
||||
stopped.set()
|
||||
for future in running:
|
||||
future.cancel()
|
||||
|
||||
if first_error is not None:
|
||||
tracker.publish()
|
||||
raise first_error
|
||||
_check_state(jid) # Especially important if a pause arrives at the last byte.
|
||||
tracker.publish()
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
part_index=len(urls), note='Pobieranie Archive.org ukończone')
|
||||
_log(jid, f'Archive.org: pobrano wszystkie {len(urls)} pliki')
|
||||
|
||||
|
||||
async def run_one(job):
|
||||
import asyncio
|
||||
urls = source_urls(job)
|
||||
if job['source_type'] == 'archive_many' and len(urls) > 1:
|
||||
await asyncio.to_thread(_transfer_many, job)
|
||||
else:
|
||||
await asyncio.to_thread(_transfer_one, job)
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Supervise anacrolix-based BitTorrent subprocesses (no aria2 for torrents).
|
||||
|
||||
Torrent payload data starts only after file selection. All subprocess state is
|
||||
written as atomic JSON so a restart can detect interrupted work and resume.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import store
|
||||
|
||||
BT_HELPER = '/usr/local/bin/x360-bt'
|
||||
_INSPECTIONS = {}
|
||||
_DOWNLOADS = {}
|
||||
|
||||
|
||||
class BitTorrentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def exe():
|
||||
candidate = os.getenv('BT_HELPER', BT_HELPER)
|
||||
if not Path(candidate).is_file():
|
||||
raise BitTorrentError(f'Brak silnika BitTorrent: {candidate} (przebuduj obraz Docker)')
|
||||
return candidate
|
||||
|
||||
|
||||
def _read(path):
|
||||
try:
|
||||
value = json.loads(Path(path).read_text(encoding='utf-8'))
|
||||
return value if isinstance(value, dict) else {}
|
||||
except (FileNotFoundError, ValueError, OSError):
|
||||
return {}
|
||||
|
||||
|
||||
def inspect_state(preview_id):
|
||||
state = _read(store.DATA / 'metadata' / preview_id / 'state.json')
|
||||
process = _INSPECTIONS.get(preview_id)
|
||||
if not state and (not process or process.returncode is not None):
|
||||
return {'status': 'error', 'error': 'Klient BitTorrent przerwał odczyt metadanych. Sprawdź engine.log podglądu lub ponów.'}
|
||||
return state
|
||||
|
||||
|
||||
def job_state(job_id):
|
||||
return _read(store.DATA / 'bt' / (job_id + '.json'))
|
||||
|
||||
|
||||
async def _spawn(args, log_path):
|
||||
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with log_path.open('ab') as log:
|
||||
try:
|
||||
return await asyncio.create_subprocess_exec(*args, stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=log, stderr=asyncio.subprocess.STDOUT)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise BitTorrentError('Nie można uruchomić silnika BT: ' + str(exc)) from exc
|
||||
|
||||
|
||||
async def start_inspection(preview_id, magnet):
|
||||
from . import torrent_meta
|
||||
folder = torrent_meta.inspect_dir(preview_id)
|
||||
args = [exe(), 'inspect', '-magnet', magnet, '-out', str(folder / 'metadata.torrent'),
|
||||
'-status', str(folder / 'state.json'), '-timeout', '5m', '-port', '0']
|
||||
process = await _spawn(args, folder / 'engine.log')
|
||||
_INSPECTIONS[preview_id] = process
|
||||
return process
|
||||
|
||||
|
||||
async def _stop(process):
|
||||
if process and process.returncode is None:
|
||||
process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), 6)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def stop_inspection(preview_id):
|
||||
await _stop(_INSPECTIONS.pop(preview_id, None))
|
||||
|
||||
|
||||
async def start_download(job):
|
||||
jid = job['id']
|
||||
old = _DOWNLOADS.get(jid)
|
||||
if old and old.returncode is None:
|
||||
return
|
||||
path = store.DATA / 'torrents' / (jid + '.torrent')
|
||||
if not path.is_file():
|
||||
raise BitTorrentError('Brak zapisanego pliku .torrent dla zadania')
|
||||
try:
|
||||
spec = json.loads(job['source'])
|
||||
selection = spec['selection']
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise BitTorrentError('Uszkodzony zapis wyboru plików torrenta') from exc
|
||||
output = store.DATA / 'raw' / jid
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
status_path = store.DATA / 'bt' / (jid + '.json')
|
||||
status_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
status_path.unlink(missing_ok=True)
|
||||
args = [exe(), 'fetch', '-torrent', str(path), '-select', selection, '-out', str(output),
|
||||
'-status', str(status_path), '-port', os.getenv('BT_PORT', '6881')]
|
||||
_DOWNLOADS[jid] = await _spawn(args, store.DATA / 'logs' / (jid + '.bt.log'))
|
||||
store.update(jid, gid='', state='downloading', note='BitTorrent: uruchomiono klienta')
|
||||
|
||||
|
||||
async def poll_download(job):
|
||||
jid = job['id']
|
||||
process = _DOWNLOADS.get(jid)
|
||||
if process is None:
|
||||
# A container restart loses the subprocess; persistently downloaded
|
||||
# pieces in data/raw are verified and resumed by the helper.
|
||||
await start_download(job)
|
||||
return
|
||||
state = job_state(jid)
|
||||
done, total = int(state.get('downloaded', 0) or 0), int(state.get('total', 0) or 0)
|
||||
speed = int(state.get('speed', 0) or 0)
|
||||
peers = int(state.get('connections', 0) or 0)
|
||||
known = int(state.get('known_peers', 0) or 0)
|
||||
if state.get('status') == 'done' and process.returncode == 0:
|
||||
await process.wait()
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
store.update(jid, state='preparing', progress=100, speed=0,
|
||||
note='BitTorrent ukończony, przygotowanie plików')
|
||||
return
|
||||
if state.get('status') == 'error' or process.returncode not in (None, 0):
|
||||
await process.wait()
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
raise BitTorrentError(state.get('error') or 'Silnik BitTorrent zakończył się niepowodzeniem; sprawdź log BT')
|
||||
if process.returncode == 0 and state.get('status') != 'done':
|
||||
_DOWNLOADS.pop(jid, None)
|
||||
raise BitTorrentError('Silnik BitTorrent zakończył pracę bez potwierdzenia ukończenia danych')
|
||||
percent = round(done / total * 100, 2) if total else 0
|
||||
store.update(jid, progress=percent, speed=speed, total=total, completed=done,
|
||||
note=f'BitTorrent: {peers} połączeń, {known} znanych peerów')
|
||||
|
||||
|
||||
async def stop_download(job_id):
|
||||
await _stop(_DOWNLOADS.pop(job_id, None))
|
||||
|
||||
|
||||
async def shutdown():
|
||||
jobs = list(_INSPECTIONS.values()) + list(_DOWNLOADS.values())
|
||||
_INSPECTIONS.clear()
|
||||
_DOWNLOADS.clear()
|
||||
await asyncio.gather(*(_stop(p) for p in jobs))
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
"""Validation, safe paths and Internet Archive integration."""
|
||||
import html
|
||||
import re
|
||||
import socket
|
||||
import ipaddress
|
||||
from urllib.parse import urlparse, quote, unquote, parse_qs
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
FORMATS = {'xex', 'god', 'raw'}
|
||||
ARCHIVE_EXTS = ('.zip', '.7z', '.rar', '.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')
|
||||
SAFE_ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9._-]{0,150}$')
|
||||
VIMM_PAGE = re.compile(r'^/vault/[0-9]+/?$')
|
||||
|
||||
|
||||
def safe_title(name):
|
||||
value = re.sub(r'[^\w .()\[\]-]+', '_', name, flags=re.UNICODE).strip(' ._')[:78]
|
||||
return value or 'Gra'
|
||||
|
||||
|
||||
def normalize_magnet(value):
|
||||
# Pasted magnet URLs often contain HTML entities, hard wraps or Markdown.
|
||||
url = html.unescape(value).strip()
|
||||
url = re.sub(r"(?i)^['\"`]*\*{0,2}magnet\*{0,2}:\?", 'magnet:?', url)
|
||||
if not url.lower().startswith('magnet:?'):
|
||||
return url
|
||||
url = re.sub(r"\\\s*", '', url)
|
||||
url = re.sub(r"\s+", '', url)
|
||||
return html.unescape(url.strip("'\"`"))
|
||||
|
||||
|
||||
def validate_url(value):
|
||||
url = normalize_magnet(value)
|
||||
if url.lower().startswith('magnet:?'):
|
||||
if len(url) > 12000:
|
||||
raise ValueError('Link magnet jest zbyt dlugi')
|
||||
params = parse_qs(urlparse(url).query, keep_blank_values=True)
|
||||
xt = params.get('xt', [])
|
||||
if not any(re.fullmatch(r'urn:btih:(?:[0-9a-fA-F]{40}|[A-Za-z2-7]{32})', item, re.I)
|
||||
or re.fullmatch(r'urn:btmh:1220[0-9a-fA-F]{64}', item, re.I)
|
||||
for item in xt):
|
||||
raise ValueError('Nieprawidlowy hash xt w magnet (BTIH: 40 znakow HEX lub 32 Base32)')
|
||||
return 'magnet', url
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
|
||||
raise ValueError('Podaj adres HTTP/HTTPS albo link magnet')
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError('Niepoprawny numer portu w URL') from exc
|
||||
if port not in (None, 80, 443):
|
||||
raise ValueError('Niestandardowy port w URL jest niedozwolony')
|
||||
host = parsed.hostname.lower().strip('.')
|
||||
if host in ('localhost',) or host.endswith('.localhost') or host.endswith('.local'):
|
||||
raise ValueError('Adres lokalny nie jest dozwolony')
|
||||
try:
|
||||
ip = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
ip = None
|
||||
if ip is not None and not ip.is_global:
|
||||
raise ValueError('Adres prywatny lub zarezerwowany jest niedozwolony')
|
||||
if host in ('vimm.net', 'www.vimm.net') and VIMM_PAGE.fullmatch(parsed.path):
|
||||
return 'vimm', url
|
||||
return 'url', url
|
||||
|
||||
|
||||
def parse_archive_id(value):
|
||||
value = value.strip()
|
||||
u = urlparse(value)
|
||||
if u.scheme in ('http', 'https') and u.hostname in ('archive.org', 'www.archive.org'):
|
||||
pieces = [unquote(x) for x in u.path.split('/') if x]
|
||||
if len(pieces) >= 2 and pieces[0] in ('details', 'download', 'metadata'):
|
||||
value = pieces[1]
|
||||
else:
|
||||
raise ValueError('Link musi wskazywac element archive.org/details/<identyfikator>')
|
||||
if not SAFE_ID.fullmatch(value):
|
||||
raise ValueError('Bledny identyfikator elementu Archive.org')
|
||||
return value
|
||||
|
||||
|
||||
def archive_download_url(identifier, filename):
|
||||
identifier = parse_archive_id(identifier)
|
||||
if (not filename or filename.startswith('/') or '\\' in filename or '\x00' in filename
|
||||
or any(p in ('..', '.') for p in PurePosixPath(filename).parts)):
|
||||
raise ValueError('Niedozwolona sciezka pliku Archive.org')
|
||||
return f'https://archive.org/download/{quote(identifier)}/{quote(filename, safe="/")}'
|
||||
|
||||
|
||||
def safe_import_file(filename, root=Path('/data/imports')):
|
||||
# User selects from server-side imports (including subdirectories).
|
||||
if not filename or '\\' in filename or filename.startswith('/') or '\x00' in filename:
|
||||
raise ValueError('Niepoprawna sciezka pliku')
|
||||
item = root.joinpath(*PurePosixPath(filename).parts)
|
||||
if not item.resolve().is_relative_to(root.resolve()):
|
||||
raise ValueError('Plik spoza katalogu imports')
|
||||
if not item.is_file() or item.is_symlink():
|
||||
raise ValueError('Wskazany plik nie istnieje')
|
||||
return item
|
||||
|
||||
|
||||
def is_archive(filename):
|
||||
name = filename.lower()
|
||||
return name.endswith(ARCHIVE_EXTS) or bool(re.search(r'\.(?:7z|zip)\.001$', name)) or bool(re.search(r'\.part0*1\.rar$', name))
|
||||
|
||||
|
||||
def is_additional_volume(filename):
|
||||
name = filename.lower()
|
||||
return bool(re.search(r'\.(?:7z|zip)\.0*0?[2-9][0-9]*$', name) or
|
||||
re.search(r'\.part0*[2-9][0-9]*\.rar$', name) or
|
||||
re.search(r'\.r\d\d$', name))
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
"""aria2 JSON-RPC. aria2 is started alongside the API server, not exposed publicly."""
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from . import archive_auth, store
|
||||
|
||||
RPC = 'http://127.0.0.1:6800/jsonrpc'
|
||||
|
||||
|
||||
async def rpc(method, *params):
|
||||
payload = {
|
||||
'jsonrpc': '2.0', 'id': 'x360', 'method': 'aria2.' + method,
|
||||
'params': ['token:' + os.environ['ARIA2_SECRET'], *params],
|
||||
}
|
||||
# On a container restart the API may start slightly before aria2 RPC.
|
||||
# Retry the connection instead of permanently failing restored jobs.
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for attempt in range(15):
|
||||
try:
|
||||
r = await client.post(RPC, json=payload)
|
||||
r.raise_for_status()
|
||||
answer = r.json()
|
||||
if 'error' in answer:
|
||||
raise RuntimeError(answer['error'].get('message', 'Bledna odpowiedz aria2'))
|
||||
return answer['result']
|
||||
except (httpx.ConnectError, httpx.ConnectTimeout):
|
||||
if attempt == 14:
|
||||
raise
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
def source_urls(job):
|
||||
if job['source_type'] == 'archive_many':
|
||||
return json.loads(job['source'])
|
||||
return [job['source']]
|
||||
|
||||
|
||||
def target_filename(uri):
|
||||
if uri.lower().startswith('magnet:'):
|
||||
return None
|
||||
name = urllib.parse.unquote(Path(urllib.parse.urlsplit(uri).path).name)
|
||||
if not name or name in ('.', '..'):
|
||||
return 'pobrany-plik.bin'
|
||||
# Do not allow URI-provided paths to escape the download directory.
|
||||
return ''.join(c if c not in '\\/:*?"<>|\x00\n\r' else '_' for c in name)[:200]
|
||||
|
||||
|
||||
async def begin(job):
|
||||
base = store.DATA / 'raw' / job['id']
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
urls = source_urls(job)
|
||||
ix = job['part_index']
|
||||
if ix >= len(urls):
|
||||
store.update(job['id'], state='preparing')
|
||||
return
|
||||
uri = urls[ix]
|
||||
opts = {
|
||||
'dir': str(base), 'continue': 'true', 'max-connection-per-server': '2',
|
||||
'split': '2', 'seed-time': '0', 'auto-file-renaming': 'false',
|
||||
'allow-overwrite': 'false', 'max-tries': '8', 'retry-wait': '30',
|
||||
}
|
||||
if job['source_type'] != 'torrent_selected':
|
||||
name = target_filename(uri)
|
||||
if name and '.' in name:
|
||||
opts['out'] = name
|
||||
elif name:
|
||||
# Let the server's Content-Disposition supply the real filename.
|
||||
opts['content-disposition'] = 'true'
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
torrent = store.DATA / 'torrents' / (job['id'] + '.torrent')
|
||||
encoded = base64.b64encode(torrent.read_bytes()).decode('ascii')
|
||||
select = json.loads(job['source'])['selection']
|
||||
opts['select-file'] = select
|
||||
opts['bt-remove-unselected-file'] = 'true'
|
||||
gid = await rpc('addTorrent', encoded, [], opts)
|
||||
note = 'Pobieranie wybranych plików torrenta'
|
||||
else:
|
||||
if archive_auth.is_archive_download(uri):
|
||||
# aria2 downloads from the local proxy. It never receives IA cookies.
|
||||
opts['header'] = [archive_auth.signed_header(job['id'], ix)]
|
||||
uri = archive_auth.internal_url(job['id'], ix)
|
||||
opts['split'] = '1'
|
||||
opts['max-connection-per-server'] = '1'
|
||||
gid = await rpc('addUri', [uri], opts)
|
||||
note = f'Pobieranie pliku {ix+1}/{len(urls)}'
|
||||
store.update(job['id'], gid=gid, state='downloading', note=note)
|
||||
|
||||
|
||||
async def poll(job):
|
||||
urls = source_urls(job)
|
||||
gid = job['gid']
|
||||
if not gid:
|
||||
await begin(job)
|
||||
return
|
||||
try:
|
||||
response = await rpc('tellStatus', gid, ['gid','status','totalLength','completedLength',
|
||||
'downloadSpeed','followedBy','errorMessage'])
|
||||
except RuntimeError as ex:
|
||||
# On restart aria2 may lose an old GID. Re-add to resume available .aria2 files.
|
||||
if 'not found' in str(ex).lower():
|
||||
store.update(job['id'], gid='', note='Odzyskiwanie transferu po restarcie')
|
||||
return
|
||||
raise
|
||||
|
||||
followed = response.get('followedBy', [])
|
||||
if followed:
|
||||
store.update(job['id'], gid=followed[0], note='Magnet: metadane gotowe')
|
||||
return
|
||||
state = response.get('status')
|
||||
length = int(response.get('totalLength', 0) or 0)
|
||||
completed = int(response.get('completedLength', 0) or 0)
|
||||
speed = int(response.get('downloadSpeed', 0) or 0)
|
||||
part_progress = (completed / length) if length else 0
|
||||
ix = job['part_index']
|
||||
pct = (ix + part_progress) / len(urls) * 100
|
||||
store.update(job['id'], progress=round(pct, 2), speed=speed, total=length,
|
||||
completed=completed, note=f'Pobieranie {ix+1}/{len(urls)}: {state}')
|
||||
if state == 'complete':
|
||||
if ix + 1 < len(urls):
|
||||
store.update(job['id'], gid='', part_index=ix+1)
|
||||
else:
|
||||
store.update(job['id'], state='preparing', progress=100, speed=0,
|
||||
note='Pobieranie ukonczone, przygotowuje pliki')
|
||||
elif state in ('error','removed'):
|
||||
message = response.get('errorMessage') or state
|
||||
if job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(urls[ix])):
|
||||
if any(x in message.lower() for x in ('authorization', '403', '401', 'forbidden')):
|
||||
message += ' — Internet Archive: sprawdź logowanie i dostęp do pliku.'
|
||||
raise RuntimeError('aria2: ' + message)
|
||||
elif state == 'paused':
|
||||
store.update(job['id'], state='paused', speed=0)
|
||||
|
||||
|
||||
async def cancel_download(gid):
|
||||
if not gid:
|
||||
return
|
||||
try:
|
||||
await rpc('forceRemove', gid)
|
||||
except (RuntimeError, httpx.HTTPError):
|
||||
pass
|
||||
|
||||
|
||||
async def pause_download(gid):
|
||||
if gid:
|
||||
await rpc('forcePause', gid)
|
||||
|
||||
|
||||
async def resume_download(gid):
|
||||
if gid:
|
||||
await rpc('unpause', gid)
|
||||
+515
@@ -0,0 +1,515 @@
|
||||
"""Self-hosted X360 Web Queue: authenticated FastAPI + one resilient worker."""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse, unquote
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import archive_auth, archive_download, bt, download, pipeline, store, torrent_meta, vimm
|
||||
from .common import (FORMATS, archive_download_url, parse_archive_id, safe_import_file,
|
||||
safe_title, validate_url)
|
||||
|
||||
|
||||
basic = HTTPBasic()
|
||||
|
||||
|
||||
def authorize(credentials: HTTPBasicCredentials = Depends(basic)):
|
||||
u = os.getenv('WEB_USER', 'admin').encode()
|
||||
p = os.getenv('WEB_PASSWORD', '').encode()
|
||||
if not p or not (secrets.compare_digest(credentials.username.encode(), u)
|
||||
and secrets.compare_digest(credentials.password.encode(), p)):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail='Nieprawidlowy login lub haslo',
|
||||
headers={'WWW-Authenticate': 'Basic'})
|
||||
|
||||
|
||||
class AddJob(BaseModel):
|
||||
url: str = Field(min_length=1, max_length=4000)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class AddImports(BaseModel):
|
||||
filename: str = Field(min_length=1, max_length=300)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class AddArchive(BaseModel):
|
||||
identifier: str = Field(min_length=1, max_length=250)
|
||||
files: list[str] = Field(min_length=1, max_length=150)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
class ItemLookup(BaseModel):
|
||||
identifier: str = Field(min_length=1, max_length=250)
|
||||
|
||||
|
||||
class ArchiveLogin(BaseModel):
|
||||
email: str = Field(min_length=3, max_length=254)
|
||||
password: str = Field(min_length=1, max_length=1024)
|
||||
|
||||
|
||||
class MagnetInspect(BaseModel):
|
||||
magnet: str = Field(min_length=20, max_length=12000)
|
||||
|
||||
|
||||
class MagnetJob(BaseModel):
|
||||
preview_id: str = Field(min_length=32, max_length=32)
|
||||
files: list[int] = Field(min_length=1, max_length=20000)
|
||||
title: str = Field(default='', max_length=160)
|
||||
format: str = 'xex'
|
||||
|
||||
|
||||
def new_job(source_type, source, title, fmt):
|
||||
if fmt not in FORMATS:
|
||||
raise HTTPException(status_code=400, detail='Format: xex, god albo raw')
|
||||
jid = str(uuid.uuid4())
|
||||
store.create({'id': jid, 'title': safe_title(title or 'Xbox 360'),
|
||||
'source_type': source_type, 'source': source, 'format': fmt})
|
||||
return store.get(jid)
|
||||
|
||||
|
||||
async def worker():
|
||||
while True:
|
||||
job = store.next_job()
|
||||
if not job:
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
jid = job['id']
|
||||
try:
|
||||
if job['state'] == 'preparing':
|
||||
await pipeline.process_job(job)
|
||||
elif job['source_type'] == 'import':
|
||||
# Local import is copied into persistent raw dir, then converted.
|
||||
src = safe_import_file(job['source'])
|
||||
target = store.DATA / 'raw' / jid
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(shutil.copy2, src, target / src.name)
|
||||
store.update(jid, state='preparing', progress=100)
|
||||
elif job['source_type'] == 'vimm':
|
||||
await vimm.browser_download(job)
|
||||
elif job['source_type'] == 'torrent_selected':
|
||||
if job['state'] == 'queued':
|
||||
await bt.start_download(job)
|
||||
else:
|
||||
await bt.poll_download(job)
|
||||
elif job['source_type'] == 'archive_many' or (job['source_type'] == 'url' and archive_auth.is_archive_download(job['source'])):
|
||||
if job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.update(jid, gid='')
|
||||
await archive_download.run_one(job)
|
||||
elif job['state'] == 'queued':
|
||||
# Resume if there is an aria2 GID instead of accidentally
|
||||
# starting a duplicate download.
|
||||
if job['gid']:
|
||||
await download.resume_download(job['gid'])
|
||||
store.update(jid, state='downloading')
|
||||
else:
|
||||
await download.begin(job)
|
||||
else:
|
||||
await download.poll(job)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except archive_download.ArchiveStopped:
|
||||
pass
|
||||
except Exception as ex:
|
||||
store.update(jid, state='error', speed=0, error=str(ex)[:1800],
|
||||
note='Błąd zadania: otwórz Log')
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
for subdir in ('raw', 'ready', 'work', 'imports', 'logs', 'metadata', 'torrents'):
|
||||
(store.DATA / subdir).mkdir(parents=True, exist_ok=True)
|
||||
store.init_db()
|
||||
task = asyncio.create_task(worker())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await task
|
||||
await bt.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(title='X360 Web Queue', version='0.1.0', lifespan=lifespan,
|
||||
docs_url=None, redoc_url=None, openapi_url=None)
|
||||
api = APIRouter(prefix='/api', dependencies=[Depends(authorize)])
|
||||
MAX_INSPECT_SECONDS = 300
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def guard_cross_origin_requests(request: Request, call_next):
|
||||
if request.method in ('POST', 'PUT', 'PATCH', 'DELETE'):
|
||||
origin = request.headers.get('origin')
|
||||
if origin and urlparse(origin).netloc != request.headers.get('host'):
|
||||
return PlainTextResponse('Nieprawidlowa domena zrodla zadania', status_code=403)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
@app.get('/')
|
||||
async def index(_: None = Depends(authorize)):
|
||||
return FileResponse(Path(__file__).parent / 'static' / 'index.html')
|
||||
|
||||
|
||||
@app.get('/healthz')
|
||||
async def healthz():
|
||||
return {'ok': True}
|
||||
|
||||
|
||||
@api.get('/jobs')
|
||||
async def jobs():
|
||||
return store.list_jobs()
|
||||
|
||||
|
||||
@api.post('/jobs')
|
||||
async def add_download(body: AddJob):
|
||||
try:
|
||||
kind, url = validate_url(body.url)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(status_code=400, detail=str(ex)) from ex
|
||||
parsed = urlparse(url)
|
||||
guessed = unquote(Path(parsed.path).name) if kind != 'magnet' else 'Torrent'
|
||||
if kind == 'vimm':
|
||||
guessed = 'Vimm-' + parsed.path.rstrip('/').split('/')[-1]
|
||||
if kind == 'magnet':
|
||||
raise HTTPException(400, 'Magnet wymaga najpierw wyboru plików (/api/magnet/inspect)')
|
||||
return new_job(kind, url, body.title or guessed, body.format)
|
||||
|
||||
|
||||
@api.get('/archive/session')
|
||||
async def archive_session():
|
||||
return archive_auth.session_status()
|
||||
|
||||
|
||||
@api.post('/archive/session')
|
||||
async def archive_login(body: ArchiveLogin):
|
||||
if '@' not in body.email:
|
||||
raise HTTPException(400, 'Niepoprawny adres e-mail')
|
||||
try:
|
||||
return await archive_auth.login(body.email.strip(), body.password)
|
||||
except archive_auth.ArchiveAuthError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
|
||||
|
||||
@api.delete('/archive/session')
|
||||
async def archive_logout():
|
||||
archive_auth.logout()
|
||||
return archive_auth.session_status()
|
||||
|
||||
|
||||
@app.api_route('/internal/archive/{job_id}/{part_index}', methods=['GET', 'HEAD'])
|
||||
async def archive_proxy(job_id: str, part_index: int, request: Request):
|
||||
if (not 0 <= part_index <= 149 or
|
||||
not archive_auth.verify_token(job_id, part_index, request.headers.get('x-x360-proxy-token'))):
|
||||
raise HTTPException(403, 'Odmowa dostępu')
|
||||
job = store.get(job_id)
|
||||
if not job or job['source_type'] not in ('url', 'archive_many'):
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
urls = download.source_urls(job)
|
||||
if part_index >= len(urls) or not archive_auth.is_archive_download(urls[part_index]):
|
||||
raise HTTPException(404, 'Nie znaleziono pliku')
|
||||
return await archive_auth.stream_archive(urls[part_index], request.method, request.headers)
|
||||
|
||||
|
||||
@api.post('/magnet/inspect')
|
||||
async def magnet_inspect(body: MagnetInspect):
|
||||
try:
|
||||
kind, magnet = validate_url(body.magnet)
|
||||
if kind != 'magnet':
|
||||
raise ValueError('Wklej link magnet')
|
||||
except ValueError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
await asyncio.to_thread(torrent_meta.cleanup_old)
|
||||
root = store.DATA / 'metadata'
|
||||
active_previews = sum(1 for item in root.iterdir() if item.is_dir() and not item.is_symlink() and
|
||||
(item / 'request.json').is_file() and
|
||||
not list(item.glob('*.torrent')) and
|
||||
time.time() - item.stat().st_mtime < MAX_INSPECT_SECONDS)
|
||||
if active_previews >= 3:
|
||||
raise HTTPException(429, 'Trzy podglądy już pobierają metadane. Poczekaj na zakończenie.')
|
||||
preview_id = uuid.uuid4().hex
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
directory.mkdir(parents=True, mode=0o700)
|
||||
try:
|
||||
await bt.start_inspection(preview_id, magnet)
|
||||
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': magnet, 'engine': 'anacrolix'})
|
||||
except (RuntimeError, httpx.HTTPError, OSError) as exc:
|
||||
await bt.stop_inspection(preview_id)
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
raise HTTPException(502, f'Nie można rozpocząć odczytu metadanych: {exc}') from exc
|
||||
return {'preview_id': preview_id, 'status': 'loading'}
|
||||
|
||||
|
||||
@api.post('/magnet/upload')
|
||||
async def torrent_upload(request: Request):
|
||||
# No multipart dependency. Transfer raw .torrent bytes with a fixed size limit.
|
||||
if request.headers.get('content-type', '').split(';', 1)[0].strip() != 'application/x-bittorrent':
|
||||
raise HTTPException(415, 'Wymagany plik .torrent')
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(raw) + len(chunk) > torrent_meta.MAX_METADATA:
|
||||
raise HTTPException(413, 'Plik .torrent przekracza limit 32 MiB')
|
||||
raw.extend(chunk)
|
||||
if not raw:
|
||||
raise HTTPException(400, 'Pusty plik .torrent')
|
||||
preview_id = uuid.uuid4().hex
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
directory.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
target = directory / 'uploaded.torrent'
|
||||
try:
|
||||
# Validate before storing the metadata. No game payload is transferred.
|
||||
decoded = torrent_meta.decode_bencode(bytes(raw))
|
||||
if b'info' not in decoded:
|
||||
raise torrent_meta.MetadataError('Plik nie zawiera pola info')
|
||||
target.write_bytes(raw)
|
||||
torrent_meta.read_torrent(target)
|
||||
torrent_meta.write_request(preview_id, {'gid': '', 'magnet': ''})
|
||||
except (torrent_meta.MetadataError, OSError) as exc:
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
return {'preview_id': preview_id, 'status': 'ready'}
|
||||
|
||||
|
||||
@api.get('/magnet/inspect/{preview_id}')
|
||||
async def magnet_inspect_status(preview_id: str):
|
||||
try:
|
||||
req = torrent_meta.load_request(preview_id)
|
||||
path = torrent_meta.torrent_file(preview_id)
|
||||
if path:
|
||||
info = torrent_meta.read_torrent(path)
|
||||
await bt.stop_inspection(preview_id)
|
||||
info.update({'status': 'ready', 'suggested': torrent_meta.suggested_indices(req['magnet'], len(info['files']))})
|
||||
return info
|
||||
elapsed = max(0, int(time.time() - req['created']))
|
||||
status = bt.inspect_state(preview_id)
|
||||
if status.get('status') == 'error':
|
||||
return {'status': 'error', 'error': status.get('error', 'Silnik BT zgłosił błąd')[:500]}
|
||||
if elapsed > MAX_INSPECT_SECONDS:
|
||||
await bt.stop_inspection(preview_id)
|
||||
return {'status': 'error', 'error': 'Brak metadanych po 5 minutach. Sprawdź dostępność peerów i DHT/UDP.'}
|
||||
return {'status': 'loading', 'elapsed': elapsed,
|
||||
'connections': int(status.get('connections') or 0),
|
||||
'seeders': 0, 'known_peers': int(status.get('known_peers') or 0),
|
||||
'aria_status': 'anacrolix'}
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
except (RuntimeError, httpx.HTTPError) as exc:
|
||||
raise HTTPException(502, 'aria2: ' + str(exc)[:200]) from exc
|
||||
|
||||
|
||||
@api.post('/magnet/inspect/{preview_id}/cancel')
|
||||
async def magnet_inspect_cancel(preview_id: str):
|
||||
try:
|
||||
req = torrent_meta.load_request(preview_id)
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
if req.get('engine') == 'anacrolix':
|
||||
await bt.stop_inspection(preview_id)
|
||||
else:
|
||||
await download.cancel_download(req['gid'])
|
||||
directory = torrent_meta.inspect_dir(preview_id)
|
||||
await asyncio.to_thread(shutil.rmtree, directory, True)
|
||||
return {'ok': True}
|
||||
|
||||
|
||||
@api.post('/magnet/jobs')
|
||||
async def magnet_selected_job(body: MagnetJob):
|
||||
if body.format not in FORMATS:
|
||||
raise HTTPException(400, 'Format: xex, god albo raw')
|
||||
try:
|
||||
torrent_meta.load_request(body.preview_id)
|
||||
path = torrent_meta.torrent_file(body.preview_id)
|
||||
if path is None:
|
||||
raise torrent_meta.MetadataError('Metadane jeszcze nie są gotowe')
|
||||
info = torrent_meta.read_torrent(path)
|
||||
await bt.stop_inspection(body.preview_id)
|
||||
valid = {item['index'] for item in info['files'] if not item['padding']}
|
||||
if len(body.files) != len(set(body.files)) or not set(body.files).issubset(valid):
|
||||
raise torrent_meta.MetadataError('Wybierz poprawne indeksy plików')
|
||||
except torrent_meta.MetadataError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
jid = str(uuid.uuid4())
|
||||
dest = store.DATA / 'torrents' / (jid + '.torrent')
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(shutil.copyfile, path, dest)
|
||||
selection = ','.join(str(i) for i in sorted(body.files))
|
||||
data = json.dumps({'selection': selection, 'count': len(body.files), 'name': info['name']})
|
||||
try:
|
||||
store.create({'id': jid, 'title': safe_title(body.title or info['name']),
|
||||
'source_type': 'torrent_selected', 'source': data, 'format': body.format})
|
||||
except Exception:
|
||||
dest.unlink(missing_ok=True)
|
||||
raise
|
||||
return store.get(jid)
|
||||
|
||||
|
||||
@api.post('/archive/lookup')
|
||||
async def archive_lookup(body: ItemLookup):
|
||||
try:
|
||||
identifier = parse_archive_id(body.identifier)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=25, follow_redirects=True, cookies=archive_auth.session_cookies()) as client:
|
||||
res = await client.get('https://archive.org/metadata/' + identifier)
|
||||
res.raise_for_status()
|
||||
data = res.json()
|
||||
except (httpx.HTTPError, ValueError) as ex:
|
||||
raise HTTPException(502, 'Archive.org metadata: ' + str(ex)) from ex
|
||||
result = []
|
||||
for entry in data.get('files', []):
|
||||
name = entry.get('name', '')
|
||||
if not name or entry.get('source') == 'metadata' or name.startswith('__'):
|
||||
continue
|
||||
# Show all file types with an extension; user chooses exact archive parts.
|
||||
if '.' not in Path(name).name:
|
||||
continue
|
||||
try:
|
||||
archive_download_url(identifier, name)
|
||||
except ValueError:
|
||||
continue
|
||||
result.append({'name': name, 'size': int(entry.get('size') or 0),
|
||||
'type': entry.get('format', '')})
|
||||
return {'identifier': identifier, 'title': data.get('metadata', {}).get('title', identifier),
|
||||
'files': result[:1000]}
|
||||
|
||||
|
||||
@api.post('/archive/jobs')
|
||||
async def add_archive(body: AddArchive):
|
||||
try:
|
||||
identifier = parse_archive_id(body.identifier)
|
||||
if len(set(body.files)) != len(body.files):
|
||||
raise ValueError('Pliki nie moga sie powtarzac')
|
||||
urls = [archive_download_url(identifier, file) for file in body.files]
|
||||
if len({Path(unquote(urlparse(u).path)).name for u in urls}) != len(urls):
|
||||
raise ValueError('Pliki maja identyczne nazwy; wybierz jeden folder na zadanie')
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
# Preserving filename order is important for multipart archives.
|
||||
return new_job('archive_many', json.dumps(urls), body.title or identifier, body.format)
|
||||
|
||||
|
||||
@api.get('/imports')
|
||||
async def imports():
|
||||
root = store.DATA / 'imports'
|
||||
result = []
|
||||
for item in root.rglob('*'):
|
||||
if len(result) >= 200:
|
||||
break
|
||||
if item.is_file() and not item.is_symlink() and item.resolve().is_relative_to(root.resolve()):
|
||||
result.append({'name': str(item.relative_to(root)), 'size': item.stat().st_size})
|
||||
return result
|
||||
|
||||
|
||||
@api.post('/imports/jobs')
|
||||
async def import_job(body: AddImports):
|
||||
try:
|
||||
path = safe_import_file(body.filename)
|
||||
except ValueError as ex:
|
||||
raise HTTPException(400, str(ex)) from ex
|
||||
return new_job('import', body.filename, path.stem, body.format)
|
||||
|
||||
|
||||
@api.post('/jobs/{job_id}/{action}')
|
||||
async def manage(job_id: str, action: str):
|
||||
job = store.get(job_id)
|
||||
if not job:
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
state = job['state']
|
||||
if action == 'pause' and state in ('queued', 'downloading'):
|
||||
if state == 'downloading':
|
||||
try:
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
else:
|
||||
await download.pause_download(job['gid'])
|
||||
except (RuntimeError, httpx.HTTPError) as ex:
|
||||
raise HTTPException(502, str(ex)) from ex
|
||||
store.update(job_id, state='paused', speed=0, note='Wstrzymano')
|
||||
elif action == 'resume' and state == 'paused':
|
||||
store.update(job_id, state='queued', note='Wznowienie zadania')
|
||||
elif action == 'cancel' and state in ('queued', 'paused', 'downloading'):
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
elif job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.update(job_id, state='cancelled', speed=0, note='Anulowano')
|
||||
elif action == 'retry' and state in ('error', 'cancelled'):
|
||||
# For already-downloaded input, retry conversion without redownloading.
|
||||
root = store.DATA / 'raw' / job_id
|
||||
good_files = ([p for p in root.rglob('*') if p.is_file() and not p.name.endswith('.aria2')]
|
||||
if root.exists() else [])
|
||||
complete = job['progress'] >= 99.9 and bool(good_files)
|
||||
if job['source_type'] == 'archive_many':
|
||||
# One finished file (or a .part segment) never means the whole
|
||||
# multi-file archive is ready to convert.
|
||||
complete = all((root / download.target_filename(url)).is_file()
|
||||
for url in download.source_urls(job))
|
||||
store.update(job_id, state='preparing' if complete else 'queued',
|
||||
gid='' if not complete else job['gid'], error='',
|
||||
note='Ponowienie zadania')
|
||||
elif action == 'remove' and state in ('error','cancelled','done','paused'):
|
||||
if job['source_type'] == 'torrent_selected':
|
||||
await bt.stop_download(job_id)
|
||||
elif state == 'paused' and job['gid']:
|
||||
await download.cancel_download(job['gid'])
|
||||
store.erase(job_id)
|
||||
return {'ok': True, 'note': 'Usunieto wpis z kolejki, pliki pozostaly na dysku'}
|
||||
else:
|
||||
raise HTTPException(409, 'Ta akcja nie jest dostepna dla obecnego stanu zadania')
|
||||
return store.get(job_id)
|
||||
|
||||
|
||||
@api.get('/jobs/{job_id}/log')
|
||||
async def job_log(job_id: str):
|
||||
if not store.get(job_id):
|
||||
raise HTTPException(404, 'Nie znaleziono zadania')
|
||||
path = store.DATA / 'logs' / f'{job_id}.log'
|
||||
job = store.get(job_id)
|
||||
lines = [f"Stan: {job['state']}", f"Etap: {job['note'] or 'brak'}"]
|
||||
if job['error']:
|
||||
lines.append('Błąd pobierania / zadania: ' + job['error'])
|
||||
bt_log = store.DATA / 'logs' / f'{job_id}.bt.log'
|
||||
if bt_log.is_file():
|
||||
lines.append('BitTorrent (ostatnie 12 KiB):')
|
||||
lines.append(bt_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
|
||||
download_log = store.DATA / 'logs' / f'{job_id}.download.log'
|
||||
if download_log.is_file():
|
||||
lines.append('Pobieranie (ostatnie 12 KiB):')
|
||||
lines.append(download_log.read_bytes()[-12000:].decode('utf-8', errors='replace'))
|
||||
lines.append('')
|
||||
lines.append(path.read_bytes()[-24000:].decode('utf-8', errors='replace')
|
||||
if path.is_file() else 'Konwersja jeszcze nie została uruchomiona.')
|
||||
return PlainTextResponse('\n'.join(lines))
|
||||
|
||||
|
||||
@api.get('/system')
|
||||
async def system():
|
||||
usage = shutil.disk_usage(store.DATA / 'raw')
|
||||
host_dir = os.getenv('XBOX_HOST_DIR', '/xbox').rstrip('/') or '/xbox'
|
||||
return {'free': usage.free, 'total': usage.total,
|
||||
'host_data_dir': host_dir,
|
||||
'formats': ['xex','god','raw'],
|
||||
'vimm_enabled': os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() in ('1','yes','true'),
|
||||
'bt_engine': 'anacrolix (Go)', 'bt_port': os.getenv('BT_PORT', '6881')}
|
||||
|
||||
|
||||
app.include_router(api)
|
||||
app.mount('/static', StaticFiles(directory=str(Path(__file__).parent / 'static')), name='static')
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
"""Automatic ISO -> XEX/GOD pipeline; no console-side patching of executables."""
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
from . import store
|
||||
from .common import is_archive, is_additional_volume, safe_title
|
||||
|
||||
READY = store.DATA / 'ready'
|
||||
WORK = store.DATA / 'work'
|
||||
RAW = store.DATA / 'raw'
|
||||
LOGS = store.DATA / 'logs'
|
||||
|
||||
|
||||
class ProcessingError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def safe_archive_path(name):
|
||||
# Archive members are always relative, independent of platform.
|
||||
norm = name.replace('\\', '/')
|
||||
if (not norm or norm.startswith('/') or '\x00' in norm or
|
||||
re.match(r'^[A-Za-z]:', norm) or
|
||||
any(x == '..' for x in PurePosixPath(norm).parts)):
|
||||
raise ProcessingError(f'Archiwum zawiera niebezpieczna sciezke: {name[:150]}')
|
||||
|
||||
|
||||
def verify_no_symlinks(directory):
|
||||
for root, dirs, files in os.walk(directory, followlinks=False):
|
||||
for name in dirs + files:
|
||||
path = Path(root) / name
|
||||
if path.is_symlink():
|
||||
raise ProcessingError('Pliki-symboliczne linki nie sa dopuszczalne: ' + name)
|
||||
|
||||
|
||||
async def command(job_id, *args):
|
||||
LOGS.mkdir(parents=True, exist_ok=True)
|
||||
log_path = LOGS / f'{job_id}.log'
|
||||
with log_path.open('ab') as log:
|
||||
log.write(('\n>>> ' + ' '.join(str(x) for x in args) + '\n').encode('utf-8'))
|
||||
process = await asyncio.create_subprocess_exec(*map(str, args), stdout=log, stderr=log,
|
||||
stdin=asyncio.subprocess.DEVNULL)
|
||||
try:
|
||||
code = await process.wait()
|
||||
except asyncio.CancelledError:
|
||||
process.terminate()
|
||||
await process.wait()
|
||||
raise
|
||||
if code:
|
||||
data = log_path.read_bytes()[-2500:].decode('utf-8', errors='replace')
|
||||
raise ProcessingError(f'Narzedzie {args[0]} zakonczone bledem {code}. Ostatni log:\n{data[-1400:]}')
|
||||
|
||||
|
||||
def list_primary_archives(source):
|
||||
files = [p for p in source.rglob('*') if p.is_file() and not p.is_symlink()]
|
||||
archives = [p for p in files if is_archive(p.name) and not is_additional_volume(p.name)]
|
||||
# Multipart .rar is detected by the first volume, subsequent parts are ignored.
|
||||
# A single .zip with .z01 volumes is also supported by 7z.
|
||||
return archives
|
||||
|
||||
|
||||
async def unpack_archive(job_id, archive, destination):
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
lower = archive.name.lower()
|
||||
store.update(job_id, state='extracting', note='Rozpakowywanie ' + archive.name[:100])
|
||||
if lower.endswith('.zip') and not any(archive.parent.glob(archive.stem + '.z[0-9][0-9]')):
|
||||
# Standard ZIP: validate entries to prevent writes outside destination.
|
||||
with zipfile.ZipFile(archive) as z:
|
||||
for member in z.infolist():
|
||||
safe_archive_path(member.filename)
|
||||
if (member.external_attr >> 16) & 0o170000 == 0o120000:
|
||||
raise ProcessingError('Archiwum ZIP zawiera link symboliczny')
|
||||
await asyncio.to_thread(z.extractall, destination)
|
||||
elif lower.endswith(('.tar', '.tar.gz', '.tgz', '.tar.xz', '.tar.bz2')):
|
||||
# tarfile's filters are version-dependent; reject links/special entries ourselves.
|
||||
with tarfile.open(archive) as tf:
|
||||
for member in tf.getmembers():
|
||||
safe_archive_path(member.name)
|
||||
if not (member.isfile() or member.isdir()):
|
||||
raise ProcessingError('Archiwum TAR zawiera link lub plik specjalny')
|
||||
await asyncio.to_thread(tf.extractall, destination, filter='data')
|
||||
else:
|
||||
# 7z/RAR/multipart. Validate file paths *before* invoking extraction.
|
||||
listing = await asyncio.create_subprocess_exec('7z','l','-slt',str(archive),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
listing_out, listing_err = await listing.communicate()
|
||||
if listing.returncode:
|
||||
if not lower.endswith('.rar'):
|
||||
raise ProcessingError('7z nie rozpoznaje archiwum: ' + listing_err[-300:].decode(errors='replace'))
|
||||
# RAR5 fallback: libunarr (unar) understands many archives missing
|
||||
# from Debian's p7zip package. Validate lsar entries first.
|
||||
lsar = await asyncio.create_subprocess_exec('lsar', '-json', '-no-recursion', str(archive),
|
||||
stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
stdout, stderr = await lsar.communicate()
|
||||
if lsar.returncode:
|
||||
raise ProcessingError('7z/unar nie rozpoznaje RAR: ' + stderr[-400:].decode(errors='replace'))
|
||||
try:
|
||||
items = json.loads(stdout).get('lsarContents', [])
|
||||
if not items:
|
||||
raise ProcessingError('Nie mozna odczytac zawartosci RAR')
|
||||
for item in items:
|
||||
safe_archive_path(item['XADFileName'])
|
||||
if any(item.get(x) for x in ('XADIsLink', 'XADIsSymbolicLink', 'XADLinkDestination')):
|
||||
raise ProcessingError('RAR zawiera link symboliczny')
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise ProcessingError('Niepoprawna lista plikow RAR') from exc
|
||||
await command(job_id, 'unar', '-no-directory', '-force-overwrite',
|
||||
'-output-directory', str(destination), str(archive))
|
||||
verify_no_symlinks(destination)
|
||||
return
|
||||
data = listing_out.decode('utf-8', errors='replace')
|
||||
entries = data.split('----------', 1)
|
||||
if len(entries) != 2:
|
||||
raise ProcessingError('Nie mozna zweryfikowac listy plikow archiwum')
|
||||
for line in entries[1].splitlines():
|
||||
if line.startswith('Path = '):
|
||||
safe_archive_path(line[7:])
|
||||
if line.startswith(('Symbolic Link = ', 'Hard Link = ')):
|
||||
raise ProcessingError('Archiwum zawiera link symboliczny lub twardy')
|
||||
await command(job_id, '7z', 'x', '-y', '-bd', str(archive), f'-o{destination}')
|
||||
verify_no_symlinks(destination)
|
||||
|
||||
|
||||
async def locate_payload(job_id, root, tmp):
|
||||
"""Find X360 ISOs or default.xex, recursively extracting up to 2 archive layers."""
|
||||
current = root
|
||||
for depth in range(3):
|
||||
verify_no_symlinks(current)
|
||||
isos = sorted(p for p in current.rglob('*') if p.is_file() and p.suffix.lower() == '.iso')
|
||||
xex_files = sorted(p for p in current.rglob('*') if p.is_file() and p.name.lower() == 'default.xex')
|
||||
if isos or xex_files:
|
||||
return isos, xex_files
|
||||
archives = list_primary_archives(current)
|
||||
if not archives:
|
||||
break
|
||||
if len(archives) != 1:
|
||||
raise ProcessingError('Znaleziono kilka niezaleznych archiwow. Dodaj jedno na zadanie lub polacz czesci gry w jedno archiwum.')
|
||||
if depth >= 2:
|
||||
raise ProcessingError('Archiwa zagniezdzone glebiej niz 3 poziomy nie sa obsługiwane')
|
||||
out = tmp / f'unpacked-{depth}'
|
||||
await unpack_archive(job_id, archives[0], out)
|
||||
current = out
|
||||
raise ProcessingError('Brak obrazu .iso ani default.xex. Sprawdz czy pobrano wszystkie czesci archiwum.')
|
||||
|
||||
|
||||
async def process_job(job):
|
||||
jid = job['id']
|
||||
fmt = job['format']
|
||||
source = RAW / jid
|
||||
temp = WORK / jid
|
||||
slug = safe_title(job['title'])[:55] + '-' + jid[:8]
|
||||
dest = READY / fmt / slug
|
||||
atomic_dest = READY / fmt / ('.' + slug + '.working')
|
||||
if not source.exists():
|
||||
raise ProcessingError('Brak pobranych plikow w katalogu raw')
|
||||
temp.mkdir(parents=True, exist_ok=True)
|
||||
if atomic_dest.exists():
|
||||
shutil.rmtree(atomic_dest)
|
||||
atomic_dest.mkdir(parents=True, exist_ok=True)
|
||||
store.update(jid, state='preparing', note='Analiza pobranych plikow', speed=0)
|
||||
try:
|
||||
verify_no_symlinks(source)
|
||||
if fmt == 'raw':
|
||||
await asyncio.to_thread(shutil.copytree, source, atomic_dest / 'files')
|
||||
else:
|
||||
isos, xex_files = await locate_payload(jid, source, temp)
|
||||
if fmt == 'xex':
|
||||
store.update(jid, state='converting', note='Tworzenie folderu XEX')
|
||||
if xex_files:
|
||||
if len(xex_files) > 1:
|
||||
raise ProcessingError('Kilka plikow default.xex; wybierz jeden obraz ISO na zadanie')
|
||||
await asyncio.to_thread(shutil.copytree, xex_files[0].parent, atomic_dest / 'Gra')
|
||||
else:
|
||||
for n, iso in enumerate(isos, 1):
|
||||
dest_dir = atomic_dest / ('Gra' if len(isos) == 1 else f'Disc_{n}')
|
||||
await command(jid, 'xdvdfs', 'unpack', str(iso), str(dest_dir))
|
||||
if not any(p.name.lower() == 'default.xex' for p in dest_dir.rglob('*') if p.is_file()):
|
||||
raise ProcessingError(f'Obraz ISO nie zawiera default.xex: {iso.name}')
|
||||
else:
|
||||
store.update(jid, state='converting', note='Konwersja ISO do GOD')
|
||||
if not isos and xex_files:
|
||||
if len(xex_files) > 1:
|
||||
raise ProcessingError('Kilka folderow default.xex; GOD wymaga jednego obrazu')
|
||||
packed = temp / 'packed.xiso.iso'
|
||||
await command(jid, 'xdvdfs', 'pack', str(xex_files[0].parent), str(packed))
|
||||
isos = [packed]
|
||||
for n, iso in enumerate(isos, 1):
|
||||
# iso2god creates <TitleID>/<ContentType>/<MediaID> inside provided root.
|
||||
# Prefix these folders with Content/0000000000000000 for Xbox internal HDD.
|
||||
disc_dir = atomic_dest if len(isos) == 1 else atomic_dest / f'Disc_{n}'
|
||||
content = disc_dir / 'Content' / '0000000000000000'
|
||||
content.mkdir(parents=True, exist_ok=True)
|
||||
await command(jid, 'iso2god', str(iso), str(content))
|
||||
if not any(p.is_dir() and re.fullmatch(r'[0-9A-Fa-f]{8}', p.name)
|
||||
for p in content.iterdir()):
|
||||
raise ProcessingError(f'Brak katalogu TitleID w wyniku ISO2GOD: {iso.name}')
|
||||
if dest.exists():
|
||||
raise ProcessingError('Folder wynikowy juz istnieje: ' + str(dest))
|
||||
atomic_dest.rename(dest)
|
||||
store.update(jid, state='done', progress=100, output=str(dest),
|
||||
note='Gotowe do skopiowania na Xbox 360')
|
||||
finally:
|
||||
# Full download is preserved; only temporary extraction is deleted.
|
||||
shutil.rmtree(temp, ignore_errors=True)
|
||||
if atomic_dest.exists():
|
||||
shutil.rmtree(atomic_dest, ignore_errors=True)
|
||||
@@ -0,0 +1,577 @@
|
||||
'use strict';
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const NS = 'http://www.w3.org/2000/svg';
|
||||
const UNITS = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
const ACTIVE = new Set(['downloading', 'preparing', 'extracting', 'converting', 'vimm_browser']);
|
||||
const LABELS = {
|
||||
queued: 'W kolejce', downloading: 'Pobieranie', paused: 'Wstrzymano',
|
||||
preparing: 'Przygotowanie', extracting: 'Rozpakowanie', converting: 'Konwersja',
|
||||
done: 'Gotowe', error: 'Błąd', cancelled: 'Anulowano', vimm_browser: 'Vimm'
|
||||
};
|
||||
const S = { archiveId: '', archiveTitle: '', archiveFiles: [], selected: new Set(),
|
||||
jobs: [], filter: 'all', connected: false, hostRoot: '/xbox', magnetPreview: null,
|
||||
magnetFiles: [], magnetSelected: new Set(), magnetName: '', magnetTitle: '', magnetFormat: 'xex' };
|
||||
|
||||
function normalizeMagnet(raw) {
|
||||
return raw.trim()
|
||||
.replace(/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i, 'magnet:?')
|
||||
.replace(/&|&|&/gi, '&')
|
||||
.replace(/\\\s*/g, '')
|
||||
.replace(/\s+/g, '')
|
||||
.replace(/[\'"`]+$/g, '');
|
||||
}
|
||||
|
||||
function bytes(n) {
|
||||
n = Math.max(0, Number(n) || 0);
|
||||
let i = 0;
|
||||
while (n >= 1024 && i < UNITS.length - 1) { n /= 1024; i++; }
|
||||
return (i ? n.toFixed(1) : n.toFixed(0)) + ' ' + UNITS[i];
|
||||
}
|
||||
|
||||
function el(tag, cls, text) {
|
||||
const node = document.createElement(tag);
|
||||
if (cls) node.className = cls;
|
||||
if (text !== undefined && text !== null) node.textContent = text;
|
||||
return node;
|
||||
}
|
||||
|
||||
function icon(name) {
|
||||
const svg = document.createElementNS(NS, 'svg');
|
||||
svg.setAttribute('class', 'icon');
|
||||
svg.setAttribute('aria-hidden', 'true');
|
||||
const use = document.createElementNS(NS, 'use');
|
||||
use.setAttribute('href', '#i-' + name);
|
||||
svg.append(use);
|
||||
return svg;
|
||||
}
|
||||
|
||||
function makeButton(label, glyph, cls = 'secondary') {
|
||||
const button = el('button', cls);
|
||||
button.type = 'button';
|
||||
button.append(icon(glyph), document.createTextNode(label));
|
||||
return button;
|
||||
}
|
||||
|
||||
async function api(path, options = {}) {
|
||||
const response = await fetch('/api' + path, {
|
||||
...options,
|
||||
headers: { 'Content-Type': 'application/json', ...(options.headers || {}) }
|
||||
});
|
||||
let result;
|
||||
try { result = await response.json(); }
|
||||
catch { result = { detail: 'Nieprawidłowa odpowiedź serwera' }; }
|
||||
if (!response.ok) {
|
||||
throw new Error(typeof result.detail === 'string' ? result.detail : JSON.stringify(result.detail || result));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function connection(online) {
|
||||
S.connected = online;
|
||||
$('connectionState').classList.toggle('online', online);
|
||||
$('connectionState').classList.toggle('offline', !online);
|
||||
$('connectionText').textContent = online ? 'Połączono' : 'Brak połączenia';
|
||||
}
|
||||
|
||||
function flash(message, bad = false) {
|
||||
const t = $('toast');
|
||||
t.textContent = message;
|
||||
t.classList.toggle('bad', bad);
|
||||
t.style.display = 'block';
|
||||
clearTimeout(window._toastTimeout);
|
||||
window._toastTimeout = setTimeout(() => { t.style.display = 'none'; }, 5000);
|
||||
}
|
||||
|
||||
function filterJobs(jobs) {
|
||||
if (S.filter === 'active') return jobs.filter(j => !['done', 'error', 'cancelled'].includes(j.state));
|
||||
if (S.filter === 'done') return jobs.filter(j => j.state === 'done');
|
||||
if (S.filter === 'error') return jobs.filter(j => j.state === 'error');
|
||||
return jobs;
|
||||
}
|
||||
|
||||
function sourceDescription(job) {
|
||||
if (job.source_type === 'archive_many') {
|
||||
try { return 'Internet Archive · ' + JSON.parse(job.source).length + ' plików'; }
|
||||
catch { return 'Internet Archive'; }
|
||||
}
|
||||
if (job.source_type === 'import') return 'Import · ' + job.source;
|
||||
if (job.source_type === 'magnet') return 'Magnet';
|
||||
if (job.source_type === 'torrent_selected') {
|
||||
try { return 'Torrent · ' + JSON.parse(job.source).count + ' plików'; }
|
||||
catch { return 'Torrent'; }
|
||||
}
|
||||
if (job.source_type === 'vimm') return 'Vimm · ' + job.source;
|
||||
return job.source;
|
||||
}
|
||||
|
||||
function jobAction(label, glyph, action, job, danger = false) {
|
||||
const button = makeButton(label, glyph, 'action' + (danger ? ' danger' : ''));
|
||||
button.addEventListener('click', async () => {
|
||||
if (action === 'remove' && !confirm('Usunąć wpis? Pobrane pliki zostaną na dysku.')) return;
|
||||
if (action === 'cancel' && !confirm('Anulować to zadanie?')) return;
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/jobs/' + encodeURIComponent(job.id) + '/' + action, { method: 'POST' });
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
return button;
|
||||
}
|
||||
|
||||
async function showLog(job) {
|
||||
try {
|
||||
const response = await fetch('/api/jobs/' + encodeURIComponent(job.id) + '/log');
|
||||
if (!response.ok) throw new Error('HTTP ' + response.status);
|
||||
$('logContent').textContent = await response.text();
|
||||
$('logDialog').showModal();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
}
|
||||
|
||||
function hostPath(path) {
|
||||
if (typeof path !== 'string') return '';
|
||||
if (path === '/data/raw') return S.hostRoot + '/downloads';
|
||||
if (path.startsWith('/data/raw/')) return S.hostRoot + '/downloads/' + path.slice('/data/raw/'.length);
|
||||
if (path === '/data') return S.hostRoot;
|
||||
if (path.startsWith('/data/')) return S.hostRoot + '/' + path.slice('/data/'.length);
|
||||
return path;
|
||||
}
|
||||
|
||||
function jobRow(job) {
|
||||
const card = el('article', 'job');
|
||||
const header = el('div', 'job-top');
|
||||
const heading = el('div', 'job-heading');
|
||||
heading.append(el('div', 'job-name', job.title), el('div', 'job-sub', sourceDescription(job)));
|
||||
header.append(heading);
|
||||
const statusClass = job.state === 'done' ? 'done' : job.state === 'error' ? 'error' : ACTIVE.has(job.state) ? 'running' : '';
|
||||
header.append(el('span', 'job-status ' + statusClass, LABELS[job.state] || job.state));
|
||||
card.append(header);
|
||||
|
||||
const pct = Math.max(0, Math.min(100, Number(job.progress) || 0));
|
||||
const unknown = job.state === 'vimm_browser';
|
||||
const progress = el('div', 'job-progress');
|
||||
const track = el('div', 'progressbar' + (job.state === 'error' ? ' is-error' : ''));
|
||||
const fill = el('span');
|
||||
fill.style.width = pct + '%';
|
||||
track.append(fill);
|
||||
progress.append(track, el('span', 'progress-text', unknown ? '—' : pct.toFixed(1) + '%'));
|
||||
card.append(progress);
|
||||
|
||||
const meta = el('div', 'job-meta');
|
||||
meta.append(el('span', 'note-text', job.note || 'Oczekiwanie'));
|
||||
meta.append(el('span', 'speed', Number(job.speed) > 0 ? bytes(job.speed) + '/s' : ''));
|
||||
card.append(meta);
|
||||
|
||||
if (job.state === 'error' && job.error) card.append(el('div', 'job-error', job.error));
|
||||
if (job.output) card.append(el('div', 'job-output', 'Zapisano: ' + hostPath(job.output)));
|
||||
|
||||
const bottom = el('div', 'job-bottom');
|
||||
const chips = el('div', 'job-chips');
|
||||
chips.append(el('span', 'job-chip', (job.format || 'raw').toUpperCase()));
|
||||
bottom.append(chips);
|
||||
const actions = el('div', 'job-actions');
|
||||
if (['queued', 'downloading'].includes(job.state)) {
|
||||
actions.append(jobAction('Pauza', 'pause', 'pause', job), jobAction('Anuluj', 'x', 'cancel', job, true));
|
||||
}
|
||||
if (job.state === 'paused') {
|
||||
actions.append(jobAction('Wznów', 'play', 'resume', job), jobAction('Anuluj', 'x', 'cancel', job, true));
|
||||
}
|
||||
if (['error', 'cancelled'].includes(job.state)) actions.append(jobAction('Ponów', 'retry', 'retry', job));
|
||||
if (['error', 'done', 'cancelled', 'paused'].includes(job.state)) actions.append(jobAction('Usuń', 'trash', 'remove', job, true));
|
||||
const log = makeButton('Log', 'file', 'action');
|
||||
log.addEventListener('click', () => showLog(job));
|
||||
actions.append(log);
|
||||
bottom.append(actions);
|
||||
card.append(bottom);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderJobs() {
|
||||
$('jobsCount').textContent = String(S.jobs.length);
|
||||
$('countQueued').textContent = String(S.jobs.filter(j => j.state === 'queued' || j.state === 'paused').length);
|
||||
$('countActive').textContent = String(S.jobs.filter(j => ACTIVE.has(j.state)).length);
|
||||
$('countDone').textContent = String(S.jobs.filter(j => j.state === 'done').length);
|
||||
const list = $('jobsList');
|
||||
list.replaceChildren();
|
||||
const filtered = filterJobs(S.jobs);
|
||||
if (!filtered.length) {
|
||||
list.append(el('div', 'empty-state', S.filter === 'all' ? 'Brak zadań. Dodaj link powyżej.' : 'Brak zadań w tym widoku.'));
|
||||
return;
|
||||
}
|
||||
for (const job of filtered) list.append(jobRow(job));
|
||||
}
|
||||
|
||||
async function refreshJobs(showError = false) {
|
||||
try {
|
||||
S.jobs = await api('/jobs');
|
||||
renderJobs();
|
||||
connection(true);
|
||||
} catch (e) {
|
||||
connection(false);
|
||||
if (showError) flash('Nie można odczytać kolejki: ' + e.message, true);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshSystem() {
|
||||
try {
|
||||
const info = await api('/system');
|
||||
$('storageNum').textContent = bytes(info.free);
|
||||
const percent = info.total > 0 ? (100 * (1 - info.free / info.total)) : 0;
|
||||
$('storageFill').style.width = Math.max(0, Math.min(100, percent)) + '%';
|
||||
$('storageText').textContent = bytes(info.total) + ' łącznie';
|
||||
const hostRoot = String(info.host_data_dir || '/xbox').replace(/\/+$/, '');
|
||||
S.hostRoot = hostRoot;
|
||||
$('importHostPath').textContent = hostRoot + '/imports/';
|
||||
$('readyXexPath').textContent = hostRoot + '/ready/xex/';
|
||||
$('readyGodPath').textContent = hostRoot + '/ready/god/';
|
||||
$('downloadHostPath').textContent = hostRoot + '/downloads/';
|
||||
$('workHostPath').textContent = hostRoot + '/work/';
|
||||
connection(true);
|
||||
} catch {
|
||||
$('storageText').textContent = 'Nie udało się odczytać dysku';
|
||||
connection(false);
|
||||
}
|
||||
}
|
||||
|
||||
for (const tab of document.querySelectorAll('.tab')) {
|
||||
tab.addEventListener('click', () => {
|
||||
for (const other of document.querySelectorAll('.tab')) {
|
||||
const selected = other === tab;
|
||||
other.classList.toggle('active', selected);
|
||||
other.setAttribute('aria-selected', String(selected));
|
||||
}
|
||||
for (const name of ['links', 'archive', 'import']) {
|
||||
$('pane-' + name).classList.toggle('hidden', name !== tab.dataset.tab);
|
||||
}
|
||||
if (tab.dataset.tab === 'import') refreshImports();
|
||||
});
|
||||
}
|
||||
|
||||
for (const filter of document.querySelectorAll('.filter')) {
|
||||
filter.addEventListener('click', () => {
|
||||
S.filter = filter.dataset.filter;
|
||||
for (const other of document.querySelectorAll('.filter')) {
|
||||
const selected = other === filter;
|
||||
other.classList.toggle('active', selected);
|
||||
other.setAttribute('aria-pressed', String(selected));
|
||||
}
|
||||
renderJobs();
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshArchiveSession() {
|
||||
try {
|
||||
const result = await api('/archive/session');
|
||||
$('archiveSessionText').textContent = result.authenticated ? ('Sesja zapisana · ' + result.email) : 'Niezalogowano';
|
||||
$('archiveLoginForm').classList.toggle('hidden', result.authenticated);
|
||||
$('archiveLogout').classList.toggle('hidden', !result.authenticated);
|
||||
} catch (e) { $('archiveSessionText').textContent = 'Błąd sprawdzania sesji'; }
|
||||
}
|
||||
|
||||
$('archiveLoginForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const button = $('archiveLoginForm').querySelector('[type=submit]');
|
||||
button.disabled = true;
|
||||
try {
|
||||
const result = await api('/archive/session', { method: 'POST', body: JSON.stringify({
|
||||
email: $('archiveEmail').value.trim(), password: $('archivePassword').value
|
||||
}) });
|
||||
$('archivePassword').value = '';
|
||||
flash('Zapisano sesję Archive.org: ' + result.email);
|
||||
await refreshArchiveSession();
|
||||
} catch (e) { $('archivePassword').value = ''; flash('Archive.org: ' + e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('archiveLogout').addEventListener('click', async () => {
|
||||
try {
|
||||
await api('/archive/session', { method: 'DELETE' });
|
||||
await refreshArchiveSession();
|
||||
flash('Usunięto zapisaną sesję Archive.org.');
|
||||
} catch (e) { flash(e.message, true); }
|
||||
});
|
||||
|
||||
function renderMagnet() {
|
||||
const list = $('magnetFiles');
|
||||
list.replaceChildren();
|
||||
const text = $('magnetFilter').value.toLowerCase();
|
||||
let displayed = 0;
|
||||
for (const file of S.magnetFiles) {
|
||||
if (!file.name.toLowerCase().includes(text)) continue;
|
||||
if (displayed >= 500) break;
|
||||
displayed++;
|
||||
const row = el('label', 'file-row');
|
||||
const box = el('input');
|
||||
box.type = 'checkbox';
|
||||
box.checked = S.magnetSelected.has(file.index);
|
||||
box.disabled = Boolean(file.padding);
|
||||
box.addEventListener('change', () => {
|
||||
if (box.checked) S.magnetSelected.add(file.index);
|
||||
else S.magnetSelected.delete(file.index);
|
||||
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
|
||||
});
|
||||
row.append(box, el('span', 'file-name', file.name),
|
||||
el('span', 'size', file.padding ? 'Systemowy' : bytes(file.size)));
|
||||
list.append(row);
|
||||
}
|
||||
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
|
||||
if (displayed >= 500) list.append(el('div', 'empty-state', 'Pierwsze 500 plików. Zawęź filtr, aby wyszukać kolejne.'));
|
||||
$('magnetSelectedCount').textContent = 'Wybrano: ' + S.magnetSelected.size;
|
||||
}
|
||||
|
||||
async function pollMagnet(previewId) {
|
||||
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
|
||||
try {
|
||||
const data = await api('/magnet/inspect/' + encodeURIComponent(previewId));
|
||||
if (S.magnetPreview !== previewId || !$('magnetDialog').open) return;
|
||||
if (data.status === 'ready') {
|
||||
S.magnetName = data.name;
|
||||
S.magnetFiles = data.files;
|
||||
S.magnetSelected = new Set((data.suggested || []).filter(i => !data.files.find(f => f.index === i)?.padding));
|
||||
$('magnetName').textContent = data.name + ' (' + data.files.length + ' plików)';
|
||||
$('magnetStatus').textContent = 'Metadane pobrane. Wybierz pliki.';
|
||||
$('magnetResults').classList.remove('hidden');
|
||||
renderMagnet();
|
||||
return;
|
||||
}
|
||||
if (data.status === 'loading') {
|
||||
const seconds = Number(data.elapsed) || 0;
|
||||
const connections = Number(data.connections) || 0;
|
||||
const known = Number(data.known_peers) || 0;
|
||||
let notice = 'Metadane: ' + seconds + ' s | połączenia: ' + connections + ' | znani peerzy: ' + known;
|
||||
if (seconds >= 25 && connections === 0)
|
||||
notice += ' | Brak połączeń. Sprawdź UDP/DHT lub wczytaj .torrent.';
|
||||
$('magnetStatus').textContent = notice;
|
||||
}
|
||||
if (data.status === 'error') {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + data.error;
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + e.message;
|
||||
return;
|
||||
}
|
||||
setTimeout(() => pollMagnet(previewId), 3000);
|
||||
}
|
||||
|
||||
async function inspectMagnet(magnet, title, format) {
|
||||
S.magnetPreview = null;
|
||||
S.magnetTitle = title;
|
||||
S.magnetFormat = format;
|
||||
S.magnetFiles = [];
|
||||
S.magnetSelected.clear();
|
||||
$('magnetStatus').textContent = 'Oczekiwanie na metadane z sieci BitTorrent…';
|
||||
$('magnetResults').classList.add('hidden');
|
||||
$('magnetFilter').value = '';
|
||||
$('magnetDialog').showModal();
|
||||
try {
|
||||
const data = await api('/magnet/inspect', { method: 'POST', body: JSON.stringify({ magnet }) });
|
||||
if (!$('magnetDialog').open) return;
|
||||
S.magnetPreview = data.preview_id;
|
||||
pollMagnet(data.preview_id);
|
||||
} catch (e) { $('magnetStatus').textContent = 'Błąd: ' + e.message; }
|
||||
}
|
||||
|
||||
async function uploadTorrent(file, fromLinks) {
|
||||
if (!file) return;
|
||||
if (file.size > 32 * 1024 * 1024) return flash('Limit pliku .torrent: 32 MiB.', true);
|
||||
if (fromLinks) {
|
||||
S.magnetTitle = $('linkTitle').value.trim();
|
||||
S.magnetFormat = $('linkFormat').value;
|
||||
}
|
||||
const previous = S.magnetPreview;
|
||||
S.magnetPreview = null;
|
||||
S.magnetFiles = [];
|
||||
S.magnetSelected.clear();
|
||||
$('magnetResults').classList.add('hidden');
|
||||
$('magnetFilter').value = '';
|
||||
$('magnetStatus').textContent = 'Wczytywanie pliku .torrent...';
|
||||
if (!$('magnetDialog').open) $('magnetDialog').showModal();
|
||||
if (previous) await api('/magnet/inspect/' + encodeURIComponent(previous) + '/cancel', { method: 'POST' }).catch(() => {});
|
||||
try {
|
||||
const response = await fetch('/api/magnet/upload', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/x-bittorrent' }, body: file
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.detail || 'HTTP ' + response.status);
|
||||
if (!$('magnetDialog').open) return;
|
||||
S.magnetPreview = data.preview_id;
|
||||
await pollMagnet(data.preview_id);
|
||||
} catch (error) {
|
||||
$('magnetStatus').textContent = 'Błąd: ' + error.message;
|
||||
}
|
||||
}
|
||||
|
||||
for (const id of ['torrentFileLinks', 'torrentFileDialog']) {
|
||||
$(id).addEventListener('change', async (event) => {
|
||||
const file = event.target.files?.[0];
|
||||
event.target.value = '';
|
||||
await uploadTorrent(file, id === 'torrentFileLinks');
|
||||
});
|
||||
}
|
||||
|
||||
$('magnetFilter').addEventListener('input', renderMagnet);
|
||||
$('magnetToggleAll').addEventListener('click', () => {
|
||||
const visible = S.magnetFiles.filter(f => !f.padding && f.name.toLowerCase().includes($('magnetFilter').value.toLowerCase())).slice(0, 500);
|
||||
const allSelected = visible.every(f => S.magnetSelected.has(f.index));
|
||||
for (const file of visible) {
|
||||
if (allSelected) S.magnetSelected.delete(file.index);
|
||||
else S.magnetSelected.add(file.index);
|
||||
}
|
||||
renderMagnet();
|
||||
});
|
||||
$('closeMagnet').addEventListener('click', () => $('magnetDialog').close());
|
||||
$('magnetDialog').addEventListener('close', () => {
|
||||
const preview = S.magnetPreview;
|
||||
S.magnetPreview = null;
|
||||
if (preview) api('/magnet/inspect/' + encodeURIComponent(preview) + '/cancel', { method: 'POST' }).catch(() => {});
|
||||
});
|
||||
$('magnetAdd').addEventListener('click', async () => {
|
||||
if (!S.magnetSelected.size) return flash('Zaznacz co najmniej jeden plik.', true);
|
||||
const button = $('magnetAdd');
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/magnet/jobs', { method: 'POST', body: JSON.stringify({
|
||||
preview_id: S.magnetPreview, files: [...S.magnetSelected],
|
||||
title: S.magnetTitle, format: S.magnetFormat
|
||||
}) });
|
||||
$('urls').value = '';
|
||||
S.magnetPreview = null;
|
||||
$('magnetDialog').close();
|
||||
flash('Wybrane pliki dodano do kolejki.');
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('linksForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const raw = $('urls').value.trim();
|
||||
if (!raw) return flash('Podaj co najmniej jeden link.', true);
|
||||
const button = $('linksForm').querySelector('[type=submit]');
|
||||
const title = $('linkTitle').value.trim();
|
||||
const format = $('linkFormat').value;
|
||||
if (/^[\'"`]*\*{0,2}magnet\*{0,2}:\?/i.test(raw)) {
|
||||
const magnet = normalizeMagnet(raw);
|
||||
if ((magnet.match(/magnet:\?/gi) || []).length !== 1)
|
||||
return flash('Magnet wklej pojedynczo, aby wybrać pliki.', true);
|
||||
await inspectMagnet(magnet, title, format);
|
||||
return;
|
||||
}
|
||||
const items = raw.split(/\r?\n/).map(s => s.trim()).filter(Boolean);
|
||||
let ok = 0;
|
||||
const errors = [];
|
||||
button.disabled = true;
|
||||
try {
|
||||
for (const url of items) {
|
||||
try { await api('/jobs', { method: 'POST', body: JSON.stringify({ url, title, format }) }); ok++; }
|
||||
catch (e) { errors.push(url.slice(0, 65) + ': ' + e.message); }
|
||||
}
|
||||
if (ok) {
|
||||
$('urls').value = '';
|
||||
flash('Dodano zadań: ' + ok);
|
||||
await refreshJobs();
|
||||
}
|
||||
if (errors.length) flash(errors.join('\n'), true);
|
||||
} finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
$('archiveForm').addEventListener('submit', async (event) => {
|
||||
event.preventDefault();
|
||||
const button = $('archiveForm').querySelector('[type=submit]');
|
||||
button.disabled = true;
|
||||
$('archiveResults').classList.add('hidden');
|
||||
try {
|
||||
const data = await api('/archive/lookup', { method: 'POST', body: JSON.stringify({ identifier: $('archiveId').value }) });
|
||||
S.archiveId = data.identifier;
|
||||
S.archiveTitle = data.title;
|
||||
S.archiveFiles = data.files;
|
||||
S.selected.clear();
|
||||
$('archiveName').textContent = data.title + ' (' + data.files.length + ')';
|
||||
$('archiveResults').classList.remove('hidden');
|
||||
renderArchive();
|
||||
} catch (e) { flash('Internet Archive: ' + e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
function renderArchive() {
|
||||
const list = $('archiveFiles');
|
||||
list.replaceChildren();
|
||||
const text = $('archiveFilter').value.toLowerCase();
|
||||
let displayed = 0;
|
||||
for (const file of S.archiveFiles) {
|
||||
if (!file.name.toLowerCase().includes(text)) continue;
|
||||
if (displayed >= 400) break;
|
||||
displayed++;
|
||||
const row = el('label', 'file-row');
|
||||
const checkbox = el('input');
|
||||
checkbox.type = 'checkbox';
|
||||
checkbox.checked = S.selected.has(file.name);
|
||||
checkbox.addEventListener('change', () => {
|
||||
if (checkbox.checked) S.selected.add(file.name);
|
||||
else S.selected.delete(file.name);
|
||||
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
|
||||
});
|
||||
row.append(checkbox, el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
|
||||
list.append(row);
|
||||
}
|
||||
if (!displayed) list.append(el('div', 'empty-state', 'Brak plików.'));
|
||||
$('selectedCount').textContent = 'Wybrano: ' + S.selected.size;
|
||||
}
|
||||
|
||||
$('archiveFilter').addEventListener('input', renderArchive);
|
||||
$('archiveAdd').addEventListener('click', async () => {
|
||||
if (!S.selected.size) return flash('Zaznacz pliki.', true);
|
||||
const button = $('archiveAdd');
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/archive/jobs', { method: 'POST', body: JSON.stringify({
|
||||
identifier: S.archiveId, files: [...S.selected], title: S.archiveTitle, format: $('archiveFormat').value
|
||||
}) });
|
||||
S.selected.clear();
|
||||
renderArchive();
|
||||
flash('Dodano do kolejki.');
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
|
||||
async function refreshImports() {
|
||||
try {
|
||||
const files = await api('/imports');
|
||||
const list = $('importList');
|
||||
list.replaceChildren();
|
||||
if (!files.length) {
|
||||
list.append(el('div', 'empty-state', 'Katalog data/imports/ jest pusty.'));
|
||||
return;
|
||||
}
|
||||
for (const file of files) {
|
||||
const row = el('div', 'file-row');
|
||||
row.append(icon('file'), el('span', 'file-name', file.name), el('span', 'size', bytes(file.size)));
|
||||
const button = makeButton('Przetwórz', 'play');
|
||||
button.addEventListener('click', async () => {
|
||||
button.disabled = true;
|
||||
try {
|
||||
await api('/imports/jobs', { method: 'POST', body: JSON.stringify({
|
||||
filename: file.name, format: $('importFormat').value
|
||||
}) });
|
||||
flash('Dodano: ' + file.name);
|
||||
await refreshJobs();
|
||||
} catch (e) { flash(e.message, true); }
|
||||
finally { button.disabled = false; }
|
||||
});
|
||||
row.append(button);
|
||||
list.append(row);
|
||||
}
|
||||
} catch (e) { flash('Import: ' + e.message, true); }
|
||||
}
|
||||
|
||||
$('refreshImports').addEventListener('click', refreshImports);
|
||||
$('refreshJobs').addEventListener('click', () => refreshJobs(true));
|
||||
$('closeLog').addEventListener('click', () => $('logDialog').close());
|
||||
refreshJobs();
|
||||
refreshSystem();
|
||||
refreshArchiveSession();
|
||||
setInterval(() => refreshJobs(), 3000);
|
||||
setInterval(refreshSystem, 30000);
|
||||
@@ -0,0 +1,4 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
|
||||
<rect width="64" height="64" rx="10" fill="#1b1e21"/>
|
||||
<path d="M17 16 32 32 47 16M17 48 32 32 47 48" fill="none" stroke="#91c99a" stroke-width="6" stroke-linecap="square"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 245 B |
@@ -0,0 +1,151 @@
|
||||
<!doctype html>
|
||||
<html lang="pl">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="dark">
|
||||
<meta name="theme-color" content="#141619">
|
||||
<title>X360 Queue</title>
|
||||
<link rel="icon" href="/static/icon.svg" type="image/svg+xml">
|
||||
<link rel="stylesheet" href="/static/style.css">
|
||||
<script src="/static/app.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<svg class="svg-defs" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
|
||||
<symbol id="i-download" viewBox="0 0 24 24"><path d="M12 3v12m-5-5 5 5 5-5M5 17v3h14v-3"/></symbol>
|
||||
<symbol id="i-link" viewBox="0 0 24 24"><path d="M10 13a5 5 0 0 0 7.4.4l2-2a5 5 0 0 0-7.1-7.1l-1.1 1.1M14 11a5 5 0 0 0-7.4-.4l-2 2a5 5 0 0 0 7.1 7.1l1.1-1.1"/></symbol>
|
||||
<symbol id="i-archive" viewBox="0 0 24 24"><rect x="3" y="4" width="18" height="4" rx="1"/><path d="M5 8v12h14V8M10 12h4"/></symbol>
|
||||
<symbol id="i-folder" viewBox="0 0 24 24"><path d="M3 7a2 2 0 0 1 2-2h5l2 2h7a2 2 0 0 1 2 2v10H3z"/></symbol>
|
||||
<symbol id="i-plus" viewBox="0 0 24 24"><path d="M12 5v14M5 12h14"/></symbol>
|
||||
<symbol id="i-refresh" viewBox="0 0 24 24"><path d="M20 7v5h-5M4 17v-5h5"/><path d="M5.6 9A7 7 0 0 1 18 6l2 6M4 12l2 6a7 7 0 0 0 12.4-3"/></symbol>
|
||||
<symbol id="i-pause" viewBox="0 0 24 24"><path d="M8 5v14M16 5v14"/></symbol>
|
||||
<symbol id="i-play" viewBox="0 0 24 24"><path d="m8 5 11 7-11 7z"/></symbol>
|
||||
<symbol id="i-x" viewBox="0 0 24 24"><path d="M6 6l12 12M18 6 6 18"/></symbol>
|
||||
<symbol id="i-retry" viewBox="0 0 24 24"><path d="M3 12a9 9 0 1 0 3-6.7M3 4v5h5"/></symbol>
|
||||
<symbol id="i-trash" viewBox="0 0 24 24"><path d="M4 7h16M10 4h4M6 7l1 13h10l1-13M10 11v5m4-5v5"/></symbol>
|
||||
<symbol id="i-file" viewBox="0 0 24 24"><path d="M6 3h8l4 4v14H6zM14 3v5h5M9 12h6M9 16h6"/></symbol>
|
||||
<symbol id="i-harddrive" viewBox="0 0 24 24"><rect x="3" y="5" width="18" height="14" rx="2"/><path d="M3 13h18M7 16h.01M11 16h.01"/></symbol>
|
||||
<symbol id="i-activity" viewBox="0 0 24 24"><path d="M3 12h4l3-7 4 14 3-7h4"/></symbol>
|
||||
<symbol id="i-check" viewBox="0 0 24 24"><path d="m5 12 5 5L19 7"/></symbol>
|
||||
<symbol id="i-clock" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6l4 2"/></symbol>
|
||||
<symbol id="i-alert" viewBox="0 0 24 24"><circle cx="12" cy="12" r="9"/><path d="M12 7v6m0 4v.1"/></symbol>
|
||||
<symbol id="i-list" viewBox="0 0 24 24"><path d="M9 6h12M9 12h12M9 18h12M4 6h.01M4 12h.01M4 18h.01"/></symbol>
|
||||
</svg>
|
||||
|
||||
<div class="shell">
|
||||
<header class="topbar">
|
||||
<div class="brand">
|
||||
<div class="brand-mark" aria-hidden="true">X</div>
|
||||
<div class="brand-text"><strong>X360 Queue</strong><span>Linux / RGH</span></div>
|
||||
</div>
|
||||
<div class="topbar-right">
|
||||
<span class="connection" id="connectionState"><span class="connection-dot"></span><span id="connectionText">Łączenie…</span></span>
|
||||
<span class="version">LOCAL</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="page-title">
|
||||
<div><h1>Pobieranie</h1><p>Dodawanie plików i konwersja XEX / GOD.</p></div>
|
||||
</div>
|
||||
|
||||
<section class="metrics" aria-label="Stan aplikacji">
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-clock"/></svg> W kolejce</span><strong id="countQueued">0</strong></div>
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-activity"/></svg> W toku</span><strong id="countActive">0</strong></div>
|
||||
<div class="metric"><span class="metric-label"><svg class="icon"><use href="#i-check"/></svg> Gotowe</span><strong id="countDone">0</strong></div>
|
||||
<div class="metric disk-metric">
|
||||
<span class="metric-label"><svg class="icon"><use href="#i-harddrive"/></svg> Wolne miejsce</span>
|
||||
<strong id="storageNum">—</strong>
|
||||
<div class="meter"><span id="storageFill" style="width:0%"></span></div>
|
||||
<small id="storageText">—</small>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" aria-labelledby="addHeading">
|
||||
<div class="panel-heading"><h2 id="addHeading">Dodaj zadanie</h2><span class="muted-info">Zadania działają po zamknięciu przeglądarki</span></div>
|
||||
<nav class="tabs" aria-label="Wybierz źródło">
|
||||
<button class="tab active" type="button" data-tab="links" aria-selected="true"><svg class="icon"><use href="#i-link"/></svg>Link / magnet</button>
|
||||
<button class="tab" type="button" data-tab="archive" aria-selected="false"><svg class="icon"><use href="#i-archive"/></svg>Internet Archive</button>
|
||||
<button class="tab" type="button" data-tab="import" aria-selected="false"><svg class="icon"><use href="#i-folder"/></svg>Import</button>
|
||||
</nav>
|
||||
|
||||
<div class="tab-body" id="pane-links">
|
||||
<form id="linksForm">
|
||||
<div class="field">
|
||||
<label for="urls">Linki <span>— jeden adres w wierszu</span></label>
|
||||
<textarea id="urls" rows="3" spellcheck="false" placeholder="https://example.org/obraz.iso magnet:?xt=urn:btih:..."></textarea>
|
||||
</div>
|
||||
<div class="input-row">
|
||||
<div class="field title-field"><label for="linkTitle">Nazwa <span>(opcjonalnie)</span></label><input id="linkTitle" placeholder="Automatyczna" autocomplete="off"></div>
|
||||
<div class="field format-field"><label for="linkFormat">Format wyjściowy</label><select id="linkFormat"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select></div>
|
||||
<button class="primary submit-button" type="submit"><svg class="icon"><use href="#i-plus"/></svg>Dodaj do kolejki</button>
|
||||
</div>
|
||||
</form>
|
||||
<div class="torrent-upload-inline"><label for="torrentFileLinks" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileLinks" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Alternatywa, gdy magnet nie zwraca metadanych.</span></div>
|
||||
<p class="note">Magnet: najpierw lista plików, potem wybór i pobieranie. Vimm: tylko dostępny przycisk pobierania. Limity i zabezpieczenia serwisu pozostają aktywne.</p>
|
||||
</div>
|
||||
|
||||
<div class="tab-body hidden" id="pane-archive">
|
||||
<section class="archive-auth" aria-label="Konto Internet Archive">
|
||||
<div class="auth-heading"><strong>Konto Archive.org</strong><span id="archiveSessionText">Sprawdzanie sesji…</span></div>
|
||||
<form id="archiveLoginForm" class="auth-login hidden" autocomplete="on">
|
||||
<div class="field"><label for="archiveEmail">E-mail</label><input id="archiveEmail" type="email" autocomplete="username" required placeholder="konto@example.org"></div>
|
||||
<div class="field"><label for="archivePassword">Hasło</label><input id="archivePassword" type="password" autocomplete="current-password" required></div>
|
||||
<button type="submit" class="secondary"><svg class="icon"><use href="#i-link"/></svg>Zaloguj</button>
|
||||
</form>
|
||||
<button id="archiveLogout" class="secondary hidden" type="button">Wyloguj</button>
|
||||
<p class="note">Hasło nie jest zapisywane. Pobieranie używa sesji Archive.org; konto nie odblokowuje plików niedostępnych dla użytkownika. Panel udostępniaj przez HTTPS lub VPN.</p>
|
||||
</section>
|
||||
<form id="archiveForm">
|
||||
<div class="field"><label for="archiveId">Adres lub identyfikator</label>
|
||||
<div class="inline-input"><input id="archiveId" placeholder="archive.org/details/identyfikator" spellcheck="false"><button class="secondary" type="submit"><svg class="icon"><use href="#i-list"/></svg>Pokaż pliki</button></div>
|
||||
</div>
|
||||
</form>
|
||||
<div class="archive-results hidden" id="archiveResults">
|
||||
<div class="file-heading"><h3 id="archiveName"></h3><input id="archiveFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
|
||||
<div class="file-table" id="archiveFiles"></div>
|
||||
<div class="file-bottom"><span id="selectedCount">Wybrano: 0</span><div class="selection-controls"><select id="archiveFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="archiveAdd" class="primary" type="button"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
|
||||
<p class="note">Części jednego archiwum zaznacz razem.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="tab-body hidden" id="pane-import">
|
||||
<div class="import-toolbar"><div><p class="import-hint">Skopiuj pliki do <code id="importHostPath">/xbox/imports/</code>.</p></div><div class="selection-controls"><select id="importFormat" aria-label="Format wyjściowy"><option value="xex">XEX</option><option value="god">GOD</option><option value="raw">Bez konwersji</option></select><button id="refreshImports" class="secondary" type="button"><svg class="icon"><use href="#i-refresh"/></svg>Odśwież</button></div></div>
|
||||
<div id="importList" class="file-table"></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="queue-section" aria-labelledby="queueHeading">
|
||||
<div class="queue-header">
|
||||
<div class="queue-title"><h2 id="queueHeading">Kolejka <span id="jobsCount" class="count">0</span></h2><span class="muted-info">Odświeżanie co 3 s</span></div>
|
||||
<div class="queue-controls">
|
||||
<div class="filters" aria-label="Filtruj zadania">
|
||||
<button type="button" class="filter active" data-filter="all" aria-pressed="true">Wszystkie</button>
|
||||
<button type="button" class="filter" data-filter="active" aria-pressed="false">W toku</button>
|
||||
<button type="button" class="filter" data-filter="done" aria-pressed="false">Gotowe</button>
|
||||
<button type="button" class="filter" data-filter="error" aria-pressed="false">Błędy</button>
|
||||
</div>
|
||||
<button id="refreshJobs" class="secondary icon-button" type="button" title="Odśwież kolejkę" aria-label="Odśwież kolejkę"><svg class="icon"><use href="#i-refresh"/></svg></button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="jobsList" class="jobs"><div class="empty-state">Brak zadań.</div></div>
|
||||
</section>
|
||||
<footer class="footer"><span>Wyniki: <code id="readyXexPath">/xbox/ready/xex/</code> lub <code id="readyGodPath">/xbox/ready/god/</code></span><span>Pobieranie: <code id="downloadHostPath">/xbox/downloads/</code> · Robocze: <code id="workHostPath">/xbox/work/</code></span></footer>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<div id="toast" role="status" aria-live="polite"></div>
|
||||
<dialog id="magnetDialog" aria-labelledby="magnetHeading">
|
||||
<div class="dialog-heading"><h2 id="magnetHeading">Pliki w torrencie</h2><button type="button" class="secondary" id="closeMagnet"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div>
|
||||
<p id="magnetStatus" class="preview-status">Pobieranie metadanych torrenta…</p>
|
||||
<div class="torrent-upload-inline"><label for="torrentFileDialog" class="secondary file-label">Wczytaj .torrent</label><input id="torrentFileDialog" type="file" accept=".torrent,application/x-bittorrent" hidden><span>Nie wymaga peerów ani oczekiwania na DHT.</span></div>
|
||||
<div id="magnetResults" class="hidden">
|
||||
<div class="file-heading"><h3 id="magnetName"></h3><input id="magnetFilter" placeholder="Filtruj pliki…" aria-label="Filtruj pliki"></div>
|
||||
<div class="file-table" id="magnetFiles"></div>
|
||||
<div class="file-bottom"><span id="magnetSelectedCount">Wybrano: 0</span><div class="selection-controls"><button type="button" id="magnetToggleAll" class="secondary">Zaznacz widoczne</button><button type="button" id="magnetAdd" class="primary"><svg class="icon"><use href="#i-download"/></svg>Pobierz wybrane</button></div></div>
|
||||
<p class="note">Pobieranie rozpocznie się dopiero po wybraniu plików. Przy archiwach wieloczęściowych zaznacz wszystkie części.</p>
|
||||
</div>
|
||||
</dialog>
|
||||
<dialog id="logDialog" aria-labelledby="logHeading"><div class="dialog-heading"><h2 id="logHeading">Log zadania</h2><button type="button" class="secondary" id="closeLog"><svg class="icon"><use href="#i-x"/></svg>Zamknij</button></div><pre id="logContent"></pre></dialog>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,214 @@
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
font-family: system-ui, -apple-system, "Segoe UI", Arial, sans-serif;
|
||||
font-size: 14px;
|
||||
font-synthesis: none;
|
||||
--bg: #121416;
|
||||
--panel: #1b1e21;
|
||||
--panel-hover: #21252a;
|
||||
--input: #151719;
|
||||
--line: #34383d;
|
||||
--line-soft: #2a2e32;
|
||||
--text: #e4e7e9;
|
||||
--muted: #a2a9af;
|
||||
--faint: #788189;
|
||||
--accent: #91c99a;
|
||||
--accent-hover: #afe0b5;
|
||||
--success-bg: #203427;
|
||||
--danger: #e8a09b;
|
||||
--danger-bg: #362324;
|
||||
}
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--text); }
|
||||
button, input, textarea, select { font: inherit; }
|
||||
button { cursor: pointer; }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
|
||||
.hidden, .svg-defs { display: none !important; }
|
||||
.icon { width: 16px; height: 16px; flex: 0 0 auto; fill: none; stroke: currentColor; stroke-width: 1.7; stroke-linecap: round; stroke-linejoin: round; }
|
||||
.shell { max-width: 1120px; margin: auto; padding: 0 24px; }
|
||||
.topbar { height: 66px; display: flex; align-items: center; justify-content: space-between; border-bottom: 1px solid var(--line); }
|
||||
.brand { display: flex; align-items: center; gap: 11px; }
|
||||
.brand-mark { display: grid; place-items: center; width: 33px; height: 33px; border: 1px solid #72977c; color: #a9d6ad; background: #1d2a21; border-radius: 5px; font-weight: 800; font-size: 19px; line-height: 1; }
|
||||
.brand-text { display: flex; flex-direction: column; gap: 2px; }
|
||||
.brand-text strong { font-size: 14px; font-weight: 700; letter-spacing: .1px; }
|
||||
.brand-text span { color: var(--faint); font-size: 11px; }
|
||||
.topbar-right { display: flex; align-items: center; gap: 17px; }
|
||||
.connection { display: inline-flex; align-items: center; gap: 7px; font-size: 12px; color: var(--muted); }
|
||||
.connection-dot { width: 7px; height: 7px; background: var(--faint); border-radius: 50%; }
|
||||
.connection.online .connection-dot { background: var(--accent); }
|
||||
.connection.offline .connection-dot { background: var(--danger); }
|
||||
.version { border: 1px solid var(--line); color: var(--faint); font: 600 10px ui-monospace, SFMono-Regular, Consolas, monospace; padding: 4px 7px; letter-spacing: .5px; border-radius: 3px; }
|
||||
.page-title { padding: 31px 0 22px; }
|
||||
h1, h2, h3, p { margin-top: 0; }
|
||||
h1 { margin-bottom: 5px; font-size: 25px; font-weight: 680; letter-spacing: -.65px; line-height: 1.3; }
|
||||
.page-title p { margin-bottom: 0; color: var(--muted); font-size: 13px; }
|
||||
.metrics { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); border: 1px solid var(--line); border-radius: 5px; background: var(--panel); overflow: hidden; }
|
||||
.metric { display: flex; flex-direction: column; gap: 9px; padding: 16px 19px; min-height: 112px; }
|
||||
.metric + .metric { border-left: 1px solid var(--line); }
|
||||
.metric-label { display: inline-flex; align-items: center; gap: 8px; color: var(--muted); font-size: 12px; }
|
||||
.metric-label .icon { width: 14px; height: 14px; color: var(--faint); }
|
||||
.metric strong { color: var(--text); font-weight: 650; font-size: 25px; font-variant-numeric: tabular-nums; line-height: 1; }
|
||||
.disk-metric { gap: 7px; }
|
||||
.disk-metric strong { font-size: 20px; }
|
||||
.disk-metric small { font-size: 10px; color: var(--faint); }
|
||||
.meter { height: 4px; background: #353a3e; overflow: hidden; width: 100%; }
|
||||
.meter span { display: block; height: 100%; background: var(--accent); transition: width .2s linear; }
|
||||
.panel { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; margin-top: 26px; }
|
||||
.panel-heading { display: flex; align-items: center; justify-content: space-between; gap: 10px; padding: 18px 20px 15px; }
|
||||
h2 { margin: 0; font-size: 16px; font-weight: 670; letter-spacing: -.2px; }
|
||||
.muted-info { font-size: 11px; color: var(--faint); }
|
||||
.tabs { display: flex; gap: 3px; padding: 0 15px; border-bottom: 1px solid var(--line); }
|
||||
.tab { display: inline-flex; align-items: center; gap: 8px; background: transparent; border: none; border-bottom: 2px solid transparent; padding: 11px 13px 12px; color: var(--muted); font-size: 12px; font-weight: 550; }
|
||||
.tab:hover { color: var(--text); background: #25292d; }
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
.tab .icon { width: 15px; height: 15px; }
|
||||
.tab-body { padding: 21px 20px 20px; }
|
||||
.field { min-width: 0; }
|
||||
label { display: block; margin-bottom: 7px; font-size: 12px; color: #cbd0d4; font-weight: 600; }
|
||||
label span { color: var(--faint); font-weight: 400; }
|
||||
input, select, textarea { display: block; width: 100%; min-width: 0; color: var(--text); background: var(--input); border: 1px solid #42474c; border-radius: 4px; padding: 9px 11px; outline: none; font-size: 12px; }
|
||||
input, select { height: 38px; }
|
||||
textarea { min-height: 89px; resize: vertical; line-height: 1.6; }
|
||||
input::placeholder, textarea::placeholder { color: #737c84; }
|
||||
select { appearance: auto; }
|
||||
.input-row { display: flex; gap: 12px; align-items: flex-end; margin-top: 13px; }
|
||||
.input-row .title-field { flex: 1 1 auto; }
|
||||
.input-row .format-field { flex: 0 0 172px; }
|
||||
button.primary, button.secondary, button.action { border-radius: 4px; display: inline-flex; align-items: center; justify-content: center; gap: 8px; font-size: 12px; font-weight: 620; white-space: nowrap; min-height: 38px; padding: 9px 12px; }
|
||||
button.primary { border: 1px solid #9ccc9e; color: #18231b; background: var(--accent); }
|
||||
button.primary:hover:not(:disabled) { background: var(--accent-hover); }
|
||||
button.secondary, button.action { border: 1px solid #444a50; background: #292d31; color: #d9dee1; }
|
||||
button.secondary:hover:not(:disabled), button.action:hover:not(:disabled) { background: #34393f; }
|
||||
button.action { min-height: 29px; padding: 5px 9px; font-size: 11px; }
|
||||
button.action.danger { color: #e3a6a4; }
|
||||
button.action.danger:hover { background: #3a2829; }
|
||||
button.icon-button { min-width: 36px; width: 36px; padding: 8px; }
|
||||
.note { color: var(--faint); font-size: 11px; line-height: 1.55; margin: 13px 0 0; }
|
||||
.inline-input { display: flex; align-items: center; gap: 9px; }
|
||||
.inline-input input { flex: 1 1 auto; }
|
||||
.file-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 24px; }
|
||||
.file-heading h3 { font-size: 13px; font-weight: 600; margin: 0; max-width: 62%; overflow-wrap: anywhere; }
|
||||
.file-heading input { max-width: 250px; }
|
||||
.file-table { max-height: 320px; overflow: auto; margin-top: 12px; border: 1px solid var(--line); border-radius: 4px; }
|
||||
.file-row { display: flex; gap: 12px; align-items: center; min-height: 44px; padding: 9px 11px; margin: 0; border-bottom: 1px solid var(--line-soft); font-size: 12px; color: #d2d7da; font-weight: 400; }
|
||||
.file-row:last-child { border-bottom: 0; }
|
||||
.file-row:hover { background: var(--panel-hover); }
|
||||
.file-row input[type=checkbox] { margin: 0; min-height: 0; width: 15px; height: 15px; accent-color: var(--accent); flex-shrink: 0; }
|
||||
.file-name { flex: 1; min-width: 0; overflow-wrap: anywhere; }
|
||||
.size { color: var(--faint); font-size: 11px; white-space: nowrap; }
|
||||
.file-bottom { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-top: 13px; color: var(--muted); font-size: 12px; }
|
||||
.selection-controls { display: flex; align-items: center; gap: 9px; }
|
||||
.selection-controls select { width: 145px; }
|
||||
.import-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 15px; }
|
||||
.import-hint { font-size: 12px; color: var(--muted); margin: 0; }
|
||||
code { color: #b7d1bd; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; font-size: 11px; }
|
||||
.queue-section { padding-top: 29px; }
|
||||
.queue-header { display: flex; justify-content: space-between; align-items: center; gap: 14px; margin-bottom: 14px; }
|
||||
.queue-title { display: flex; align-items: baseline; gap: 13px; }
|
||||
.queue-title h2 { display: flex; align-items: center; gap: 9px; }
|
||||
.count { font-size: 11px; color: #bdc4c9; background: #2c3034; border: 1px solid #3d4247; min-width: 21px; padding: 2px 6px; text-align: center; border-radius: 3px; font-variant-numeric: tabular-nums; }
|
||||
.queue-controls { display: flex; gap: 9px; align-items: center; }
|
||||
.filters { display: flex; border: 1px solid var(--line); border-radius: 4px; overflow: hidden; }
|
||||
.filter { background: #1c1f22; color: var(--muted); border: 0; border-right: 1px solid var(--line); padding: 9px 13px; font-size: 11px; min-height: 36px; }
|
||||
.filter:last-child { border-right: 0; }
|
||||
.filter:hover { color: var(--text); background: #272b30; }
|
||||
.filter.active { color: var(--accent); background: #29342c; font-weight: 650; }
|
||||
.jobs { display: flex; flex-direction: column; gap: 8px; }
|
||||
.job { background: var(--panel); border: 1px solid var(--line); border-radius: 5px; padding: 14px 17px; }
|
||||
.job-top { display: flex; justify-content: space-between; gap: 20px; align-items: flex-start; }
|
||||
.job-heading { min-width: 0; flex: 1; }
|
||||
.job-name { font-weight: 650; font-size: 13px; line-height: 1.45; overflow-wrap: anywhere; }
|
||||
.job-sub { color: var(--faint); font-size: 11px; margin-top: 3px; overflow-wrap: anywhere; max-height: 2.8em; overflow: hidden; }
|
||||
.job-status { flex-shrink: 0; display: inline-flex; align-items: center; gap: 6px; border: 1px solid #4a5156; color: #bbc4ca; background: #272c30; border-radius: 3px; font-size: 10px; line-height: 1; padding: 6px 8px; white-space: nowrap; }
|
||||
.job-status::before { content: ''; width: 6px; height: 6px; border-radius: 50%; background: #9ca5ac; }
|
||||
.job-status.running { background: #233126; color: #add7af; border-color: #39563e; }
|
||||
.job-status.running::before { background: var(--accent); }
|
||||
.job-status.done { background: var(--success-bg); color: #a8d9ac; border-color: #3b5942; }
|
||||
.job-status.done::before { background: #8fcd95; }
|
||||
.job-status.error { background: var(--danger-bg); color: #f1aaa3; border-color: #704446; }
|
||||
.job-status.error::before { background: var(--danger); }
|
||||
.job-progress { display: flex; align-items: center; gap: 12px; margin-top: 12px; }
|
||||
.progressbar { width: 100%; height: 5px; background: #373d41; overflow: hidden; }
|
||||
.progressbar span { display: block; height: 100%; background: #8aba94; transition: width .25s linear; }
|
||||
.progressbar.is-error span { background: #c47e78; }
|
||||
.progress-text { color: #c5cbd0; text-align: right; font-variant-numeric: tabular-nums; font-size: 11px; flex: 0 0 50px; }
|
||||
.job-meta { display: flex; align-items: center; justify-content: space-between; gap: 20px; margin-top: 8px; font-size: 11px; color: var(--faint); }
|
||||
.job-meta .note-text { overflow-wrap: anywhere; }
|
||||
.job-meta .speed { flex-shrink: 0; font-variant-numeric: tabular-nums; }
|
||||
.job-bottom { display: flex; justify-content: space-between; gap: 10px; align-items: center; flex-wrap: wrap; margin-top: 11px; }
|
||||
.job-chips { display: inline-flex; gap: 7px; align-items: center; }
|
||||
.job-chip { color: #aab3ba; font: 11px ui-monospace, SFMono-Regular, Consolas, monospace; border: 1px solid #3c4449; padding: 3px 7px; border-radius: 3px; }
|
||||
.job-actions { display: flex; flex-wrap: wrap; gap: 6px; margin-left: auto; }
|
||||
.job-error { margin-top: 11px; padding: 9px 11px; border-left: 2px solid #c6857e; color: #ebaba6; background: #2b2122; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.5; }
|
||||
.job-output { color: #accdb5; overflow-wrap: anywhere; margin-top: 9px; font-size: 11px; }
|
||||
.empty-state { color: var(--faint); text-align: center; border: 1px dashed #42484e; background: var(--panel); padding: 34px 12px; border-radius: 4px; font-size: 12px; }
|
||||
.footer { display: flex; justify-content: space-between; gap: 20px; padding: 23px 0 36px; color: var(--faint); font-size: 11px; }
|
||||
#toast { display: none; position: fixed; right: 20px; bottom: 20px; max-width: min(420px, calc(100vw - 40px)); z-index: 20; padding: 13px 16px; border: 1px solid #6e9e75; border-radius: 4px; background: #24402a; color: #d3f1d6; font-size: 12px; line-height: 1.45; white-space: pre-line; }
|
||||
#toast.bad { background: #48282a; color: #ffd7d3; border-color: #955b60; }
|
||||
dialog { width: min(760px, calc(100vw - 30px)); max-height: 85vh; background: var(--panel); color: var(--text); border: 1px solid #545a60; border-radius: 5px; padding: 18px; }
|
||||
dialog::backdrop { background: rgba(0,0,0,.75); }
|
||||
.dialog-heading { display: flex; justify-content: space-between; align-items: center; gap: 20px; padding-bottom: 15px; border-bottom: 1px solid var(--line); }
|
||||
.dialog-heading h2 { font-size: 15px; }
|
||||
pre { overflow: auto; max-height: 66vh; white-space: pre-wrap; overflow-wrap: anywhere; font-size: 11px; line-height: 1.65; color: #c8cdd0; }
|
||||
@media (max-width: 760px) {
|
||||
.shell { padding: 0 15px; }
|
||||
.topbar { height: 59px; }
|
||||
.version { display: none; }
|
||||
.page-title { padding: 22px 0 16px; }
|
||||
.metrics { grid-template-columns: repeat(2, minmax(0,1fr)); }
|
||||
.metric:nth-child(3) { border-left: 0; border-top: 1px solid var(--line); }
|
||||
.metric:nth-child(4) { border-top: 1px solid var(--line); }
|
||||
.metric { min-height: 99px; padding: 15px; }
|
||||
.panel { margin-top: 19px; }
|
||||
.panel-heading { padding: 16px 15px 13px; }
|
||||
.panel-heading .muted-info { display: none; }
|
||||
.tabs { padding: 0 6px; }
|
||||
.tab { padding: 11px 9px; font-size: 11px; gap: 6px; }
|
||||
.tab-body { padding: 16px 15px; }
|
||||
.input-row { flex-wrap: wrap; }
|
||||
.input-row .title-field { flex: 1 1 100%; }
|
||||
.input-row .format-field { flex: 1 1 110px; }
|
||||
.input-row .submit-button { flex: 1 1 auto; }
|
||||
.file-heading { align-items: stretch; flex-direction: column; }
|
||||
.file-heading h3 { max-width: 100%; }
|
||||
.file-heading input { max-width: none; }
|
||||
.queue-header { align-items: stretch; flex-direction: column; gap: 13px; }
|
||||
.queue-controls { justify-content: space-between; }
|
||||
.queue-title { justify-content: space-between; }
|
||||
.filter { padding: 9px; }
|
||||
.job { padding: 13px; }
|
||||
.footer { flex-direction: column; gap: 7px; }
|
||||
}
|
||||
@media (max-width: 440px) {
|
||||
.inline-input, .import-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.selection-controls { width: 100%; }
|
||||
.selection-controls select { flex: 1; min-width: 0; }
|
||||
.file-bottom { align-items: stretch; flex-direction: column; }
|
||||
.job-top { gap: 9px; }
|
||||
.job-status { max-width: 130px; white-space: normal; }
|
||||
.queue-title .muted-info { display: none; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*, *::before, *::after { transition-duration: 0ms !important; animation-duration: 0ms !important; }
|
||||
}
|
||||
|
||||
/* Archive login and torrent metadata selection: compact, flat controls. */
|
||||
.archive-auth { border: 1px solid var(--line); padding: 13px; margin-bottom: 20px; background: var(--input); border-radius: 4px; }
|
||||
.auth-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; font-size: 12px; }
|
||||
.auth-heading span { color: var(--muted); overflow-wrap: anywhere; font-size: 11px; }
|
||||
.auth-login { display: flex; align-items: end; gap: 10px; flex-wrap: wrap; margin-top: 13px; }
|
||||
.auth-login .field { flex: 1 1 190px; }
|
||||
.archive-auth .note { margin-top: 10px; }
|
||||
#archiveLogout { margin-top: 12px; }
|
||||
#magnetDialog { width: min(840px, calc(100vw - 24px)); }
|
||||
.preview-status { color: var(--muted); font-size: 12px; margin: 14px 0; }
|
||||
#magnetDialog .file-table { max-height: 45vh; }
|
||||
#magnetDialog .file-row input:disabled { opacity: .3; }
|
||||
@media(max-width: 540px) { .auth-login .field, .auth-login button { flex: 1 1 100%; } }
|
||||
|
||||
/* Flat torrent metadata fallback controls. */
|
||||
.torrent-upload-inline {display:flex; align-items:center; flex-wrap:wrap; gap:10px; margin:10px 0; color:var(--muted, #8d939d); font-size:12px;}
|
||||
.file-label {display:inline-flex; align-items:center; justify-content:center; cursor:pointer; border:1px solid #434a51; background:#292e33; color:#d7dce1; padding:7px 10px; border-radius:3px; font-size:12px; font-weight:600;}
|
||||
.file-label:hover {background:#32383e;}
|
||||
#torrentFileLinks, #torrentFileDialog {display:none !important;}
|
||||
@@ -0,0 +1,95 @@
|
||||
"""SQLite-backed durable job queue. Single-worker, multiple API connections."""
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
DATA = Path('/data')
|
||||
DB_PATH = DATA / 'queue.sqlite3'
|
||||
|
||||
FIELDS = {
|
||||
'title', 'source_type', 'source', 'format', 'state', 'progress', 'speed',
|
||||
'total', 'completed', 'part_index', 'gid', 'error', 'output', 'note', 'updated',
|
||||
}
|
||||
|
||||
|
||||
def connection():
|
||||
DATA.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(DB_PATH, timeout=30)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.execute('PRAGMA busy_timeout=30000')
|
||||
con.execute('PRAGMA journal_mode=WAL')
|
||||
return con
|
||||
|
||||
|
||||
def init_db():
|
||||
with connection() as con:
|
||||
con.execute('''CREATE TABLE IF NOT EXISTS jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
source_type TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
format TEXT NOT NULL,
|
||||
state TEXT NOT NULL DEFAULT 'queued',
|
||||
progress REAL NOT NULL DEFAULT 0,
|
||||
speed INTEGER NOT NULL DEFAULT 0,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
completed INTEGER NOT NULL DEFAULT 0,
|
||||
part_index INTEGER NOT NULL DEFAULT 0,
|
||||
gid TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
output TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
created REAL NOT NULL,
|
||||
updated REAL NOT NULL
|
||||
)''')
|
||||
con.execute('CREATE INDEX IF NOT EXISTS jobs_created_idx ON jobs(created)')
|
||||
# Zadania przerwane w trakcie rozpakowania/konwersji moga byc ponowione.
|
||||
con.execute("UPDATE jobs SET state='preparing', note='Wznowiono konwersje po restarcie' "
|
||||
"WHERE state IN ('extracting','converting')")
|
||||
con.execute("UPDATE jobs SET state='queued', note='Przywrocono zadanie po restarcie' "
|
||||
"WHERE state='vimm_browser'")
|
||||
|
||||
|
||||
def create(job):
|
||||
now = time.time()
|
||||
with connection() as con:
|
||||
con.execute('''INSERT INTO jobs(id,title,source_type,source,format,state,created,updated)
|
||||
VALUES(?,?,?,?,?,'queued',?,?)''',
|
||||
(job['id'], job['title'], job['source_type'],
|
||||
job['source'], job['format'], now, now))
|
||||
|
||||
|
||||
def get(job_id):
|
||||
with connection() as con:
|
||||
row = con.execute('SELECT * FROM jobs WHERE id=?', (job_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def list_jobs():
|
||||
with connection() as con:
|
||||
rows = con.execute('SELECT * FROM jobs ORDER BY created DESC LIMIT 300').fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def next_job():
|
||||
with connection() as con:
|
||||
row = con.execute('''SELECT * FROM jobs
|
||||
WHERE state IN ('queued','downloading','preparing')
|
||||
ORDER BY created ASC LIMIT 1''').fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def update(job_id, **values):
|
||||
if not values:
|
||||
return
|
||||
if not set(values).issubset(FIELDS):
|
||||
raise ValueError('Niepoprawne pola aktualizacji bazy')
|
||||
values['updated'] = time.time()
|
||||
with connection() as con:
|
||||
con.execute('UPDATE jobs SET ' + ','.join(f'{k}=?' for k in values)
|
||||
+ ' WHERE id=?', (*values.values(), job_id))
|
||||
|
||||
|
||||
def erase(job_id):
|
||||
with connection() as con:
|
||||
con.execute('DELETE FROM jobs WHERE id=?', (job_id,))
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Safe, small bencode parser and ephemeral magnet metadata inspection state."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import store
|
||||
|
||||
ID = re.compile(r'^[a-f0-9]{32}$')
|
||||
MAX_METADATA = 32 * 1024 * 1024
|
||||
|
||||
|
||||
class MetadataError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def decode_bencode(raw):
|
||||
if len(raw) > MAX_METADATA:
|
||||
raise MetadataError('Metadane torrenta są zbyt duże')
|
||||
position = 0
|
||||
count = 0
|
||||
|
||||
def decode(depth=0):
|
||||
nonlocal position, count
|
||||
if depth > 24:
|
||||
raise MetadataError('Metadane mają zbyt duże zagnieżdżenie')
|
||||
count += 1
|
||||
if count > 600000 or position >= len(raw):
|
||||
raise MetadataError('Uszkodzone metadane torrenta')
|
||||
start = raw[position]
|
||||
if start == 105: # i<number>e
|
||||
end = raw.find(b'e', position + 1)
|
||||
if end < 0 or end - position > 25:
|
||||
raise MetadataError('Błędna liczba bencode')
|
||||
try:
|
||||
number = int(raw[position + 1:end])
|
||||
except ValueError as exc:
|
||||
raise MetadataError('Błędna liczba bencode') from exc
|
||||
position = end + 1
|
||||
return number
|
||||
if start in (108, 100): # l, d
|
||||
position += 1
|
||||
output = [] if start == 108 else {}
|
||||
while position < len(raw) and raw[position] != 101:
|
||||
if start == 108:
|
||||
output.append(decode(depth + 1))
|
||||
else:
|
||||
key = decode(depth + 1)
|
||||
if not isinstance(key, bytes):
|
||||
raise MetadataError('Niepoprawny klucz bencode')
|
||||
output[key] = decode(depth + 1)
|
||||
if position >= len(raw):
|
||||
raise MetadataError('Nieukończone metadane')
|
||||
position += 1
|
||||
return output
|
||||
if 48 <= start <= 57: # <len>:<bytes>
|
||||
colon = raw.find(b':', position, min(len(raw), position + 22))
|
||||
if colon < 0:
|
||||
raise MetadataError('Błędny ciąg bencode')
|
||||
try:
|
||||
length = int(raw[position:colon])
|
||||
except ValueError as exc:
|
||||
raise MetadataError('Błędna długość bencode') from exc
|
||||
position = colon + 1
|
||||
if length < 0 or length > len(raw) - position:
|
||||
raise MetadataError('Niepoprawna długość pola')
|
||||
result = raw[position:position + length]
|
||||
position += length
|
||||
return result
|
||||
raise MetadataError('Uszkodzone metadane torrenta')
|
||||
|
||||
result = decode()
|
||||
if position != len(raw) or not isinstance(result, dict):
|
||||
raise MetadataError('Niepoprawny format torrenta')
|
||||
return result
|
||||
|
||||
|
||||
def _segment(raw):
|
||||
if not isinstance(raw, bytes) or not raw or len(raw) > 255 or raw in (b'.', b'..'):
|
||||
raise MetadataError('Niedozwolona nazwa w torrencie')
|
||||
if any(char in raw for char in (b'/', b'\\', b'\0')):
|
||||
raise MetadataError('Niebezpieczna ścieżka w torrencie')
|
||||
return raw.decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
def read_torrent(path):
|
||||
try:
|
||||
data = decode_bencode(Path(path).read_bytes())
|
||||
info = data[b'info']
|
||||
name = _segment(info.get(b'name.utf-8') or info[b'name'])
|
||||
result = []
|
||||
if b'files' in info:
|
||||
files = info[b'files']
|
||||
if not isinstance(files, list) or len(files) > 50000:
|
||||
raise MetadataError('Nieprawidłowa lista plików torrenta')
|
||||
for index, file in enumerate(files, start=1):
|
||||
parts = file.get(b'path.utf-8') or file[b'path']
|
||||
if not isinstance(parts, list) or not parts:
|
||||
raise MetadataError('Błędna ścieżka pliku')
|
||||
length = file[b'length']
|
||||
if not isinstance(length, int) or length < 0:
|
||||
raise MetadataError('Nieprawidłowy rozmiar pliku')
|
||||
result.append({'index': index, 'name': '/'.join([name] + [_segment(x) for x in parts]),
|
||||
'size': length, 'padding': b'p' in file.get(b'attr', b'')})
|
||||
else:
|
||||
length = info[b'length']
|
||||
if not isinstance(length, int) or length < 0:
|
||||
raise MetadataError('Nieprawidłowy rozmiar pliku')
|
||||
result.append({'index': 1, 'name': name, 'size': length, 'padding': False})
|
||||
if not result:
|
||||
raise MetadataError('Torrent nie zawiera plików')
|
||||
return {'name': name, 'files': result}
|
||||
except (KeyError, TypeError, OSError, ValueError) as exc:
|
||||
if isinstance(exc, MetadataError):
|
||||
raise
|
||||
raise MetadataError('Nie można odczytać plików z torrenta') from exc
|
||||
|
||||
|
||||
def suggested_indices(magnet, max_index):
|
||||
"""magnet so indexes are zero-based; aria2 select-file indexes are one-based."""
|
||||
so = parse_qs(urlsplit(magnet).query).get('so', [''])[0]
|
||||
suggested = set()
|
||||
for chunk in so.split(',')[:1000]:
|
||||
try:
|
||||
if '-' in chunk:
|
||||
a, b = (int(x) for x in chunk.split('-', 1))
|
||||
if b < a or b - a > 1000:
|
||||
continue
|
||||
values = range(a, b + 1)
|
||||
else:
|
||||
values = [int(chunk)]
|
||||
suggested.update(n + 1 for n in values if 0 <= n < max_index)
|
||||
except ValueError:
|
||||
continue
|
||||
return sorted(suggested)
|
||||
|
||||
|
||||
def inspect_dir(preview_id):
|
||||
if not ID.fullmatch(preview_id):
|
||||
raise MetadataError('Nieprawidłowy identyfikator metadanych')
|
||||
return store.DATA / 'metadata' / preview_id
|
||||
|
||||
|
||||
def write_request(preview_id, data):
|
||||
path = inspect_dir(preview_id)
|
||||
path.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
filename = path / 'request.json'
|
||||
flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
|
||||
with os.fdopen(os.open(filename, flags, 0o600), 'w', encoding='utf-8') as f:
|
||||
json.dump({'created': time.time(), **data}, f)
|
||||
|
||||
|
||||
def load_request(preview_id):
|
||||
try:
|
||||
info = json.loads((inspect_dir(preview_id) / 'request.json').read_text(encoding='utf-8'))
|
||||
if not isinstance(info['gid'], str) or not isinstance(info['magnet'], str):
|
||||
raise MetadataError('Błędne dane podglądu')
|
||||
return info
|
||||
except FileNotFoundError as exc:
|
||||
raise MetadataError('Nie znaleziono podglądu magneta') from exc
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise MetadataError('Uszkodzone dane podglądu') from exc
|
||||
|
||||
|
||||
def torrent_file(preview_id):
|
||||
matches = list(inspect_dir(preview_id).glob('*.torrent'))
|
||||
return matches[0] if len(matches) == 1 and matches[0].stat().st_size <= MAX_METADATA else None
|
||||
|
||||
|
||||
def cleanup_old(max_age=86400):
|
||||
root = store.DATA / 'metadata'
|
||||
if not root.exists():
|
||||
return
|
||||
now = time.time()
|
||||
for child in root.iterdir():
|
||||
if (ID.fullmatch(child.name) and child.is_dir() and not child.is_symlink()
|
||||
and now - child.stat().st_mtime > max_age):
|
||||
shutil.rmtree(child)
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
|
||||
import os
|
||||
import re
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
import httpx
|
||||
|
||||
from . import store
|
||||
|
||||
|
||||
async def browser_download(job):
|
||||
if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
|
||||
raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
|
||||
url = job['source']
|
||||
u = urlparse(url)
|
||||
if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
|
||||
raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
|
||||
r = await c.get('https://vimm.net/robots.txt')
|
||||
if r.status_code == 200:
|
||||
parser = RobotFileParser()
|
||||
parser.parse(r.text.splitlines())
|
||||
if not parser.can_fetch('x360-web-queue', url):
|
||||
raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
|
||||
elif r.status_code not in (404, 410):
|
||||
raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
|
||||
except httpx.HTTPError as exc:
|
||||
raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
|
||||
|
||||
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
|
||||
|
||||
store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
|
||||
out = store.DATA / 'raw' / job['id']
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True, executable_path='/usr/bin/chromium',
|
||||
args=['--no-sandbox', '--disable-dev-shm-usage'])
|
||||
try:
|
||||
context = await browser.new_context(accept_downloads=True)
|
||||
page = await context.new_page()
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=60000)
|
||||
# Only the original, visible button is actionable. Missing/taken-down
|
||||
# buttons and challenges are not bypassed.
|
||||
controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
|
||||
usable = None
|
||||
for i in range(min(await controls.count(), 10)):
|
||||
c = controls.nth(i)
|
||||
if await c.is_visible() and await c.is_enabled():
|
||||
usable = c
|
||||
break
|
||||
if usable is None:
|
||||
raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
|
||||
store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
|
||||
timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
|
||||
try:
|
||||
async with page.expect_download(timeout=timeout) as d:
|
||||
await usable.click(timeout=15000)
|
||||
download = await d.value
|
||||
except PlaywrightTimeout as exc:
|
||||
raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
|
||||
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
|
||||
if not name:
|
||||
name = 'vimm-download.bin'
|
||||
store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
|
||||
await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
|
||||
failure = await download.failure()
|
||||
if failure:
|
||||
raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
|
||||
store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
|
||||
finally:
|
||||
await browser.close()
|
||||
@@ -0,0 +1,5 @@
|
||||
module x360-bt-helper
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require github.com/anacrolix/torrent v1.61.0
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
// x360-bt: non-interactive BitTorrent helper based on the anacrolix/torrent
|
||||
// library, also used by the user's minerva-pick tool. It emits machine-readable
|
||||
// status and never starts downloading payload data while inspecting a magnet.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/anacrolix/torrent"
|
||||
)
|
||||
|
||||
type state struct {
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Connections int `json:"connections"`
|
||||
KnownPeers int `json:"known_peers"`
|
||||
Downloaded int64 `json:"downloaded,omitempty"`
|
||||
Total int64 `json:"total,omitempty"`
|
||||
Speed int64 `json:"speed,omitempty"`
|
||||
Elapsed int64 `json:"elapsed,omitempty"`
|
||||
}
|
||||
|
||||
func writeState(path string, s state) {
|
||||
if path == "" {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return
|
||||
}
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".bt-state-*")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err = f.Write(b); err != nil {
|
||||
f.Close()
|
||||
return
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.Rename(f.Name(), path)
|
||||
}
|
||||
|
||||
func newClient(dataDir string, port int) (*torrent.Client, error) {
|
||||
cfg := torrent.NewDefaultClientConfig()
|
||||
cfg.DisableIPv6 = true
|
||||
cfg.DataDir = dataDir
|
||||
cfg.Seed = false
|
||||
cfg.NoDefaultPortForwarding = true
|
||||
cfg.ListenPort = port
|
||||
return torrent.NewClient(cfg)
|
||||
}
|
||||
|
||||
func inspect(ctx context.Context, magnet, output, statusPath string, port int, timeout time.Duration) error {
|
||||
if !strings.HasPrefix(strings.ToLower(magnet), "magnet:?") {
|
||||
return errors.New("nieprawidlowy magnet")
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(output), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newClient(filepath.Dir(output), port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("klient BT: %w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
t, err := client.AddMagnet(magnet)
|
||||
if err != nil {
|
||||
return fmt.Errorf("magnet: %w", err)
|
||||
}
|
||||
started := time.Now()
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
ticker := time.NewTicker(3 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
snapshot := state{Status: "loading", Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm()), Elapsed: int64(time.Since(started).Seconds())}
|
||||
writeState(statusPath, snapshot)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("brak metadanych po %s (polaczenia=%d, znani peerzy=%d)", timeout, snapshot.Connections, snapshot.KnownPeers)
|
||||
case <-t.GotInfo():
|
||||
mi := t.Metainfo()
|
||||
f, err := os.CreateTemp(filepath.Dir(output), ".bt-meta-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
temp := f.Name()
|
||||
defer os.Remove(temp)
|
||||
if err = mi.Write(f); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err = f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if info, err := os.Stat(temp); err != nil || info.Size() > 32*1024*1024 {
|
||||
return errors.New("metadane przekraczaja 32 MiB")
|
||||
}
|
||||
if err = os.Rename(temp, output); err != nil {
|
||||
return err
|
||||
}
|
||||
writeState(statusPath, state{Status: "ready", Connections: snapshot.Connections, KnownPeers: snapshot.KnownPeers, Elapsed: snapshot.Elapsed})
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseSelected(selection string, files []*torrent.File) ([]*torrent.File, error) {
|
||||
if selection == "" {
|
||||
return nil, errors.New("brak wybranych plikow")
|
||||
}
|
||||
used := make(map[int]bool)
|
||||
out := make([]*torrent.File, 0)
|
||||
for _, s := range strings.Split(selection, ",") {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil || n < 1 || n > len(files) || used[n] {
|
||||
return nil, fmt.Errorf("niepoprawny indeks pliku %q", s)
|
||||
}
|
||||
used[n] = true
|
||||
out = append(out, files[n-1])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func progress(files []*torrent.File) (done, total int64) {
|
||||
for _, f := range files {
|
||||
total += f.Length()
|
||||
n := f.BytesCompleted()
|
||||
if n > f.Length() {
|
||||
n = f.Length()
|
||||
}
|
||||
done += n
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func fetch(ctx context.Context, input, selection, outDir, statusPath string, port int) error {
|
||||
if err := os.MkdirAll(outDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
client, err := newClient(outDir, port)
|
||||
if err != nil {
|
||||
return fmt.Errorf("klient BT: %w", err)
|
||||
}
|
||||
defer client.Close()
|
||||
t, err := client.AddTorrentFromFile(input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("plik torrent: %w", err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-t.GotInfo():
|
||||
}
|
||||
selected, err := parseSelected(selection, t.Files())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range selected {
|
||||
f.Download()
|
||||
}
|
||||
started := time.Now()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
lastDone := int64(0)
|
||||
lastAt := time.Now()
|
||||
for {
|
||||
done, total := progress(selected)
|
||||
now := time.Now()
|
||||
seconds := now.Sub(lastAt).Seconds()
|
||||
var speed int64
|
||||
if seconds > 0 {
|
||||
speed = int64(float64(done-lastDone) / seconds)
|
||||
if speed < 0 {
|
||||
speed = 0
|
||||
}
|
||||
}
|
||||
lastDone = done
|
||||
lastAt = now
|
||||
if total == 0 || done >= total {
|
||||
writeState(statusPath, state{Status: "done", Total: total, Downloaded: done, Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm())})
|
||||
return nil
|
||||
}
|
||||
writeState(statusPath, state{Status: "downloading", Total: total, Downloaded: done, Speed: speed, Connections: len(t.PeerConns()), KnownPeers: len(t.KnownSwarm()), Elapsed: int64(time.Since(started).Seconds())})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func run(ctx context.Context) (string, error) {
|
||||
if len(os.Args) < 2 {
|
||||
return "", errors.New("tryb inspect albo fetch")
|
||||
}
|
||||
command := os.Args[1]
|
||||
flags := flag.NewFlagSet(command, flag.ContinueOnError)
|
||||
magnet := flags.String("magnet", "", "magnet link")
|
||||
input := flags.String("torrent", "", "plik .torrent")
|
||||
selection := flags.String("select", "", "indeksy 1-based oddzielone przecinkiem")
|
||||
out := flags.String("out", "", "sciezka wyjsciowa")
|
||||
statusPath := flags.String("status", "", "sciezka JSON statusu")
|
||||
port := flags.Int("port", 0, "port TCP/UDP (0 = losowy)")
|
||||
timeout := flags.Duration("timeout", 5*time.Minute, "timeout metadanych")
|
||||
if err := flags.Parse(os.Args[2:]); err != nil {
|
||||
return *statusPath, err
|
||||
}
|
||||
if *port < 0 || *port > 65535 {
|
||||
return *statusPath, errors.New("niepoprawny port")
|
||||
}
|
||||
var err error
|
||||
switch command {
|
||||
case "inspect":
|
||||
if *out == "" || *magnet == "" {
|
||||
err = errors.New("podaj -magnet i -out")
|
||||
} else {
|
||||
err = inspect(ctx, *magnet, *out, *statusPath, *port, *timeout)
|
||||
}
|
||||
case "fetch":
|
||||
if *input == "" || *out == "" {
|
||||
err = errors.New("podaj -torrent i -out")
|
||||
} else {
|
||||
err = fetch(ctx, *input, *selection, *out, *statusPath, *port)
|
||||
}
|
||||
default:
|
||||
err = errors.New("nieznany tryb: " + command)
|
||||
}
|
||||
return *statusPath, err
|
||||
}
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
path, err := run(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "x360-bt:", err)
|
||||
writeState(path, state{Status: "error", Error: err.Error()})
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
services:
|
||||
x360-web:
|
||||
build: .
|
||||
container_name: x360-web-queue
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
ports:
|
||||
- "${WEB_BIND:-0.0.0.0}:${WEB_PORT:-8088}:8000"
|
||||
- "${BT_PORT:-6881}:${BT_PORT:-6881}/tcp"
|
||||
- "${BT_PORT:-6881}:${BT_PORT:-6881}/udp"
|
||||
environment:
|
||||
WEB_USER: ${WEB_USER:?Run ./setup.sh}
|
||||
WEB_PASSWORD: ${WEB_PASSWORD:?Run ./setup.sh}
|
||||
ARIA2_SECRET: ${ARIA2_SECRET:?Run ./setup.sh}
|
||||
BT_PORT: ${BT_PORT:-6881}
|
||||
ARCHIVE_CONNECTIONS: ${ARCHIVE_CONNECTIONS:-50}
|
||||
ENABLE_VIMM_BROWSER: ${ENABLE_VIMM_BROWSER:-true}
|
||||
VIMM_DOWNLOAD_TIMEOUT: ${VIMM_DOWNLOAD_TIMEOUT:-7200}
|
||||
XBOX_HOST_DIR: ${XBOX_HOST_DIR:-/xbox}
|
||||
# Python, Playwright and converter temporary files use host /xbox/work.
|
||||
TMPDIR: /data/work/tmp
|
||||
XDG_CACHE_HOME: /data/work/cache
|
||||
volumes:
|
||||
# Application state, metadata, work, ready files and logs on the host.
|
||||
- type: bind
|
||||
source: ${XBOX_HOST_DIR:-/xbox}
|
||||
target: /data
|
||||
bind:
|
||||
create_host_path: false
|
||||
# Preserve existing /data/raw paths for resumed jobs and aria2 sessions.
|
||||
# Downloads are stored on the host as /xbox/downloads, not /xbox/raw.
|
||||
- type: bind
|
||||
source: ${XBOX_HOST_DIR:-/xbox}/downloads
|
||||
target: /data/raw
|
||||
bind:
|
||||
create_host_path: false
|
||||
shm_size: '512mb'
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${WEB_PASSWORD:?WEB_PASSWORD must be set}"
|
||||
: "${ARIA2_SECRET:?ARIA2_SECRET must be set}"
|
||||
: "${BT_PORT:=6881}"
|
||||
case "$WEB_PASSWORD" in CHANGE_ME*|"") echo "Zmien WEB_PASSWORD w .env" >&2; exit 1;; esac
|
||||
case "$ARIA2_SECRET" in CHANGE_ME*|"") echo "Zmien ARIA2_SECRET w .env" >&2; exit 1;; esac
|
||||
mkdir -p /data/raw /data/ready /data/work /data/work/tmp /data/work/cache /data/imports /data/aria2
|
||||
if [ ! -f /data/aria2/session.txt ]; then touch /data/aria2/session.txt; fi
|
||||
aria2c \
|
||||
--enable-rpc=true --rpc-listen-all=false --rpc-listen-port=6800 \
|
||||
--rpc-secret="$ARIA2_SECRET" \
|
||||
--dir=/data/raw \
|
||||
--input-file=/data/aria2/session.txt \
|
||||
--save-session=/data/aria2/session.txt --save-session-interval=30 \
|
||||
--continue=true --max-concurrent-downloads=2 \
|
||||
--max-connection-per-server=2 --split=2 --min-split-size=8M \
|
||||
--seed-time=0 --bt-stop-timeout=900 \
|
||||
--enable-dht=false --listen-port=6882 \
|
||||
--file-allocation=none --auto-file-renaming=false \
|
||||
--allow-overwrite=false --max-tries=8 --retry-wait=30 \
|
||||
--console-log-level=warn \
|
||||
> /data/aria2/aria2.log 2>&1 &
|
||||
exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --no-access-log
|
||||
@@ -0,0 +1,4 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
httpx==0.28.1
|
||||
playwright==1.52.0
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
. ./xbox-paths.sh
|
||||
|
||||
XBOX_DIR=$(xbox_host_directory)
|
||||
xbox_prepare_directories "$XBOX_DIR"
|
||||
|
||||
if [ -f ./data/queue.sqlite3 ] && [ ! -f "$XBOX_DIR/queue.sqlite3" ]; then
|
||||
echo 'Wykryto poprzednia kolejke w ./data/queue.sqlite3.' >&2
|
||||
echo 'Zanim uruchomisz nowa wersje, wykonaj: ./migrate-to-xbox.sh' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -f .env ]; then
|
||||
echo '.env juz istnieje; nie nadpisuje hasel.'
|
||||
else
|
||||
umask 077
|
||||
PASSWORD=$(head -c 48 /dev/urandom | od -An -tx1 | tr -d ' \n')
|
||||
SECRET=$(head -c 48 /dev/urandom | od -An -tx1 | tr -d ' \n')
|
||||
sed -e "s/^WEB_PASSWORD=.*/WEB_PASSWORD=$PASSWORD/" -e "s/^ARIA2_SECRET=.*/ARIA2_SECRET=$SECRET/" .env.example > .env
|
||||
# Persist a customized host directory so future `docker compose up` uses it.
|
||||
if [ "$XBOX_DIR" != /xbox ]; then
|
||||
grep -v '^XBOX_HOST_DIR=' .env > .env.tmp
|
||||
printf 'XBOX_HOST_DIR=%s\n' "$XBOX_DIR" >> .env.tmp
|
||||
mv .env.tmp .env
|
||||
fi
|
||||
chmod 600 .env
|
||||
echo 'Utworzono .env (login admin, losowe haslo).'
|
||||
echo "Haslo panelu: $PASSWORD"
|
||||
fi
|
||||
|
||||
echo "Dane hosta: $XBOX_DIR (pobrane: $XBOX_DIR/downloads, robocze: $XBOX_DIR/work)"
|
||||
if command -v docker >/dev/null 2>&1 && [ "${XBOX_SKIP_DOCKER:-0}" != 1 ]; then
|
||||
docker compose up -d --build
|
||||
echo "Panel: http://ADRES_IP_WIRTUALKI:${WEB_PORT:-8088}"
|
||||
else
|
||||
echo 'Uruchom Docker Engine z Compose i wykonaj: docker compose up -d --build'
|
||||
fi
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Multi-file Archive.org scheduling, fair budgets and safe restarts."""
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import archive_download, main, store
|
||||
|
||||
|
||||
class BytesStream(httpx.SyncByteStream):
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
def __iter__(self):
|
||||
yield self.data
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def setup(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '50')
|
||||
monkeypatch.setenv('WEB_USER', 'admin')
|
||||
monkeypatch.setenv('WEB_PASSWORD', 'unit-test-password')
|
||||
monkeypatch.setattr(archive_download, 'MIN_SEGMENT_BYTES', 16)
|
||||
store.init_db()
|
||||
return tmp_path
|
||||
|
||||
|
||||
def multi_job(names=('A.iso', 'B.iso', 'C.iso')):
|
||||
urls = [f'https://archive.org/download/test/{name}' for name in names]
|
||||
return main.new_job('archive_many', json.dumps(urls), 'Multi', 'raw')
|
||||
|
||||
|
||||
def mock_archive(monkeypatch, files, first_request_barrier=None, delay=0):
|
||||
lock = threading.Lock()
|
||||
active = 0
|
||||
max_active = 0
|
||||
seen = {}
|
||||
requests = []
|
||||
|
||||
def handler(req):
|
||||
nonlocal active, max_active
|
||||
name = req.url.path.rsplit('/', 1)[-1]
|
||||
data = files[name]
|
||||
with lock:
|
||||
active += 1
|
||||
max_active = max(max_active, active)
|
||||
first = name not in seen
|
||||
seen[name] = seen.get(name, 0) + 1
|
||||
requests.append((name, req.headers.get('range')))
|
||||
try:
|
||||
if first and first_request_barrier:
|
||||
first_request_barrier.wait(timeout=5)
|
||||
if delay:
|
||||
time.sleep(delay)
|
||||
span = req.headers.get('range')
|
||||
if span:
|
||||
match = re.fullmatch(r'bytes=(\d+)-(\d*)', span)
|
||||
assert match, span
|
||||
begin = int(match.group(1))
|
||||
end = int(match.group(2)) if match.group(2) else len(data)-1
|
||||
if begin >= len(data):
|
||||
return httpx.Response(416, headers={'Content-Range': f'bytes */{len(data)}'})
|
||||
assert 0 <= begin <= end < len(data)
|
||||
return httpx.Response(206, stream=BytesStream(data[begin:end+1]), headers={
|
||||
'Content-Range': f'bytes {begin}-{end}/{len(data)}',
|
||||
'Content-Length': str(end - begin + 1), 'ETag': '"r1"'})
|
||||
return httpx.Response(200, stream=BytesStream(data), headers={'Content-Length': str(len(data))})
|
||||
finally:
|
||||
with lock:
|
||||
active -= 1
|
||||
|
||||
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
|
||||
transport=httpx.MockTransport(handler), follow_redirects=False))
|
||||
return requests, lambda: max_active
|
||||
|
||||
|
||||
def test_three_files_start_together_and_share_fifty_connections(setup, monkeypatch):
|
||||
names = ('A.iso', 'B.iso', 'C.iso')
|
||||
contents = {name: bytes([i+10]) * 1024 for i, name in enumerate(names)}
|
||||
barrier = threading.Barrier(3)
|
||||
requests, max_connections = mock_archive(monkeypatch, contents, barrier, delay=0.005)
|
||||
job = multi_job(names)
|
||||
asyncio.run(archive_download.run_one(job))
|
||||
for name in names:
|
||||
assert (setup / 'raw' / job['id'] / name).read_bytes() == contents[name]
|
||||
data = store.get(job['id'])
|
||||
assert data['state'] == 'preparing'
|
||||
assert data['progress'] == 100
|
||||
assert data['part_index'] == 3
|
||||
assert data['total'] == 3 * 1024
|
||||
assert data['completed'] == 3 * 1024
|
||||
assert max_connections() <= 50
|
||||
assert max_connections() > 1
|
||||
assert sum(1 for _, item in requests if item == 'bytes=0-0') == 3
|
||||
log = (setup / 'logs' / f'{job["id"]}.download.log').read_text()
|
||||
assert 'start 3 plików równolegle' in log
|
||||
assert '17, 17, 16' in log
|
||||
|
||||
|
||||
def test_existing_fifty_segment_manifests_do_not_exceed_global_limit(setup, monkeypatch):
|
||||
files = {'A.iso': b'a' * 1024, 'B.iso': b'b' * 1024, 'C.iso': b'c' * 1024}
|
||||
requests, max_connections = mock_archive(monkeypatch, files, delay=0.002)
|
||||
job = multi_job()
|
||||
root = setup / 'raw' / job['id']
|
||||
root.mkdir(parents=True)
|
||||
for url in json.loads(job['source']):
|
||||
target = root / (url.rsplit('/', 1)[-1] + '.part')
|
||||
_, manifest = archive_download._prepare_segments(target, url, 1024, '"r1"', 50)
|
||||
assert len(manifest['ranges']) == 50
|
||||
archive_download._transfer_many(job)
|
||||
assert sum(bool(item and item != 'bytes=0-0') for _, item in requests) == 150
|
||||
assert max_connections() <= 50
|
||||
assert store.get(job['id'])['state'] == 'preparing'
|
||||
for name, data in files.items():
|
||||
assert (root / name).read_bytes() == data
|
||||
|
||||
|
||||
def test_skip_completed_files_and_resume_incomplete_file(setup, monkeypatch):
|
||||
files = {'A.iso': b'a' * 128, 'B.iso': b'b' * 128, 'C.iso': b'c' * 128}
|
||||
requests, _ = mock_archive(monkeypatch, files)
|
||||
job = multi_job()
|
||||
raw = setup / 'raw' / job['id']
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'A.iso').write_bytes(files['A.iso'])
|
||||
(raw / 'B.iso.part').write_bytes(files['B.iso'][:32])
|
||||
archive_download._transfer_many(job)
|
||||
assert store.get(job['id'])['state'] == 'preparing'
|
||||
assert not any(name == 'A.iso' for name, _ in requests)
|
||||
assert not any(name == 'B.iso' and value == 'bytes=0-31' for name, value in requests)
|
||||
for name, data in files.items():
|
||||
assert (raw / name).read_bytes() == data
|
||||
|
||||
|
||||
def test_pause_stops_all_files_and_resume_keeps_segments(setup, monkeypatch):
|
||||
files = {'A.iso': b'a' * 1024, 'B.iso': b'b' * 1024}
|
||||
gate = threading.Event()
|
||||
release = threading.Event()
|
||||
original = httpx.MockTransport
|
||||
|
||||
def handler(req):
|
||||
name = req.url.path.rsplit('/', 1)[-1]
|
||||
data = files[name]
|
||||
header = req.headers.get('range', '')
|
||||
match = re.fullmatch(r'bytes=(\d+)-(\d*)', header)
|
||||
assert match
|
||||
start = int(match.group(1))
|
||||
end = int(match.group(2)) if match.group(2) else len(data)-1
|
||||
if (start, end) != (0, 0) and not gate.is_set():
|
||||
gate.set()
|
||||
assert release.wait(5), 'download thread was not released'
|
||||
return httpx.Response(206, stream=BytesStream(data[start:end+1]), headers={
|
||||
'ETag': '"r1"', 'Content-Range': f'bytes {start}-{end}/{len(data)}'})
|
||||
|
||||
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
|
||||
transport=original(handler), follow_redirects=False))
|
||||
job = multi_job(('A.iso', 'B.iso'))
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
fut = executor.submit(archive_download._transfer_many, job)
|
||||
assert gate.wait(5)
|
||||
store.update(job['id'], state='paused')
|
||||
release.set()
|
||||
with pytest.raises(archive_download.ArchiveStopped):
|
||||
fut.result(timeout=10)
|
||||
assert store.get(job['id'])['state'] == 'paused'
|
||||
store.update(job['id'], state='queued')
|
||||
archive_download._transfer_many(store.get(job['id']))
|
||||
assert store.get(job['id'])['state'] == 'preparing'
|
||||
assert (setup / 'raw' / job['id'] / 'A.iso').read_bytes() == files['A.iso']
|
||||
assert (setup / 'raw' / job['id'] / 'B.iso').read_bytes() == files['B.iso']
|
||||
|
||||
|
||||
def test_duplicate_filenames_fail_without_writing(setup):
|
||||
j = multi_job(('A/Game.iso', 'B/Game.iso'))
|
||||
with pytest.raises(archive_download.ArchiveDownloadError, match='powtarzające'):
|
||||
archive_download._transfer_many(j)
|
||||
assert not (setup / 'raw' / j['id']).exists()
|
||||
|
||||
|
||||
def test_retry_partial_archive_not_mistaken_for_ready_conversion(setup, monkeypatch):
|
||||
j = multi_job()
|
||||
raw = setup / 'raw' / j['id']
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'A.iso').write_bytes(b'a')
|
||||
store.update(j['id'], state='error', progress=99.99)
|
||||
|
||||
async def idle_worker():
|
||||
await asyncio.sleep(1000)
|
||||
monkeypatch.setattr(main, 'worker', idle_worker)
|
||||
with TestClient(main.app) as client:
|
||||
response = client.post(f'/api/jobs/{j["id"]}/retry', auth=('admin', 'unit-test-password'))
|
||||
assert response.status_code == 200
|
||||
assert response.json()['state'] == 'queued'
|
||||
|
||||
|
||||
def test_legacy_aria2_file_is_not_accepted_as_finished(setup, monkeypatch):
|
||||
job = multi_job(('A.iso', 'B.iso'))
|
||||
raw = setup / 'raw' / job['id']
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'A.iso').write_bytes(b'old incomplete file')
|
||||
(raw / 'A.iso.aria2').write_bytes(b'aria2 control file')
|
||||
files = {'A.iso': b'A' * 128, 'B.iso': b'B' * 128}
|
||||
requests, _ = mock_archive(monkeypatch, files)
|
||||
archive_download._transfer_many(job)
|
||||
assert ('A.iso', 'bytes=0-0') in requests
|
||||
assert (raw / 'A.iso').read_bytes() == files['A.iso']
|
||||
assert (raw / 'B.iso').read_bytes() == files['B.iso']
|
||||
|
||||
|
||||
def test_aggregate_progress_is_weighted_by_bytes_not_file_count(setup):
|
||||
j = multi_job()
|
||||
tracker = archive_download._BatchProgress(j['id'], ['A.iso', 'B.iso', 'C.iso'], 50)
|
||||
tracker.report(0, 100, 100)
|
||||
tracker.report(1, 10, 1000)
|
||||
tracker.report(2, 20, 2000)
|
||||
tracker.publish()
|
||||
data = store.get(j['id'])
|
||||
assert data['total'] == 3100
|
||||
assert data['completed'] == 130
|
||||
assert data['progress'] == 4.19
|
||||
assert 'limit 50 połączeń' in data['note']
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Deterministic tests for multi-range authenticated Archive.org downloads."""
|
||||
import re
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import archive_download, main, store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '4')
|
||||
monkeypatch.setattr(archive_download, 'MIN_SEGMENT_BYTES', 16)
|
||||
monkeypatch.setattr(archive_download.time, 'sleep', lambda _: None)
|
||||
store.init_db()
|
||||
return tmp_path
|
||||
|
||||
|
||||
def job():
|
||||
return main.new_job('url', 'https://archive.org/download/demo/Game.iso', 'Demo', 'raw')
|
||||
|
||||
|
||||
class BytesStream(httpx.SyncByteStream):
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
def __iter__(self):
|
||||
yield self.data
|
||||
|
||||
|
||||
def server(monkeypatch, data, custom=None, etag='"release-1"'):
|
||||
requests = []
|
||||
|
||||
def handler(req):
|
||||
requests.append(req)
|
||||
if custom:
|
||||
res = custom(req)
|
||||
if res is not None:
|
||||
return res
|
||||
header = req.headers.get('range')
|
||||
if not header:
|
||||
return httpx.Response(200, stream=BytesStream(data), headers={'Content-Length': str(len(data))})
|
||||
match = re.fullmatch(r'bytes=(\d+)-(\d*)', header)
|
||||
assert match, header
|
||||
start = int(match.group(1))
|
||||
end = int(match.group(2)) if match.group(2) else len(data) - 1
|
||||
assert end < len(data) and 0 <= start <= end
|
||||
return httpx.Response(206, stream=BytesStream(data[start:end + 1]), headers={
|
||||
'Content-Range': f'bytes {start}-{end}/{len(data)}',
|
||||
'Content-Length': str(end - start + 1), 'ETag': etag})
|
||||
|
||||
transport = httpx.MockTransport(handler)
|
||||
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
|
||||
transport=transport, follow_redirects=False))
|
||||
return requests
|
||||
|
||||
|
||||
def final_path(env, j):
|
||||
return env / 'raw' / j['id'] / 'Game.iso'
|
||||
|
||||
|
||||
def test_segments_reconstruct_in_order_and_show_ranges(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
requests = server(monkeypatch, data)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert store.get(j['id'])['state'] == 'preparing'
|
||||
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
|
||||
spans = [req.headers['range'] for req in requests]
|
||||
assert spans[0] == 'bytes=0-0'
|
||||
assert sorted(spans[1:]) == ['bytes=0-31', 'bytes=32-63', 'bytes=64-95', 'bytes=96-127']
|
||||
assert all(req.headers.get('if-range') == '"release-1"' for req in requests[1:])
|
||||
assert '4 połączeń HTTP Range' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
|
||||
|
||||
|
||||
def test_existing_serial_prefix_is_reused(env, monkeypatch):
|
||||
data = bytes(range(200))
|
||||
requests = server(monkeypatch, data)
|
||||
j = job()
|
||||
folder = env / 'raw' / j['id']
|
||||
folder.mkdir(parents=True)
|
||||
(folder / 'Game.iso.part').write_bytes(data[:40])
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert len(requests) == 5
|
||||
assert all(int(r.headers['range'].split('=')[1].split('-')[0]) >= 40 for r in requests[1:])
|
||||
|
||||
|
||||
def test_interrupted_segment_resumes_only_missing_bytes(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
failed = {'once': False}
|
||||
monkeypatch.setattr(archive_download, 'CHUNK_BYTES', 4)
|
||||
|
||||
class BrokenStream(httpx.SyncByteStream):
|
||||
def __iter__(self):
|
||||
yield data[:5]
|
||||
raise httpx.ReadError('connection reset')
|
||||
|
||||
def break_once(req):
|
||||
if req.headers.get('range') == 'bytes=0-31' and not failed['once']:
|
||||
failed['once'] = True
|
||||
return httpx.Response(206, stream=BrokenStream(), headers={
|
||||
'Content-Range': 'bytes 0-31/128', 'ETag': '"release-1"'})
|
||||
return None
|
||||
|
||||
requests = server(monkeypatch, data, custom=break_once)
|
||||
monkeypatch.setattr(archive_download, 'MAX_ATTEMPTS', 1)
|
||||
j = job()
|
||||
with pytest.raises(archive_download.ArchiveDownloadError, match='Segment'):
|
||||
archive_download._transfer_one(j)
|
||||
assert (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
|
||||
assert failed['once']
|
||||
requests.clear()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert any(r.headers.get('range') == 'bytes=4-31' for r in requests)
|
||||
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
|
||||
|
||||
|
||||
def test_server_ignores_segment_range_and_serial_fallback(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
def ignore(req):
|
||||
if req.headers.get('range') not in (None, 'bytes=0-0'):
|
||||
return httpx.Response(200, stream=BytesStream(data))
|
||||
return None
|
||||
|
||||
requests = server(monkeypatch, data, custom=ignore)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert any('range' not in r.headers for r in requests)
|
||||
assert not (env / 'raw' / j['id'] / 'Game.iso.part.segments').exists()
|
||||
|
||||
|
||||
def test_http_503_one_segment_retries_without_losing_other_parts(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
failed = {'once': False}
|
||||
def temporary(req):
|
||||
if req.headers.get('range') == 'bytes=32-63' and not failed['once']:
|
||||
failed['once'] = True
|
||||
return httpx.Response(503, headers={'Retry-After': '1'})
|
||||
return None
|
||||
|
||||
requests = server(monkeypatch, data, custom=temporary)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert sum(req.headers.get('range') == 'bytes=32-63' for req in requests) == 2
|
||||
assert 'HTTP 503' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
|
||||
|
||||
|
||||
def test_interrupted_assembly_is_rebuilt_safely(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
server(monkeypatch, data)
|
||||
original = archive_download._assemble_segments
|
||||
failed = {'once': False}
|
||||
|
||||
def broken_assembly(jid, partial, destination, folder, manifest):
|
||||
if not failed['once']:
|
||||
failed['once'] = True
|
||||
manifest['assembling'] = True
|
||||
archive_download._write_manifest(folder, manifest)
|
||||
partial.write_bytes(b'partial-assembly')
|
||||
raise OSError('simulated crash')
|
||||
return original(jid, partial, destination, folder, manifest)
|
||||
|
||||
monkeypatch.setattr(archive_download, '_assemble_segments', broken_assembly)
|
||||
j = job()
|
||||
with pytest.raises(OSError, match='simulated crash'):
|
||||
archive_download._transfer_one(j)
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
|
||||
|
||||
def test_small_file_uses_single_transfer(env, monkeypatch):
|
||||
data = b'hello world'
|
||||
requests = server(monkeypatch, data)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert len(requests) == 2
|
||||
assert requests[0].headers['range'] == 'bytes=0-0'
|
||||
assert 'range' not in requests[1].headers
|
||||
|
||||
|
||||
def test_archive_connections_can_be_disabled(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '1')
|
||||
requests = server(monkeypatch, data)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert len(requests) == 1
|
||||
assert 'range' not in requests[0].headers
|
||||
|
||||
|
||||
def test_changed_etag_forces_clean_serial_restart(env, monkeypatch):
|
||||
data = bytes(range(128))
|
||||
def changed(req):
|
||||
if req.headers.get('range') not in (None, 'bytes=0-0'):
|
||||
value = req.headers['range'].split('=', 1)[1]
|
||||
lo, hi = [int(x) for x in value.split('-')]
|
||||
return httpx.Response(206, stream=BytesStream(data[lo:hi + 1]), headers={
|
||||
'Content-Range': f'bytes {lo}-{hi}/128', 'ETag': '"changed"'})
|
||||
return None
|
||||
|
||||
requests = server(monkeypatch, data, custom=changed)
|
||||
j = job()
|
||||
# This prefix must be discarded when the origin presents conflicting validators.
|
||||
raw = env / 'raw' / j['id']
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'Game.iso.part').write_bytes(b'old-data')
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
assert any('range' not in req.headers for req in requests)
|
||||
|
||||
|
||||
def test_archive_rejects_unexpected_compression(env, monkeypatch):
|
||||
def compressed(_request):
|
||||
return httpx.Response(206, headers={
|
||||
'Content-Encoding': 'gzip', 'Content-Range': 'bytes 0-0/128'})
|
||||
server(monkeypatch, bytes(range(128)), custom=compressed)
|
||||
j = job()
|
||||
with pytest.raises(archive_download.ArchiveDownloadError, match='Accept-Encoding'):
|
||||
archive_download._transfer_one(j)
|
||||
assert not final_path(env, j).exists()
|
||||
|
||||
|
||||
def test_invalid_connection_settings_clamped(env, monkeypatch):
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '100')
|
||||
assert archive_download._connection_limit() == 50
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '-1')
|
||||
assert archive_download._connection_limit() == 1
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', 'invalid')
|
||||
assert archive_download._connection_limit() == 50
|
||||
|
||||
|
||||
def test_default_fifty_parallel_ranges_complete(env, monkeypatch):
|
||||
"""The new default really runs 50 transfers, not only displaying that value."""
|
||||
monkeypatch.delenv('ARCHIVE_CONNECTIONS', raising=False)
|
||||
assert archive_download._connection_limit() == 50
|
||||
data = bytes(range(256)) * 4 # Enough to split into 50 ranges in this test
|
||||
requests = server(monkeypatch, data)
|
||||
j = job()
|
||||
archive_download._transfer_one(j)
|
||||
assert final_path(env, j).read_bytes() == data
|
||||
spans = [request.headers['range'] for request in requests]
|
||||
assert spans[0] == 'bytes=0-0'
|
||||
assert len(spans) == 51 # Range probe plus 50 segments
|
||||
assert len(set(spans[1:])) == 50
|
||||
assert store.get(j['id'])['state'] == 'preparing'
|
||||
assert '50 połączeń HTTP Range' in (env / 'logs' / (j['id'] + '.download.log')).read_text()
|
||||
@@ -0,0 +1,124 @@
|
||||
import asyncio
|
||||
import io
|
||||
import os
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import common, store, pipeline, download
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
|
||||
monkeypatch.setattr(pipeline, 'RAW', tmp_path / 'raw')
|
||||
monkeypatch.setattr(pipeline, 'READY', tmp_path / 'ready')
|
||||
monkeypatch.setattr(pipeline, 'WORK', tmp_path / 'work')
|
||||
monkeypatch.setattr(pipeline, 'LOGS', tmp_path / 'logs')
|
||||
store.init_db()
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_uris():
|
||||
assert common.validate_url('https://archive.org/download/demo/game.iso')[0] == 'url'
|
||||
assert common.validate_url('https://vimm.net/vault/12345')[0] == 'vimm'
|
||||
assert common.validate_url('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d')[0] == 'magnet'
|
||||
for url in ('file:///etc/passwd', 'http://127.0.0.1/admin',
|
||||
'https://user:password@example.org', 'https://localhost/file',
|
||||
'ftp://example.org/file'):
|
||||
with pytest.raises(ValueError):
|
||||
common.validate_url(url)
|
||||
|
||||
|
||||
def test_archive_id_and_escape():
|
||||
assert common.parse_archive_id('https://archive.org/details/some-item') == 'some-item'
|
||||
assert common.archive_download_url('some-item', 'Game (1).7z').endswith('Game%20%281%29.7z')
|
||||
for name in ('../target.iso', '/etc/password', 'foo/../../abc', 'a\\b', ''):
|
||||
with pytest.raises(ValueError):
|
||||
common.archive_download_url('item', name)
|
||||
for name in ('../../secret', '/absolute/file', 'C:\\Windows\\System32\\x', 'folder/../x'):
|
||||
with pytest.raises(pipeline.ProcessingError):
|
||||
pipeline.safe_archive_path(name)
|
||||
|
||||
|
||||
def test_database_queue(isolated):
|
||||
for t in ('Test1', 'Test2'):
|
||||
store.create({'id': t, 'title': t, 'source_type': 'url',
|
||||
'source': 'https://example.org/test.zip', 'format': 'xex'})
|
||||
assert len(store.list_jobs()) == 2
|
||||
assert store.next_job()['id'] == 'Test1'
|
||||
store.update('Test1', state='done', progress=100)
|
||||
assert store.next_job()['id'] == 'Test2'
|
||||
|
||||
|
||||
def test_process_xex_zip(isolated):
|
||||
jid = str(uuid.uuid4())
|
||||
store.create({'id': jid, 'title': 'Demo Game', 'source_type': 'import',
|
||||
'source': 'game.zip', 'format': 'xex'})
|
||||
raw = isolated / 'raw' / jid
|
||||
raw.mkdir(parents=True)
|
||||
with zipfile.ZipFile(raw / 'game.zip', 'w') as archive:
|
||||
archive.writestr('folder/default.xex', b'This is a fake XEX for queue testing')
|
||||
archive.writestr('folder/data/readme.txt', b'hello')
|
||||
asyncio.run(pipeline.process_job(store.get(jid)))
|
||||
job = store.get(jid)
|
||||
assert job['state'] == 'done'
|
||||
assert (Path(job['output']) / 'Gra' / 'default.xex').exists()
|
||||
assert (Path(job['output']) / 'Gra' / 'data' / 'readme.txt').read_bytes() == b'hello'
|
||||
|
||||
|
||||
def test_zip_traversal_rejected(isolated):
|
||||
jid = 'traversal'
|
||||
store.create({'id': jid, 'title': 'Bad', 'source_type': 'import',
|
||||
'source': 'bad.zip', 'format': 'xex'})
|
||||
raw = isolated / 'raw' / jid
|
||||
raw.mkdir(parents=True)
|
||||
with zipfile.ZipFile(raw / 'bad.zip', 'w') as archive:
|
||||
archive.writestr('../../pwned', b'nope')
|
||||
with pytest.raises(pipeline.ProcessingError):
|
||||
asyncio.run(pipeline.process_job(store.get(jid)))
|
||||
assert not (isolated / 'pwned').exists()
|
||||
|
||||
|
||||
def test_raw_copy(isolated):
|
||||
jid = 'simple'
|
||||
store.create({'id': jid, 'title': 'Keep File', 'source_type': 'import',
|
||||
'source': 'anything.bin', 'format': 'raw'})
|
||||
raw = isolated / 'raw' / jid
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'anything.bin').write_bytes(b'abc')
|
||||
asyncio.run(pipeline.process_job(store.get(jid)))
|
||||
assert (Path(store.get(jid)['output']) / 'files' / 'anything.bin').read_bytes() == b'abc'
|
||||
|
||||
|
||||
def test_multipart_urls():
|
||||
job={'source_type':'archive_many', 'source':'["https://host/A.part1.rar", "https://host/A.part2.rar"]'}
|
||||
assert len(download.source_urls(job)) == 2
|
||||
assert download.target_filename('https://example.com/Gra%20XYZ.iso') == 'Gra XYZ.iso'
|
||||
|
||||
|
||||
def test_api_auth_and_add(isolated, monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
from app import main
|
||||
monkeypatch.setenv('WEB_USER', 'admin')
|
||||
monkeypatch.setenv('WEB_PASSWORD', 'secretsecret1234')
|
||||
async def idle_worker():
|
||||
await asyncio.sleep(10000)
|
||||
monkeypatch.setattr(main, 'worker', idle_worker)
|
||||
with TestClient(main.app) as client:
|
||||
assert client.get('/').status_code == 401
|
||||
assert client.get('/api/jobs').status_code == 401
|
||||
auth = ('admin', 'secretsecret1234')
|
||||
assert client.get('/', auth=auth).status_code == 200
|
||||
r = client.post('/api/jobs', auth=auth,
|
||||
json={'url': 'https://example.com/DEMO.iso','format': 'xex'})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()['title'] == 'DEMO.iso'
|
||||
assert client.get('/api/jobs', auth=auth).json()[0]['id'] == r.json()['id']
|
||||
assert client.post('/api/jobs', auth=auth,
|
||||
json={'url':'file:///etc/passwd','format':'xex'}).status_code == 400
|
||||
assert client.post('/api/jobs', auth=auth,
|
||||
json={'url':'https://example.org/foo.iso','format':'oops'}).status_code == 400
|
||||
@@ -0,0 +1,255 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app import archive_auth, archive_download, bt, common, download, main, store, torrent_meta
|
||||
|
||||
|
||||
def bencode(item):
|
||||
if isinstance(item, int):
|
||||
return b'i' + str(item).encode() + b'e'
|
||||
if isinstance(item, str):
|
||||
item = item.encode()
|
||||
if isinstance(item, bytes):
|
||||
return str(len(item)).encode() + b':' + item
|
||||
if isinstance(item, (list, tuple)):
|
||||
return b'l' + b''.join(bencode(x) for x in item) + b'e'
|
||||
if isinstance(item, dict):
|
||||
return b'd' + b''.join(bencode(k) + bencode(v) for k, v in sorted(item.items())) + b'e'
|
||||
raise TypeError(item)
|
||||
|
||||
|
||||
def sample_torrent():
|
||||
return bencode({b'announce': b'https://tracker.example/announce', b'info': {
|
||||
b'name': b'MyGame', b'files': [
|
||||
{b'length': 100, b'path': [b'Game.iso']},
|
||||
{b'length': 40, b'path': [b'readme.txt']},
|
||||
{b'length': 12, b'path': [b'..padding', b'pad'], b'attr': b'p'},
|
||||
]}})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
|
||||
monkeypatch.setenv('WEB_USER', 'admin')
|
||||
monkeypatch.setenv('WEB_PASSWORD', 'mysecretpassword')
|
||||
monkeypatch.setenv('ARIA2_SECRET', 'secret-rpc-key')
|
||||
store.init_db()
|
||||
async def idle_worker():
|
||||
await asyncio.sleep(10000)
|
||||
monkeypatch.setattr(main, 'worker', idle_worker)
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_torrent_metadata_parser_and_magnet_selection(tmp_path):
|
||||
file = tmp_path / 'test.torrent'
|
||||
file.write_bytes(sample_torrent())
|
||||
meta = torrent_meta.read_torrent(file)
|
||||
assert [row['name'] for row in meta['files']] == [
|
||||
'MyGame/Game.iso', 'MyGame/readme.txt', 'MyGame/..padding/pad']
|
||||
assert meta['files'][0]['index'] == 1
|
||||
assert meta['files'][2]['padding'] is True
|
||||
assert torrent_meta.suggested_indices('magnet:?xt=urn:btih:123&so=15,0-1', 20) == [1, 2, 16]
|
||||
|
||||
|
||||
def test_torrent_path_traversal_rejected(tmp_path):
|
||||
file = tmp_path / 'bad.torrent'
|
||||
file.write_bytes(bencode({b'info': {b'name': b'Game', b'files': [
|
||||
{b'length': 5, b'path': [b'..', b'secret']}]}}))
|
||||
with pytest.raises(torrent_meta.MetadataError):
|
||||
torrent_meta.read_torrent(file)
|
||||
with pytest.raises(torrent_meta.MetadataError):
|
||||
torrent_meta.decode_bencode(b'd3:fooi1eeTRAILING')
|
||||
|
||||
|
||||
def test_archive_session_secrets_and_logout(isolated):
|
||||
archive_auth._save_session('test@archive.org', {
|
||||
'logged-in-user': 'member%40archive.org', 'logged-in-sig': 'signedsession'})
|
||||
path = archive_auth.session_path()
|
||||
assert oct(path.stat().st_mode & 0o777) == '0o600'
|
||||
assert archive_auth.session_status() == {'authenticated': True, 'email': 'test@archive.org'}
|
||||
assert archive_auth.session_cookies().get('logged-in-sig') == 'signedsession'
|
||||
archive_auth.logout()
|
||||
assert archive_auth.session_status()['authenticated'] is False
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
def test_archive_xauthn_login_no_password_saved(isolated, monkeypatch):
|
||||
original = httpx.AsyncClient
|
||||
def handler(request):
|
||||
assert request.method == 'POST'
|
||||
assert request.url.host == 'archive.org'
|
||||
assert b'password=not-saved' in request.content
|
||||
return httpx.Response(200, json={'success': True, 'values': {'cookies': {
|
||||
'logged-in-user': 'user%40example.org; Path=/',
|
||||
'logged-in-sig': 'sig%3Dabc; Path=/'}}})
|
||||
transport = httpx.MockTransport(handler)
|
||||
monkeypatch.setattr(archive_auth.httpx, 'AsyncClient', lambda **kw: original(**kw, transport=transport))
|
||||
status = asyncio.run(archive_auth.login('user@example.org', 'not-saved'))
|
||||
assert status['authenticated'] is True
|
||||
content = archive_auth.session_path().read_text()
|
||||
assert 'not-saved' not in content
|
||||
assert 'sig%3Dabc' in content
|
||||
|
||||
|
||||
def test_archive_relay_redirect_cookie_scoping_and_range(isolated, monkeypatch):
|
||||
archive_auth._save_session('user@example.org', {
|
||||
'logged-in-user': 'member', 'logged-in-sig': 'secret-cookie'})
|
||||
originals = httpx.AsyncClient
|
||||
seen = []
|
||||
def handler(request):
|
||||
seen.append((request.url.host, request.headers.get('cookie'), request.headers.get('range')))
|
||||
if request.url.host == 'archive.org':
|
||||
return httpx.Response(302, headers={'Location': 'https://ia808.us.archive.org/download/asset.zip'})
|
||||
class FakeStream(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield b'bcd'
|
||||
return httpx.Response(206, headers={'Content-Length': '3', 'Content-Range': 'bytes 1-3/4',
|
||||
'Content-Type': 'application/octet-stream'}, stream=FakeStream())
|
||||
transport = httpx.MockTransport(handler)
|
||||
monkeypatch.setattr(archive_auth.httpx, 'AsyncClient', lambda **kw: originals(**kw, transport=transport))
|
||||
|
||||
async def call():
|
||||
response = await archive_auth.stream_archive('https://archive.org/download/item/asset.zip', 'GET',
|
||||
{'range': 'bytes=1-3'})
|
||||
data = b''
|
||||
async for part in response.body_iterator:
|
||||
data += part
|
||||
return response, data
|
||||
response, data = asyncio.run(call())
|
||||
assert response.status_code == 206
|
||||
assert data == b'bcd'
|
||||
assert len(seen) == 2
|
||||
assert all('secret-cookie' in (item[1] or '') for item in seen)
|
||||
assert all(item[2] == 'bytes=1-3' for item in seen)
|
||||
assert archive_auth.allowed_redirect('https://attacker.example/') is False
|
||||
assert archive_auth.allowed_redirect('http://archive.org/') is False
|
||||
|
||||
|
||||
def test_magnet_inspection_selection_and_signed_archive_proxy(isolated, monkeypatch):
|
||||
started = []
|
||||
stopped = []
|
||||
async def fake_start(preview_id, magnet):
|
||||
started.append((preview_id, magnet))
|
||||
async def fake_stop(preview_id):
|
||||
stopped.append(preview_id)
|
||||
monkeypatch.setattr(bt, 'start_inspection', fake_start)
|
||||
monkeypatch.setattr(bt, 'stop_inspection', fake_stop)
|
||||
monkeypatch.setattr(bt, 'inspect_state', lambda pid: {'connections': 2, 'known_peers': 5})
|
||||
with TestClient(main.app) as client:
|
||||
auth = ('admin', 'mysecretpassword')
|
||||
magnet = 'magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&dn=Test&so=1'
|
||||
r = client.post('/api/magnet/inspect', auth=auth, json={'magnet': magnet})
|
||||
assert r.status_code == 200, r.text
|
||||
pid = r.json()['preview_id']
|
||||
status = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
|
||||
assert status['status'] == 'loading'
|
||||
assert status['connections'] == 2 and status['known_peers'] == 5
|
||||
assert started == [(pid, magnet)]
|
||||
folder = torrent_meta.inspect_dir(pid)
|
||||
(folder / 'hash.torrent').write_bytes(sample_torrent())
|
||||
info = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
|
||||
assert info['status'] == 'ready'
|
||||
assert info['suggested'] == [2]
|
||||
assert info['files'][1]['name'] == 'MyGame/readme.txt'
|
||||
assert client.post('/api/magnet/jobs', auth=auth,
|
||||
json={'preview_id': pid, 'files': [4], 'format': 'raw'}).status_code == 400
|
||||
assert client.post('/api/magnet/jobs', auth=auth,
|
||||
json={'preview_id': pid, 'files': [3], 'format': 'raw'}).status_code == 400
|
||||
r = client.post('/api/magnet/jobs', auth=auth,
|
||||
json={'preview_id': pid, 'files': [2], 'format': 'raw'})
|
||||
assert r.status_code == 200, r.text
|
||||
job = r.json()
|
||||
assert job['source_type'] == 'torrent_selected'
|
||||
assert (isolated / 'torrents' / (job['id'] + '.torrent')).read_bytes() == sample_torrent()
|
||||
assert json.loads(job['source'])['selection'] == '2'
|
||||
assert pid in stopped
|
||||
|
||||
archive_job = main.new_job('archive_many', json.dumps(['https://archive.org/download/test/game.iso']), 'Demo', 'raw')
|
||||
uri = '/internal/archive/' + archive_job['id'] + '/0'
|
||||
assert client.get(uri).status_code == 403
|
||||
assert client.get(uri, headers={'X-X360-Proxy-Token': 'invalid'}).status_code == 403
|
||||
async def fake_stream(url, method, headers):
|
||||
assert url == 'https://archive.org/download/test/game.iso'
|
||||
return PlainTextResponse('proxy-ok')
|
||||
monkeypatch.setattr(archive_auth, 'stream_archive', fake_stream)
|
||||
token = archive_auth.signed_header(archive_job['id'], 0).split(': ', 1)[1]
|
||||
assert client.get(uri, headers={'X-X360-Proxy-Token': token}).text == 'proxy-ok'
|
||||
assert client.post('/api/jobs', auth=auth,
|
||||
json={'url': magnet, 'format': 'raw'}).status_code == 400
|
||||
|
||||
|
||||
def test_wrapped_and_html_escaped_magnet():
|
||||
hash_value = 'c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d'
|
||||
pasted = "'**magnet**:?xt=urn:btih:" + hash_value + "&dn=Minerva_Myrient\\\n&so=15&tr=udp%3A%2F%2Ftracker.ope\\\nnbittorrent.com%3A6969%2Fannounce'"
|
||||
kind, cleaned = common.validate_url(pasted)
|
||||
assert kind == 'magnet'
|
||||
assert cleaned.startswith('magnet:?xt=urn:btih:' + hash_value + '&dn=')
|
||||
assert '&so=15&tr=' in cleaned
|
||||
assert 'tracker.openbittorrent.com' in cleaned
|
||||
assert '\\' not in cleaned and '&' not in cleaned and '\n' not in cleaned
|
||||
assert torrent_meta.suggested_indices(cleaned, 20) == [16]
|
||||
with pytest.raises(ValueError):
|
||||
common.validate_url('magnet:?xt=urn:btih:abcdef')
|
||||
|
||||
|
||||
def test_upload_torrent_and_queue_selection(isolated, monkeypatch):
|
||||
async def fake_rpc(method, *args):
|
||||
if method == 'addTorrent':
|
||||
return '123456'
|
||||
if method == 'tellStatus':
|
||||
return {'status': 'active', 'connections': '0'}
|
||||
return 'ok'
|
||||
monkeypatch.setattr(download, 'rpc', fake_rpc)
|
||||
with TestClient(main.app) as client:
|
||||
auth = ('admin', 'mysecretpassword')
|
||||
uploaded = client.post('/api/magnet/upload', auth=auth,
|
||||
content=sample_torrent(), headers={'Content-Type': 'application/x-bittorrent'})
|
||||
assert uploaded.status_code == 200, uploaded.text
|
||||
pid = uploaded.json()['preview_id']
|
||||
info = client.get('/api/magnet/inspect/' + pid, auth=auth).json()
|
||||
assert info['status'] == 'ready'
|
||||
assert info['files'][0]['name'] == 'MyGame/Game.iso'
|
||||
assert info['suggested'] == []
|
||||
job = client.post('/api/magnet/jobs', auth=auth,
|
||||
json={'preview_id': pid, 'files': [1], 'format': 'xex'})
|
||||
assert job.status_code == 200, job.text
|
||||
assert (isolated / 'torrents' / (job.json()['id'] + '.torrent')).exists()
|
||||
assert client.post('/api/magnet/upload', auth=auth,
|
||||
content=b'garbage', headers={'Content-Type': 'application/x-bittorrent'}).status_code == 400
|
||||
assert client.post('/api/magnet/upload', auth=auth,
|
||||
content=sample_torrent(), headers={'Content-Type': 'application/json'}).status_code == 415
|
||||
|
||||
|
||||
def test_metadata_status_includes_connections_and_timeout(isolated, monkeypatch):
|
||||
stopped = []
|
||||
async def fake_start(preview_id, magnet):
|
||||
pass
|
||||
async def fake_stop(preview_id):
|
||||
stopped.append(preview_id)
|
||||
monkeypatch.setattr(bt, 'start_inspection', fake_start)
|
||||
monkeypatch.setattr(bt, 'stop_inspection', fake_stop)
|
||||
monkeypatch.setattr(bt, 'inspect_state', lambda pid: {'connections': 0, 'known_peers': 0})
|
||||
magnet='magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=15'
|
||||
with TestClient(main.app) as client:
|
||||
auth=('admin','mysecretpassword')
|
||||
resp=client.post('/api/magnet/inspect',auth=auth,json={'magnet':magnet})
|
||||
assert resp.status_code == 200
|
||||
pid=resp.json()['preview_id']
|
||||
status=client.get('/api/magnet/inspect/'+pid,auth=auth).json()
|
||||
assert status['status']=='loading' and status['connections']==0
|
||||
request_path=torrent_meta.inspect_dir(pid)/'request.json'
|
||||
state=json.loads(request_path.read_text())
|
||||
state['created'] -= 310
|
||||
request_path.write_text(json.dumps(state))
|
||||
expired=client.get('/api/magnet/inspect/'+pid,auth=auth).json()
|
||||
assert expired['status']=='error'
|
||||
assert pid in stopped
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Regression checks for host-side /xbox storage and safe upgrade migration."""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app import main
|
||||
|
||||
PROJECT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _copy_scripts(dst):
|
||||
for name in ('xbox-paths.sh', 'migrate-to-xbox.sh', 'setup.sh', '.env.example'):
|
||||
shutil.copy2(PROJECT / name, dst / name)
|
||||
|
||||
|
||||
def _run_script(project, name, root, **extra):
|
||||
env = os.environ.copy()
|
||||
env.update(XBOX_HOST_DIR=str(root), XBOX_MIGRATION_OFFLINE='1', XBOX_SKIP_DOCKER='1', **extra)
|
||||
return subprocess.run(['sh', name], cwd=project, env=env,
|
||||
capture_output=True, text=True, timeout=30)
|
||||
|
||||
|
||||
def test_config_uses_host_and_dedicated_downloads():
|
||||
compose = (PROJECT / 'compose.yaml').read_text()
|
||||
assert 'source: ${XBOX_HOST_DIR:-/xbox}' in compose
|
||||
assert 'source: ${XBOX_HOST_DIR:-/xbox}/downloads' in compose
|
||||
assert 'target: /data/raw' in compose
|
||||
assert 'target: /data' in compose
|
||||
assert 'create_host_path: false' in compose
|
||||
assert 'TMPDIR: /data/work/tmp' in compose
|
||||
assert './data:/data' not in compose
|
||||
assert 'XBOX_HOST_DIR=/xbox' in (PROJECT / '.env.example').read_text()
|
||||
assert '--dir=/data/raw' in (PROJECT / 'entrypoint.sh').read_text()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not shutil.which('rsync'), reason='rsync not installed on test host')
|
||||
def test_migrate_preserves_original_data_and_queue(tmp_path):
|
||||
project = tmp_path / 'old project'
|
||||
project.mkdir()
|
||||
_copy_scripts(project)
|
||||
host = tmp_path / 'host volume'
|
||||
host.mkdir()
|
||||
data = project / 'data'
|
||||
(data / 'raw' / 'task-1').mkdir(parents=True)
|
||||
(data / 'raw' / 'task-1' / 'demo.iso.part').write_bytes(b'12345')
|
||||
(data / 'work' / 'task-1').mkdir(parents=True)
|
||||
(data / 'work' / 'task-1' / 'unpacked.iso').write_bytes(b'unpacked')
|
||||
(data / 'ready' / 'xex' / 'demo').mkdir(parents=True)
|
||||
(data / 'ready' / 'xex' / 'demo' / 'default.xex').write_bytes(b'xex')
|
||||
(data / 'private').mkdir()
|
||||
(data / 'private' / 'archive-session.json').write_text('{"session": "example"}')
|
||||
(data / 'queue.sqlite3').write_bytes(b'queue')
|
||||
(data / 'queue.sqlite3-wal').write_bytes(b'wal')
|
||||
(data / 'aria2').mkdir()
|
||||
(data / 'aria2' / 'session.txt').write_text('/data/raw/task-1/demo.iso')
|
||||
|
||||
result = _run_script(project, 'migrate-to-xbox.sh', host)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (host / 'downloads' / 'task-1' / 'demo.iso.part').read_bytes() == b'12345'
|
||||
assert (host / 'work' / 'task-1' / 'unpacked.iso').read_bytes() == b'unpacked'
|
||||
assert (host / 'ready' / 'xex' / 'demo' / 'default.xex').read_bytes() == b'xex'
|
||||
assert (host / 'queue.sqlite3').read_bytes() == b'queue'
|
||||
assert (host / 'queue.sqlite3-wal').read_bytes() == b'wal'
|
||||
assert (host / 'private' / 'archive-session.json').is_file()
|
||||
assert (host / 'aria2' / 'session.txt').read_text() == '/data/raw/task-1/demo.iso'
|
||||
assert not (host / 'raw').exists()
|
||||
assert not (host / '.x360-migration-in-progress').exists()
|
||||
assert (data / 'raw' / 'task-1' / 'demo.iso.part').read_bytes() == b'12345'
|
||||
assert (data / 'queue.sqlite3').read_bytes() == b'queue'
|
||||
|
||||
|
||||
@pytest.mark.skipif(not shutil.which('rsync'), reason='rsync not installed on test host')
|
||||
def test_migration_does_not_replace_existing_queue(tmp_path):
|
||||
project = tmp_path / 'project'
|
||||
project.mkdir()
|
||||
_copy_scripts(project)
|
||||
host = tmp_path / 'xbox'
|
||||
host.mkdir()
|
||||
(project / 'data').mkdir()
|
||||
(project / 'data' / 'queue.sqlite3').write_bytes(b'old')
|
||||
(host / 'queue.sqlite3').write_bytes(b'new')
|
||||
result = _run_script(project, 'migrate-to-xbox.sh', host)
|
||||
assert result.returncode != 0
|
||||
assert (host / 'queue.sqlite3').read_bytes() == b'new'
|
||||
assert not (host / '.x360-migration-in-progress').exists()
|
||||
|
||||
|
||||
def test_setup_does_not_silently_drop_existing_database(tmp_path):
|
||||
project = tmp_path / 'project'
|
||||
project.mkdir()
|
||||
_copy_scripts(project)
|
||||
host = tmp_path / 'xbox'
|
||||
host.mkdir()
|
||||
(project / 'data').mkdir()
|
||||
(project / 'data' / 'queue.sqlite3').write_bytes(b'important')
|
||||
result = _run_script(project, 'setup.sh', host)
|
||||
assert result.returncode != 0
|
||||
assert 'migrate-to-xbox.sh' in result.stderr
|
||||
assert not (project / '.env').exists()
|
||||
|
||||
|
||||
def test_setup_persists_custom_host_path(tmp_path):
|
||||
project = tmp_path / 'project'
|
||||
project.mkdir()
|
||||
_copy_scripts(project)
|
||||
host = tmp_path / 'external-host-share'
|
||||
host.mkdir()
|
||||
result = _run_script(project, 'setup.sh', host)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert f'XBOX_HOST_DIR={host}' in (project / '.env').read_text()
|
||||
assert (host / 'downloads').is_dir()
|
||||
assert (host / 'work' / 'tmp').is_dir()
|
||||
assert (host / 'ready').is_dir()
|
||||
|
||||
|
||||
def test_missing_host_directory_fails_without_creating(tmp_path):
|
||||
project = tmp_path / 'project'
|
||||
project.mkdir()
|
||||
_copy_scripts(project)
|
||||
host = tmp_path / 'not-mounted'
|
||||
result = _run_script(project, 'setup.sh', host)
|
||||
assert result.returncode != 0
|
||||
assert not host.exists()
|
||||
assert not (project / '.env').exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_shows_actual_host_path(tmp_path, monkeypatch):
|
||||
from app import store
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
(tmp_path / 'raw').mkdir(parents=True, exist_ok=True)
|
||||
monkeypatch.setenv('XBOX_HOST_DIR', '/xbox')
|
||||
result = await main.system()
|
||||
assert result['host_data_dir'] == '/xbox'
|
||||
assert result['total'] > 0
|
||||
@@ -0,0 +1,164 @@
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import archive_auth, archive_download, bt, main, store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(store, 'DATA', tmp_path)
|
||||
monkeypatch.setattr(store, 'DB_PATH', tmp_path / 'queue.sqlite3')
|
||||
monkeypatch.setenv('ARIA2_SECRET', 'test')
|
||||
monkeypatch.setenv('ARCHIVE_CONNECTIONS', '1') # Old single-stream behavior is covered separately.
|
||||
store.init_db()
|
||||
return tmp_path
|
||||
|
||||
|
||||
def create_archive(url='https://archive.org/download/item/Game.iso'):
|
||||
return main.new_job('url', url, 'Game', 'raw')
|
||||
|
||||
|
||||
def fake_http_client(monkeypatch, handler):
|
||||
transport = httpx.MockTransport(handler)
|
||||
monkeypatch.setattr(archive_download, '_client', lambda: httpx.Client(
|
||||
transport=transport, cookies=archive_auth.session_cookies(), follow_redirects=False))
|
||||
monkeypatch.setattr(archive_download.time, 'sleep', lambda delay: None)
|
||||
|
||||
|
||||
class BytesStream(httpx.SyncByteStream):
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
|
||||
def __iter__(self):
|
||||
yield self.data
|
||||
|
||||
|
||||
def test_archive_transient_http_500_is_retried_and_logged(isolated, monkeypatch):
|
||||
archive_auth._save_session('user@example.org', {
|
||||
'logged-in-user': 'user', 'logged-in-sig': 'cookie-secret'})
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append(request)
|
||||
if len(requests) < 3:
|
||||
return httpx.Response(500, text='upstream temporarily broken')
|
||||
return httpx.Response(200, stream=BytesStream(b'ISO-CONTENT'), headers={'Content-Length': '11'})
|
||||
|
||||
fake_http_client(monkeypatch, handler)
|
||||
job = create_archive()
|
||||
archive_download._transfer_one(job)
|
||||
assert store.get(job['id'])['state'] == 'preparing'
|
||||
assert (isolated / 'raw' / job['id'] / 'Game.iso').read_bytes() == b'ISO-CONTENT'
|
||||
assert len(requests) == 3
|
||||
assert all('cookie-secret' in req.headers.get('cookie', '') for req in requests)
|
||||
text = (isolated / 'logs' / (job['id'] + '.download.log')).read_text()
|
||||
assert 'HTTP 500' in text and 'Ukończono' in text
|
||||
assert 'cookie-secret' not in text
|
||||
|
||||
|
||||
def test_archive_range_resume_with_redirect(isolated, monkeypatch):
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append((request.url.host, request.headers.get('range')))
|
||||
if request.url.host == 'archive.org':
|
||||
return httpx.Response(302, headers={'Location': 'https://ia444.us.archive.org/download/item/Game.iso'})
|
||||
return httpx.Response(206, stream=BytesStream(b'DEF'), headers={'Content-Range': 'bytes 3-5/6',
|
||||
'Content-Length': '3'})
|
||||
|
||||
fake_http_client(monkeypatch, handler)
|
||||
job = create_archive()
|
||||
raw = isolated / 'raw' / job['id']
|
||||
raw.mkdir(parents=True)
|
||||
(raw / 'Game.iso.part').write_bytes(b'ABC')
|
||||
archive_download._transfer_one(job)
|
||||
assert (raw / 'Game.iso').read_bytes() == b'ABCDEF'
|
||||
assert not (raw / 'Game.iso.part').exists()
|
||||
assert requests == [('archive.org', 'bytes=3-'), ('ia444.us.archive.org', 'bytes=3-')]
|
||||
|
||||
|
||||
def test_archive_403_is_explicit_and_not_retried(isolated, monkeypatch):
|
||||
seen = []
|
||||
def handler(request):
|
||||
seen.append(1)
|
||||
return httpx.Response(403, text='denied')
|
||||
fake_http_client(monkeypatch, handler)
|
||||
job = create_archive()
|
||||
with pytest.raises(archive_download.ArchiveDownloadError, match='HTTP 403'):
|
||||
archive_download._transfer_one(job)
|
||||
assert len(seen) == 1
|
||||
|
||||
|
||||
def test_archive_persistent_500_reports_origin_failure(isolated, monkeypatch):
|
||||
seen = []
|
||||
def handler(request):
|
||||
seen.append(1)
|
||||
return httpx.Response(500)
|
||||
fake_http_client(monkeypatch, handler)
|
||||
job = create_archive()
|
||||
with pytest.raises(archive_download.ArchiveDownloadError, match='HTTP 500 po 5 próbach'):
|
||||
archive_download._transfer_one(job)
|
||||
assert len(seen) == 5
|
||||
|
||||
|
||||
def test_archive_cancel_keeps_partial_files(isolated, monkeypatch):
|
||||
job = create_archive()
|
||||
store.update(job['id'], state='paused')
|
||||
fake_http_client(monkeypatch, lambda req: pytest.fail('Do not download paused job'))
|
||||
with pytest.raises(archive_download.ArchiveStopped):
|
||||
archive_download._transfer_one(job)
|
||||
|
||||
|
||||
def test_bt_helper_subprocess_inspection_and_fetch(isolated, monkeypatch):
|
||||
"""Exercise process supervisor and status semantics without real peers."""
|
||||
from tests.test_features import sample_torrent
|
||||
|
||||
helper = isolated / 'fake-helper'
|
||||
# Tests the Python supervisor lifecycle, not the compiled Go / network engine.
|
||||
helper.write_text('#!' + sys.executable + '\n' + '''
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
args = sys.argv[1:]
|
||||
mode = args[0]
|
||||
def param(x): return args[args.index(x)+1]
|
||||
state = Path(param('-status'))
|
||||
state.parent.mkdir(parents=True, exist_ok=True)
|
||||
if mode == 'inspect':
|
||||
Path(param('-out')).write_bytes(bytes.fromhex(os.environ['FAKE_TORRENT_HEX']))
|
||||
state.write_text(json.dumps({'status':'ready','connections':2}))
|
||||
else:
|
||||
output=Path(param('-out'))
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
(output/'Game.iso').write_bytes(b'foo')
|
||||
state.write_text(json.dumps({'status':'done','downloaded':3,'total':3}))
|
||||
''')
|
||||
helper.chmod(0o755)
|
||||
monkeypatch.setenv('FAKE_TORRENT_HEX', sample_torrent().hex())
|
||||
monkeypatch.setenv('BT_HELPER', str(helper))
|
||||
monkeypatch.setattr(bt, '_DOWNLOADS', {})
|
||||
monkeypatch.setattr(bt, '_INSPECTIONS', {})
|
||||
|
||||
async def check():
|
||||
pid = 'a' * 32
|
||||
folder = isolated / 'metadata' / pid
|
||||
folder.mkdir(parents=True)
|
||||
proc = await bt.start_inspection(pid, 'magnet:?xt=urn:btih:' + 'a'*40)
|
||||
assert await proc.wait() == 0
|
||||
assert bt.inspect_state(pid)['status'] == 'ready'
|
||||
await bt.stop_inspection(pid)
|
||||
job = main.new_job('torrent_selected', json.dumps({'selection':'1'}), 'Game', 'raw')
|
||||
(isolated/'torrents').mkdir()
|
||||
(isolated/'torrents'/(job['id']+'.torrent')).write_bytes(sample_torrent())
|
||||
await bt.start_download(job)
|
||||
proc = bt._DOWNLOADS[job['id']]
|
||||
assert await proc.wait() == 0
|
||||
await bt.poll_download(store.get(job['id']))
|
||||
assert store.get(job['id'])['state'] == 'preparing'
|
||||
await bt.shutdown()
|
||||
asyncio.run(check())
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Browser test: pasted magnet cleanup, live diagnostics and .torrent upload."""
|
||||
from pathlib import Path
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
root = Path(__file__).resolve().parents[1] / 'app' / 'static'
|
||||
html = (root / 'index.html').read_text().replace('<link rel="stylesheet" href="/static/style.css">', '').replace('<script src="/static/app.js" defer></script>', '')
|
||||
css = (root / 'style.css').read_text()
|
||||
js = (root / 'app.js').read_text()
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(executable_path='/usr/bin/chromium', args=['--no-sandbox', '--disable-dev-shm-usage'])
|
||||
page = browser.new_page(viewport={'width': 1280, 'height': 900})
|
||||
errors = []
|
||||
page.on('pageerror', lambda err: errors.append(str(err)))
|
||||
page.set_content(html)
|
||||
page.add_style_tag(content=css)
|
||||
page.evaluate('''() => {
|
||||
window.__seenMagnet = null;
|
||||
window.fetch = async (url, options = {}) => {
|
||||
const u = String(url), m = options.method || 'GET';
|
||||
let reply = {};
|
||||
if (u === '/api/system') reply = {free:1000000000,total:2000000000};
|
||||
else if (u === '/api/jobs' && m === 'GET') reply = [];
|
||||
else if (u === '/api/archive/session') reply = {authenticated:false};
|
||||
else if (u === '/api/magnet/inspect' && m === 'POST') {
|
||||
window.__seenMagnet = JSON.parse(options.body).magnet;
|
||||
reply = {status:'loading',preview_id:'a'.repeat(32)};
|
||||
}
|
||||
else if (u === '/api/magnet/upload') reply = {status:'ready',preview_id:'b'.repeat(32)};
|
||||
else if (u.includes('/magnet/inspect/' + 'a'.repeat(32)) && m === 'GET')
|
||||
reply = {status:'loading',elapsed:35,connections:0,aria_status:'active'};
|
||||
else if (u.includes('/magnet/inspect/' + 'b'.repeat(32)) && m === 'GET')
|
||||
reply = {status:'ready',name:'Demo',suggested:[],files:[{index:1,name:'Demo/Game.iso',size:1024,padding:false}]};
|
||||
else if (u.startsWith('/api/magnet/inspect/') && m === 'POST') reply = {ok:true};
|
||||
return new Response(JSON.stringify(reply), {status:200,headers:{'Content-Type':'application/json'}});
|
||||
};
|
||||
}''')
|
||||
page.add_script_tag(content=js)
|
||||
raw = "'**magnet**:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=15\\\n&tr=udp%3A%2F%2Fexample.com%2Fannounce'"
|
||||
page.locator('#urls').fill(raw)
|
||||
page.locator('#linksForm button[type=submit]').click()
|
||||
page.locator('#magnetDialog').wait_for(state='visible')
|
||||
page.locator('#magnetStatus').get_by_text('Brak połączeń', exact=False).wait_for()
|
||||
parsed = page.evaluate('window.__seenMagnet')
|
||||
assert parsed.startswith('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=15&tr='), parsed
|
||||
assert '\n' not in parsed and '&' not in parsed
|
||||
page.screenshot(path='/mnt/data/x360-magnet-diagnostics-preview.png', full_page=True)
|
||||
page.locator('#torrentFileDialog').set_input_files({'name':'demo.torrent','mimeType':'application/x-bittorrent','buffer':b'd4:infod4:name4:Demoe e'})
|
||||
page.locator('#magnetResults').wait_for(state='visible')
|
||||
assert page.locator('#magnetFiles input[type=checkbox]').count() == 1
|
||||
page.screenshot(path='/mnt/data/x360-magnet-upload-preview.png', full_page=True)
|
||||
assert not errors, errors
|
||||
browser.close()
|
||||
print('UI magnet fallback PASS')
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Browser UI smoke test with mocked fetch; Chromium policies block navigation."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1] / 'app' / 'static'
|
||||
html = (ROOT / 'index.html').read_text()
|
||||
html = html.replace('<link rel="stylesheet" href="/static/style.css">', '')
|
||||
html = html.replace('<script src="/static/app.js" defer></script>', '')
|
||||
css = (ROOT / 'style.css').read_text()
|
||||
js = (ROOT / 'app.js').read_text()
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(executable_path='/usr/bin/chromium',
|
||||
args=['--no-sandbox', '--disable-dev-shm-usage'])
|
||||
page = browser.new_page(viewport={'width': 1250, 'height': 950})
|
||||
errors = []
|
||||
page.on('pageerror', lambda error: errors.append(str(error)))
|
||||
page.set_content(html)
|
||||
page.add_style_tag(content=css)
|
||||
page.evaluate(r'''() => {
|
||||
window.__submitted = [];
|
||||
window.fetch = async (url, opts = {}) => {
|
||||
const u = String(url);
|
||||
const m = opts.method || 'GET';
|
||||
let value = {ok:true};
|
||||
if (u === '/api/system') value = {free:100000000000, total:200000000000};
|
||||
else if (u === '/api/jobs' && m === 'GET') value = [];
|
||||
else if (u === '/api/archive/session' && m === 'GET') value = {authenticated:window.__loggedIn === true,email:window.__loggedIn ? 'demo@example.org' : ''};
|
||||
else if (u === '/api/archive/session' && m === 'POST') {window.__loggedIn=true;value={authenticated:true,email:'demo@example.org'};}
|
||||
else if (u === '/api/magnet/inspect' && m === 'POST') value = {preview_id:'a'.repeat(32),status:'loading'};
|
||||
else if (u.startsWith('/api/magnet/inspect/') && m === 'GET') value = {status:'ready',name:'Sample',suggested:[2],files:[
|
||||
{index:1,name:'Sample/Game.iso',size:7210000000,padding:false},
|
||||
{index:2,name:'Sample/Manual.pdf',size:15313,padding:false},
|
||||
{index:3,name:'Sample/.pad',size:700,padding:true}
|
||||
]};
|
||||
else if (u === '/api/magnet/jobs' && m === 'POST') {window.__submitted.push(JSON.parse(opts.body));value={ok:true};}
|
||||
return new Response(JSON.stringify(value), {status:200,headers:{'Content-Type':'application/json'}});
|
||||
};
|
||||
}''')
|
||||
page.add_script_tag(content=js)
|
||||
page.locator('button[data-tab="archive"]').click()
|
||||
assert page.locator('#archiveEmail').is_visible()
|
||||
page.locator('#archiveEmail').fill('demo@example.org')
|
||||
page.locator('#archivePassword').fill('dummy-password')
|
||||
page.locator('#archiveLoginForm button[type=submit]').click()
|
||||
page.locator('#archiveLogout').wait_for(state='visible')
|
||||
page.screenshot(path='/mnt/data/x360-archive-auth-preview.png', full_page=True)
|
||||
page.locator('button[data-tab="links"]').click()
|
||||
page.locator('#urls').fill('magnet:?xt=urn:btih:c98fbaa2a36ee48e3d9423be02cdbefd9281bc7d&so=1')
|
||||
page.locator('#linksForm [type=submit]').click()
|
||||
page.locator('#magnetResults').wait_for(state='visible')
|
||||
assert page.locator('#magnetFiles input[type="checkbox"]').count() == 3
|
||||
assert page.locator('#magnetFiles input[type="checkbox"]').nth(1).is_checked()
|
||||
page.screenshot(path='/mnt/data/x360-magnet-select-preview.png', full_page=True)
|
||||
page.locator('#magnetFiles input[type="checkbox"]').nth(0).check()
|
||||
page.locator('#magnetAdd').click()
|
||||
page.locator('#magnetDialog').wait_for(state='hidden')
|
||||
assert page.evaluate('window.__submitted') == [
|
||||
{'preview_id':'a'*32,'files':[2,1],'title':'','format':'xex'}]
|
||||
assert errors == [], errors
|
||||
browser.close()
|
||||
print('UI smoke PASS: Archive login form, torrent modal, file selection, submit')
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/bin/sh
|
||||
# Shared POSIX shell helpers. Must be sourced from the project directory.
|
||||
|
||||
xbox_host_directory() {
|
||||
if [ -n "${XBOX_HOST_DIR:-}" ]; then
|
||||
root=$XBOX_HOST_DIR
|
||||
elif [ -f .env ]; then
|
||||
root=$(sed -n 's/^XBOX_HOST_DIR=//p' .env | tail -n 1)
|
||||
# Handle normal quoted .env values without executing file contents.
|
||||
case "$root" in
|
||||
'"'*'"') root=${root#\"}; root=${root%\"} ;;
|
||||
"'"*"'") root=${root#\'}; root=${root%\'} ;;
|
||||
esac
|
||||
root=${root:-/xbox}
|
||||
else
|
||||
root=/xbox
|
||||
fi
|
||||
case "$root" in
|
||||
/*) ;;
|
||||
*) echo "XBOX_HOST_DIR musi byc sciezka bezwzgledna: $root" >&2; return 1 ;;
|
||||
esac
|
||||
if [ ! -d "$root" ]; then
|
||||
echo "Nie znaleziono $root na hoscie Linux." >&2
|
||||
echo "Utworz go przed uruchomieniem, np.: sudo mkdir -p '$root'" >&2
|
||||
echo "Jesli to montowany dysk, upewnij sie, ze jest zamontowany." >&2
|
||||
return 1
|
||||
fi
|
||||
root=$(cd "$root" && pwd -P) || return 1
|
||||
if [ "$root" = / ]; then
|
||||
echo 'Odmowa uzycia / jako katalogu danych.' >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$root"
|
||||
}
|
||||
|
||||
xbox_prepare_directories() {
|
||||
root=$1
|
||||
for rel in downloads work work/tmp work/cache ready imports logs metadata torrents bt aria2; do
|
||||
if [ -L "$root/$rel" ]; then
|
||||
echo "Odmowa uzycia dowiazania symbolicznego: $root/$rel" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! mkdir -p "$root/$rel"; then
|
||||
echo "Brak uprawnien zapisu w $root. Ustaw prawa katalogu na hoscie." >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
if [ -L "$root/private" ]; then
|
||||
echo "Odmowa uzycia dowiazania symbolicznego: $root/private" >&2
|
||||
return 1
|
||||
fi
|
||||
if [ ! -d "$root/private" ]; then
|
||||
mkdir -m 700 "$root/private" || return 1
|
||||
fi
|
||||
}
|
||||
Reference in new issue
Block a user