first commit

app/archive_download.py

@@ -0,0 +1,760 @@
+"""Direct, authenticated Internet Archive downloader.
+
+Avoids the old aria2 -> localhost HTTP relay (which hid origin errors behind
+HTTP 500). Supports Range resumes, transient 5xx/429 retries and pause/cancel.
+Only forwards IA cookies via httpx's domain-scoped cookie jar.
+"""
+import hashlib
+import json
+import os
+import re
+import shutil
+import time
+from collections import deque
+from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
+from contextlib import closing
+from threading import Event, Lock
+from pathlib import Path
+from urllib.parse import urljoin
+
+import httpx
+
+from . import archive_auth, store
+from .download import source_urls, target_filename
+
+RETRY_CODES = {408, 425, 429, 500, 502, 503, 504}
+MAX_ATTEMPTS = 5
+MAX_CONNECTIONS = 50
+MIN_SEGMENT_BYTES = 8 * 1024 * 1024
+CHUNK_BYTES = 256 * 1024
+RANGE = re.compile(r'^bytes (\d+)-(\d+)/(\d+|\*)$', re.I)
+FINISH_RANGE = re.compile(r'^bytes \*/(\d+)$', re.I)
+LOG_LOCK = Lock()
+
+
+class ArchiveDownloadError(RuntimeError):
+    pass
+
+
+class ArchiveStopped(Exception):
+    """Pause or cancellation was requested, not a transfer error."""
+
+
+def _client():
+    return httpx.Client(timeout=httpx.Timeout(90, connect=25), cookies=archive_auth.session_cookies(),
+                        follow_redirects=False, trust_env=True)
+
+
+def _check_state(jid, stop=None):
+    if stop is not None and stop.is_set():
+        raise ArchiveStopped()
+    current = store.get(jid)
+    if not current or current['state'] in ('paused', 'cancelled', 'error'):
+        raise ArchiveStopped()
+
+
+def _response(client, url, headers):
+    current = url
+    for _ in range(8):
+        if not archive_auth.allowed_redirect(current):
+            raise ArchiveDownloadError('Archive.org: nieprawidłowe przekierowanie poza domenę Archive.org')
+        request = client.build_request('GET', current, headers=headers)
+        response = client.send(request, stream=True)
+        if response.status_code in (301, 302, 303, 307, 308) and 'location' in response.headers:
+            next_url = urljoin(current, response.headers['location'])
+            response.close()
+            current = next_url
+            continue
+        return response
+    raise ArchiveDownloadError('Archive.org: zbyt wiele przekierowań')
+
+
+def _log(job_id, message):
+    target = store.DATA / 'logs' / (job_id + '.download.log')
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with LOG_LOCK:
+        with target.open('a', encoding='utf-8') as fp:
+            fp.write(time.strftime('%Y-%m-%d %H:%M:%S') + ' ' + message + '\n')
+
+
+def _transfer_one(job, *, batch=None, connections=None, stop=None):
+    jid = job['id']
+    ix = job['part_index']
+    urls = source_urls(job)
+    if ix >= len(urls):
+        store.update(jid, state='preparing')
+        return
+    uri = urls[ix]
+    if not archive_auth.is_archive_download(uri):
+        raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
+    root = store.DATA / 'raw' / jid
+    root.mkdir(parents=True, exist_ok=True)
+    filename = target_filename(uri)
+    if not filename:
+        raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
+    destination = root / filename
+    partial = root / (filename + '.part')
+    _check_state(jid, stop)
+    if batch is None:
+        store.update(jid, state='downloading', gid='', note=f'Archive.org: plik {ix+1}/{len(urls)}')
+    _log(jid, f'Rozpoczęto plik {ix+1}/{len(urls)}: {filename}')
+
+    if connections is None:
+        connections = _connection_limit()
+    # An existing manifest may contain many segments even if the new budget
+    # only allows one active connection. Keep those downloaded bytes.
+    if (connections > 1 or _segment_dir(partial).exists()) and _parallel_transfer(
+            job, ix, urls, uri, filename, partial, destination, connections, batch=batch, stop=stop):
+        return
+    # If the server cannot serve ranges (or parallel transfer was disabled),
+    # keep the verified single-stream prefix, not an unfinished assembly.
+    _discard_parallel_state(partial)
+    # Existing aria2 control files are not compatible with this downloader.
+    # Start from our own .part file rather than assuming an aria2 file is complete.
+    with _client() as client:
+        for attempt in range(MAX_ATTEMPTS):
+            _check_state(jid, stop)
+            start = partial.stat().st_size if partial.exists() else 0
+            headers = {'User-Agent': 'X360-Web-Queue/2.0', 'Accept-Encoding': 'identity'}
+            if start:
+                headers['Range'] = f'bytes={start}-'
+            try:
+                with closing(_response(client, uri, headers)) as response:
+                    code = response.status_code
+                    if code in RETRY_CODES:
+                        _log(jid, f'HTTP {code} z Archive.org, próba {attempt+1}/{MAX_ATTEMPTS}')
+                        reason = f'Archive.org HTTP {code}'
+                        if attempt == MAX_ATTEMPTS - 1:
+                            raise ArchiveDownloadError(f'{reason} po {MAX_ATTEMPTS} próbach; możliwy problem serwera Archive.org')
+                    elif code in (401, 403):
+                        raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia do pliku.')
+                    elif code == 404:
+                        raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono wybranego pliku')
+                    elif code == 416 and start:
+                        m = FINISH_RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
+                        if m and int(m.group(1)) == start:
+                            _check_state(jid, stop)
+                            partial.replace(destination)
+                            _finish(job, ix, urls, batch=batch)
+                            _log(jid, f'Ukończono {filename} (HTTP 416 potwierdziło rozmiar)')
+                            return
+                        partial.unlink(missing_ok=True)
+                        continue
+                    elif code not in (200, 206):
+                        raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
+                    else:
+                        if 'text/html' in response.headers.get('Content-Type', '').lower():
+                            raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
+                        if code == 206:
+                            match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
+                            if not match or int(match.group(1)) != start:
+                                raise ArchiveDownloadError('Archive.org zwróciło niezgodny zakres HTTP Range')
+                            total = int(match.group(3)) if match.group(3) != '*' else 0
+                        else:
+                            # Server ignored Range: do not append duplicate bytes.
+                            start = 0
+                            total = int(response.headers.get('Content-Length') or 0)
+                        written = start
+                        last_update = 0.0
+                        prev = written
+                        prev_at = time.monotonic()
+                        mode = 'ab' if start else 'wb'
+                        with partial.open(mode) as fp:
+                            for chunk in response.iter_raw(chunk_size=256 * 1024):
+                                _check_state(jid, stop)
+                                if not chunk:
+                                    continue
+                                fp.write(chunk)
+                                written += len(chunk)
+                                now = time.monotonic()
+                                if now - last_update >= 0.8:
+                                    elapsed = max(0.001, now - prev_at)
+                                    speed = int((written - prev) / elapsed)
+                                    prev, prev_at, last_update = written, now, now
+                                    portion = written / total if total else 0
+                                    if batch is not None:
+                                        batch.report(ix, written, total)
+                                    else:
+                                        store.update(jid, progress=round((ix + min(portion, 1)) / len(urls) * 100, 2),
+                                                     speed=speed, total=total, completed=written,
+                                                     note=f'Archive.org: plik {ix+1}/{len(urls)}, próba {attempt+1}')
+                        if total and written != total:
+                            raise httpx.ReadError(f'Przerwany transfer: pobrano {written} z {total} bajtów')
+                        _check_state(jid, stop)
+                        partial.replace(destination)
+                        _finish(job, ix, urls, batch=batch)
+                        _log(jid, f'Ukończono {filename}: {written} bajtów')
+                        return
+            except ArchiveStopped:
+                raise
+            except ArchiveDownloadError:
+                raise
+            except (httpx.HTTPError, OSError) as exc:
+                _log(jid, f'Błąd transferu {type(exc).__name__}: {str(exc)[:180]}')
+                if attempt == MAX_ATTEMPTS - 1:
+                    raise ArchiveDownloadError(f'Archive.org: przerwane połączenie po {MAX_ATTEMPTS} próbach: {type(exc).__name__}: {exc}') from exc
+            # Applies to transient HTTP codes and connection exceptions.
+            if batch is None:
+                store.update(jid, note=f'Archive.org: ponawianie po HTTP/błędzie sieci ({attempt+2}/{MAX_ATTEMPTS})', speed=0)
+            for _ in range(min(2 ** attempt, 12) * 2):
+                _check_state(jid, stop)
+                time.sleep(0.5)
+
+
+def _finish(job, ix, urls, *, batch=None):
+    jid = job['id']
+    if batch is not None:
+        batch.complete(ix)
+        return
+    if ix + 1 < len(urls):
+        store.update(jid, part_index=ix+1, state='queued', gid='', speed=0,
+                     progress=round((ix+1)/len(urls) * 100, 2), note='Pobrano część archiwum')
+    else:
+        store.update(jid, state='preparing', progress=100, speed=0,
+                     note='Pobieranie Archive.org ukończone')
+
+
+
+class _RangeUnsupported(Exception):
+    """A server stopped honoring byte ranges; caller can use the serial path."""
+    def __init__(self, message, reset_prefix=False):
+        super().__init__(message)
+        self.reset_prefix = reset_prefix
+
+
+def _connection_limit():
+    try:
+        return max(1, min(MAX_CONNECTIONS, int(os.getenv('ARCHIVE_CONNECTIONS', '50'))))
+    except (ValueError, OverflowError):
+        return 50
+
+
+def _segment_dir(partial):
+    return Path(str(partial) + '.segments')
+
+
+def _read_manifest(folder):
+    try:
+        item = json.loads((folder / 'manifest.json').read_text(encoding='utf-8'))
+        if not isinstance(item, dict):
+            return None
+        return item
+    except (OSError, ValueError):
+        return None
+
+
+def _write_manifest(folder, manifest):
+    folder.mkdir(mode=0o700, parents=True, exist_ok=True)
+    temp = folder / 'manifest.json.tmp'
+    with temp.open('w', encoding='utf-8') as f:
+        json.dump(manifest, f, sort_keys=True)
+        f.flush()
+        os.fsync(f.fileno())
+    temp.replace(folder / 'manifest.json')
+
+
+def _discard_parallel_state(partial, reset_prefix=False):
+    """Discard stale segment files without interpreting an assembled prefix as a resume."""
+    folder = _segment_dir(partial)
+    if not folder.exists():
+        return
+    manifest = _read_manifest(folder)
+    if reset_prefix:
+        partial.unlink(missing_ok=True)
+    elif manifest and manifest.get('assembling') and partial.exists():
+        prefix = manifest.get('prefix')
+        if isinstance(prefix, int) and 0 <= prefix <= partial.stat().st_size:
+            with partial.open('r+b') as f:
+                f.truncate(prefix)
+        else:
+            partial.unlink(missing_ok=True)
+    shutil.rmtree(folder)
+
+
+def _validate_ranges(ranges, prefix, total):
+    if not isinstance(ranges, list) or len(ranges) < 2 or len(ranges) > MAX_CONNECTIONS:
+        return False
+    cursor = prefix
+    for item in ranges:
+        if (not isinstance(item, list) or len(item) != 2 or
+                not all(isinstance(x, int) and not isinstance(x, bool) for x in item)):
+            return False
+        start, end = item
+        if start != cursor or end < start or end >= total:
+            return False
+        cursor = end + 1
+    return cursor == total
+
+
+def _prepare_segments(partial, uri, total, validator, connections):
+    folder = _segment_dir(partial)
+    source_hash = hashlib.sha256(uri.encode('utf-8')).hexdigest()
+    manifest = _read_manifest(folder)
+    valid = False
+    if manifest:
+        prefix = manifest.get('prefix')
+        numeric_prefix = isinstance(prefix, int) and not isinstance(prefix, bool)
+        prefix_present = numeric_prefix and (prefix == 0 or
+                                             (partial.exists() and partial.stat().st_size >= prefix))
+        valid = (manifest.get('version') == 1 and manifest.get('source_hash') == source_hash and
+                 manifest.get('total') == total and manifest.get('validator') == validator and
+                 isinstance(prefix, int) and not isinstance(prefix, bool) and 0 <= prefix < total and
+                 _validate_ranges(manifest.get('ranges'), prefix, total) and prefix_present)
+        if valid and partial.exists() and partial.stat().st_size != prefix:
+            # A crash during assembly is safe to recover from: parts remain intact.
+            if manifest.get('assembling') and partial.stat().st_size > prefix:
+                with partial.open('r+b') as f:
+                    f.truncate(prefix)
+            else:
+                valid = False
+    if valid:
+        return folder, manifest
+
+    if folder.exists():
+        # Invalid or mismatched metadata cannot establish which bytes are safe.
+        _discard_parallel_state(partial, reset_prefix=True)
+    prefix = partial.stat().st_size if partial.exists() else 0
+    if prefix > total:
+        partial.unlink(missing_ok=True)
+        prefix = 0
+    if prefix >= total:
+        return None, {'prefix': prefix, 'total': total}
+    remaining = total - prefix
+    n = min(connections, max(1, remaining // MIN_SEGMENT_BYTES))
+    if n < 2:
+        return None, None
+    quotient, extra = divmod(remaining, n)
+    cursor = prefix
+    ranges = []
+    for i in range(n):
+        end = cursor + quotient + (1 if i < extra else 0) - 1
+        ranges.append([cursor, end])
+        cursor = end + 1
+    manifest = {'version': 1, 'source_hash': source_hash, 'total': total,
+                'validator': validator, 'prefix': prefix, 'ranges': ranges,
+                'assembling': False}
+    _write_manifest(folder, manifest)
+    return folder, manifest
+
+
+def _retry_wait(job_id, attempt, stop, retry_after=None, external_stop=None):
+    delay = min(2 ** attempt, 16)
+    if retry_after and retry_after.isdigit():
+        delay = min(max(1, int(retry_after)), 60)
+    for _ in range(delay * 2):
+        if stop.is_set():
+            raise ArchiveStopped()
+        _check_state(job_id, external_stop)
+        time.sleep(0.5)
+
+
+def _check_content(response):
+    if 'text/html' in response.headers.get('Content-Type', '').lower():
+        raise ArchiveDownloadError('Archive.org zwróciło HTML zamiast pliku. Sprawdź dostęp i logowanie.')
+    if response.headers.get('Content-Encoding', 'identity').lower() not in ('', 'identity'):
+        raise ArchiveDownloadError('Archive.org zignorowało Accept-Encoding: identity; transfer w tym kodowaniu jest niebezpieczny')
+
+
+def _probe_range(job_id, uri, stop=None):
+    """Only a real 206 with a numeric total enables parallel mode."""
+    headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity', 'Range': 'bytes=0-0'}
+    with _client() as client:
+        for attempt in range(MAX_ATTEMPTS):
+            _check_state(job_id, stop)
+            try:
+                with closing(_response(client, uri, headers)) as response:
+                    code = response.status_code
+                    if code in RETRY_CODES:
+                        _log(job_id, f'Archive.org: sprawdzenie Range HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
+                        if attempt == MAX_ATTEMPTS - 1:
+                            raise ArchiveDownloadError(f'Archive.org HTTP {code} podczas sprawdzania Range po {MAX_ATTEMPTS} próbach')
+                        delay = response.headers.get('Retry-After')
+                    elif code in (401, 403):
+                        raise ArchiveDownloadError(f'Archive.org HTTP {code}: brak dostępu. Sprawdź sesję i uprawnienia.')
+                    elif code == 404:
+                        raise ArchiveDownloadError('Archive.org HTTP 404: nie znaleziono pliku')
+                    elif code == 200:
+                        _check_content(response)
+                        return None
+                    elif code == 206:
+                        _check_content(response)
+                        match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
+                        if not match or (int(match.group(1)), int(match.group(2))) != (0, 0) or match.group(3) == '*':
+                            return None
+                        total = int(match.group(3))
+                        if total <= 0:
+                            return None
+                        etag = response.headers.get('ETag', '').strip()
+                        if etag.startswith('W/'):
+                            etag = ''
+                        validator = etag or response.headers.get('Last-Modified', '').strip()
+                        if not validator or len(validator) > 256 or '\n' in validator:
+                            validator = ''
+                        return total, validator
+                    else:
+                        raise ArchiveDownloadError(f'Archive.org HTTP {code}: odmowa pobrania pliku')
+            except ArchiveDownloadError:
+                raise
+            except (httpx.HTTPError, OSError) as exc:
+                _log(job_id, f'Archive.org: sprawdzenie Range - {type(exc).__name__}: {str(exc)[:150]}')
+                if attempt == MAX_ATTEMPTS - 1:
+                    raise ArchiveDownloadError(f'Archive.org: brak odpowiedzi na żądanie Range po {MAX_ATTEMPTS} próbach') from exc
+                delay = None
+            _retry_wait(job_id, attempt, Event(), delay, stop)
+    return None
+
+
+def _segment_worker(job_id, uri, folder, manifest, index, stop, external_stop=None):
+    begin, end = manifest['ranges'][index]
+    total = manifest['total']
+    limit = end - begin + 1
+    target = folder / f'{index:02d}.part'
+    if target.exists() and target.stat().st_size > limit:
+        target.unlink()
+    for attempt in range(MAX_ATTEMPTS):
+        if stop.is_set():
+            raise ArchiveStopped()
+        _check_state(job_id, external_stop)
+        present = target.stat().st_size if target.exists() else 0
+        if present == limit:
+            return
+        headers = {'User-Agent': 'X360-Web-Queue/2.1', 'Accept-Encoding': 'identity',
+                   'Range': f'bytes={begin + present}-{end}'}
+        if manifest['validator']:
+            headers['If-Range'] = manifest['validator']
+        try:
+            with _client() as client:
+                with closing(_response(client, uri, headers)) as response:
+                    code = response.status_code
+                    if code in RETRY_CODES:
+                        _log(job_id, f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code}, próba {attempt+1}/{MAX_ATTEMPTS}')
+                        if attempt == MAX_ATTEMPTS - 1:
+                            raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: HTTP {code} po {MAX_ATTEMPTS} próbach')
+                        delay = response.headers.get('Retry-After')
+                    elif code in (401, 403):
+                        raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}: brak dostępu')
+                    elif code == 200:
+                        raise _RangeUnsupported('Archive.org zignorowało zakres HTTP Range / If-Range', reset_prefix=True)
+                    elif code != 206:
+                        raise ArchiveDownloadError(f'Segment {index+1}: Archive.org HTTP {code}')
+                    else:
+                        _check_content(response)
+                        match = RANGE.fullmatch(response.headers.get('Content-Range', '').strip())
+                        if not match or match.group(3) == '*' or (int(match.group(1)), int(match.group(2)), int(match.group(3))) != (begin + present, end, total):
+                            raise _RangeUnsupported('Archive.org zwróciło niezgodny Content-Range', reset_prefix=True)
+                        if manifest['validator']:
+                            header = 'ETag' if manifest['validator'].startswith('"') else 'Last-Modified'
+                            received_validator = response.headers.get(header, '').strip()
+                            if received_validator and received_validator != manifest['validator']:
+                                raise _RangeUnsupported('Archive.org zmieniło plik podczas pobierania', reset_prefix=True)
+                        last_check = time.monotonic()
+                        with target.open('ab') as f:
+                            for chunk in response.iter_raw(chunk_size=CHUNK_BYTES):
+                                if stop.is_set():
+                                    raise ArchiveStopped()
+                                if time.monotonic() - last_check >= 1:
+                                    _check_state(job_id, external_stop)
+                                    last_check = time.monotonic()
+                                if not chunk:
+                                    continue
+                                if f.tell() + len(chunk) > limit:
+                                    raise _RangeUnsupported('Archive.org wysłało zbyt dużo danych w zakresie', reset_prefix=True)
+                                f.write(chunk)
+                            f.flush()
+                            os.fsync(f.fileno())
+                        if target.stat().st_size != limit:
+                            raise httpx.ReadError('Przerwana transmisja segmentu')
+                        return
+        except (ArchiveStopped, ArchiveDownloadError, _RangeUnsupported):
+            raise
+        except (httpx.HTTPError, OSError) as exc:
+            _log(job_id, f'Segment {index+1}: {type(exc).__name__}: {str(exc)[:150]}, próba {attempt+1}/{MAX_ATTEMPTS}')
+            if attempt == MAX_ATTEMPTS - 1:
+                raise ArchiveDownloadError(f'Segment {index+1}/{len(manifest["ranges"])}: przerwany transfer po {MAX_ATTEMPTS} próbach: {type(exc).__name__}') from exc
+            delay = None
+        _retry_wait(job_id, attempt, stop, delay, external_stop)
+
+
+def _report_parallel(job_id, ix, url_count, partial, folder, manifest, last, batch=None):
+    prefix = manifest['prefix']
+    downloaded = prefix
+    for index, (start, end) in enumerate(manifest['ranges']):
+        path = folder / f'{index:02d}.part'
+        if path.exists():
+            downloaded += min(end - start + 1, path.stat().st_size)
+    now = time.monotonic()
+    prev_size, prev_at = last
+    speed = max(0, int((downloaded - prev_size) / max(0.001, now - prev_at)))
+    total = manifest['total']
+    if batch is None:
+        store.update(job_id, completed=downloaded, total=total, speed=speed,
+                     progress=round((ix + min(downloaded / total, 1)) / url_count * 100, 2),
+                     note=f'Archive.org: plik {ix+1}/{url_count}, {len(manifest["ranges"])} połączeń')
+    else:
+        batch.report(ix, downloaded, total)
+    return downloaded, now
+
+
+def _assemble_segments(job_id, partial, destination, folder, manifest, stop=None):
+    prefix = manifest['prefix']
+    total = manifest['total']
+    for index, (start, end) in enumerate(manifest['ranges']):
+        part = folder / f'{index:02d}.part'
+        if not part.is_file() or part.stat().st_size != end - start + 1:
+            raise ArchiveDownloadError('Niekompletne segmenty; nie można złożyć pliku')
+    manifest['assembling'] = True
+    _write_manifest(folder, manifest)
+    with partial.open('r+b' if partial.exists() else 'w+b') as output:
+        output.truncate(prefix)
+        output.seek(prefix)
+        for index in range(len(manifest['ranges'])):
+            with (folder / f'{index:02d}.part').open('rb') as source:
+                while True:
+                    _check_state(job_id, stop)
+                    block = source.read(1024 * 1024)
+                    if not block:
+                        break
+                    output.write(block)
+        output.flush()
+        os.fsync(output.fileno())
+    if partial.stat().st_size != total:
+        raise ArchiveDownloadError('Błąd scalania pliku: niezgodny rozmiar końcowy')
+    partial.replace(destination)
+    shutil.rmtree(folder)
+
+
+def _parallel_transfer(job, ix, urls, uri, filename, partial, destination, connections, *, batch=None, stop=None):
+    jid = job['id']
+    batch_stop = stop
+    probe = _probe_range(jid, uri, stop)
+    if probe is None:
+        _log(jid, 'Archive.org: serwer nie obsługuje Range; pobieranie pojedynczym połączeniem')
+        return False
+    total, validator = probe
+    folder, manifest = _prepare_segments(partial, uri, total, validator, connections)
+    if folder is None and manifest is None:
+        _log(jid, 'Archive.org: mały plik lub końcówka; pobieranie pojedynczym połączeniem')
+        return False
+    if folder is None and manifest['prefix'] == total:
+        partial.replace(destination)
+        _finish(job, ix, urls, batch=batch)
+        return True
+    _log(jid, f'Archive.org: {len(manifest["ranges"])} połączeń HTTP Range, rozmiar {total} B, wznowienie od {manifest["prefix"]} B')
+    local_stop = Event()
+    current = (manifest['prefix'] + sum(min((folder / f'{idx:02d}.part').stat().st_size, end - start + 1)
+                                        for idx, (start, end) in enumerate(manifest['ranges'])
+                                        if (folder / f'{idx:02d}.part').exists()), time.monotonic())
+    try:
+        with ThreadPoolExecutor(max_workers=min(connections, len(manifest['ranges']))) as pool:
+            pending = {pool.submit(_segment_worker, jid, uri, folder, manifest, i, local_stop, batch_stop)
+                       for i in range(len(manifest['ranges']))}
+            try:
+                while pending:
+                    _check_state(jid, batch_stop)
+                    done, pending = wait(pending, timeout=0.8, return_when=FIRST_COMPLETED)
+                    current = _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
+                    for task in done:
+                        task.result()
+            finally:
+                local_stop.set()
+                for task in pending:
+                    task.cancel()
+    except _RangeUnsupported as exc:
+        _log(jid, 'Archive.org: niezgodna obsługa Range, przełączam na jedno połączenie')
+        _discard_parallel_state(partial, reset_prefix=exc.reset_prefix)
+        return False
+    _check_state(jid, batch_stop)
+    _report_parallel(jid, ix, len(urls), partial, folder, manifest, current, batch)
+    _log(jid, 'Archive.org: scalanie segmentów')
+    if batch is None:
+        store.update(jid, note='Archive.org: scalanie segmentów', speed=0)
+    if batch_stop is None:
+        _assemble_segments(jid, partial, destination, folder, manifest)
+    else:
+        _assemble_segments(jid, partial, destination, folder, manifest, batch_stop)
+    _finish(job, ix, urls, batch=batch)
+    _log(jid, f'Ukończono {filename}: {total} bajtów, {len(manifest["ranges"])} połączeń')
+    return True
+
+
+class _BatchProgress:
+    """Collect progress from file threads; only the scheduler updates SQLite.
+
+    A single stream/segment never overwrites the total progress or total speed
+    from another file. No extra database table is needed for old installations.
+    """
+
+    def __init__(self, job_id, filenames, limit):
+        self.job_id = job_id
+        self.filenames = filenames
+        self.limit = limit
+        self.lock = Lock()
+        self.sizes = [0] * len(filenames)
+        self.totals = [0] * len(filenames)
+        self.seen = [False] * len(filenames)
+        self.done = set()
+        self.active = set()
+        self.transferred = 0
+        self.last_publish = time.monotonic()
+
+    def start(self, index):
+        with self.lock:
+            self.active.add(index)
+
+    def report(self, index, downloaded, total):
+        with self.lock:
+            downloaded = max(0, int(downloaded))
+            if self.seen[index]:
+                self.transferred += max(0, downloaded - self.sizes[index])
+            else:
+                # Existing .part and .segments from an earlier run are not
+                # counted as freshly transferred bytes per second.
+                self.seen[index] = True
+            self.sizes[index] = downloaded
+            self.totals[index] = max(0, int(total))
+
+    def complete(self, index, size=None):
+        if size is None:
+            size = (store.DATA / 'raw' / self.job_id / self.filenames[index]).stat().st_size
+        with self.lock:
+            if self.seen[index]:
+                self.transferred += max(0, size - self.sizes[index])
+            self.seen[index] = True
+            self.sizes[index] = size
+            self.totals[index] = size
+            self.done.add(index)
+            self.active.discard(index)
+
+    def publish(self):
+        now = time.monotonic()
+        with self.lock:
+            elapsed = max(0.001, now - self.last_publish)
+            speed = max(0, int(self.transferred / elapsed))
+            self.transferred = 0
+            self.last_publish = now
+            total_files = len(self.filenames)
+            fractions = []
+            for index in range(total_files):
+                if index in self.done:
+                    fractions.append(1.0)
+                elif self.totals[index]:
+                    fractions.append(min(self.sizes[index] / self.totals[index], 1))
+                else:
+                    fractions.append(0.0)
+            completed_bytes = sum(self.sizes)
+            total_bytes = sum(self.totals)
+            if total_bytes and all(self.totals):
+                # A 20 KiB NFO must not count the same as an 8 GiB ISO.
+                progress = round(100 * min(completed_bytes, total_bytes) / total_bytes, 2)
+            else:
+                # Until all content lengths are known, show the fraction of
+                # completed files/known segments as a provisional estimate.
+                progress = round(sum(fractions) / total_files * 100, 2)
+            finished = len(self.done)
+            active = len(self.active)
+            contiguous = next((i for i in range(total_files) if i not in self.done), total_files)
+        store.update(self.job_id, part_index=contiguous, progress=progress,
+                     completed=completed_bytes, total=total_bytes, speed=speed,
+                     note=f'Archive.org: {finished}/{total_files} gotowe, {active} równolegle, limit {self.limit} połączeń')
+
+
+def _transfer_many(job):
+    """Download multiple Archive.org files at once with ONE connection budget.
+
+    Each slot owns a fixed connection allowance. Reusing the slot for the next
+    file preserves the global cap even with 50-segment old resume manifests.
+    """
+    jid = job['id']
+    urls = source_urls(job)
+    if not urls:
+        raise ArchiveDownloadError('Brak plików Archive.org do pobrania')
+    names = []
+    for uri in urls:
+        if not archive_auth.is_archive_download(uri):
+            raise ArchiveDownloadError('Nieprawidłowy URL pobierania Archive.org')
+        name = target_filename(uri)
+        if not name:
+            raise ArchiveDownloadError('Brak nazwy pliku Archive.org')
+        names.append(name)
+    if len({name.casefold() for name in names}) != len(names):
+        raise ArchiveDownloadError('Wybrane pliki mają powtarzające się nazwy. Wybierz pliki o różnych nazwach.')
+
+    root = store.DATA / 'raw' / jid
+    root.mkdir(parents=True, exist_ok=True)
+    _check_state(jid)
+    limit = _connection_limit()
+    tracker = _BatchProgress(jid, names, limit)
+    waiting = deque()
+    for ix, name in enumerate(names):
+        dest = root / name
+        if dest.is_file() and not Path(str(dest) + '.aria2').exists():
+            # A completed destination was atomically renamed by the old or
+            # new downloader. A legacy aria2 control file means its sibling
+            # destination may only contain an incomplete preallocated file.
+            tracker.complete(ix, dest.stat().st_size)
+            _log(jid, f'Plik {ix+1}/{len(urls)} już gotowy: {name}')
+        else:
+            waiting.append(ix)
+
+    store.update(jid, state='downloading', gid='', note=f'Archive.org: start równoległego pobierania {len(waiting)} plików')
+    tracker.publish()
+    if not waiting:
+        _check_state(jid)
+        store.update(jid, state='preparing', progress=100, speed=0,
+                     note='Pobieranie Archive.org ukończone')
+        return
+
+    slots = min(limit, len(waiting))
+    budgets = [limit // slots + (1 if i < limit % slots else 0) for i in range(slots)]
+    _log(jid, f'Archive.org: start {slots} plików równolegle; budżet {limit} połączeń HTTP '
+              f'(przydział na plik: {", ".join(map(str, budgets))})')
+    stopped = Event()
+    first_error = None
+
+    def launch(pool, slot):
+        ix = waiting.popleft()
+        item = dict(job, part_index=ix)
+        tracker.start(ix)
+        return pool.submit(_transfer_one, item, batch=tracker, connections=budgets[slot], stop=stopped)
+
+    with ThreadPoolExecutor(max_workers=slots, thread_name_prefix='archive-file') as pool:
+        running = {launch(pool, slot): slot for slot in range(slots)}
+        try:
+            while running:
+                _check_state(jid, stopped)
+                finished, _ = wait(running, timeout=0.8, return_when=FIRST_COMPLETED)
+                tracker.publish()
+                for future in finished:
+                    slot = running.pop(future)
+                    try:
+                        future.result()
+                    except Exception as exc:
+                        first_error = first_error or exc
+                        stopped.set()
+                    else:
+                        if waiting and not stopped.is_set():
+                            running[launch(pool, slot)] = slot
+                if first_error is not None:
+                    break
+        finally:
+            stopped.set()
+            for future in running:
+                future.cancel()
+
+    if first_error is not None:
+        tracker.publish()
+        raise first_error
+    _check_state(jid)  # Especially important if a pause arrives at the last byte.
+    tracker.publish()
+    store.update(jid, state='preparing', progress=100, speed=0,
+                 part_index=len(urls), note='Pobieranie Archive.org ukończone')
+    _log(jid, f'Archive.org: pobrano wszystkie {len(urls)} pliki')
+
+
+async def run_one(job):
+    import asyncio
+    urls = source_urls(job)
+    if job['source_type'] == 'archive_many' and len(urls) > 1:
+        await asyncio.to_thread(_transfer_many, job)
+    else:
+        await asyncio.to_thread(_transfer_one, job)