"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass.""" import os import re import asyncio from pathlib import Path from urllib.parse import urlparse from urllib.robotparser import RobotFileParser import httpx from . import store async def browser_download(job): if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'): raise RuntimeError('Adapter Vimm jest wylaczony przez administratora') url = job['source'] u = urlparse(url) if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path): raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/') try: async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c: r = await c.get('https://vimm.net/robots.txt') if r.status_code == 200: parser = RobotFileParser() parser.parse(r.text.splitlines()) if not parser.can_fetch('x360-web-queue', url): raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony') elif r.status_code not in (404, 410): raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm') except httpx.HTTPError as exc: raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen') out = store.DATA / 'raw' / job['id'] out.mkdir(parents=True, exist_ok=True) async with async_playwright() as p: browser = await p.chromium.launch( headless=True, executable_path='/usr/bin/chromium', args=['--no-sandbox', '--disable-dev-shm-usage']) try: context = await browser.new_context(accept_downloads=True) page = await context.new_page() await page.goto(url, wait_until='domcontentloaded', timeout=60000) # Only the original, visible button is actionable. Missing/taken-down # buttons and challenges are not bypassed. controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])') usable = None for i in range(min(await controls.count(), 10)): c = controls.nth(i) if await c.is_visible() and await c.is_enabled(): usable = c break if usable is None: raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)') store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)') timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000 try: async with page.expect_download(timeout=timeout) as d: await usable.click(timeout=15000) download = await d.value except PlaywrightTimeout as exc: raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180] if not name: name = 'vimm-download.bin' store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce') await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000) failure = await download.failure() if failure: raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure) store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone') finally: await browser.close()