first commit

app/vimm.py

@@ -0,0 +1,76 @@
+"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
+import os
+import re
+import asyncio
+from pathlib import Path
+from urllib.parse import urlparse
+from urllib.robotparser import RobotFileParser
+
+import httpx
+
+from . import store
+
+
+async def browser_download(job):
+    if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
+        raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
+    url = job['source']
+    u = urlparse(url)
+    if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
+        raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
+    try:
+        async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
+            r = await c.get('https://vimm.net/robots.txt')
+        if r.status_code == 200:
+            parser = RobotFileParser()
+            parser.parse(r.text.splitlines())
+            if not parser.can_fetch('x360-web-queue', url):
+                raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
+        elif r.status_code not in (404, 410):
+            raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
+    except httpx.HTTPError as exc:
+        raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
+
+    from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
+
+    store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
+    out = store.DATA / 'raw' / job['id']
+    out.mkdir(parents=True, exist_ok=True)
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(
+            headless=True, executable_path='/usr/bin/chromium',
+            args=['--no-sandbox', '--disable-dev-shm-usage'])
+        try:
+            context = await browser.new_context(accept_downloads=True)
+            page = await context.new_page()
+            await page.goto(url, wait_until='domcontentloaded', timeout=60000)
+            # Only the original, visible button is actionable. Missing/taken-down
+            # buttons and challenges are not bypassed.
+            controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
+            usable = None
+            for i in range(min(await controls.count(), 10)):
+                c = controls.nth(i)
+                if await c.is_visible() and await c.is_enabled():
+                    usable = c
+                    break
+            if usable is None:
+                raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
+            store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
+            timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
+            try:
+                async with page.expect_download(timeout=timeout) as d:
+                    await usable.click(timeout=15000)
+                download = await d.value
+            except PlaywrightTimeout as exc:
+                raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
+            name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
+            if not name:
+                name = 'vimm-download.bin'
+            store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
+            await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
+            failure = await download.failure()
+            if failure:
+                raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
+            store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
+        finally:
+            await browser.close()