77 lines
3.9 KiB
Python
77 lines
3.9 KiB
Python
"""Optional best-effort Vimm page adapter. No CAPTCHA/challenge or rights restrictions bypass."""
|
|
import os
|
|
import re
|
|
import asyncio
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
from urllib.robotparser import RobotFileParser
|
|
|
|
import httpx
|
|
|
|
from . import store
|
|
|
|
|
|
async def browser_download(job):
|
|
if os.getenv('ENABLE_VIMM_BROWSER', 'true').lower() not in ('1', 'yes', 'true'):
|
|
raise RuntimeError('Adapter Vimm jest wylaczony przez administratora')
|
|
url = job['source']
|
|
u = urlparse(url)
|
|
if u.hostname not in ('vimm.net', 'www.vimm.net') or not re.fullmatch(r'/vault/\d+/?', u.path):
|
|
raise RuntimeError('Adapter Vimm przyjmuje tylko /vault/<id>')
|
|
try:
|
|
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as c:
|
|
r = await c.get('https://vimm.net/robots.txt')
|
|
if r.status_code == 200:
|
|
parser = RobotFileParser()
|
|
parser.parse(r.text.splitlines())
|
|
if not parser.can_fetch('x360-web-queue', url):
|
|
raise RuntimeError('Vimm robots.txt zabrania automatycznego otwarcia tej strony')
|
|
elif r.status_code not in (404, 410):
|
|
raise RuntimeError('Nie moge sprawdzic zasad robots.txt Vimm')
|
|
except httpx.HTTPError as exc:
|
|
raise RuntimeError('Nie mozna sprawdzic robots.txt Vimm: ' + str(exc)) from exc
|
|
|
|
from playwright.async_api import async_playwright, TimeoutError as PlaywrightTimeout
|
|
|
|
store.update(job['id'], state='vimm_browser', note='Otwieram oryginalna strone Vimm; jedna sesja, bez obchodzenia zabezpieczen')
|
|
out = store.DATA / 'raw' / job['id']
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(
|
|
headless=True, executable_path='/usr/bin/chromium',
|
|
args=['--no-sandbox', '--disable-dev-shm-usage'])
|
|
try:
|
|
context = await browser.new_context(accept_downloads=True)
|
|
page = await context.new_page()
|
|
await page.goto(url, wait_until='domcontentloaded', timeout=60000)
|
|
# Only the original, visible button is actionable. Missing/taken-down
|
|
# buttons and challenges are not bypassed.
|
|
controls = page.locator('#dl_form button[type="submit"], #dl_form input[type="submit"], #dl_form button:not([type])')
|
|
usable = None
|
|
for i in range(min(await controls.count(), 10)):
|
|
c = controls.nth(i)
|
|
if await c.is_visible() and await c.is_enabled():
|
|
usable = c
|
|
break
|
|
if usable is None:
|
|
raise RuntimeError('Brak dostepnego przycisku pobierania na Vimm (niedostepny plik, limit lub zmieniona strona)')
|
|
store.update(job['id'], note='Czekam na pobieranie/kolejke serwisu Vimm (bez omijania limitow)')
|
|
timeout = min(max(int(os.getenv('VIMM_DOWNLOAD_TIMEOUT', '7200')), 120), 21600) * 1000
|
|
try:
|
|
async with page.expect_download(timeout=timeout) as d:
|
|
await usable.click(timeout=15000)
|
|
download = await d.value
|
|
except PlaywrightTimeout as exc:
|
|
raise RuntimeError('Vimm nie rozpoczal pobierania w limicie czasu. Sprawdz kolejke albo przycisk.') from exc
|
|
name = re.sub(r'[\\/:*?"<>|\x00-\x1f]', '_', Path(download.suggested_filename).name)[:180]
|
|
if not name:
|
|
name = 'vimm-download.bin'
|
|
store.update(job['id'], note='Pobieram plik z Vimm; brak API postepu w przegladarce')
|
|
await asyncio.wait_for(download.save_as(out / name), timeout=timeout / 1000)
|
|
failure = await download.failure()
|
|
if failure:
|
|
raise RuntimeError('Pobieranie Vimm nie powiodlo sie: ' + failure)
|
|
store.update(job['id'], state='preparing', progress=100, note='Vimm: pobieranie ukonczone')
|
|
finally:
|
|
await browser.close()
|