Start worker #28

Merged
gru merged 2 commits from start_worker into master 2026-06-14 23:00:46 +02:00
5 changed files with 339 additions and 31 deletions
Showing only changes of commit 005867999f - Show all commits
+77 -12
View File
@@ -1,9 +1,12 @@
#!/usr/bin/env python3
from __future__ import annotations
import os
import re
import time
from pathlib import Path
from urllib.parse import urljoin, urlparse
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
@@ -31,6 +34,40 @@ GOOGLE_FONT_FAMILIES = (
"Source Sans 3",
)
GOOGLE_FONT_WEIGHTS = "400;500;600;700;800"
DOWNLOAD_RETRIES = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRIES", "4"))
DOWNLOAD_RETRY_DELAY = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRY_DELAY", "10"))
DOWNLOAD_TIMEOUT = int(os.environ.get("PYTORRENT_DOWNLOAD_TIMEOUT", "180"))
def retry_countdown(seconds: int) -> None:
for remaining in range(seconds, 0, -1):
print(f"Retrying in {remaining}s...", end="\r", flush=True)
time.sleep(1)
if seconds > 0:
print(" " * 40, end="\r", flush=True)
def candidate_urls(url: str) -> list[str]:
candidates = [url]
replacements = (
("https://cdn.jsdelivr.net/npm/bootstrap@", "https://unpkg.com/bootstrap@"),
("https://cdn.jsdelivr.net/npm/bootswatch@", "https://unpkg.com/bootswatch@"),
("https://cdn.jsdelivr.net/npm/swagger-ui-dist@", "https://unpkg.com/swagger-ui-dist@"),
("https://cdn.jsdelivr.net/gh/lipis/flag-icons@", "https://cdn.jsdelivr.net/npm/flag-icons@"),
("https://cdn.jsdelivr.net/gh/DevExpress/bootstrap-themes@master/", "https://raw.githubusercontent.com/DevExpress/bootstrap-themes/master/"),
("https://cdn.socket.io/", "https://cdnjs.cloudflare.com/ajax/libs/socket.io/"),
("https://cdnjs.cloudflare.com/ajax/libs/font-awesome/", "https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@"),
)
for old, new in replacements:
if url.startswith(old):
candidates.append(url.replace(old, new, 1))
# font-awesome has a different path layout on npm/jsDelivr.
candidates = [item.replace("/css/all.min.css", "/css/all.min.css") for item in candidates]
unique = []
for item in candidates:
if item not in unique:
unique.append(item)
return unique
def google_fonts_css_url() -> str:
@@ -147,15 +184,31 @@ def bootstrap_css_asset(theme: str) -> dict[str, str]:
def download(url: str, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
req = Request(url, headers={"User-Agent": "pyTorrent installer"})
with urlopen(req, timeout=60) as response:
data = response.read()
if not data:
raise RuntimeError(f"Empty response for {url}")
tmp = dest.with_suffix(dest.suffix + ".tmp")
tmp.write_bytes(data)
tmp.replace(dest)
print(f"OK {dest.relative_to(ROOT)}")
last_error: Exception | None = None
for candidate in candidate_urls(url):
for attempt in range(1, DOWNLOAD_RETRIES + 1):
try:
req = Request(candidate, headers={"User-Agent": "pyTorrent installer"})
with urlopen(req, timeout=DOWNLOAD_TIMEOUT) as response:
data = response.read()
if not data:
raise RuntimeError(f"Empty response for {candidate}")
tmp = dest.with_suffix(dest.suffix + ".tmp")
tmp.write_bytes(data)
tmp.replace(dest)
if candidate != url:
print(f"OK {dest.relative_to(ROOT)} from fallback {candidate}")
else:
print(f"OK {dest.relative_to(ROOT)}")
return
except (HTTPError, URLError, TimeoutError, OSError, RuntimeError) as exc:
last_error = exc
print(f"Download failed ({attempt}/{DOWNLOAD_RETRIES}) for {candidate}: {exc}")
if attempt < DOWNLOAD_RETRIES:
retry_countdown(DOWNLOAD_RETRY_DELAY)
if candidate != candidate_urls(url)[-1]:
print(f"Trying alternative source: {candidate_urls(url)[candidate_urls(url).index(candidate) + 1]}")
raise RuntimeError(f"Failed to download {url}: {last_error}")
def download_css_with_assets(url: str, dest: Path) -> None:
@@ -184,10 +237,22 @@ def download_google_fonts_css(url: str, dest: Path) -> None:
"Accept": "text/css,*/*;q=0.1",
},
)
with urlopen(req, timeout=60) as response:
css = response.read().decode("utf-8", errors="ignore")
last_error: Exception | None = None
css = ""
for attempt in range(1, DOWNLOAD_RETRIES + 1):
try:
with urlopen(req, timeout=DOWNLOAD_TIMEOUT) as response:
css = response.read().decode("utf-8", errors="ignore")
if not css.strip():
raise RuntimeError(f"Empty response for {url}")
break
except (HTTPError, URLError, TimeoutError, OSError, RuntimeError) as exc:
last_error = exc
print(f"Download failed ({attempt}/{DOWNLOAD_RETRIES}) for {url}: {exc}")
if attempt < DOWNLOAD_RETRIES:
retry_countdown(DOWNLOAD_RETRY_DELAY)
if not css.strip():
raise RuntimeError(f"Empty response for {url}")
raise RuntimeError(f"Failed to download {url}: {last_error}")
def replace_url(match: re.Match[str]) -> str:
quote = match.group(1) or ""
+62 -6
View File
@@ -24,6 +24,10 @@ REPO_URL="${PYTORRENT_REPO_URL:-https://github.com/zdzichu6969/pyTorrent}"
REPO_BRANCH="${PYTORRENT_REPO_BRANCH:-master}"
WORK_DIR="${PYTORRENT_BOOTSTRAP_DIR:-/tmp/pytorrent-only-installer}"
KEEP_WORK_DIR="${PYTORRENT_KEEP_BOOTSTRAP_DIR:-0}"
DOWNLOAD_RETRIES="${PYTORRENT_DOWNLOAD_RETRIES:-4}"
DOWNLOAD_RETRY_DELAY="${PYTORRENT_DOWNLOAD_RETRY_DELAY:-10}"
DOWNLOAD_CONNECT_TIMEOUT="${PYTORRENT_DOWNLOAD_CONNECT_TIMEOUT:-30}"
DOWNLOAD_MAX_TIME="${PYTORRENT_DOWNLOAD_MAX_TIME:-600}"
default_archive_url() {
case "${REPO_URL%/}" in
@@ -61,13 +65,65 @@ prepare_downloader() {
fail "curl or wget is required."
}
retry_countdown() {
local seconds="$1"
local remaining
for ((remaining=seconds; remaining>0; remaining--)); do
printf 'Retrying in %ss...\r' "${remaining}"
sleep 1
done
[[ "${seconds}" -gt 0 ]] && printf '%*s\r' 40 ''
}
archive_url_candidates() {
local url="$1"
printf '%s\n' "${url}"
case "${url}" in
https://github.com/*/archive/refs/heads/*.tar.gz)
local rest owner repo branch
rest="${url#https://github.com/}"
owner="${rest%%/*}"
rest="${rest#*/}"
repo="${rest%%/*}"
branch="${url##*/}"
branch="${branch%.tar.gz}"
printf 'https://codeload.github.com/%s/%s/tar.gz/refs/heads/%s\n' "${owner}" "${repo}" "${branch}"
;;
https://github.com/*/archive/*.tar.gz)
local rest owner repo ref
rest="${url#https://github.com/}"
owner="${rest%%/*}"
rest="${rest#*/}"
repo="${rest%%/*}"
ref="${url##*/}"
ref="${ref%.tar.gz}"
printf 'https://codeload.github.com/%s/%s/tar.gz/%s\n' "${owner}" "${repo}" "${ref}"
;;
esac
}
download_file() {
local url="$1" destination="$2"
if [[ "${DOWNLOADER}" == "curl" ]]; then
curl -fL "${url}" -o "${destination}"
else
wget -O "${destination}" "${url}"
fi
local url="$1"
local destination="$2"
local candidate attempt status
while IFS= read -r candidate; do
[[ -n "${candidate}" ]] || continue
for ((attempt=1; attempt<=DOWNLOAD_RETRIES; attempt++)); do
if [[ "${DOWNLOADER}" == "curl" ]]; then
curl -fL --connect-timeout "${DOWNLOAD_CONNECT_TIMEOUT}" --max-time "${DOWNLOAD_MAX_TIME}" "${candidate}" -o "${destination}" && return 0
status=$?
else
wget --timeout="${DOWNLOAD_CONNECT_TIMEOUT}" --read-timeout="${DOWNLOAD_MAX_TIME}" --tries=1 -O "${destination}" "${candidate}" && return 0
status=$?
fi
log "Download failed (${attempt}/${DOWNLOAD_RETRIES}) from ${candidate} (exit ${status})."
if [[ "${attempt}" -lt "${DOWNLOAD_RETRIES}" ]]; then
retry_countdown "${DOWNLOAD_RETRY_DELAY}"
fi
done
log "Trying alternative source if available after: ${candidate}"
done < <(archive_url_candidates "${url}")
return 1
}
cleanup() {
+60 -5
View File
@@ -17,6 +17,10 @@ REPO_URL="${PYTORRENT_REPO_URL:-https://github.com/zdzichu6969/pyTorrent}"
REPO_BRANCH="${PYTORRENT_REPO_BRANCH:-master}"
WORK_DIR="${PYTORRENT_BOOTSTRAP_DIR:-/tmp/pytorrent-stack-installer}"
KEEP_WORK_DIR="${PYTORRENT_KEEP_BOOTSTRAP_DIR:-0}"
DOWNLOAD_RETRIES="${PYTORRENT_DOWNLOAD_RETRIES:-4}"
DOWNLOAD_RETRY_DELAY="${PYTORRENT_DOWNLOAD_RETRY_DELAY:-10}"
DOWNLOAD_CONNECT_TIMEOUT="${PYTORRENT_DOWNLOAD_CONNECT_TIMEOUT:-30}"
DOWNLOAD_MAX_TIME="${PYTORRENT_DOWNLOAD_MAX_TIME:-600}"
default_archive_url() {
case "${REPO_URL%/}" in
@@ -105,14 +109,65 @@ prepare_downloader() {
fail "curl or wget is required and no supported package manager was found."
}
retry_countdown() {
local seconds="$1"
local remaining
for ((remaining=seconds; remaining>0; remaining--)); do
printf 'Retrying in %ss...\r' "${remaining}"
sleep 1
done
[[ "${seconds}" -gt 0 ]] && printf '%*s\r' 40 ''
}
archive_url_candidates() {
local url="$1"
printf '%s\n' "${url}"
case "${url}" in
https://github.com/*/archive/refs/heads/*.tar.gz)
local rest owner repo branch
rest="${url#https://github.com/}"
owner="${rest%%/*}"
rest="${rest#*/}"
repo="${rest%%/*}"
branch="${url##*/}"
branch="${branch%.tar.gz}"
printf 'https://codeload.github.com/%s/%s/tar.gz/refs/heads/%s\n' "${owner}" "${repo}" "${branch}"
;;
https://github.com/*/archive/*.tar.gz)
local rest owner repo ref
rest="${url#https://github.com/}"
owner="${rest%%/*}"
rest="${rest#*/}"
repo="${rest%%/*}"
ref="${url##*/}"
ref="${ref%.tar.gz}"
printf 'https://codeload.github.com/%s/%s/tar.gz/%s\n' "${owner}" "${repo}" "${ref}"
;;
esac
}
download_file() {
local url="$1"
local destination="$2"
if [[ "${DOWNLOADER}" == "curl" ]]; then
curl -fL "${url}" -o "${destination}"
else
wget -O "${destination}" "${url}"
fi
local candidate attempt status
while IFS= read -r candidate; do
[[ -n "${candidate}" ]] || continue
for ((attempt=1; attempt<=DOWNLOAD_RETRIES; attempt++)); do
if [[ "${DOWNLOADER}" == "curl" ]]; then
curl -fL --connect-timeout "${DOWNLOAD_CONNECT_TIMEOUT}" --max-time "${DOWNLOAD_MAX_TIME}" "${candidate}" -o "${destination}" && return 0
status=$?
else
wget --timeout="${DOWNLOAD_CONNECT_TIMEOUT}" --read-timeout="${DOWNLOAD_MAX_TIME}" --tries=1 -O "${destination}" "${candidate}" && return 0
status=$?
fi
log "Download failed (${attempt}/${DOWNLOAD_RETRIES}) from ${candidate} (exit ${status})."
if [[ "${attempt}" -lt "${DOWNLOAD_RETRIES}" ]]; then
retry_countdown "${DOWNLOAD_RETRY_DELAY}"
fi
done
log "Trying alternative source if available after: ${candidate}"
done < <(archive_url_candidates "${url}")
return 1
}
detect_os_family() {
+70 -4
View File
@@ -24,6 +24,54 @@ DEFAULT_CURL_REF = "8.19.0"
DEFAULT_SERVICE_PATH = "/etc/systemd/system/rtorrent@.service"
DEFAULT_SCGI_PORT = 5000
DEFAULT_TORRENT_PORT = 51300
DOWNLOAD_RETRIES = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRIES", "4"))
DOWNLOAD_RETRY_DELAY = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRY_DELAY", "10"))
DOWNLOAD_CONNECT_TIMEOUT = int(os.environ.get("PYTORRENT_DOWNLOAD_CONNECT_TIMEOUT", "30"))
DOWNLOAD_MAX_TIME = int(os.environ.get("PYTORRENT_DOWNLOAD_MAX_TIME", "600"))
def retry_countdown(seconds):
for remaining in range(seconds, 0, -1):
print(f"Retrying in {remaining}s...", end="\r", flush=True)
time.sleep(1)
if seconds > 0:
print(" " * 40, end="\r", flush=True)
def run_with_retry(cmd, *, retries=DOWNLOAD_RETRIES, retry_delay=DOWNLOAD_RETRY_DELAY, retry_label=None, **kwargs):
last_error = None
label = retry_label or " ".join(str(x) for x in cmd[:3])
for attempt in range(1, retries + 1):
try:
return run(cmd, **kwargs)
except InstallError as exc:
last_error = exc
print(f"{label} failed ({attempt}/{retries}): {exc}")
if attempt < retries:
retry_countdown(retry_delay)
raise last_error
def download_url_candidates(url):
candidates = [url]
if url.startswith("https://github.com/c-ares/c-ares/releases/download/v") and url.endswith(".tar.gz"):
version = url.rsplit("/c-ares-", 1)[-1].removesuffix(".tar.gz")
candidates.append(f"https://codeload.github.com/c-ares/c-ares/tar.gz/refs/tags/v{version}")
if url.startswith("https://curl.se/download/curl-") and url.endswith(".tar.gz"):
version = url.rsplit("/curl-", 1)[-1].removesuffix(".tar.gz")
tag = "curl-" + version.replace(".", "_")
candidates.append(f"https://github.com/curl/curl/releases/download/{tag}/curl-{version}.tar.gz")
candidates.append(f"https://codeload.github.com/curl/curl/tar.gz/refs/tags/{tag}")
if "sourceforge.net/projects/xmlrpc-c/files/latest/download" in url:
candidates.append("https://downloads.sourceforge.net/project/xmlrpc-c/latest/download")
if url.startswith("https://downloads.sourceforge.net/project/xmlrpc-c/"):
candidates.append(url.replace("https://downloads.sourceforge.net/", "https://sourceforge.net/projects/").replace("project/xmlrpc-c/", "xmlrpc-c/files/"))
unique = []
for candidate in candidates:
if candidate not in unique:
unique.append(candidate)
return unique
class InstallError(Exception):
@@ -220,17 +268,35 @@ def clone_or_update_repo(repo_url, repo_dir, ref, *, debug=False):
repo_dir = Path(repo_dir)
if not repo_dir.exists():
with Spinner(f"Cloning {repo_dir.name}", enabled=not debug):
run(["git", "clone", repo_url, str(repo_dir)], debug=debug)
run_with_retry(["git", "clone", repo_url, str(repo_dir)], debug=debug, retry_label=f"git clone {repo_url}")
else:
print(f"Repository already exists: {repo_dir}")
with Spinner(f"Checking out {repo_dir.name} -> {ref}", enabled=not debug):
run(["git", "fetch", "--all", "--tags"], cwd=str(repo_dir), debug=debug)
run_with_retry(["git", "fetch", "--all", "--tags"], cwd=str(repo_dir), debug=debug, retry_label=f"git fetch {repo_dir.name}")
run(["git", "checkout", ref], cwd=str(repo_dir), debug=debug)
run(["git", "pull", "--ff-only"], cwd=str(repo_dir), check=False, debug=debug)
run_with_retry(["git", "pull", "--ff-only"], cwd=str(repo_dir), check=False, debug=debug, retry_label=f"git pull {repo_dir.name}")
def download_file(url, destination, *, debug=False):
run(["curl", "-fL", url, "-o", str(destination)], debug=debug)
last_error = None
for candidate in download_url_candidates(url):
for attempt in range(1, DOWNLOAD_RETRIES + 1):
try:
return run([
"curl",
"-fL",
"--connect-timeout", str(DOWNLOAD_CONNECT_TIMEOUT),
"--max-time", str(DOWNLOAD_MAX_TIME),
candidate,
"-o", str(destination),
], debug=debug)
except InstallError as exc:
last_error = exc
print(f"Download failed ({attempt}/{DOWNLOAD_RETRIES}) from {candidate}: {exc}")
if attempt < DOWNLOAD_RETRIES:
retry_countdown(DOWNLOAD_RETRY_DELAY)
print(f"Trying alternative source if available after: {candidate}")
raise last_error or InstallError(f"Download failed: {url}")
def extract_tarball(tarball, destination, *, debug=False):
@@ -24,6 +24,54 @@ DEFAULT_CURL_REF = "8.19.0"
DEFAULT_SERVICE_PATH = "/etc/systemd/system/rtorrent@.service"
DEFAULT_SCGI_PORT = 5000
DEFAULT_TORRENT_PORT = 51300
DOWNLOAD_RETRIES = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRIES", "4"))
DOWNLOAD_RETRY_DELAY = int(os.environ.get("PYTORRENT_DOWNLOAD_RETRY_DELAY", "10"))
DOWNLOAD_CONNECT_TIMEOUT = int(os.environ.get("PYTORRENT_DOWNLOAD_CONNECT_TIMEOUT", "30"))
DOWNLOAD_MAX_TIME = int(os.environ.get("PYTORRENT_DOWNLOAD_MAX_TIME", "600"))
def retry_countdown(seconds):
for remaining in range(seconds, 0, -1):
print(f"Retrying in {remaining}s...", end="\r", flush=True)
time.sleep(1)
if seconds > 0:
print(" " * 40, end="\r", flush=True)
def run_with_retry(cmd, *, retries=DOWNLOAD_RETRIES, retry_delay=DOWNLOAD_RETRY_DELAY, retry_label=None, **kwargs):
last_error = None
label = retry_label or " ".join(str(x) for x in cmd[:3])
for attempt in range(1, retries + 1):
try:
return run(cmd, **kwargs)
except InstallError as exc:
last_error = exc
print(f"{label} failed ({attempt}/{retries}): {exc}")
if attempt < retries:
retry_countdown(retry_delay)
raise last_error
def download_url_candidates(url):
candidates = [url]
if url.startswith("https://github.com/c-ares/c-ares/releases/download/v") and url.endswith(".tar.gz"):
version = url.rsplit("/c-ares-", 1)[-1].removesuffix(".tar.gz")
candidates.append(f"https://codeload.github.com/c-ares/c-ares/tar.gz/refs/tags/v{version}")
if url.startswith("https://curl.se/download/curl-") and url.endswith(".tar.gz"):
version = url.rsplit("/curl-", 1)[-1].removesuffix(".tar.gz")
tag = "curl-" + version.replace(".", "_")
candidates.append(f"https://github.com/curl/curl/releases/download/{tag}/curl-{version}.tar.gz")
candidates.append(f"https://codeload.github.com/curl/curl/tar.gz/refs/tags/{tag}")
if "sourceforge.net/projects/xmlrpc-c/files/latest/download" in url:
candidates.append("https://downloads.sourceforge.net/project/xmlrpc-c/latest/download")
if url.startswith("https://downloads.sourceforge.net/project/xmlrpc-c/"):
candidates.append(url.replace("https://downloads.sourceforge.net/", "https://sourceforge.net/projects/").replace("project/xmlrpc-c/", "xmlrpc-c/files/"))
unique = []
for candidate in candidates:
if candidate not in unique:
unique.append(candidate)
return unique
class InstallError(Exception):
@@ -221,17 +269,35 @@ def clone_or_update_repo(repo_url, repo_dir, ref, *, debug=False):
repo_dir = Path(repo_dir)
if not repo_dir.exists():
with Spinner(f"Cloning {repo_dir.name}", enabled=not debug):
run(["git", "clone", repo_url, str(repo_dir)], debug=debug)
run_with_retry(["git", "clone", repo_url, str(repo_dir)], debug=debug, retry_label=f"git clone {repo_url}")
else:
print(f"Repository already exists: {repo_dir}")
with Spinner(f"Checking out {repo_dir.name} -> {ref}", enabled=not debug):
run(["git", "fetch", "--all", "--tags"], cwd=str(repo_dir), debug=debug)
run_with_retry(["git", "fetch", "--all", "--tags"], cwd=str(repo_dir), debug=debug, retry_label=f"git fetch {repo_dir.name}")
run(["git", "checkout", ref], cwd=str(repo_dir), debug=debug)
run(["git", "pull", "--ff-only"], cwd=str(repo_dir), check=False, debug=debug)
run_with_retry(["git", "pull", "--ff-only"], cwd=str(repo_dir), check=False, debug=debug, retry_label=f"git pull {repo_dir.name}")
def download_file(url, destination, *, debug=False):
run(["curl", "-fL", url, "-o", str(destination)], debug=debug)
last_error = None
for candidate in download_url_candidates(url):
for attempt in range(1, DOWNLOAD_RETRIES + 1):
try:
return run([
"curl",
"-fL",
"--connect-timeout", str(DOWNLOAD_CONNECT_TIMEOUT),
"--max-time", str(DOWNLOAD_MAX_TIME),
candidate,
"-o", str(destination),
], debug=debug)
except InstallError as exc:
last_error = exc
print(f"Download failed ({attempt}/{DOWNLOAD_RETRIES}) from {candidate}: {exc}")
if attempt < DOWNLOAD_RETRIES:
retry_countdown(DOWNLOAD_RETRY_DELAY)
print(f"Trying alternative source if available after: {candidate}")
raise last_error or InstallError(f"Download failed: {url}")
def extract_tarball(tarball, destination, *, debug=False):