From 8b632f1229c7071eb0aa87f8ce1dccf301e967c3 Mon Sep 17 00:00:00 2001 From: Dmitry Kokorin Date: Tue, 15 Sep 2026 13:23:18 +0300 Subject: [PATCH] lgdl.py: stall detection + fresh-start wipe + speed guard; MANIFEST queue status --- downloads/04-pictures/MANIFEST.md | 17 +++++++++++++++++ tools/lgdl.py | 20 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/downloads/04-pictures/MANIFEST.md b/downloads/04-pictures/MANIFEST.md index e0da423..084df2f 100644 --- a/downloads/04-pictures/MANIFEST.md +++ b/downloads/04-pictures/MANIFEST.md @@ -77,3 +77,20 @@ RU: no PDF → tier fb2 (2 files). «Полный цветовой тест Лю EN: no PDF → no fb2 → tier epub (azw twin skipped). | 38-ottmann-color-symbolism-en-eranos.epub | 480 962 | 149898445/107212960 | epub TOC verified: "edited by KLAUS OTTMANN", 7 lectures (Benz, Corbin, Huyghe, Izutsu, Portmann, Scholem, Zahan) — matches Касталия RU edition ✓ | RU «Символизм цвета» (Клуб Касталия, ISBN 9785908110549): title/ISBN/author/series queries → 0. + +## Queue status (2026-07-15) +Single serial master queue (/tmp/master_q.sh, log /tmp/master_q.log), hardened lgdl.py +(stall-detect + resume wipe, speed guard, 3600s timeout, 16 tries). ~30 KB/s link. +Queued in order: +| f_id | target | size | +|---|---|---| +| 91819237 | 03-red-book-en-facsimile-norton-2009 | 154 MB | +| 91545973 | 03-red-book-en-facsimile-2009-b | 177 MB | +| 93127234 | 05-abt-territory-symbol-ru-kastalia-2013 | 155 MB | +| 91481666 | 28-man-symbols-en-doubleday-1969-a | 91 MB | +| 91577995 | 28-man-symbols-en-doubleday-1969-b | 62 MB | +| 98995698 | 28-man-symbols-en-doubleday-1969-c | 35 MB | +| 96629120 | 28-man-symbols-en-anchor-1988 | 22 MB | +| 94051405 | 30-kandinsky-spiritual-ru-buksmart-2020-t1 | 154 MB | +| 94051663 | 30-kandinsky-spiritual-ru-buksmart-2020-t2 | 161 MB | +ETA total ≈ 10 h. All 12 items fully searched; items 11/24/26/31/32/38 complete. diff --git a/tools/lgdl.py b/tools/lgdl.py index f2f5ed8..a3fa9e7 100644 --- a/tools/lgdl.py +++ b/tools/lgdl.py @@ -98,13 +98,14 @@ def sniff(data: bytes) -> str: return "unknown" -def download(f_id, out_dir, name, tries=8): +def download(f_id, out_dir, name, tries=16): fi = file_info(f_id) md5, size, ext = fi["md5"], int(fi.get("filesize") or 0), fi.get("extension") or "bin" if not name.lower().endswith("." + ext.lower()): name = f"{name}.{ext}" os.makedirs(out_dir, exist_ok=True) out = os.path.join(out_dir, name) + stall = 0 for t in range(1, tries + 1): card = None for c in range(5): # card fetch with backoff (libgen DB is flaky) @@ -125,13 +126,28 @@ def download(f_id, out_dir, name, tries=8): continue url = m.group(1) tmp = out + ".part" + prev = os.path.getsize(tmp) if os.path.exists(tmp) else 0 try: rc = curl(url, out=tmp, referer=f"{BASE}/ads.php?md5={md5}", binary=True, - timeout=900, extra=["-C", "-"]) + timeout=3600, + # abort hung connections: <500B/s for 300s + extra=["-C", "-", "--speed-time", "300", "--speed-limit", "500"]) except RuntimeError as e: print(f" try {t}: fetch error ({e}) — back off") time.sleep(45) continue + cur = os.path.getsize(tmp) if os.path.exists(tmp) else 0 + # resume-hang detection: two consecutive zero-progress resumes at the + # same offset => this CDN offset is wedged; wipe and start fresh + if 0 < prev == cur: + stall += 1 + if stall >= 2: + print(f" try {t}: resume stalled 2x at {prev}b — wiping part, fresh start") + os.remove(tmp) + stall = 0 + cur = 0 + else: + stall = 0 data = open(tmp, "rb").read() kind = sniff(data) ok_size = (size == 0) or (len(data) == size)