sec05: EN downloads complete (30 files) + SUMMARY at-a-glance + lgdl queue infra

- 30/31 EN candidates downloaded via libgen, all content-verified
  (magic+pages+first-page text); 08b 37-pp fragment discarded
- EN gaps: 04/05/15/16/23 not found (rare academic), 03/14 ia-restricted
- MANIFEST.md: EN sections per item + not-downloadable extended
- 26 cards: Downloads tables w/ EN rows + EN NOT FOUND notes (6 cards)
- SUMMARY.md created (canon v1): at-a-glance table 29 items, EN->RU authors
- tools/lgdl.py: speed-probe + CDN-edge re-draw (per-request get.php
  302s to different storage hosts), connect timeout, stall wipe on first stall
- AGENTS.md: Lesson 9 (EN-queue infra: ads.php throttle, watchdog,
  unbuffered logs, dual-queue trap, AA 403, i2p verdict)
This commit is contained in:
Dmitry Kokorin 2026-09-24 00:04:13 +03:00
parent 67fac97559
commit 2a2699d61a
60 changed files with 411 additions and 37 deletions

View file

@ -36,7 +36,8 @@ BASE = "https://libgen.vg"
JAR = "/tmp/lgdl_cookies.txt"
def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=None):
def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=None,
connect_timeout=None):
cmd = ["curl", "-s", "--fail", "--max-time", str(timeout),
"-c", JAR, "-b", JAR,
"-H", "User-Agent: " + UA,
@ -44,6 +45,8 @@ def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=
"-H", "Accept-Language: ru,en;q=0.9",
"-H", "Referer: " + (referer or BASE + "/"),
"-L"]
if connect_timeout:
cmd += ["--connect-timeout", str(connect_timeout)]
if extra:
cmd += extra
if out:
@ -118,12 +121,15 @@ def download(f_id, out_dir, name, tries=16):
stall = 0
for t in range(1, tries + 1):
card = None
print(f" try {t}: card fetch start")
for c in range(5): # card fetch with backoff (libgen DB is flaky)
try:
card = curl(f"{BASE}/ads.php?md5={md5}",
referer=f"{BASE}/file.php?id={f_id}", tries=1)
referer=f"{BASE}/file.php?id={f_id}", tries=1,
timeout=90, connect_timeout=20)
break
except RuntimeError:
print(f" try {t}/{c+1}: card fetch failed — back off")
time.sleep(8 * (c + 1))
if card is None:
print(f" try {t}: card fetch failed 5x — back off harder")
@ -135,11 +141,47 @@ def download(f_id, out_dir, name, tries=16):
time.sleep(4)
continue
url = m.group(1)
# speed probe: 6s range fetch on the fresh CDN edge; <15KB/s => this
# edge is wedged, re-draw a new get.php link (libgen 302s to different
# storage hosts per request)
probe_ok = False
for p in range(3):
try:
pr = subprocess.run(["curl", "-s", "--fail", "-o", "/dev/null",
"-r", "0-300000", "--max-time", "6",
"--connect-timeout", "10",
"-H", "User-Agent: " + UA,
"-H", "Referer: " + f"{BASE}/ads.php?md5={md5}",
"-w", "%{size_download} %{speed_download}",
url], capture_output=True, text=True, timeout=20)
if pr.returncode == 0:
parts = pr.stdout.split()
spd = float(parts[1]) if len(parts) > 1 else 0
print(f" try {t}: probe edge {p+1}: {spd/1024:.0f} KB/s")
if spd >= 15 * 1024:
probe_ok = True
break
except Exception:
pass
print(f" try {t}: edge {p+1} slow/wedged — re-drawing get.php link")
time.sleep(3)
try:
card2 = curl(f"{BASE}/ads.php?md5={md5}",
referer=f"{BASE}/file.php?id={f_id}", tries=1,
timeout=90, connect_timeout=20)
m2 = re.search(r'(https?://[^"\']*get\.php\?md5=' + md5 + r'&key=[^"\']+)', card2)
if m2:
url = m2.group(1)
except RuntimeError:
pass
if not probe_ok:
print(f" try {t}: all 3 edges slow — waiting out the CDN")
tmp = out + ".part"
prev = os.path.getsize(tmp) if os.path.exists(tmp) else 0
print(f" try {t}: data fetch start ({prev}b resumed)")
try:
rc = curl(url, out=tmp, referer=f"{BASE}/ads.php?md5={md5}", binary=True,
timeout=3600,
timeout=3600, connect_timeout=30,
# abort hung connections: <500B/s for 300s
extra=["-C", "-", "--speed-time", "300", "--speed-limit", "500"])
except RuntimeError as e:
@ -147,17 +189,13 @@ def download(f_id, out_dir, name, tries=16):
time.sleep(45)
continue
cur = os.path.getsize(tmp) if os.path.exists(tmp) else 0
# resume-hang detection: two consecutive zero-progress resumes at the
# same offset => this CDN offset is wedged; wipe and start fresh
# resume-hang detection: zero-progress resume at the same offset =>
# this CDN offset is wedged; wipe and start fresh (first stall is enough)
if 0 < prev == cur:
stall += 1
if stall >= 2:
print(f" try {t}: resume stalled 2x at {prev}b — wiping part, fresh start")
os.remove(tmp)
stall = 0
cur = 0
else:
print(f" try {t}: resume stalled at {prev}b — wiping part, fresh start")
os.remove(tmp)
stall = 0
cur = 0
data = open(tmp, "rb").read()
kind = sniff(data)
ok_size = (size == 0) or (len(data) == size)