sec05: EN downloads complete (30 files) + SUMMARY at-a-glance + lgdl queue infra
- 30/31 EN candidates downloaded via libgen, all content-verified (magic+pages+first-page text); 08b 37-pp fragment discarded - EN gaps: 04/05/15/16/23 not found (rare academic), 03/14 ia-restricted - MANIFEST.md: EN sections per item + not-downloadable extended - 26 cards: Downloads tables w/ EN rows + EN NOT FOUND notes (6 cards) - SUMMARY.md created (canon v1): at-a-glance table 29 items, EN->RU authors - tools/lgdl.py: speed-probe + CDN-edge re-draw (per-request get.php 302s to different storage hosts), connect timeout, stall wipe on first stall - AGENTS.md: Lesson 9 (EN-queue infra: ads.php throttle, watchdog, unbuffered logs, dual-queue trap, AA 403, i2p verdict)
This commit is contained in:
parent
67fac97559
commit
2a2699d61a
60 changed files with 411 additions and 37 deletions
|
|
@ -36,7 +36,8 @@ BASE = "https://libgen.vg"
|
|||
JAR = "/tmp/lgdl_cookies.txt"
|
||||
|
||||
|
||||
def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=None):
|
||||
def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=None,
|
||||
connect_timeout=None):
|
||||
cmd = ["curl", "-s", "--fail", "--max-time", str(timeout),
|
||||
"-c", JAR, "-b", JAR,
|
||||
"-H", "User-Agent: " + UA,
|
||||
|
|
@ -44,6 +45,8 @@ def curl(url, out=None, referer=None, binary=False, timeout=240, tries=3, extra=
|
|||
"-H", "Accept-Language: ru,en;q=0.9",
|
||||
"-H", "Referer: " + (referer or BASE + "/"),
|
||||
"-L"]
|
||||
if connect_timeout:
|
||||
cmd += ["--connect-timeout", str(connect_timeout)]
|
||||
if extra:
|
||||
cmd += extra
|
||||
if out:
|
||||
|
|
@ -118,12 +121,15 @@ def download(f_id, out_dir, name, tries=16):
|
|||
stall = 0
|
||||
for t in range(1, tries + 1):
|
||||
card = None
|
||||
print(f" try {t}: card fetch start")
|
||||
for c in range(5): # card fetch with backoff (libgen DB is flaky)
|
||||
try:
|
||||
card = curl(f"{BASE}/ads.php?md5={md5}",
|
||||
referer=f"{BASE}/file.php?id={f_id}", tries=1)
|
||||
referer=f"{BASE}/file.php?id={f_id}", tries=1,
|
||||
timeout=90, connect_timeout=20)
|
||||
break
|
||||
except RuntimeError:
|
||||
print(f" try {t}/{c+1}: card fetch failed — back off")
|
||||
time.sleep(8 * (c + 1))
|
||||
if card is None:
|
||||
print(f" try {t}: card fetch failed 5x — back off harder")
|
||||
|
|
@ -135,11 +141,47 @@ def download(f_id, out_dir, name, tries=16):
|
|||
time.sleep(4)
|
||||
continue
|
||||
url = m.group(1)
|
||||
# speed probe: 6s range fetch on the fresh CDN edge; <15KB/s => this
|
||||
# edge is wedged, re-draw a new get.php link (libgen 302s to different
|
||||
# storage hosts per request)
|
||||
probe_ok = False
|
||||
for p in range(3):
|
||||
try:
|
||||
pr = subprocess.run(["curl", "-s", "--fail", "-o", "/dev/null",
|
||||
"-r", "0-300000", "--max-time", "6",
|
||||
"--connect-timeout", "10",
|
||||
"-H", "User-Agent: " + UA,
|
||||
"-H", "Referer: " + f"{BASE}/ads.php?md5={md5}",
|
||||
"-w", "%{size_download} %{speed_download}",
|
||||
url], capture_output=True, text=True, timeout=20)
|
||||
if pr.returncode == 0:
|
||||
parts = pr.stdout.split()
|
||||
spd = float(parts[1]) if len(parts) > 1 else 0
|
||||
print(f" try {t}: probe edge {p+1}: {spd/1024:.0f} KB/s")
|
||||
if spd >= 15 * 1024:
|
||||
probe_ok = True
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
print(f" try {t}: edge {p+1} slow/wedged — re-drawing get.php link")
|
||||
time.sleep(3)
|
||||
try:
|
||||
card2 = curl(f"{BASE}/ads.php?md5={md5}",
|
||||
referer=f"{BASE}/file.php?id={f_id}", tries=1,
|
||||
timeout=90, connect_timeout=20)
|
||||
m2 = re.search(r'(https?://[^"\']*get\.php\?md5=' + md5 + r'&key=[^"\']+)', card2)
|
||||
if m2:
|
||||
url = m2.group(1)
|
||||
except RuntimeError:
|
||||
pass
|
||||
if not probe_ok:
|
||||
print(f" try {t}: all 3 edges slow — waiting out the CDN")
|
||||
tmp = out + ".part"
|
||||
prev = os.path.getsize(tmp) if os.path.exists(tmp) else 0
|
||||
print(f" try {t}: data fetch start ({prev}b resumed)")
|
||||
try:
|
||||
rc = curl(url, out=tmp, referer=f"{BASE}/ads.php?md5={md5}", binary=True,
|
||||
timeout=3600,
|
||||
timeout=3600, connect_timeout=30,
|
||||
# abort hung connections: <500B/s for 300s
|
||||
extra=["-C", "-", "--speed-time", "300", "--speed-limit", "500"])
|
||||
except RuntimeError as e:
|
||||
|
|
@ -147,17 +189,13 @@ def download(f_id, out_dir, name, tries=16):
|
|||
time.sleep(45)
|
||||
continue
|
||||
cur = os.path.getsize(tmp) if os.path.exists(tmp) else 0
|
||||
# resume-hang detection: two consecutive zero-progress resumes at the
|
||||
# same offset => this CDN offset is wedged; wipe and start fresh
|
||||
# resume-hang detection: zero-progress resume at the same offset =>
|
||||
# this CDN offset is wedged; wipe and start fresh (first stall is enough)
|
||||
if 0 < prev == cur:
|
||||
stall += 1
|
||||
if stall >= 2:
|
||||
print(f" try {t}: resume stalled 2x at {prev}b — wiping part, fresh start")
|
||||
os.remove(tmp)
|
||||
stall = 0
|
||||
cur = 0
|
||||
else:
|
||||
print(f" try {t}: resume stalled at {prev}b — wiping part, fresh start")
|
||||
os.remove(tmp)
|
||||
stall = 0
|
||||
cur = 0
|
||||
data = open(tmp, "rb").read()
|
||||
kind = sniff(data)
|
||||
ok_size = (size == 0) or (len(data) == size)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue