From 67fac97559297e802c99f1f4d21bbb47a3024365 Mon Sep 17 00:00:00 2001 From: Dmitry Kokorin Date: Wed, 23 Sep 2026 00:49:32 +0300 Subject: [PATCH] =?UTF-8?q?tools/lgdl.py:=20precheck=20=20=20[pages]=20=E2=80=94=20mislinked-file=20gate=20(reverse?= =?UTF-8?q?-map=20editions=20+=20locator=20batch=20path=20+=20size=20sanit?= =?UTF-8?q?y;=20exit=200/1/2);=20dl=20with=204th=20arg=20auto-prechecks=20?= =?UTF-8?q?and=20aborts=20on=20RED.=20Tested:=20caught=203=20bad=20files?= =?UTF-8?q?=20(=D0=9E=D0=B4=D0=B5=D1=80=D0=B6=D0=B8=D0=BC=D1=8B=D0=B9=20?= =?UTF-8?q?=D0=BC=D0=B0=D0=B3=D0=B8=D0=B5=D0=B9=20/=20Laso=20/=20Captain?= =?UTF-8?q?=20Action=20comic);=20OK=20path=20verified?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 5 +++ tools/lgdl.py | 92 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b137420..612df77 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -469,6 +469,11 @@ sections/0N-/ # one .md per BOOK (chapter-level list items are merged int 8. **Content-verify gate (2026-09-22, after the sec05 mislinked batch)**: every downloaded file gets a two-second check — magic bytes + `pdfinfo` pages + first-page/head text containing the expected RU title/author. A file that fails is discarded and re-sourced (usually flib). + **Pre-download gate: `tools/lgdl.py precheck [pages]`** (exit 0 OK / + 1 SUSPECT / 2 BAD) — checks the file record's `editions` reverse-map (must contain the target + edition), `locator` batch path, size sanity; `dl ` runs it + automatically and aborts on BAD. Mislink = skip that f_id → next file of the same edition → + next candidate edition → flib; log rejected f_ids in MANIFEST notes. ## Catalog caching convention diff --git a/tools/lgdl.py b/tools/lgdl.py index a3fa9e7..2af88f6 100644 --- a/tools/lgdl.py +++ b/tools/lgdl.py @@ -11,12 +11,22 @@ Pipeline (curl engine — Python's DNS is flaky on this box, curl is not): Usage: tools/lgdl.py ed [more_ids...] - tools/lgdl.py dl + tools/lgdl.py dl [edition_id] tools/lgdl.py md5 + tools/lgdl.py precheck [expected_pages] + +precheck (no download, ~2 libgen calls): file record vs the target edition. + RED (exit 2) — reverse-map `editions` non-empty and does NOT contain the + target edition (file belongs to another book), or locator path is a + foreign-language batch (spa/English/fiction) for a Cyrillic book. + YELLOW(exit 1) — reverse map empty, or size implausible vs expected pages. + OK (exit 0) — reverse map matches target, or empty with plausible locator. +`dl` with the optional 4th arg runs precheck first and aborts on RED. Verification on dl: final size must equal libgen filesize; magic-byte sniff (fb2/pdf/epub/doc) must not be HTML. Card+key are re-fetched up to 3x. -Politeness: ~4s between libgen requests. +Content-verify after dl (AGENTS.md lesson 8): pdfinfo pages vs RSL + first-page +text must contain the RU title/author. Politeness: ~4s between libgen requests. """ import json, os, re, subprocess, sys, time @@ -164,6 +174,72 @@ def download(f_id, out_dir, name, tries=16): return None +def precheck(f_id, e_id, expected_pages=None): + """Mislinked-file gate. Returns 0=OK, 1=SUSPECT, 2=BAD (see module doc).""" + fi = file_info(f_id) + time.sleep(2) + ext, size = fi.get("extension"), int(fi.get("filesize") or 0) + locator = fi.get("locator") or "" + rev = {eid: int(x.get("e_id") or 0) for eid, x in (fi.get("editions") or {}).items()} + rev = {k: v for k, v in rev.items() if v} + verdict, reasons = "OK", [] + + # 1) reverse map: who does this file BELONG to? + rev_match = None + if rev: + if int(e_id) in rev.values(): + rev_match = True + reasons.append(f"reverse-map matches target e_id {e_id}") + else: + verdict = "BAD" + owners = [] + for owner_eid in set(rev.values()): + try: + oe = jget(f"json.php?object=e&addkeys=title,author,year&ids={owner_eid}") + o = oe.get(str(owner_eid), {}) + owners.append(f"e_id {owner_eid} «{o.get('title')}» {o.get('author')} {o.get('year')}") + time.sleep(2) + except Exception: + owners.append(f"e_id {owner_eid} (?)") + reasons.append("reverse-map points to ANOTHER edition: " + "; ".join(owners)) + else: + rev_match = False + verdict = "SUSPECT" if verdict == "OK" else verdict + reasons.append("reverse-map EMPTY (old file) — rely on locator + post-verify") + + # 2) locator: foreign-language batch path. Only meaningful when the reverse + # map does NOT confirm the target (matching map = the edition itself comes + # from that batch — an origin note, not a mislink). + low = locator.lower() + foreign = re.search(r"(spa\\|spanish|english|epublibre|0day|\\fiction\\)", low) + cyrillic = re.search(r"[\u0400-\u04FF]", locator) + if locator and foreign and not cyrillic: + if rev_match is True: + reasons.append(f"origin note: edition comes from a foreign batch: {locator[:120]}") + else: + verdict = "BAD" if verdict == "OK" else verdict + reasons.append(f"locator is a foreign batch path: {locator[:120]}") + elif locator and rev_match is not True: + reasons.append(f"locator ok: {locator[:120]}") + + # 3) size sanity vs expected pages (text formats are tiny: ~200B/pp; + # scans: 50KB..15MB/pp) + if expected_pages and size: + per_lo = 200 if ext in ("fb2", "epub", "txt", "html") else 10_000 + lo, hi = expected_pages * per_lo, expected_pages * 15_000_000 + if size < lo: + verdict = "SUSPECT" if verdict == "OK" else verdict + reasons.append(f"size {size}b < {lo}b for ~{expected_pages} pp (too small — fragment?)") + elif size > hi: + verdict = "SUSPECT" if verdict == "OK" else verdict + reasons.append(f"size {size}b > {hi}b for ~{expected_pages} pp (too big — scan mismatch?)") + + print(f"precheck f_id={f_id} vs e_id={e_id}: {verdict} [{ext}, {size}b]") + for r in reasons: + print(f" - {r}") + return {"OK": 0, "SUSPECT": 1, "BAD": 2}[verdict] + + def main(): if len(sys.argv) < 2: sys.exit(__doc__) @@ -179,7 +255,19 @@ def main(): time.sleep(1) elif cmd == "dl": f_id, out_dir, name = sys.argv[2], sys.argv[3], sys.argv[4] + if len(sys.argv) > 5 and sys.argv[5].isdigit(): + rc = precheck(f_id, sys.argv[5]) + if rc == 2: + print(" ABORT: precheck RED (mislinked file) — pick another f_id/edition or flib") + sys.exit(2) + if rc == 1: + print(" precheck SUSPECT — downloading, MUST post-verify content") + time.sleep(3) download(f_id, out_dir, name) + elif cmd == "precheck": + f_id, e_id = sys.argv[2], sys.argv[3] + pages = int(sys.argv[4]) if len(sys.argv) > 4 else None + sys.exit(precheck(f_id, e_id, pages)) elif cmd == "md5": print(file_info(sys.argv[2])["md5"]) else: