tools/lgdl.py: precheck <f_id> <edition_id> [pages] — mislinked-file gate (reverse-map editions + locator batch path + size sanity; exit 0/1/2); dl with 4th arg auto-prechecks and aborts on RED. Tested: caught 3 bad files (Одержимый магией / Laso / Captain Action comic); OK path verified
This commit is contained in:
parent
b43e8f763c
commit
67fac97559
2 changed files with 95 additions and 2 deletions
|
|
@ -469,6 +469,11 @@ sections/0N-<name>/ # one .md per BOOK (chapter-level list items are merged int
|
||||||
8. **Content-verify gate (2026-09-22, after the sec05 mislinked batch)**: every downloaded file
|
8. **Content-verify gate (2026-09-22, after the sec05 mislinked batch)**: every downloaded file
|
||||||
gets a two-second check — magic bytes + `pdfinfo` pages + first-page/head text containing the
|
gets a two-second check — magic bytes + `pdfinfo` pages + first-page/head text containing the
|
||||||
expected RU title/author. A file that fails is discarded and re-sourced (usually flib).
|
expected RU title/author. A file that fails is discarded and re-sourced (usually flib).
|
||||||
|
**Pre-download gate: `tools/lgdl.py precheck <f_id> <edition_id> [pages]`** (exit 0 OK /
|
||||||
|
1 SUSPECT / 2 BAD) — checks the file record's `editions` reverse-map (must contain the target
|
||||||
|
edition), `locator` batch path, size sanity; `dl <f_id> <dir> <name> <edition_id>` runs it
|
||||||
|
automatically and aborts on BAD. Mislink = skip that f_id → next file of the same edition →
|
||||||
|
next candidate edition → flib; log rejected f_ids in MANIFEST notes.
|
||||||
|
|
||||||
## Catalog caching convention
|
## Catalog caching convention
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,12 +11,22 @@ Pipeline (curl engine — Python's DNS is flaky on this box, curl is not):
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
tools/lgdl.py ed <edition_id> [more_ids...]
|
tools/lgdl.py ed <edition_id> [more_ids...]
|
||||||
tools/lgdl.py dl <file_id> <out_dir> <target_name>
|
tools/lgdl.py dl <file_id> <out_dir> <target_name> [edition_id]
|
||||||
tools/lgdl.py md5 <file_id>
|
tools/lgdl.py md5 <file_id>
|
||||||
|
tools/lgdl.py precheck <file_id> <edition_id> [expected_pages]
|
||||||
|
|
||||||
|
precheck (no download, ~2 libgen calls): file record vs the target edition.
|
||||||
|
RED (exit 2) — reverse-map `editions` non-empty and does NOT contain the
|
||||||
|
target edition (file belongs to another book), or locator path is a
|
||||||
|
foreign-language batch (spa/English/fiction) for a Cyrillic book.
|
||||||
|
YELLOW(exit 1) — reverse map empty, or size implausible vs expected pages.
|
||||||
|
OK (exit 0) — reverse map matches target, or empty with plausible locator.
|
||||||
|
`dl` with the optional 4th arg runs precheck first and aborts on RED.
|
||||||
|
|
||||||
Verification on dl: final size must equal libgen filesize; magic-byte sniff
|
Verification on dl: final size must equal libgen filesize; magic-byte sniff
|
||||||
(fb2/pdf/epub/doc) must not be HTML. Card+key are re-fetched up to 3x.
|
(fb2/pdf/epub/doc) must not be HTML. Card+key are re-fetched up to 3x.
|
||||||
Politeness: ~4s between libgen requests.
|
Content-verify after dl (AGENTS.md lesson 8): pdfinfo pages vs RSL + first-page
|
||||||
|
text must contain the RU title/author. Politeness: ~4s between libgen requests.
|
||||||
"""
|
"""
|
||||||
import json, os, re, subprocess, sys, time
|
import json, os, re, subprocess, sys, time
|
||||||
|
|
||||||
|
|
@ -164,6 +174,72 @@ def download(f_id, out_dir, name, tries=16):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def precheck(f_id, e_id, expected_pages=None):
|
||||||
|
"""Mislinked-file gate. Returns 0=OK, 1=SUSPECT, 2=BAD (see module doc)."""
|
||||||
|
fi = file_info(f_id)
|
||||||
|
time.sleep(2)
|
||||||
|
ext, size = fi.get("extension"), int(fi.get("filesize") or 0)
|
||||||
|
locator = fi.get("locator") or ""
|
||||||
|
rev = {eid: int(x.get("e_id") or 0) for eid, x in (fi.get("editions") or {}).items()}
|
||||||
|
rev = {k: v for k, v in rev.items() if v}
|
||||||
|
verdict, reasons = "OK", []
|
||||||
|
|
||||||
|
# 1) reverse map: who does this file BELONG to?
|
||||||
|
rev_match = None
|
||||||
|
if rev:
|
||||||
|
if int(e_id) in rev.values():
|
||||||
|
rev_match = True
|
||||||
|
reasons.append(f"reverse-map matches target e_id {e_id}")
|
||||||
|
else:
|
||||||
|
verdict = "BAD"
|
||||||
|
owners = []
|
||||||
|
for owner_eid in set(rev.values()):
|
||||||
|
try:
|
||||||
|
oe = jget(f"json.php?object=e&addkeys=title,author,year&ids={owner_eid}")
|
||||||
|
o = oe.get(str(owner_eid), {})
|
||||||
|
owners.append(f"e_id {owner_eid} «{o.get('title')}» {o.get('author')} {o.get('year')}")
|
||||||
|
time.sleep(2)
|
||||||
|
except Exception:
|
||||||
|
owners.append(f"e_id {owner_eid} (?)")
|
||||||
|
reasons.append("reverse-map points to ANOTHER edition: " + "; ".join(owners))
|
||||||
|
else:
|
||||||
|
rev_match = False
|
||||||
|
verdict = "SUSPECT" if verdict == "OK" else verdict
|
||||||
|
reasons.append("reverse-map EMPTY (old file) — rely on locator + post-verify")
|
||||||
|
|
||||||
|
# 2) locator: foreign-language batch path. Only meaningful when the reverse
|
||||||
|
# map does NOT confirm the target (matching map = the edition itself comes
|
||||||
|
# from that batch — an origin note, not a mislink).
|
||||||
|
low = locator.lower()
|
||||||
|
foreign = re.search(r"(spa\\|spanish|english|epublibre|0day|\\fiction\\)", low)
|
||||||
|
cyrillic = re.search(r"[\u0400-\u04FF]", locator)
|
||||||
|
if locator and foreign and not cyrillic:
|
||||||
|
if rev_match is True:
|
||||||
|
reasons.append(f"origin note: edition comes from a foreign batch: {locator[:120]}")
|
||||||
|
else:
|
||||||
|
verdict = "BAD" if verdict == "OK" else verdict
|
||||||
|
reasons.append(f"locator is a foreign batch path: {locator[:120]}")
|
||||||
|
elif locator and rev_match is not True:
|
||||||
|
reasons.append(f"locator ok: {locator[:120]}")
|
||||||
|
|
||||||
|
# 3) size sanity vs expected pages (text formats are tiny: ~200B/pp;
|
||||||
|
# scans: 50KB..15MB/pp)
|
||||||
|
if expected_pages and size:
|
||||||
|
per_lo = 200 if ext in ("fb2", "epub", "txt", "html") else 10_000
|
||||||
|
lo, hi = expected_pages * per_lo, expected_pages * 15_000_000
|
||||||
|
if size < lo:
|
||||||
|
verdict = "SUSPECT" if verdict == "OK" else verdict
|
||||||
|
reasons.append(f"size {size}b < {lo}b for ~{expected_pages} pp (too small — fragment?)")
|
||||||
|
elif size > hi:
|
||||||
|
verdict = "SUSPECT" if verdict == "OK" else verdict
|
||||||
|
reasons.append(f"size {size}b > {hi}b for ~{expected_pages} pp (too big — scan mismatch?)")
|
||||||
|
|
||||||
|
print(f"precheck f_id={f_id} vs e_id={e_id}: {verdict} [{ext}, {size}b]")
|
||||||
|
for r in reasons:
|
||||||
|
print(f" - {r}")
|
||||||
|
return {"OK": 0, "SUSPECT": 1, "BAD": 2}[verdict]
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
if len(sys.argv) < 2:
|
if len(sys.argv) < 2:
|
||||||
sys.exit(__doc__)
|
sys.exit(__doc__)
|
||||||
|
|
@ -179,7 +255,19 @@ def main():
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
elif cmd == "dl":
|
elif cmd == "dl":
|
||||||
f_id, out_dir, name = sys.argv[2], sys.argv[3], sys.argv[4]
|
f_id, out_dir, name = sys.argv[2], sys.argv[3], sys.argv[4]
|
||||||
|
if len(sys.argv) > 5 and sys.argv[5].isdigit():
|
||||||
|
rc = precheck(f_id, sys.argv[5])
|
||||||
|
if rc == 2:
|
||||||
|
print(" ABORT: precheck RED (mislinked file) — pick another f_id/edition or flib")
|
||||||
|
sys.exit(2)
|
||||||
|
if rc == 1:
|
||||||
|
print(" precheck SUSPECT — downloading, MUST post-verify content")
|
||||||
|
time.sleep(3)
|
||||||
download(f_id, out_dir, name)
|
download(f_id, out_dir, name)
|
||||||
|
elif cmd == "precheck":
|
||||||
|
f_id, e_id = sys.argv[2], sys.argv[3]
|
||||||
|
pages = int(sys.argv[4]) if len(sys.argv) > 4 else None
|
||||||
|
sys.exit(precheck(f_id, e_id, pages))
|
||||||
elif cmd == "md5":
|
elif cmd == "md5":
|
||||||
print(file_info(sys.argv[2])["md5"])
|
print(file_info(sys.argv[2])["md5"])
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue