dima/jung
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

sec08 MANIFEST rebuilt to sec01-07 cancon (per-item 'N files' headers, Cross-references before Not downloadable); queue scripts generalized to tools/lgdl-queue.sh + lgdl-rounds.sh, logs/queue removed from downloads dir

Browse source
This commit is contained in:
Dmitry Kokorin 2026-09-26 22:45:03 +03:00
parent 8e664c7099
commit 4bc2bfb6e4
8 changed files with 129 additions and 1031 deletions
Download patch file Download diff file Expand all files Collapse all files

27
tools/lgdl-queue.sh Executable file
View file

@ -0,0 +1,27 @@
#!/bin/bash
# Libgen EN-download queue for a section (sec08 pattern, 2026-09-26).
# Usage: tools/lgdl-queue.sh <section-dir> <queue-file>
# <section-dir> e.g. downloads/09-<name>
# <queue-file> lines: "<f_id> <edition_id> <file-name-base>"
# Per-file: skip-if-exists, timeout 1500, tries=3 (fast-fail under ads.php throttle),
# unbuffered logs. Pair with lgdl-rounds.sh for cooldown rounds.
cd /home/dima/jung
D="$1"; Q="$2"; LOG="$D/queue.log"
[ -z "$D" ] || [ -z "$Q" ] && { echo "usage: lgdl-queue.sh <section-dir> <queue-file>"; exit 1; }
echo "=== queue start $(date)" >> "$LOG"
while read -r f_id ed_id name; do
[ -z "$f_id" ] && continue
if ls "$D/$name"*.pdf "$D/$name"*.epub "$D/$name"*.fb2 "$D/$name"*.doc "$D/$name"*.rtf "$D/$name"*.djvu >/dev/null 2>&1; then
echo "SKIP (exists): $name" >> "$LOG"
continue
fi
rm -f "$D/$name"*.part
echo "=== $name (f/$f_id) $(date)" >> "$LOG"
timeout 1500 python3 -u -c "
import sys; sys.path.insert(0,'tools'); import lgdl
r = lgdl.download(sys.argv[1], sys.argv[2], sys.argv[3], tries=3)
sys.exit(0 if r else 1)" "$f_id" "$D" "$name" >> "$LOG" 2>&1
echo "rc=$? $name" >> "$LOG"
sleep 2
done < "$Q"
echo "=== queue round DONE $(date)" >> "$LOG"

20
tools/lgdl-rounds.sh Executable file
View file

@ -0,0 +1,20 @@
#!/bin/bash
# Cooldown-round runner for lgdl-queue.sh (ads.php throttle: 25-min pauses between rounds).
# Usage: tools/lgdl-rounds.sh <section-dir> <queue-file> [max_rounds]
cd /home/dima/jung
D="$1"; Q="$2"; MAX="${3:-6}"; RLOG="$D/rounds.log"
n=1
while [ "$n" -le "$MAX" ]; do
echo "=== ROUND $n $(date)" >> "$RLOG"
bash tools/lgdl-queue.sh "$D" "$Q"
remaining=0
while read -r f_id ed_id name; do
[ -z "$f_id" ] && continue
ls "$D/$name"*.pdf "$D/$name"*.epub "$D/$name"*.fb2 "$D/$name"*.doc "$D/$name"*.rtf "$D/$name"*.djvu >/dev/null 2>&1 || remaining=$((remaining+1))
done < "$Q"
echo "round $n done, remaining=$remaining $(date)" >> "$RLOG"
[ "$remaining" -eq 0 ] && { echo "ALL DONE" >> "$RLOG"; break; }
sleep 1500
n=$((n+1))
done
echo "=== ROUNDS COMPLETE $(date)" >> "$RLOG"