root
/
kmftools
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: root:main
Branches Tags
root:main
root:feature/pipeline-finalization
root:feature/bookstate-progress-fix
root:feat/dashboard-upgrade
root:feat/redis-state-model
root:feat/audiotasks
root:feat/cover-download
root:feat/download-progress-abort
root:celery-integration
..
compare: root:feat/dashboard-upgrade
Branches Tags
root:main
root:feature/pipeline-finalization
root:feature/bookstate-progress-fix
root:feat/dashboard-upgrade
root:feat/redis-state-model
root:feat/audiotasks
root:feat/cover-download
root:feat/download-progress-abort
root:celery-integration

No commits in common. 'main' and 'feat/dashboard-upgrade' have entirely different histories.
main ... feat/dashb

82 changed files with 1647 additions and 4973 deletions
Show Stats

2
.gitignore vendored
Unescape View File

@ -12,5 +12,3 @@
.env
**/.env
log.txt
**/static/covers/

23
bookscraper/README.md
Unescape View File

@ -125,8 +125,7 @@ docker run \
```
docker compose down --remove-orphans
docker image prune -f
docker compose down
docker builder prune -af
docker volume prune -f
docker compose build --no-cache
@ -136,20 +135,6 @@ docker compose down
docker compose build
docker compose up
docker compose up -d
docker compose build --no-cache web && docker compose up web
docker compose build worker_download && docker compose up worker_download
docker compose down --remove-orphans
docker compose build --no-cache worker_m4b
docker compose up -d worker_m4b
docker compose up web
docker compose build web
docker compose restart web
tar \
--exclude="**pycache**" \
--exclude="_/**pycache**/_" \
@ -157,9 +142,3 @@ tar \
--exclude=".venv" \
--exclude="venv" \
-czvf project.tar.gz .
docker compose down
docker image rm bookscraper-worker_m4b || true
docker builder prune -af
docker compose build --no-cache worker_m4b
docker compose up -d worker_m4b

566
bookscraper/app.py
Unescape View File

@ -1,319 +1,126 @@
# ============================================
# File: bookscraper/app.py (ASYNC SCRAPING)
# ============================================
from dotenv import load_dotenv
load_dotenv()
import os
from flask import (
Flask,
render_template,
request,
jsonify,
send_from_directory,
redirect,
url_for,
)
print(">>> [WEB] Importing celery_app …")
from celery_app import celery_app
from celery.result import AsyncResult
from db.db import init_db
from db.repository import (
get_registered_books,
fetch_book,
fetch_all_books,
get_progress,
)
from logbus.publisher import log
init_db() # ensure DB schema exists before Flask starts
from flask import Flask, render_template, request, jsonify
from scraper.logger import log_debug
# Abort + Progress (per book_id)
from scraper.abort import set_abort
from scraper.progress import get_progress
# UI LOGS (GLOBAL — no book_id)
from scraper.ui_log import get_ui_logs, reset_ui_logs
from celery.result import AsyncResult
from scraper.state import state as r
from scraper.logger_decorators import logcall
from scraper.utils.state_sync import sync_books_from_redis
from scraper.services.init_service import InitService
# Cover serving
from flask import send_from_directory
import os
# INIT DB
init_db()
import redis
# Flask
app = Flask(__name__)
# =====================================================
# STATIC FILE SERVING
# STATIC FILE SERVING FOR OUTPUT
# =====================================================
OUTPUT_ROOT = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
@app.route("/output/<path:filename>")
@logcall
def serve_output(filename):
return send_from_directory(OUTPUT_ROOT, filename, as_attachment=False)
# =====================================================
# SECTION 1 — NAVIGATION / HTML PAGES
# HOME PAGE
# =====================================================
@app.route("/", methods=["GET"])
@logcall
def index():
return redirect(url_for("dashboard"))
@app.route("/dashboard", methods=["GET"])
@logcall
def dashboard():
logs_list = get_ui_logs() or []
registered_books = get_registered_books()
log(f"[WEB] Registered books: {registered_books}")
from db.repository import fetch_all_books
from pprint import pprint
pprint(fetch_all_books())
pprint(get_registered_books())
# reg = [b for b in get_registered_books() if b.get("status") != "hidden"]
return render_template(
"dashboard/dashboard.html",
books=list_active_books(),
registered=registered_books,
logs=logs_list,
)
@app.route("/book/<book_idx>")
@logcall
def book_detail(book_idx):
title = r.get(f"book:{book_idx}:title") or book_idx
return render_template(
"dashboard/book_detail.html",
book_id=book_idx,
title=title,
logs=get_ui_logs(),
)
return render_template("index.html")
# =====================================================
# SECTION 2 — ACTION ROUTES (INIT, START, ABORT, HIDE)
# START SCRAPING (async via Celery)
# =====================================================
@app.route("/start", methods=["POST"])
def start_scraping():
url = request.form.get("url", "").strip()
@app.route("/init", methods=["POST"])
@logcall
def init_book():
# -------------------------------------------------
# Accept single URL (legacy) OR multi-line URLs
# -------------------------------------------------
raw_urls = request.form.get("urls") or request.form.get("url") or ""
urls = [line.strip() for line in raw_urls.splitlines() if line.strip()]
if not urls:
if not url:
# ★ FIX: dashboard moet altijd books + logs meekrijgen
return render_template(
"dashboard/dashboard.html",
error="Geen URL(s) opgegeven.",
error="Geen URL opgegeven.",
books=list_active_books(),
registered=get_registered_books(),
logs=get_ui_logs(),
)
# -------------------------------------------------
# Duplicate check: existing book_ids
# -------------------------------------------------
existing_books = {b["book_idx"] for b in fetch_all_books()}
results = []
# -------------------------------------------------
# Process each URL independently
# -------------------------------------------------
for url in urls:
try:
book_id = InitService.derive_book_id(url)
if book_id in existing_books:
results.append(
{
"url": url,
"status": "skipped",
"book_id": book_id,
"message": "Al geregistreerd",
}
)
continue
result = InitService.execute(url)
results.append(
{
"url": url,
"status": "registered",
"book_id": result.get("book_id"),
"title": result.get("title"),
}
)
except Exception as e:
log_debug(f"[INIT] ERROR for url={url}: {e}")
results.append(
{
"url": url,
"status": "error",
"error": str(e),
}
)
# -------------------------------------------------
# Summary message
# -------------------------------------------------
ok = sum(1 for r in results if r["status"] == "registered")
skipped = sum(1 for r in results if r["status"] == "skipped")
failed = sum(1 for r in results if r["status"] == "error")
message = f"Geregistreerd: {ok}, overgeslagen: {skipped}, fouten: {failed}"
reg = [b for b in get_registered_books() if b.get("status") != "hidden"]
return render_template(
"dashboard/dashboard.html",
message=message,
init_results=results, # optioneel voor UI-weergave
books=list_active_books(),
registered=reg,
logs=get_ui_logs(),
)
@app.route("/hide/<book_idx>", methods=["POST"])
@logcall
def hide_registered_book(book_idx):
# intentionally left disabled
pass
@app.route("/start", methods=["POST"])
@logcall
def start_scraping():
# 1) Form field: book_idx
book_idx = request.form.get("book_idx")
log(f"[WEB][START] Received start request for book_idx={book_idx}")
if not book_idx:
msg = "book_idx ontbreekt in formulier"
log(f"[WEB][START] ERROR: {msg}")
return jsonify({"status": "error", "message": msg}), 400
# 2) Fetch boek uit SQLite
try:
book = fetch_book(book_idx)
log(f"[WEB][START] Fetched book from DB: {book}")
except Exception as e:
log(f"[WEB][START] DB ERROR: {e}")
return jsonify({"status": "error", "message": "DB fout"}), 500
if not book:
msg = f"Boek '{book_idx}' niet gevonden in DB"
log(f"[WEB][START] ERROR: {msg}")
return jsonify({"status": "error", "message": msg}), 404
# 3) Boek moet een URL hebben
url = book.get("book_url")
if not url:
msg = f"Boek '{book_idx}' heeft geen book_url in DB"
log(f"[WEB][START] ERROR: {msg}")
return jsonify({"status": "error", "message": msg}), 500
# 4) Reset UI logs
reset_ui_logs()
log_debug(f"[WEB] Scraping via Celery: {url}")
# 5) Logging
log(f"[WEB][START] Starting full scrape book_idx={book_idx}, url={url}")
log_debug(f"[WEB][START] DEBUG: book data = {book}")
# 6) Celery controller taak starten
try:
async_result = celery_app.send_task(
"scraper.tasks.controller_tasks.start_full_scrape",
args=[book_idx],
queue="controller",
)
except Exception as e:
log(f"[WEB][START] Celery ERROR: {e}")
return jsonify({"status": "error", "message": f"Celery fout: {e}"}), 500
# 7) Successfully dispatched task
log(f"[WEB][START] Task dispatched: {async_result.id}")
reg = [b for b in get_registered_books() if b.get("status") != "hidden"]
async_result = celery_app.send_task(
"scraper.tasks.scraping.start_scrape_book",
args=[url],
queue="scraping",
)
# ★ FIX: direct dashboard tonen met actuele data
return render_template(
"dashboard/dashboard.html",
scraping_task_id=async_result.id,
books=list_active_books(),
registered=reg,
logs=get_ui_logs(),
)
@app.route("/abort/<book_idx>", methods=["POST"])
@logcall
def abort_download(book_idx):
log_debug(f"[WEB] Abort requested for book: {book_idx}")
set_abort(book_idx)
return jsonify({"status": "ok", "aborted": book_idx})
# =====================================================
# SECTION 3 — API ROUTES (JSON)
# CLEAR UI LOGS
# =====================================================
@app.route("/clear-logs", methods=["POST"])
def clear_logs():
reset_ui_logs()
return jsonify({"status": "ok", "message": "UI logs cleared"})
@app.route("/api/state/all", methods=["GET"])
@logcall
def api_state_all():
"""
Returns the merged SQL + Redis state for all books
(same logic as /debug/inspect_state but JSON-only).
"""
from scraper.utils.state_sync import inspect_books_state
return jsonify(inspect_books_state())
@app.route("/api/books")
@logcall
def api_books():
return jsonify(list_active_books())
@app.route("/api/book/<book_idx>/status")
@logcall
def api_book_status(book_idx):
return jsonify(getStatus(book_idx))
@app.route("/api/book/<book_idx>/logs")
@logcall
def api_book_logs(book_idx):
logs = r.lrange(f"logs:{book_idx}", 0, -1) or []
return jsonify(logs)
# =====================================================
# ABORT (per book_id)
# =====================================================
@app.route("/abort/<book_id>", methods=["POST"])
def abort_download(book_id):
log_debug(f"[WEB] Abort requested for book: {book_id}")
set_abort(book_id)
return jsonify({"status": "ok", "aborted": book_id})
@app.route("/progress/<book_idx>")
@logcall
def progress(book_idx):
return jsonify(get_progress(book_idx))
# =====================================================
# PROGRESS (per book_id)
# =====================================================
@app.route("/progress/<book_id>", methods=["GET"])
def progress(book_id):
return jsonify(get_progress(book_id))
@app.route("/celery-result/<task_id>")
@logcall
# =====================================================
# CELERY RESULT → return book_id
# =====================================================
@app.route("/celery-result/<task_id>", methods=["GET"])
def celery_result(task_id):
result = AsyncResult(task_id, app=celery_app)
if result.successful():
return jsonify({"ready": True, "result": result.get()})
if result.failed():
@ -321,78 +128,119 @@ def celery_result(task_id):
return jsonify({"ready": False})
@app.route("/clear-logs", methods=["POST"])
@logcall
def clear_logs():
reset_ui_logs()
return jsonify({"status": "ok"})
# =====================================================
# API: book status new model
# =====================================================
def getStatus(book_id):
@app.route("/logs", methods=["GET"])
@logcall
def logs():
try:
last_index = int(request.args.get("last_index", -1))
except:
last_index = -1
state = r.hgetall(f"book:{book_id}:state")
status = state.get("status") or "unknown"
dl_done = int(state.get("chapters_download_done", 0))
dl_skipped = int(state.get("chapters_download_skipped", 0))
dl_total = int(state.get("chapters_total", 0))
au_done = int(state.get("audio_done") or 0)
title = state.get("title") or book_id
all_logs = get_ui_logs() or []
au_total = dl_total
new_lines = []
new_last = last_index
return {
"book_id": book_id,
"title": title,
"status": status,
"download_done": dl_done,
"download_skipped": dl_skipped,
"download_total": dl_total,
"audio_done": au_done,
"audio_total": au_total,
}
for idx, line in enumerate(all_logs):
if idx > last_index:
new_lines.append(line)
new_last = idx
return jsonify({"lines": new_lines, "last": new_last})
# =====================================================
# REDIS BACKEND — BOOK STATE MODEL
# =====================================================
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
from flask import render_template
from scraper.services.status_check_service import StatusCheckService
from logbus.publisher import log
def list_active_booksold():
"""Return list of active books from Redis Book State Model."""
keys = r.keys("book:*:state")
books = []
for key in keys:
book_id = key.split(":")[1]
print(book_id)
books.append(getStatus(book_id))
from db.repository import get_book_state
return books
@app.route("/inspect/statuscheck/<book_idx>", methods=["POST"])
@logcall
def inspect_statuscheck(book_idx):
try:
StatusCheckService.run(book_idx)
return ("", 204) # background action, geen UI
except Exception as e:
log(f"[STATUSCHECK] ERROR book_idx={book_idx}: {e}")
return jsonify({"error": str(e)}), 500
def list_active_books():
books = []
for key in r.scan_iter(match="book:*:state", count=1000):
first = key.find(":")
second = key.find(":", first + 1)
book_id = key[first + 1 : second]
books.append(getStatus(book_id))
return books
# =====================================================
# SECTION 4 — DEBUG ROUTES
# API: list all active books
# =====================================================
@app.route("/api/books")
def api_books():
return jsonify(list_active_books())
@app.route("/api/book/<book_id>/status")
def api_book_status(book_id):
return jsonify(getStatus(book_id))
@app.route("/debug/sync_state", methods=["GET"])
def debug_sync_state():
results = sync_books_from_redis()
return {"status": "ok", "synced": results}
# =====================================================
# API: book logs
# =====================================================
@app.route("/api/book/<book_id>/logs")
def api_book_logs(book_id):
logs = r.lrange(f"logs:{book_id}", 0, -1) or []
return jsonify(logs)
from scraper.utils.state_sync import inspect_books_state
# =====================================================
# VIEW: DASHBOARD
# =====================================================
@app.route("/dashboard")
def dashboard():
logs_list = get_ui_logs() or []
# ★ FIX: dashboard moet altijd books + logs krijgen
return render_template(
"dashboard/dashboard.html",
books=list_active_books(),
logs=logs_list, # dashboard krijgt LIST, geen dict
)
@app.route("/debug/inspect_state", methods=["GET"])
def debug_inspect_state():
results = inspect_books_state()
return render_template("debug/inspect_state.html", results=results)
# =====================================================
# VIEW: BOOK DETAIL PAGE
# =====================================================
@app.route("/book/<book_id>")
def book_detail(book_id):
title = r.get(f"book:{book_id}:title") or book_id
return render_template(
"dashboard/book_detail.html",
book_id=book_id,
title=title,
logs=get_ui_logs(),
)
@app.route("/debug/redis-keys")
@logcall
def debug_redis_keys():
cursor = 0
results = {}
while True:
cursor, keys = r.scan(cursor, match="*", count=200)
for k in keys:
@ -402,127 +250,41 @@ def debug_redis_keys():
results[k] = "<non-string value>"
if cursor == 0:
break
return jsonify(results)
return jsonify(results)
# =====================================================
# DB DEBUG
# =====================================================
@app.route("/api/db/books")
@logcall
def api_db_books():
try:
books = fetch_all_books()
return jsonify({"status": "ok", "books": books})
except Exception as e:
return jsonify({"status": "error", "message": str(e)}), 500
# =============================================
# DEBUG QUEUE VIEW (HTML)
# =============================================
from flask import render_template
from urllib.parse import urlparse
import redis
from celery_app import celery_app
@app.route("/debug/queues")
def debug_queues():
insp = celery_app.control.inspect()
workers_active = insp.active() or {}
workers_scheduled = insp.scheduled() or {}
workers_reserved = insp.reserved() or {}
redis_url = os.getenv("REDIS_BROKER")
parsed = urlparse(redis_url)
r2 = redis.Redis(
host=parsed.hostname,
port=parsed.port,
db=int(parsed.path.strip("/") or 0),
decode_responses=True,
)
queue_names = ["scraping", "controller", "download", "parse", "save", "audio"]
queues = []
for q in queue_names:
key = f"celery:{q}"
try:
queues.append(
{
"name": q,
"redis_key": key,
"length": r2.llen(key),
"items": r2.lrange(key, 0, 30),
}
)
except Exception as e:
queues.append(
{
"name": q,
"redis_key": key,
"length": "ERR",
"items": [str(e)],
}
)
return render_template(
"debug/queues.html",
queues=queues,
workers_active=workers_active,
workers_reserved=workers_reserved,
workers_scheduled=workers_scheduled,
)
# ============================================================
# Rolling log endpoint (no new file)
# ============================================================
from flask import jsonify, request
# =====================================================
# SECTION 5 — INTERNAL HELPERS
# ROLLING LOG ENDPOINT — DELTA POLLING VIA ui_log
# =====================================================
from scraper.ui_log import get_ui_logs_delta
@logcall
def getStatus(book_idx):
state = r.hgetall(f"book:{book_idx}:state")
status = state.get("status") or "unknown"
dl_done = int(state.get("chapters_download_done", 0))
dl_skipped = int(state.get("chapters_download_skipped", 0))
dl_total = int(state.get("chapters_total", 0))
au_done = int(state.get("audio_done") or 0)
title = state.get("title") or book_idx
return {
"book_id": book_idx,
"title": title,
"status": status,
"download_done": dl_done,
"download_skipped": dl_skipped,
"download_total": dl_total,
"audio_done": au_done,
"audio_total": dl_total,
}
@app.route("/logs", methods=["GET"])
def logs():
"""
Delta log delivery for WebGUI.
Browser sends ?last_index=N, we return only new lines.
"""
try:
last_index = int(request.args.get("last_index", -1))
except:
last_index = -1
new_lines, total = get_ui_logs_delta(last_index)
@logcall
def list_active_books():
books = []
for key in r.scan_iter(match="book:*:state", count=1000):
first = key.find(":")
second = key.find(":", first + 1)
book_idx = key[first + 1 : second]
books.append(getStatus(book_idx))
return books
return jsonify({"lines": new_lines, "total": total})
# =====================================================
# SECTION 6 — FLASK RUNNER
# RUN FLASK
# =====================================================
if __name__ == "__main__":
debug = os.getenv("FLASK_DEBUG", "0") == "1"
host = os.getenv("HOST", "0.0.0.0")

0
bookscraper/app/routes/init_book.py
Unescape View File

1
bookscraper/audio_worker_local.py
Unescape View File

@ -54,7 +54,6 @@ def main():
"-l",
"INFO",
"--pool=prefork",
"--concurrency=2",
]
print("[AUDIO-LOCAL] Launching Celery via subprocess…")

11
bookscraper/celery_app.py
Unescape View File

@ -32,17 +32,6 @@ celery_app = Celery(
],
)
# >>>>> PLAATS DIT HIER <<<<<
celery_app.conf.update(
worker_redirect_stdouts_level="WARNING",
task_send_sent_event=False,
resultrepr_maxsize=0,
worker_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
worker_task_log_format="[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
)
# >>>>> TOT HIER <<<<<
celery_app.conf.task_routes = {
"scraper.tasks.scraping.*": {"queue": "scraping"},
"scraper.tasks.controller_tasks.*": {"queue": "controller"},

81
bookscraper/db/db.py
Unescape View File

@ -1,11 +1,14 @@
# ============================================================
# File: db/db.py (UPDATED for book_idx-only architecture)
# File: db/db.py
# Purpose:
# Raw SQLite engine for BookScraper.
# - Connection management
# - init_db() schema creation + safe schema upgrade
# - upsert_book() atomic write (now uses book_idx)
# - raw fetch helpers
# Provides ONLY low-level DB primitives.
# - Connection management (WAL mode)
# - init_db() schema creation
# - upsert_book() atomic write
# - raw fetch helpers (private)
#
# All business logic belongs in repository.py.
# ============================================================
import os
@ -45,24 +48,19 @@ def enable_wal_mode(conn):
# ------------------------------------------------------------
# Schema creation + SAFE schema upgrades
# Schema creation
# ------------------------------------------------------------
def init_db():
conn = get_db()
# --------------------------------------------------------
# BASE SCHEMA — book_idx is now PRIMARY KEY
# --------------------------------------------------------
conn.execute(
"""
CREATE TABLE IF NOT EXISTS books (
book_idx INTEGER PRIMARY KEY,
book_id TEXT PRIMARY KEY,
title TEXT,
author TEXT,
description TEXT,
cover_url TEXT,
cover_path TEXT,
book_url TEXT,
chapters_total INTEGER,
@ -72,61 +70,24 @@ def init_db():
audio_done INTEGER DEFAULT 0,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
processdate DATETIME,
last_update DATETIME
);
"""
)
conn.commit()
# --------------------------------------------------------
# SCHEMA UPGRADE UTILITY
# --------------------------------------------------------
def add_column(name, type_):
try:
conn.execute(f"ALTER TABLE books ADD COLUMN {name} {type_};")
except:
pass # column already exists
cols = conn.execute("PRAGMA table_info(books);").fetchall()
colnames = [c[1] for c in cols]
# --------------------------------------------------------
# UPGRADE NEW FIELDS — future-proof, matched with Redis state model
# --------------------------------------------------------
# (book_idx already exists as PRIMARY KEY — no need to add)
add_column("description", "TEXT")
add_column("cover_path", "TEXT")
add_column("book_url", "TEXT")
# Download counters
add_column("chapters_download_done", "INTEGER DEFAULT 0")
add_column("chapters_download_skipped", "INTEGER DEFAULT 0")
# Audio counters
add_column("audio_skipped", "INTEGER DEFAULT 0")
# Optional future fields
add_column("audio_total", "INTEGER DEFAULT 0")
conn.commit()
# ------------------------------------------------------------
# WRITE OPERATIONS (book_idx-based UPSERT)
# WRITE OPERATIONS
# ------------------------------------------------------------
def upsert_book(book_idx, **fields):
def upsert_book(book_id, **fields):
"""
UPSERT by book_idx.
Replaces old upsert that used book_id.
Raw upsert primitive. Repository layer should call this.
"""
conn = get_db()
keys = ["book_idx"] + list(fields.keys())
values = [book_idx] + list(fields.values())
keys = ["book_id"] + list(fields.keys())
values = [book_id] + list(fields.values())
placeholders = ",".join(["?"] * len(values))
updates = ", ".join([f"{k} = excluded.{k}" for k in fields.keys()])
@ -134,7 +95,7 @@ def upsert_book(book_idx, **fields):
sql = f"""
INSERT INTO books ({','.join(keys)})
VALUES ({placeholders})
ON CONFLICT(book_idx)
ON CONFLICT(book_id)
DO UPDATE SET {updates},
last_update = CURRENT_TIMESTAMP;
"""
@ -144,13 +105,11 @@ def upsert_book(book_idx, **fields):
# ------------------------------------------------------------
# RAW READ OPERATIONS
# RAW READ OPERATIONS (PRIVATE)
# ------------------------------------------------------------
def _raw_get_book(book_idx):
def _raw_get_book(book_id):
conn = get_db()
row = conn.execute(
"SELECT * FROM books WHERE book_idx = ?;", (book_idx,)
).fetchone()
row = conn.execute("SELECT * FROM books WHERE book_id = ?;", (book_id,)).fetchone()
return dict(row) if row else None

0
bookscraper/db/light_fetch.py
Unescape View File

371
bookscraper/db/repository.py
Unescape View File

@ -1,320 +1,97 @@
# ============================================================
# File: db/repository.py
# Purpose:
# Unified façade for BookScraper database state.
# High-level BookScraper database interface.
# This is the ONLY module Celery tasks and Flask should use.
#
# Responsibilities:
# - Route metadata → SQLite
# - Route counters → Redis (live) + SQLite (snapshot)
# - Provide a clean API for tasks and Flask UI
# ============================================================
# ============================================================
# UPDATED — canonical read model via get_book_state
# ============================================================
from scraper.logger_decorators import logcall
from logbus.publisher import log
import redis
import os
# ============================================================
# SQL low-level engines (snapshot storage)
# ============================================================
from db.state_sql import (
sql_fetch_book,
sql_fetch_all_books,
sql_set_status,
sql_set_chapters_total,
sql_register_book,
sql_update_book,
# Uses low-level primitives from db.db, but exposes
# domain-level operations:
# - fetch_book / fetch_all_books
# - create_or_update_book
# - set_status
# - incrementing counters
# ============================================================
from db.db import (
upsert_book,
_raw_get_book,
_raw_get_all_books,
)
# ============================================================
# REDIS low-level engines (live counters)
# ============================================================
from db.state_redis import (
redis_set_status,
redis_set_chapters_total,
redis_inc_download_done,
redis_inc_download_skipped,
redis_inc_parsed_done,
redis_inc_audio_done,
redis_inc_audio_skipped,
)
# ============================================================
# Redis client (read-only for legacy + guards)
# ============================================================
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
_r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
# ------------------------------------------------------------
# FETCH OPERATIONS
# ------------------------------------------------------------
def fetch_book(book_id):
"""Return a single book dict or None."""
return _raw_get_book(book_id)
# ============================================================
# LEGACY PROGRESS (UI only, unchanged)
# ============================================================
def _legacy_get_progress(book_idx):
return {
"book_idx": book_idx,
"total": int(_r.get(f"progress:{book_idx}:total") or 0),
"completed": int(_r.get(f"progress:{book_idx}:completed") or 0),
"skipped": int(_r.get(f"progress:{book_idx}:skipped") or 0),
"failed": int(_r.get(f"progress:{book_idx}:failed") or 0),
"abort": _r.exists(f"abort:{book_idx}") == 1,
"failed_list": _r.lrange(f"progress:{book_idx}:failed_list", 0, -1),
}
@logcall
def get_progress(book_idx):
return _legacy_get_progress(book_idx)
# ============================================================
# FETCH (SQLite snapshot)
# ============================================================
@logcall
def fetch_book(book_idx):
return sql_fetch_book(book_idx)
@logcall
def fetch_all_books():
return sql_fetch_all_books()
"""Return all books ordered newest → oldest."""
return _raw_get_all_books()
# ============================================================
# INIT / UPDATE METADATA
# ============================================================
@logcall
def register_book(
book_idx,
title,
author=None,
description=None,
cover_url=None,
cover_path=None,
book_url=None,
):
sql_register_book(
book_idx,
{
"book_idx": book_idx,
"title": title,
"author": author,
"description": description,
"cover_url": cover_url,
"cover_path": cover_path,
"book_url": book_url,
"chapters_total": 0,
"status": "registered",
},
)
@logcall
def update_book_after_full_scrape(
book_idx,
# ------------------------------------------------------------
# BOOK CREATION / METADATA
# ------------------------------------------------------------
def create_or_update_book(
book_id,
title=None,
author=None,
description=None,
cover_url=None,
chapters_total=None,
cover_url=None,
cover_path=None,
status=None,
):
fields = {}
if title is not None:
fields["title"] = title
if author is not None:
fields["author"] = author
if description is not None:
fields["description"] = description
if cover_url is not None:
fields["cover_url"] = cover_url
if chapters_total is not None:
fields["chapters_total"] = chapters_total
fields["status"] = "active"
sql_update_book(book_idx, fields)
# ============================================================
# STATUS
# ============================================================
@logcall
def set_status(book_idx, status):
redis_set_status(book_idx, status)
sql_set_status(book_idx, status)
# ============================================================
# TOTALS
# ============================================================
@logcall
def set_chapters_total(book_idx, total):
redis_set_chapters_total(book_idx, total)
sql_set_chapters_total(book_idx, total)
# ============================================================
# COUNTERS — WRITE ONLY
# ============================================================
@logcall
def inc_download_done(book_idx, amount=1):
redis_inc_download_done(book_idx, amount)
@logcall
def inc_download_skipped(book_idx, amount=1):
redis_inc_download_skipped(book_idx, amount)
@logcall
def inc_parsed_done(book_idx, amount=1):
redis_inc_parsed_done(book_idx, amount)
@logcall
def inc_audio_done(book_idx, amount=1):
redis_inc_audio_done(book_idx, amount)
@logcall
def inc_audio_skipped(book_idx, amount=1):
redis_inc_audio_skipped(book_idx, amount)
# ============================================================
# CANONICAL READ MODEL
# ============================================================
@logcall
def get_book_state(book_idx):
"""
Canonical merged read model.
Rules:
- SQL = snapshot baseline
- Redis = live counters
- merged = max(sql, redis)
- capped at chapters_total
"""
sqlite_row = sql_fetch_book(book_idx) or {}
redis_state = _r.hgetall(f"book:{book_idx}:state") or {}
def _int(v):
try:
return int(v)
except Exception:
return 0
chapters_total = _int(sqlite_row.get("chapters_total"))
# SQL snapshot
sql_downloaded = _int(sqlite_row.get("downloaded"))
sql_audio_done = _int(sqlite_row.get("audio_done"))
sql_audio_skipped = _int(sqlite_row.get("audio_skipped"))
# Redis live
redis_downloaded = _int(redis_state.get("chapters_download_done")) + _int(
redis_state.get("chapters_download_skipped")
)
redis_audio_done = _int(redis_state.get("audio_done"))
redis_audio_skipped = _int(redis_state.get("audio_skipped"))
# Merge
merged_downloaded = max(sql_downloaded, redis_downloaded)
merged_audio_done = max(sql_audio_done, redis_audio_done)
merged_audio_skipped = max(sql_audio_skipped, redis_audio_skipped)
if chapters_total > 0:
merged_downloaded = min(merged_downloaded, chapters_total)
merged_audio_done = min(merged_audio_done, chapters_total)
merged_audio_skipped = min(merged_audio_skipped, chapters_total)
audio_completed = merged_audio_done + merged_audio_skipped
# Build state
state = dict(sqlite_row)
state.update(
{
"downloaded": merged_downloaded,
"audio_done": merged_audio_done,
"audio_skipped": merged_audio_skipped,
"chapters_total": chapters_total,
}
)
# Derived status
status = sqlite_row.get("status") or "unknown"
if chapters_total > 0:
if merged_downloaded < chapters_total:
status = "downloading"
elif merged_downloaded == chapters_total and audio_completed < chapters_total:
status = "audio"
elif audio_completed >= chapters_total:
status = "done"
state["status"] = status
return state
# ============================================================
# READ HELPERS (VIA get_book_state ONLY)
# ============================================================
@logcall
def get_chapters_total(book_idx):
return int(get_book_state(book_idx).get("chapters_total", 0))
@logcall
def get_audio_done(book_idx):
return int(get_book_state(book_idx).get("audio_done", 0))
@logcall
def get_audio_completed_total(book_idx):
state = get_book_state(book_idx)
return int(state.get("audio_done", 0)) + int(state.get("audio_skipped", 0))
# ============================================================
# STATUSCHECK GUARD (INTENTIONAL DIRECT REDIS)
# ============================================================
@logcall
def try_trigger_statuscheck(book_idx):
return bool(_r.set(f"book:{book_idx}:statuscheck:triggered", "1", nx=True))
# ============================================================
# ACTIVE / REGISTERED BOOK LISTS (UI API)
# ============================================================
@logcall
def get_registered_books():
"""
Books visible in the 'registered' list in the UI.
"""
all_books = sql_fetch_all_books()
HIDDEN_STATES = {"hidden"}
return [b for b in all_books if b.get("status") not in HIDDEN_STATES]
@logcall
def get_active_books():
"""
Books currently active in the dashboard.
"""
all_books = sql_fetch_all_books()
HIDDEN_STATES = {"hidden", "done"}
return [b for b in all_books if b.get("status") not in HIDDEN_STATES]
@logcall
def store_m4b_error(book_idx: str, volume: str, error_text: str):
"""
Passive storage of m4b errors.
No logic, no retries, no state transitions.
"""
key = f"book:{book_idx}:m4b:errors"
entry = f"{volume}: {error_text}"
_r.rpush(key, entry)
if cover_url is not None:
fields["cover_url"] = cover_url
if cover_path is not None:
fields["cover_path"] = cover_path
if status is not None:
fields["status"] = status
if fields:
upsert_book(book_id, **fields)
# ------------------------------------------------------------
# STATUS MANAGEMENT
# ------------------------------------------------------------
def set_status(book_id, status):
upsert_book(book_id, status=status)
# ------------------------------------------------------------
# INCREMENTING COUNTERS (atomic)
# ------------------------------------------------------------
def inc_downloaded(book_id, amount=1):
book = _raw_get_book(book_id)
if not book:
return
cur = book.get("downloaded", 0) or 0
upsert_book(book_id, downloaded=cur + amount)
def inc_parsed(book_id, amount=1):
book = _raw_get_book(book_id)
if not book:
return
cur = book.get("parsed", 0) or 0
upsert_book(book_id, parsed=cur + amount)
def inc_audio_done(book_id, amount=1):
book = _raw_get_book(book_id)
if not book:
return
cur = book.get("audio_done", 0) or 0
upsert_book(book_id, audio_done=cur + amount)

0
bookscraper/db/schema.py
Unescape View File

130
bookscraper/db/state_redis.py
Unescape View File

@ -1,130 +0,0 @@
# ============================================================
# File: db/state_redis.py (UPDATED for book_idx-only architecture)
# Purpose:
# Low-level Redis counters/state for BookScraper.
# Used ONLY by db.repository façade.
# ============================================================
import os
import time
import redis
from logbus.publisher import log
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
# ------------------------------------------------------------
# INTERNAL KEY BUILDER
# ------------------------------------------------------------
def _key(book_idx: str) -> str:
return f"book:{book_idx}:state"
# ------------------------------------------------------------
# STATUS
# ------------------------------------------------------------
def redis_set_status(book_idx: str, status: str):
log(f"[DB-REDIS] Setting status for {book_idx} to {status}")
key = _key(book_idx)
r.hset(key, "status", status)
r.hset(key, "last_update", int(time.time()))
# ------------------------------------------------------------
# TOTAL CHAPTERS
# ------------------------------------------------------------
def redis_set_chapters_total(book_idx: str, total: int):
key = _key(book_idx)
r.hset(key, "chapters_total", total)
r.hset(key, "last_update", int(time.time()))
# ------------------------------------------------------------
# DOWNLOAD COUNTERS
# ------------------------------------------------------------
def redis_inc_download_done(book_idx: str, amount: int = 1):
log(f"[DB-REDIS] Incrementing download done for {book_idx} by {amount}")
key = _key(book_idx)
r.hincrby(key, "chapters_download_done", amount)
r.hset(key, "last_update", int(time.time()))
def redis_inc_download_skipped(book_idx: str, amount: int = 1):
log(f"[DB-REDIS] Incrementing download skipped for {book_idx} by {amount}")
key = _key(book_idx)
r.hincrby(key, "chapters_download_skipped", amount)
r.hset(key, "last_update", int(time.time()))
# ------------------------------------------------------------
# PARSE COUNTERS
# ------------------------------------------------------------
def redis_inc_parsed_done(book_idx: str, amount: int = 1):
log(f"[DB-REDIS] Incrementing parsed done for {book_idx} by {amount}")
key = _key(book_idx)
r.hincrby(key, "chapters_parsed_done", amount)
r.hset(key, "last_update", int(time.time()))
# ------------------------------------------------------------
# AUDIO COUNTERS
# ------------------------------------------------------------
def redis_inc_audio_done(book_idx: str, amount: int = 1):
log(f"[DB-REDIS] Incrementing audio done for {book_idx} by {amount}")
key = _key(book_idx)
r.hincrby(key, "audio_done", amount)
r.hset(key, "last_update", int(time.time()))
def redis_inc_audio_skipped(book_idx: str, amount: int = 1):
log(f"[DB-REDIS] Incrementing audio skipped for {book_idx} by {amount}")
key = _key(book_idx)
r.hincrby(key, "audio_skipped", amount)
r.hset(key, "last_update", int(time.time()))
# ------------------------------------------------------------
# INITIALISE BOOK STATE
# ------------------------------------------------------------
def init_book_state(book_idx: str, title: str, url: str, chapters_total: int):
"""
Initialiseert de complete Redis state voor een nieuw boek.
LET OP:
- Als een key al bestaat → NIET resetten (progress behouden).
- Alleen missende velden worden toegevoegd.
"""
key = f"book:{book_idx}:state"
# Bestaat al? Dan vullen we alleen missende velden aan.
exists = r.exists(key)
pipeline = r.pipeline()
# Basis metadata
pipeline.hsetnx(key, "book_id", book_idx)
pipeline.hsetnx(key, "title", title or "")
pipeline.hsetnx(key, "url", url or "")
# State
pipeline.hsetnx(key, "status", "registered")
# Counters
pipeline.hsetnx(key, "chapters_total", chapters_total)
pipeline.hsetnx(key, "chapters_download_done", 0)
pipeline.hsetnx(key, "chapters_download_skipped", 0)
pipeline.hsetnx(key, "chapters_parsed_done", 0)
pipeline.hsetnx(key, "audio_done", 0)
pipeline.hsetnx(key, "audio_skipped", 0)
# Timestamp
pipeline.hset(key, "last_update", int(time.time()))
pipeline.execute()
if exists:
log(f"[DB-REDIS] init_book_state(): UPDATED existing state for {book_idx}")
else:
log(f"[DB-REDIS] init_book_state(): CREATED new state for {book_idx}")

178
bookscraper/db/state_sql.py
Unescape View File

@ -1,178 +0,0 @@
# ============================================================
# File: db/state_sql.py (UPDATED for book_idx-only architecture)
# Purpose:
# Low-level SQLite snapshot layer for BookScraper metadata.
# Used ONLY through db.repository façade.
# ============================================================
import sqlite3
import os
from logbus.publisher import log
# Must match db/db.py
DB_PATH = os.getenv("BOOKSCRAPER_DB", "/app/data/books.db")
# ------------------------------------------------------------
# INTERNAL HELPERS
# ------------------------------------------------------------
def _connect():
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
return conn
# ------------------------------------------------------------
# FETCH
# ------------------------------------------------------------
def sql_fetch_book(book_idx):
conn = _connect()
cur = conn.cursor()
cur.execute("SELECT * FROM books WHERE book_idx = ?", (book_idx,))
row = cur.fetchone()
conn.close()
return dict(row) if row else None
def sql_fetch_all_books():
conn = _connect()
cur = conn.cursor()
cur.execute("SELECT * FROM books ORDER BY created_at DESC")
rows = cur.fetchall()
conn.close()
return [dict(r) for r in rows]
# ------------------------------------------------------------
# REGISTER / UPDATE
# ------------------------------------------------------------
def sql_register_book(book_idx, fields: dict):
"""
Insert or replace entire book record.
book_idx is the PRIMARY KEY.
"""
conn = _connect()
cur = conn.cursor()
cols = ", ".join(["book_idx"] + list(fields.keys()))
placeholders = ", ".join(["?"] * (1 + len(fields)))
values = [book_idx] + list(fields.values())
cur.execute(
f"INSERT OR REPLACE INTO books ({cols}) VALUES ({placeholders})",
values,
)
conn.commit()
conn.close()
def sql_update_book(book_idx, fields: dict):
if not fields:
return
conn = _connect()
cur = conn.cursor()
set_clause = ", ".join([f"{k} = ?" for k in fields])
params = list(fields.values()) + [book_idx]
cur.execute(
f"UPDATE books SET {set_clause} WHERE book_idx = ?",
params,
)
conn.commit()
conn.close()
# ------------------------------------------------------------
# STATUS
# ------------------------------------------------------------
def sql_set_status(book_idx, status: str):
conn = _connect()
cur = conn.cursor()
cur.execute(
"UPDATE books SET status = ? WHERE book_idx = ?",
(status, book_idx),
)
conn.commit()
conn.close()
# ------------------------------------------------------------
# CHAPTER TOTAL (snapshot)
# ------------------------------------------------------------
def sql_set_chapters_total(book_idx, total: int):
conn = _connect()
cur = conn.cursor()
cur.execute(
"UPDATE books SET chapters_total = ? WHERE book_idx = ?",
(total, book_idx),
)
conn.commit()
conn.close()
# ------------------------------------------------------------
# COUNTERS (SNAPSHOT-ONLY)
# ------------------------------------------------------------
def sql_inc_downloaded(book_idx, amount=1):
conn = _connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE books
SET downloaded = COALESCE(downloaded,0) + ?
WHERE book_idx = ?
""",
(amount, book_idx),
)
conn.commit()
conn.close()
def sql_inc_parsed(book_idx, amount=1):
conn = _connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE books
SET parsed = COALESCE(parsed,0) + ?
WHERE book_idx = ?
""",
(amount, book_idx),
)
conn.commit()
conn.close()
def sql_inc_audio_done(book_idx, amount=1):
log(f"[DB-SQL] Incrementing audio_done for {book_idx} by {amount}")
conn = _connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE books
SET audio_done = COALESCE(audio_done,0) + ?
WHERE book_idx = ?
""",
(amount, book_idx),
)
conn.commit()
conn.close()
def sql_inc_audio_skipped(book_idx, amount=1):
log(f"[DB-SQL] Incrementing audio_skipped for {book_idx} by {amount}")
conn = _connect()
cur = conn.cursor()
cur.execute(
"""
UPDATE books
SET audio_skipped = COALESCE(audio_skipped,0) + ?
WHERE book_idx = ?
""",
(amount, book_idx),
)
conn.commit()
conn.close()

19
bookscraper/docker-compose.yml
Unescape View File

@ -149,22 +149,3 @@ services:
- .env
command: celery -A celery_app worker -Q scraping -n scraping@%h -l INFO
restart: "no"
# ----------------------------------------------------------
# M4B Worker (Finalization)
# ----------------------------------------------------------
worker_m4b:
build:
context: .
dockerfile: docker/Dockerfile.m4b
container_name: worker_m4b
command: celery -A celery_app worker -Q m4b -n m4b@%h -l INFO
depends_on:
redis:
condition: service_healthy
env_file:
- .env
volumes:
- .:/app
- /Users/peter/mnt/asustor/Sync/bookscraper/books:/Users/peter/mnt/asustor/Sync/bookscraper/books
- /Users/peter/mnt/asustor/Sync/bookscraper/db:/Users/peter/mnt/asustor/Sync/bookscraper/db
restart: "no"

70
bookscraper/docker/Dockerfile.m4b
Unescape View File

@ -1,70 +0,0 @@
FROM debian:12
ENV DEBIAN_FRONTEND=noninteractive
# ----------------------------------------------------------
# System + PHP (PHP 8.2 native)
# ----------------------------------------------------------
RUN apt-get update && apt-get install -y \
ffmpeg \
curl \
ca-certificates \
bash \
php-cli \
php-intl \
php-json \
php-mbstring \
php-xml \
php-curl \
php-zip \
python3 \
python3-pip \
python3-venv \
\
# build deps for mp4v2
git \
build-essential \
autoconf \
automake \
libtool \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
# ----------------------------------------------------------
# Python venv (PEP 668 compliant)
# ----------------------------------------------------------
RUN python3 -m venv /opt/venv
ENV PATH="/opt/venv/bin:/usr/local/bin:$PATH"
# ----------------------------------------------------------
# Build & install mp4v2 (mp4info)
# ----------------------------------------------------------
WORKDIR /tmp
RUN git clone https://github.com/sandreas/mp4v2 \
&& cd mp4v2 \
&& ./configure \
&& make -j$(nproc) \
&& make install \
&& echo "/usr/local/lib" > /etc/ld.so.conf.d/mp4v2.conf \
&& ldconfig \
&& cd / \
&& rm -rf /tmp/mp4v2
# ----------------------------------------------------------
# Install m4b-tool
# ----------------------------------------------------------
RUN curl -L https://github.com/sandreas/m4b-tool/releases/latest/download/m4b-tool.phar \
-o /usr/local/bin/m4b-tool \
&& chmod +x /usr/local/bin/m4b-tool
# ----------------------------------------------------------
# App
# ----------------------------------------------------------
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
COPY . /app
CMD ["bash"]

22
bookscraper/logbus/publisher.py
Unescape View File

@ -1,31 +1,9 @@
# logbus/publisher.py
import logging
import os
logger = logging.getLogger("logbus")
logger.setLevel(logging.WARNING)
# ============================================================
# FILE LOGGER — log.txt in BOOKSCRAPER_OUTPUT_DIR
# ============================================================
try:
root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
os.makedirs(root, exist_ok=True)
file_path = os.path.join(root, "log.txt")
file_handler = logging.FileHandler(file_path, mode="a", encoding="utf-8")
file_formatter = logging.Formatter("%(message)s") # exact zoals input
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
except Exception:
# Logging naar file mag nooit de app laten crashen
pass
def log(message: str):
"""

1
bookscraper/mp4v2

@ -1 +0,0 @@
Subproject commit 480a73324f53d0d24bea4931c3902097f8e2a663

BIN
bookscraper/project.zip
View File

Binary file not shown.

68
bookscraper/scraper/abort.py
Unescape View File

@ -1,7 +1,7 @@
import os
import redis
from scraper.logger_decorators import logcall
# GUI log (non-breaking)
from scraper.ui_log import push_ui
# ---------------------------------------------------------
@ -13,58 +13,55 @@ r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
# Debug mode (optional)
ABORT_DEBUG = os.getenv("ABORT_DEBUG", "1") == "1"
# Avoid duplicate spam
# Internal flag to avoid spamming the same message
_seen_debug_keys = set()
# =========================================================
# INTERNAL DEBUGGING
# ABORT FLAG
# =========================================================
def _debug(msg: str):
"""Print + GUI log (non-breaking, minimal noise)."""
print(msg)
push_ui(msg)
# =========================================================
# ABORT FLAG — unified book_idx
# =========================================================
def set_abort(book_idx: str):
"""Enable abort mode for book_idx."""
key = f"abort:{book_idx}"
def set_abort(book_id: str):
"""Enable abort mode for this book."""
key = f"abort:{book_id}"
r.set(key, "1")
if ABORT_DEBUG:
_debug(f"[ABORT] SET {key}")
def clear_abort(book_idx: str):
def clear_abort(book_id: str):
"""Clear abort flag."""
key = f"abort:{book_idx}"
key = f"abort:{book_id}"
r.delete(key)
if ABORT_DEBUG:
_debug(f"[ABORT] CLEAR {key}")
def abort_requested(book_idx: str, redis_client=None) -> bool:
def abort_requested(book_id: str, redis_client=None) -> bool:
"""
Check whether abort flag is active for book_idx.
Return True if abort flag is set.
redis_client:
- Docker workers → None → use default Redis (r)
- Local macOS audio worker → passes Redis(host=127.0.0.1)
- Local macOS audio → passes Redis(host=127.0.0.1)
"""
client = redis_client or r
key = f"abort:{book_idx}"
key = f"abort:{book_id}"
try:
exists = client.exists(key)
if ABORT_DEBUG:
# Log only once per book
# Log once per key
if key not in _seen_debug_keys:
try:
conn = client.connection_pool.connection_kwargs
@ -72,53 +69,54 @@ def abort_requested(book_idx: str, redis_client=None) -> bool:
port = conn.get("port")
db = conn.get("db")
_debug(
# f"[ABORT_DEBUG] first check book_idx={book_idx} "
f"[ABORT_DEBUG] first check book_id={book_id} "
f"redis={host}:{port} db={db}"
)
except Exception:
_debug(f"[ABORT_DEBUG] first check book_idx={book_idx}")
_debug(f"[ABORT_DEBUG] first check book_id={book_id}")
_seen_debug_keys.add(key)
# Log ACTIVE state
# Only log abort ACTIVE
if exists == 1:
_debug(f"[ABORT] ACTIVE for {book_idx}")
_debug(f"[ABORT] ACTIVE for {book_id}")
return exists == 1
except Exception as e:
if ABORT_DEBUG:
_debug(f"[ABORT_DEBUG] ERROR checking {key}: {e}")
return False
# =========================================================
# PER-CHAPTER STATE — unified book_idx
# PER-CHAPTER STATE
# =========================================================
def mark_chapter_started(book_idx: str, chapter_num: int):
key = f"started:{book_idx}:{chapter_num}"
def mark_chapter_started(book_id: str, chapter_num: int):
key = f"started:{book_id}:{chapter_num}"
r.set(key, "1")
def chapter_started(book_idx: str, chapter_num: int) -> bool:
key = f"started:{book_idx}:{chapter_num}"
def chapter_started(book_id: str, chapter_num: int) -> bool:
key = f"started:{book_id}:{chapter_num}"
return r.exists(key) == 1
# =========================================================
# RESET STATE FOR BOOK_IDX
# UTILITY: RESET FOR A BOOK
# =========================================================
def reset_book_state(book_idx: str):
def reset_book_state(book_id: str):
"""
Remove abort flag and all per-chapter started markers.
Remove abort flag and all chapter-start markers.
"""
# abort flag
r.delete(f"abort:{book_idx}")
key = f"abort:{book_id}"
r.delete(key)
# chapter markers
pattern = f"started:{book_idx}:*"
pattern = f"started:{book_id}:*"
for k in r.scan_iter(pattern):
r.delete(k)

227
bookscraper/scraper/book_scraper.py
Unescape View File

@ -1,55 +1,202 @@
# ============================================================
# File: scraper/book_scraper.py
# Purpose:
# Backwards-compatible wrapper giving the SAME public API
# as the old BookScraper, but internally uses ScrapeEngine.
#
# execute() → full metadata + chapterlist (NO book_idx creation)
#
# ID management is now handled exclusively by InitService.
# ============================================================
# scraper/book_scraper.py
from scraper.logger_decorators import logcall
from scraper.services.scrape_engine import ScrapeEngine
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import re
from scraper.logger import log_debug
from scraper.utils import clean_text, load_replacements
from scraper.models.book_state import Chapter
class BookScraper:
"""
Backwards-compatible BookScraper façade.
Minimal scraper: only metadata + chapter list.
The DownloadController handles Celery pipelines for:
- download
- parse
- save
"""
Old responsibilities (metadata, chapters, covers, downloads)
are now split:
def __init__(self, site, url):
self.site = site
self.url = url
ScrapeEngine → metadata + chapterlist
Download tasks → handle download/parse/save
InitService → determines book_idx (single source of truth)
self.book_title = ""
self.book_author = ""
self.book_description = ""
self.cover_url = ""
self.chapter_base = None
This wrapper intentionally does NOT generate a book_idx or book_id.
It only returns metadata/chapters in legacy-compatible dict format.
"""
self.chapters = []
@logcall
def __init__(self, site_scraper, url: str):
self.site = site_scraper
self.url = url
# Load custom replacements
extra = load_replacements("replacements.txt")
self.site.replacements.update(extra)
@logcall
# ------------------------------------------------------------
def execute(self):
"""
Legacy public API:
Return metadata + chapter list EXACTLY as before,
but without generating any book_id.
"""
"""Main entry point. Returns metadata + chapter URLs."""
soup = self._fetch(self.url)
self._parse_title(soup)
self._parse_author(soup)
self._parse_description(soup)
self._parse_cover(soup)
data = ScrapeEngine.fetch_metadata_and_chapters(self.site, self.url)
chapter_page = self.get_chapter_page(soup)
self.parse_chapter_links(chapter_page)
log_debug(f"[BookScraper] Completed metadata parse")
# Legacy structure preserved, unchanged:
return {
"title": data.get("title"),
"author": data.get("author"),
"description": data.get("description"),
"cover_url": data.get("cover_url"),
"chapters": data.get("chapters", []),
"chapters_total": data.get("chapters_total", 0),
"book_url": data.get("book_url"), # used later by parse/save tasks
"title": self.book_title,
"author": self.book_author,
"description": self.book_description,
"cover_url": self.cover_url, # ← used by DownloadController
"book_url": self.url,
"chapters": [
{"num": ch.number, "title": ch.title, "url": ch.url}
for ch in self.chapters
],
}
# ------------------------------------------------------------
def _fetch(self, url):
log_debug(f"[BookScraper] Fetch: {url}")
resp = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10)
resp.encoding = self.site.encoding
return BeautifulSoup(resp.text, "lxml")
# ------------------------------------------------------------
def _parse_title(self, soup):
h1 = soup.find("h1")
self.book_title = h1.get_text(strip=True) if h1 else "UnknownTitle"
log_debug(f"[BookScraper] Title = {self.book_title}")
def _parse_author(self, soup):
td = soup.find("td", string=lambda t: t and "作" in t)
raw = td.get_text(strip=True) if td else ""
self.book_author = raw.split("：")[1] if "：" in raw else "UnknownAuthor"
log_debug(f"[BookScraper] Author = {self.book_author}")
def _parse_description(self, soup):
span = soup.find("span", string=lambda t: t and "内容简介" in t)
if not span:
self.book_description = ""
log_debug("[BookScraper] Description not found")
return
parts = []
for sib in span.next_siblings:
if getattr(sib, "name", None) == "span":
break
text = (
sib.get_text(strip=True)
if hasattr(sib, "get_text")
else str(sib).strip()
)
if text:
parts.append(text)
self.book_description = clean_text("\n".join(parts), self.site.replacements)
log_debug(f"[BookScraper] Description length = {len(self.book_description)}")
# ------------------------------------------------------------
def _parse_cover(self, soup):
"""
Extract correct cover based on book_id path logic.
1. primary: match "/files/article/image/{vol}/{book_id}/"
2. fallback: endswith "/{book_id}s.jpg"
"""
# Extract book_id from URL
m = re.search(r"/(\d+)\.html$", self.url)
if not m:
log_debug("[BookScraper] No book_id found in URL → cannot match cover")
return
book_id = m.group(1)
# Extract vol folder from URL (bookinfo/<vol>/<id>.html)
m2 = re.search(r"/bookinfo/(\d+)/", self.url)
volume = m2.group(1) if m2 else None
log_debug(f"[BookScraper] Book ID = {book_id}, Volume = {volume}")
imgs = soup.find_all("img", src=True)
chosen = None
# --------------------------------------------------------
# PRIORITY 1: Path-match
# /files/article/image/{vol}/{book_id}/
# --------------------------------------------------------
if volume:
target_path = f"/files/article/image/{volume}/{book_id}/"
for img in imgs:
src = img["src"]
if target_path in src:
chosen = src
log_debug(f"[BookScraper] Cover matched by PATH: {src}")
break
# --------------------------------------------------------
# PRIORITY 2: endswith "/{book_id}s.jpg"
# --------------------------------------------------------
if not chosen:
target_suffix = f"/{book_id}s.jpg"
for img in imgs:
src = img["src"]
if src.endswith(target_suffix):
chosen = src
log_debug(f"[BookScraper] Cover matched by SUFFIX: {src}")
break
# --------------------------------------------------------
# No match
# --------------------------------------------------------
if not chosen:
log_debug("[BookScraper] No matching cover found")
return
self.cover_url = urljoin(self.site.root, chosen)
log_debug(f"[BookScraper] Cover URL = {self.cover_url}")
# ------------------------------------------------------------
def get_chapter_page(self, soup):
"""Return BeautifulSoup of the main chapter list page."""
node = soup.select_one(
"html > body > div:nth-of-type(6) > div:nth-of-type(2) > div > table"
)
href = node.select_one("a").get("href")
chapter_url = urljoin(self.site.root, href)
# base for chapter links
parts = chapter_url.rsplit("/", 1)
self.chapter_base = parts[0] + "/"
return self._fetch(chapter_url)
# ------------------------------------------------------------
def parse_chapter_links(self, soup):
cont = soup.select_one(self.site.chapter_list_selector)
items = cont.select("ul li a[href]")
self.chapters = []
idx = 1
for a in items:
href = a.get("href")
if not href.endswith(".html"):
continue
title = a.get_text(strip=True)
full = urljoin(self.chapter_base, href)
self.chapters.append(Chapter(idx, title, full))
idx += 1
log_debug(f"[BookScraper] Found {len(self.chapters)} chapters")

213
bookscraper/scraper/download_controller.py
Unescape View File

@ -1,54 +1,63 @@
# =========================================================
# File: scraper/download_controller.py
# Purpose:
# Build Celery pipelines for all chapters using book_idx
# Handles:
# • volume assignment
# • cover download + replication
# • script generation
# • Redis Book State Model init
# • abort tracking
# Build Celery pipelines for all chapters
# and pass book_id for abort/progress/log functionality.
# + Download and replicate cover image to all volume folders
# + Generate scripts (allinone.txt, makebook, say)
# + Initialize Redis Book State Model (status + counters)
# =========================================================
from celery import group
from scraper.tasks.pipeline import build_chapter_pipeline
# ❗ IMPORTANT:
# generate_all_scripts MUST NOT import DownloadController, otherwise circular import.
# We keep the import, but scriptgen must be clean.
from scraper import scriptgen
from scraper.scriptgen import generate_all_scripts
from logbus.publisher import log
import os
import requests
import shutil
from scraper.abort import abort_requested # DEBUG allowed
from db.repository import create_or_update_book
from scraper.abort import abort_requested
from db.state_redis import init_book_state
from db.repository import set_status, set_chapters_total
# NEW: Redis State Model (C&U)
from scraper.progress import (
init_book_state,
set_status,
set_chapter_total,
)
class DownloadController:
"""
Coordinates all chapter pipelines (download → parse → save).
Coordinates all chapter pipelines (download → parse → save),
including:
- volume splitting
- consistent meta propagation
- book_id-based abort + progress tracking
- cover download + volume replication
- script generation (allinone.txt, makebook, say)
- Redis book state initialisation and status updates
"""
def __init__(self, book_idx: str, scrape_result: dict):
self.book_idx = str(book_idx)
def __init__(self, book_id: str, scrape_result: dict):
self.book_id = book_id
self.scrape_result = scrape_result
# Metadata
# Core metadata
self.title = scrape_result.get("title", "UnknownBook")
self.chapters = scrape_result.get("chapters", []) or []
self.cover_url = scrape_result.get("cover_url")
# Output folder
# Output base dir
root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
# Volume size
self.max_vol = int(os.getenv("MAX_VOL_SIZE", "200"))
# Base folder for the whole book
self.book_base = os.path.join(root, self.title)
os.makedirs(self.book_base, exist_ok=True)
# Meta passed downstream
# Meta passed to parse/save stage
self.meta = {
"title": self.title,
"author": scrape_result.get("author"),
@ -56,120 +65,200 @@ class DownloadController:
"book_url": scrape_result.get("book_url"),
}
log(f"[CTRL_DEBUG] Controller init book_idx={self.book_idx}")
# -------------------------------------------------
# DEBUG — bevestig dat controller correct book_id ziet
# -------------------------------------------------
log(f"[CTRL_DEBUG] Controller init book_id={book_id} title='{self.title}'")
# Init Redis Book State Model
try:
abort_state = abort_requested(book_id)
log(f"[CTRL_DEBUG] abort_requested(book_id={book_id}) → {abort_state}")
except Exception as e:
log(f"[CTRL_DEBUG] abort_requested ERROR: {e}")
# -------------------------------------------------
# NEW: Initialize Redis Book State Model
# -------------------------------------------------
try:
init_book_state(
book_id=self.book_idx,
book_id=self.book_id,
title=self.title,
url=self.meta["book_url"],
url=self.scrape_result.get("book_url"),
chapters_total=len(self.chapters),
)
log(f"[CTRL_STATE] init_book_state() completed for {self.title}")
except Exception as e:
log(f"[CTRL_STATE] init_book_state FAILED: {e}")
# ---------------------------------------------------------
# Cover Download
# ---------------------------------------------------------
def download_cover(self):
"""Download one cover image into the root of the book folder."""
if not self.cover_url:
return log(f"[CTRL] No cover URL for '{self.title}'")
log(f"[CTRL] No cover URL found for '{self.title}'")
return
cover_path = os.path.join(self.book_base, "cover.jpg")
headers = {
"User-Agent": "Mozilla/5.0",
"Referer": self.scrape_result.get("book_url") or "",
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:118.0) "
"Gecko/20100101 Firefox/118.0"
),
"Referer": self.scrape_result.get("book_url") or "https://www.piaotia.com/",
}
try:
log(f"[CTRL] Downloading cover: {self.cover_url}")
resp = requests.get(self.cover_url, timeout=10, headers=headers)
resp.raise_for_status()
with open(cover_path, "wb") as f:
f.write(resp.content)
log(f"[CTRL] Cover saved: {cover_path}")
log(f"[CTRL] Cover saved to: {cover_path}")
except Exception as e:
log(f"[CTRL] Cover download failed: {e}")
log(f"[CTRL] Cover download failed: {e} (url={self.cover_url})")
# ---------------------------------------------------------
# Cover Replication to Volumes
# ---------------------------------------------------------
def replicate_cover_to_volumes(self):
"""Copy cover.jpg into each existing Volume_xxx directory."""
src = os.path.join(self.book_base, "cover.jpg")
if not os.path.exists(src):
log("[CTRL] No cover.jpg found, replication skipped")
return
for entry in os.listdir(self.book_base):
if entry.lower().startswith("volume_"):
dst = os.path.join(self.book_base, entry, "cover.jpg")
try:
try:
for entry in os.listdir(self.book_base):
if entry.lower().startswith("volume_"):
vol_dir = os.path.join(self.book_base, entry)
dst = os.path.join(vol_dir, "cover.jpg")
shutil.copyfile(src, dst)
log(f"[CTRL] Cover replicated → {dst}")
except Exception as e:
log(f"[CTRL] Cover replication failed: {e}")
log(f"[CTRL] Cover replicated into: {dst}")
except Exception as e:
log(f"[CTRL] Cover replication failed: {e}")
# ---------------------------------------------------------
def store_cover_in_static(self):
"""
Copy the main cover.jpg from book_base into static/covers/<book_id>.jpg.
This allows the Flask web UI to serve the cover directly.
"""
src = os.path.join(self.book_base, "cover.jpg")
if not os.path.exists(src):
log("[CTRL] No cover.jpg found, cannot store in static/covers")
return
os.makedirs("static/covers", exist_ok=True)
dst = os.path.join("static/covers", f"{self.book_idx}.jpg")
# static/covers/<book_id>.jpg
static_dir = os.path.join("static", "covers")
os.makedirs(static_dir, exist_ok=True)
dst = os.path.join(static_dir, f"{self.book_id}.jpg")
try:
shutil.copyfile(src, dst)
log(f"[CTRL] Cover stored for UI: {dst}")
except Exception as e:
log(f"[CTRL] Failed storing cover: {e}")
log(f"[CTRL] Failed to store cover in static: {e}")
# ---------------------------------------------------------
# Volume isolation
# ---------------------------------------------------------
def get_volume_path(self, chapter_num: int) -> str:
"""Returns the correct volume directory for a chapter."""
vol_index = (chapter_num - 1) // self.max_vol + 1
vol_name = f"Volume_{vol_index:03d}"
vol_path = os.path.join(self.book_base, vol_name)
os.makedirs(vol_path, exist_ok=True)
return vol_path
# ---------------------------------------------------------
# Pipeline launcher
# ---------------------------------------------------------
def start(self):
total = len(self.chapters)
log(f"[CTRL] Starting pipeline for '{self.title}' ({total} chapters)")
# Update Redis/SQLite state
log(
f"[CTRL] Initialising pipeline for '{self.title}' "
f"(book_id={self.book_id}, chapters={total}, max_vol={self.max_vol})"
)
log(f"[CTRL] Output root: {self.book_base}")
# -------------------------------------
# NEW: Redis state update
# -------------------------------------
try:
set_status(self.book_idx, "downloading")
set_chapters_total(self.book_idx, total)
set_status(self.book_id, "downloading")
set_chapter_total(self.book_id, total)
log(f"[CTRL_STATE] Status set to 'downloading' for {self.book_id}")
except Exception as e:
log(f"[CTRL_STATE] Unable to set state: {e}")
log(f"[CTRL_STATE] set_status/set_chapter_total FAILED: {e}")
# Download cover
# -------------------------------------
# 1) Download cover
# -------------------------------------
self.download_cover()
# Build pipeline tasks
tasks = []
for ch in self.chapters:
num = ch["num"]
chapter_info = {
"num": num,
"url": ch["url"],
"title": ch.get("title"),
"volume_path": self.get_volume_path(num),
# Build chapter_dict (NEW)
chapter_num = ch["num"]
chapter_url = ch["url"]
chapter_title = ch.get("title")
volume_path = self.get_volume_path(chapter_num)
chapter_dict = {
"num": chapter_num,
"url": chapter_url,
"title": chapter_title,
"volume_path": volume_path,
}
tasks.append(build_chapter_pipeline(self.book_idx, chapter_info, self.meta))
# Dispatch pipeline with chapter_dict
tasks.append(
build_chapter_pipeline(
self.book_id,
chapter_dict,
self.meta,
)
)
async_result = group(tasks).apply_async()
# Replicate cover + place in static
log(
f"[CTRL] Pipelines dispatched for '{self.title}' "
f"(book_id={self.book_id}, group_id={async_result.id})"
)
# Debug abort state
try:
abort_state = abort_requested(self.book_id)
log(f"[CTRL_DEBUG] After-dispatch abort state: {abort_state}")
except Exception as e:
log(f"[CTRL_DEBUG] abort_requested error after dispatch: {e}")
# -------------------------------------------------------
self.replicate_cover_to_volumes()
self.store_cover_in_static()
# Generate scripts (LATE IMPORT to avoid circular)
# -------------------------------------------------------
try:
scriptgen.generate_all_scripts(
self.book_base, self.title, self.meta["author"]
generate_all_scripts(
self.book_base,
self.title,
self.meta.get("author"),
)
log("[CTRL] Scripts generated")
log(f"[CTRL] Scripts generated for '{self.title}'")
except Exception as e:
log(f"[CTRL] Script generation failed: {e}")

0
bookscraper/scraper/engine/__init__.py
Unescape View File

27
bookscraper/scraper/engine/fetcher.py
Unescape View File

@ -1,27 +0,0 @@
# ============================================================
# File: scraper/engine/fetcher.py
# Purpose:
# Low-level HTML fetch utility shared by all site scrapers.
# Replaces scattered _fetch() logic inside BookScraper.
# ============================================================
import requests
from bs4 import BeautifulSoup
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:118.0) "
"Gecko/20100101 Firefox/118.0"
)
}
def fetch_html(url: str, encoding: str = "utf-8", timeout: int = 10) -> BeautifulSoup:
"""
Fetch HTML with a consistent user-agent and encoding.
Returns BeautifulSoup(lxml).
"""
resp = requests.get(url, headers=HEADERS, timeout=timeout)
resp.encoding = encoding
return BeautifulSoup(resp.text, "lxml")

65
bookscraper/scraper/engine/parser.py
Unescape View File

@ -1,65 +0,0 @@
# ============================================================
# File: scraper/engine/parser.py
# Purpose:
# High-level scraping API coordinating metadata extraction
# and chapter extraction using pluggable SiteScraper classes.
#
# This is the new central engine:
# - extract_metadata_only() used by INIT flow
# - extract_metadata_full() used by full scraping pipeline
# ============================================================
from scraper.engine.fetcher import fetch_html
def extract_metadata_only(url: str, site_scraper):
"""
Extract ONLY lightweight metadata:
- title
- author
- description
- cover_url
- chapters_total = 0
"""
soup = fetch_html(url, site_scraper.encoding)
title = site_scraper.parse_title(soup)
author = site_scraper.parse_author(soup)
description = site_scraper.parse_description(soup)
cover_url = site_scraper.parse_cover(soup, url)
return {
"title": title,
"author": author,
"description": description,
"cover_url": cover_url,
"chapters_total": 0,
"book_url": url,
}
def extract_metadata_full(url: str, site_scraper):
"""
Full scraping (metadata + chapterlist).
Used by the scraping Celery pipeline.
"""
soup = fetch_html(url, site_scraper.encoding)
# metadata
meta = extract_metadata_only(url, site_scraper)
# chapter list
chapter_page_url = site_scraper.extract_chapter_page_url(soup)
chapter_page_soup = fetch_html(chapter_page_url, site_scraper.encoding)
chapters = site_scraper.parse_chapter_list(chapter_page_soup)
meta["chapters"] = chapters
return meta
def build_book_id(title: str) -> str:
"""
Canonical book_id generator.
SCRAPE currently uses title as ID → preserve that behavior.
"""
return title

33
bookscraper/scraper/logger_decorators.py
Unescape View File

@ -1,33 +0,0 @@
# ============================================================
# File: scraper/logger_decorators.py
# Purpose: Function-call logging decorator
# ============================================================
from functools import wraps
from scraper.logger import log_debug
def logcall(func):
"""
Decorator: log function name + arguments every time it's called.
Usage: @logcall above any function.
"""
@wraps(func)
def wrapper(*args, **kwargs):
# Naam van de functie
name = func.__qualname__
# Eerste logregel vóór uitvoering
# log_debug(f"[CALL] {name} args={args} kwargs={kwargs}")
log_debug(f"[CALL] {name} args={args}")
# log_debug(f"[CALL] {name}")
result = func(*args, **kwargs)
# Log ná uitvoering
# log_debug(f"[RETURN] {name} → {result}")
return result
return wrapper

0
bookscraper/scraper/_dep_progress.py.bsk → bookscraper/scraper/progress.py
Unescape View File

2
bookscraper/scraper/replacements/junk.txt
Unescape View File

@ -54,8 +54,6 @@ Copyright=
章节出错=
点此举报=
举报原因=
求收藏=
推荐票=
www.piaotia.com=
www.piaotian.com=
www.=

41
bookscraper/scraper/scriptgen.py
Unescape View File

@ -5,7 +5,6 @@
import os
import stat
from logbus.publisher import log
from scraper.logger_decorators import logcall
TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates")
@ -44,40 +43,13 @@ def detect_volumes(book_base: str):
# ------------------------------------------------------------
def build_merge_block(title: str, author: str, volumes):
lines = []
# --------------------------------------------------------
# Normalize input (defensive)
# --------------------------------------------------------
title = (title or "").strip()
author = (author or "").strip()
total_vols = len(volumes)
# Padding-regel:
# - altijd minimaal 2 (01, 02)
# - 3 bij >=100
if total_vols >= 100:
pad = 3
else:
pad = 2
for num, dirname in volumes:
vol_num = f"{num:0{pad}d}" # voor filename
series_part = f"{num:0{pad}d}" # voor series-part (string!)
line = (
f"m4b-tool merge --jobs=4 "
f'--writer="{author}" '
f'--sortalbum="{title}" '
f'--albumartist="{author}" '
f'--album="{title}" '
f'--name="{title}" '
f'--series="{title}" '
f'--series-part="{series_part}" '
f'--output-file="{title}-{vol_num}.m4b" '
f'm4b-tool merge --jobs=4 --writer="{author}" '
f'--albumartist="{author}" --album="{title}" '
f'--name="{title}" --output-file="{title}-{num}.m4b" '
f'"{dirname}" -vvv'
)
lines.append(line)
if not lines:
@ -89,14 +61,7 @@ def build_merge_block(title: str, author: str, volumes):
# ------------------------------------------------------------
# Main generator
# ------------------------------------------------------------
@logcall
def generate_all_scripts(book_base: str, title: str, author: str):
# --------------------------------------------------------
# Defensive normalize
# --------------------------------------------------------
title = (title or "").strip()
author = (author or "").strip()
log(f"[SCRIPTGEN] Generating scripts in {book_base}")
# Load templates

0
bookscraper/scraper/services/__init__.py
Unescape View File

94
bookscraper/scraper/services/audio_completion.py
Unescape View File

@ -1,94 +0,0 @@
# ============================================================
# File: scraper/services/audio_completion.py
# Purpose:
# Orchestration hook after audio completion.
#
# Rules (STRICT):
# - ALWAYS read via get_book_state()
# - Use ONLY merged counters from repository
# - NO usage of derived status field
# - Completion rule:
# audio_completed < chapters_total → NOT DONE
# ============================================================
from logbus.publisher import log
from scraper.logger_decorators import logcall
from db.repository import (
get_book_state,
try_trigger_statuscheck,
)
from scraper.services.status_check_service import StatusCheckService
from scraper.tasks.m4b_tasks import queue_m4b_for_book
@logcall
def trigger_audio_completion_check(book_idx: str):
"""
Called after inc_audio_done() OR inc_audio_skipped().
Flow:
1. Fetch canonical merged state from repository
2. Evaluate completion via merged counters ONLY
3. Run filesystem validation (authoritative)
4. Apply idempotency guard
5. Queue m4b exactly once
"""
try:
# ----------------------------------------------------
# STEP 1 — CANONICAL MERGED STATE
# ----------------------------------------------------
state = get_book_state(book_idx)
chapters_total = int(state.get("chapters_total", 0))
audio_done = int(state.get("audio_done", 0))
audio_skipped = int(state.get("audio_skipped", 0))
audio_completed = audio_done + audio_skipped
log(
f"[AUDIO-COMPLETION] book={book_idx} "
f"audio_completed={audio_completed} chapters_total={chapters_total}"
)
# ----------------------------------------------------
# STEP 2 — FAST REJECT (MERGED COUNTERS ONLY)
# ----------------------------------------------------
if chapters_total <= 0 or audio_completed < chapters_total:
log(f"[AUDIO-COMPLETION] not yet complete for book={book_idx}")
return
# ----------------------------------------------------
# STEP 3 — FILESYSTEM VALIDATION (AUTHORITATIVE)
# ----------------------------------------------------
result = StatusCheckService.run(book_idx)
fs = result.get("filesystem", {})
audio_files = fs.get("audio_files", 0)
chapters_txt = fs.get("chapters_txt", 0)
effective_audio = audio_files + audio_skipped
if effective_audio < chapters_txt:
log(
f"[AUDIO-COMPLETION] FS validation failed "
f"(audio_files={audio_files}, skipped={audio_skipped}, txt={chapters_txt})"
)
return
# ----------------------------------------------------
# STEP 4 — IDEMPOTENCY GUARD (AFTER FS CONFIRMATION)
# ----------------------------------------------------
if not try_trigger_statuscheck(book_idx):
log(f"[AUDIO-COMPLETION] statuscheck already triggered for {book_idx}")
return
# ----------------------------------------------------
# STEP 5 — FINAL ACTION
# ----------------------------------------------------
log(f"[AUDIO-COMPLETION] DONE → queue m4b for book={book_idx}")
queue_m4b_for_book(book_idx)
except Exception as exc:
# MUST NEVER break audio workers
log(f"[AUDIO-COMPLETION][ERROR] book={book_idx} error={exc}")

45
bookscraper/scraper/services/cover_service.py
Unescape View File

@ -1,45 +0,0 @@
# ============================================================
# File: scraper/services/cover_service.py
# ============================================================
import os
import requests
from logbus.publisher import log
from typing import Optional
class CoverService:
@staticmethod
def download_main_cover(cover_url: str, book_id: str) -> Optional[str]:
"""
Downloads cover image into: static/covers/<book_id>.jpg.
Returns local path or None.
"""
if not cover_url:
log(f"[COVER] No cover URL for book={book_id}")
return None
static_dir = os.path.join("static", "covers")
os.makedirs(static_dir, exist_ok=True)
dst_path = os.path.join(static_dir, f"{book_id}.jpg")
try:
log(f"[COVER] Downloading: {cover_url}")
resp = requests.get(
cover_url, timeout=10, headers={"User-Agent": "Mozilla/5.0"}
)
resp.raise_for_status()
with open(dst_path, "wb") as f:
f.write(resp.content)
log(f"[COVER] Stored: {dst_path}")
return dst_path
except Exception as e:
log(f"[COVER] FAILED ({cover_url}) → {e}")
return None

95
bookscraper/scraper/services/init_service.py
Unescape View File

@ -1,95 +0,0 @@
# ============================================================
# File: scraper/services/init_service.py
# Purpose:
# Orchestrate INIT-flow:
# - resolve site
# - fetch minimal metadata
# - derive book_idx
# - register in SQLite
# - store main cover
# ============================================================
import re
from scraper.services.site_resolver import SiteResolver
from scraper.services.scrape_engine import ScrapeEngine
from scraper.services.cover_service import CoverService
from db.repository import register_book
from scraper.logger_decorators import logcall
class InitService:
# ------------------------------------------------------------
# BOOK IDX DERIVATION
# ------------------------------------------------------------
@staticmethod
@logcall
def derive_book_id(url: str) -> str:
"""
PTWXZ URL format ends with /{id}.html.
If no match → fallback to sanitized URL.
Returns:
book_idx (string)
"""
m = re.search(r"/(\d+)\.html$", url)
if m:
return m.group(1)
# Fallback — ensures deterministic ID for unknown formats
return url.replace("/", "_").replace(":", "_")
# ------------------------------------------------------------
# MAIN INIT FLOW
# ------------------------------------------------------------
@staticmethod
@logcall
def execute(url: str) -> dict:
"""
INIT entry point.
Returns complete metadata + registration result.
"""
# 1) Resolve site handler
site = SiteResolver.resolve(url)
# 2) Create unified book_idx
book_idx = InitService.derive_book_id(url)
# Some site objects historically expect .book_id — we support it but DO NOT rely on it.
site.book_id = book_idx
# 3) Fetch initial metadata (title/author/description/cover)
meta = ScrapeEngine.fetch_metadata_only(site, url)
title = meta.get("title") or "Unknown"
author = meta.get("author")
description = meta.get("description")
cover_url = meta.get("cover_url")
# 4) Download & store main cover for UI
cover_path = CoverService.download_main_cover(cover_url, book_idx)
# 5) Register in SQLite (book_idx is the SOLE primary ID)
register_book(
book_idx=book_idx,
title=title,
author=author,
description=description,
cover_url=cover_url,
cover_path=cover_path,
book_url=url,
)
# 6) Return metadata for UI / API
return {
"book_idx": book_idx,
"title": title,
"author": author,
"description": description,
"cover_url": cover_url,
"cover_path": cover_path,
"status": "registered",
}

286
bookscraper/scraper/services/scrape_engine.py
Unescape View File

@ -1,286 +0,0 @@
# ============================================================
# File: scraper/services/scrape_engine.py (C&U — no circular import)
# Purpose:
# Unified scraping engine for INIT-flow and Celery tasks.
# ScrapeEngine does NOT determine book_idx itself.
# ============================================================
import os
import time
import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
from logbus.publisher import log
from scraper.logger import log_debug
from scraper.logger_decorators import logcall
from scraper.utils.utils import load_replacements
class ScrapeEngine:
"""
Central scraping engine.
Metadata + chapterlist scraping.
All methods logged with @logcall.
IMPORTANT:
- ScrapeEngine NEVER decides book_idx.
- No dependency on InitService (prevents circular import).
"""
# ------------------------------------------------------------
# REPLACEMENTS LOADER
# ------------------------------------------------------------
@staticmethod
@logcall
def _apply_replacements(site):
fp = os.path.join(os.getcwd(), "replacements.txt")
extra = load_replacements(fp)
if not hasattr(site, "replacements"):
site.replacements = {}
site.replacements.update(extra)
return True
# ------------------------------------------------------------
# RATE LIMITER
# ------------------------------------------------------------
MIN_DELAY = 1.0 / float(os.getenv("MAX_DOWNLOADS_PER_SEC", "1"))
@staticmethod
@logcall
def _throttle(last_time=[0]):
now = time.time()
elapsed = now - last_time[0]
if elapsed < ScrapeEngine.MIN_DELAY:
time.sleep(ScrapeEngine.MIN_DELAY - elapsed)
last_time[0] = time.time()
return True
# ------------------------------------------------------------
# HTTP GET
# ------------------------------------------------------------
@staticmethod
@logcall
def _get_doc(url: str, site):
attempt = 1
while True:
ScrapeEngine._throttle()
log_debug(f"[SCRAPER] GET {url} (attempt {attempt})")
try:
resp = requests.get(
url,
headers={"User-Agent": "Mozilla/5.0"},
timeout=10,
)
except Exception as e:
log_debug(f"Network error {e} → retry {attempt + 1}s")
time.sleep(attempt + 1)
attempt += 1
continue
code = resp.status_code
if code == 200:
resp.encoding = getattr(site, "encoding", "utf-8")
return BeautifulSoup(resp.text, "lxml")
if code == 429:
cooldown = 60
log_debug("429 detected — cooldown 60s")
for i in range(cooldown, 0, -1):
log_debug(f" cooldown {i}s…")
time.sleep(1)
attempt += 1
continue
if code in (403, 500):
wait = min(5 * attempt, 30)
log_debug(f"HTTP {code} → retry in {wait}s")
time.sleep(wait)
attempt += 1
continue
wait = attempt + 1
log_debug(f"Unexpected HTTP {code} → sleep {wait}s")
time.sleep(wait)
attempt += 1
# ------------------------------------------------------------
# PARSER HELPERS
# ------------------------------------------------------------
@staticmethod
@logcall
def _parse_title(soup):
h1 = soup.find("h1")
return h1.get_text(strip=True) if h1 else "UnknownTitle"
@staticmethod
@logcall
def _parse_author(soup):
td = soup.find("td", string=lambda t: t and "作" in t)
if td and "：" in td.get_text():
return td.get_text(strip=True).split("：")[1]
return "UnknownAuthor"
@staticmethod
@logcall
def _parse_description(soup):
span = soup.find("span", string=lambda t: t and "内容简介" in t)
if not span:
return ""
parts = []
for sib in span.next_siblings:
if getattr(sib, "name", None) == "span":
break
txt = (
sib.get_text(strip=True)
if hasattr(sib, "get_text")
else str(sib).strip()
)
if txt:
parts.append(txt)
return "\n".join(parts)
# ------------------------------------------------------------
# COVER PARSER (NO InitService dependency)
# ------------------------------------------------------------
@staticmethod
@logcall
def _parse_cover(soup, site):
"""
Extract book index from URL heuristically instead of InitService
(prevents circular import).
"""
# Typical Chinese novel sites embed numeric ID in URL path
try:
parsed = urlparse(site.url)
digits = re.findall(r"\d+", parsed.path)
book_idx = digits[-1] if digits else None
except Exception:
book_idx = None
imgs = soup.find_all("img", src=True)
candidates = []
for img in imgs:
src = img["src"].strip()
filename = os.path.basename(src)
if book_idx and book_idx in filename:
candidates.append((filename, src))
if not candidates:
return None
candidates.sort(key=lambda t: len(t[0])) # smallest filename
return urljoin(site.root, candidates[0][1])
# ------------------------------------------------------------
# RESOLVE CHAPTER PAGE
# ------------------------------------------------------------
@staticmethod
@logcall
def _resolve_chapter_page(soup, site):
node = soup.select_one(
"html > body > div:nth-of-type(6) > div:nth-of-type(2) > div > table"
)
if not node:
raise ValueError("Could not locate chapter list base node")
href = node.select_one("a").get("href")
url = urljoin(site.root, href)
parsed = urlparse(url)
basepath = parsed.path.rsplit("/", 1)[0] + "/"
chapter_base = f"{parsed.scheme}://{parsed.netloc}{basepath}"
return url, chapter_base
# ------------------------------------------------------------
# PARSE CHAPTER LINKS
# ------------------------------------------------------------
@staticmethod
@logcall
def _parse_chapter_links(soup, chapter_base, selector):
cont = soup.select_one(selector)
if not cont:
return []
items = cont.select("ul li a[href]")
chapters = []
idx = 1
for a in items:
href = a.get("href")
if not href.endswith(".html"):
continue
title = a.get_text(strip=True)
full = urljoin(chapter_base, href)
chapters.append({"num": idx, "title": title, "url": full})
idx += 1
return chapters
# ============================================================
# PUBLIC APIS
# ============================================================
@staticmethod
@logcall
def fetch_metadata_only(site, url: str) -> dict:
ScrapeEngine._apply_replacements(site)
soup = ScrapeEngine._get_doc(url, site)
site.url = url # needed for cover parsing
return {
"title": ScrapeEngine._parse_title(soup),
"author": ScrapeEngine._parse_author(soup),
"description": ScrapeEngine._parse_description(soup),
"cover_url": ScrapeEngine._parse_cover(soup, site),
"book_url": url,
}
@staticmethod
@logcall
def fetch_metadata_and_chapters(site, url: str) -> dict:
ScrapeEngine._apply_replacements(site)
soup = ScrapeEngine._get_doc(url, site)
site.url = url
title = ScrapeEngine._parse_title(soup)
author = ScrapeEngine._parse_author(soup)
desc = ScrapeEngine._parse_description(soup)
cover = ScrapeEngine._parse_cover(soup, site)
chapter_page_url, chapter_base = ScrapeEngine._resolve_chapter_page(soup, site)
chapter_soup = ScrapeEngine._get_doc(chapter_page_url, site)
chapters = ScrapeEngine._parse_chapter_links(
chapter_soup, chapter_base, site.chapter_list_selector
)
return {
"title": title,
"author": author,
"description": desc,
"cover_url": cover,
"chapters": chapters,
"chapters_total": len(chapters),
"book_url": url,
}
@staticmethod
@logcall
def fetch_chapterlist(site, url: str):
ScrapeEngine._apply_replacements(site)
soup = ScrapeEngine._get_doc(url, site)
chapter_page_url, chapter_base = ScrapeEngine._resolve_chapter_page(soup, site)
chapter_soup = ScrapeEngine._get_doc(chapter_page_url, site)
return ScrapeEngine._parse_chapter_links(
chapter_soup, chapter_base, site.chapter_list_selector
)

20
bookscraper/scraper/services/site_resolver.py
Unescape View File

@ -1,20 +0,0 @@
# ============================================================
# File: scraper/services/site_resolver.py
# Purpose:
# Determine which BookSite implementation applies for a given URL.
# This keeps INIT-flow and SCRAPE-flow site-agnostic.
# ============================================================
from scraper.sites import BookSite # current PTWXZ implementation
class SiteResolver:
"""
Resolves the correct BookSite class based on URL.
Currently only PTWXZ/Piaotian is supported.
"""
@staticmethod
def resolve(url: str):
# Later: add more domain rules for other sources
return BookSite()

135
bookscraper/scraper/services/status_check_service.py
Unescape View File

@ -1,135 +0,0 @@
# ============================================================
# File: scraper/services/status_check_service.py
# Purpose:
# Handmatige, idempotente statuscheck per boek.
#
# Bepaalt op basis van het filesystem:
# - aantal gedownloade chapters (.txt)
# - aantal gegenereerde audiofiles (.m4b)
#
# En schrijft deze gevalideerde werkelijkheid naar SQL.
#
# LET OP:
# - Geen Redis
# - Geen Celery
# - Geen status-transities
# - Geen pipeline-logica
# ============================================================
import os
from datetime import datetime
from typing import Dict, Any
from logbus.publisher import log
from scraper.logger_decorators import logcall
from db.state_sql import sql_fetch_book, sql_update_book
class StatusCheckService:
"""
Statuscheck op basis van filesystem.
Single source of truth = disk.
"""
@staticmethod
@logcall
def run(book_idx: str) -> Dict[str, Any]:
"""
Voer statuscheck uit voor één boek.
Returns een inspecteerbaar dict met:
- filesystem tellingen
- SQL before / after snapshot
"""
# ----------------------------------------------------
# 1. SQL fetch (bestaat het boek?)
# ----------------------------------------------------
sql_before = sql_fetch_book(book_idx)
if not sql_before:
raise ValueError(f"[STATUSCHECK] Book not found in SQL: {book_idx}")
# ----------------------------------------------------
# 2. Bepaal filesystem root
# ----------------------------------------------------
output_root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
title = sql_before.get("title")
book_dir = os.path.join(output_root, title)
if not os.path.isdir(book_dir):
log(
f"[STATUSCHECK] No output directory for book_idx={book_idx} : title='{title}')"
)
chapters_txt = 0
audio_files = 0
volumes = 0
else:
chapters_txt = 0
audio_files = 0
volumes = 0
# ------------------------------------------------
# 3. Scan volumes
# ------------------------------------------------
for entry in os.listdir(book_dir):
if not entry.lower().startswith("volume_"):
continue
volumes += 1
volume_path = os.path.join(book_dir, entry)
if not os.path.isdir(volume_path):
continue
# ---- TXT chapters ----
for fname in os.listdir(volume_path):
if fname.lower().endswith(".txt"):
chapters_txt += 1
# ---- Audio ----
audio_dir = os.path.join(volume_path, "Audio")
if os.path.isdir(audio_dir):
for fname in os.listdir(audio_dir):
if fname.lower().endswith(".m4b"):
audio_files += 1
# ----------------------------------------------------
# 4. SQL update (snapshot)
# ----------------------------------------------------
now = datetime.utcnow().isoformat(timespec="seconds")
update_fields = {
"downloaded": chapters_txt,
"audio_done": audio_files,
"last_update": now,
}
sql_update_book(book_idx, update_fields)
sql_after = sql_fetch_book(book_idx)
# ----------------------------------------------------
# 5. Resultaat voor inspect/debug
# ----------------------------------------------------
result = {
"book_idx": book_idx,
"filesystem": {
"book_dir": book_dir,
"exists": os.path.isdir(book_dir),
"volumes": volumes,
"chapters_txt": chapters_txt,
"audio_files": audio_files,
},
"sql_before": sql_before,
"sql_after": sql_after,
"notes": [],
}
log(
f"[STATUSCHECK] book_idx={book_idx} "
f"chapters={chapters_txt} audio={audio_files}"
)
return result

28
bookscraper/scraper/sites/__init__.py
Unescape View File

@ -1,28 +0,0 @@
# ============================================================
# File: scraper/sites/__init__.py
# Purpose:
# Site autodetection based on URL.
# ============================================================
from scraper.sites.piaotian import PiaotianScraper
def get_scraper_for_url(url: str):
"""
Return the correct scraper instance for a given URL.
Later: add more site implementations.
"""
if "ptwxz" in url or "piaotian" in url:
return PiaotianScraper()
raise ValueError(f"No scraper available for URL: {url}")
# ============================================================
# Backwards-compatibility export for legacy BookScraper
# ============================================================
# Old code expects:
# from scraper.sites import BookSite
# We map that to our new PiaotianScraper implementation.
BookSite = PiaotianScraper

52
bookscraper/scraper/sites/base.py
Unescape View File

@ -1,52 +0,0 @@
# ============================================================
# File: scraper/sites/base.py
# Purpose:
# Abstract interface that every site-specific scraper must implement.
# ============================================================
from abc import ABC, abstractmethod
from bs4 import BeautifulSoup
from typing import Optional
class SiteScraper(ABC):
"""
Defines the interface for site-specific scrapers.
Each concrete scraper (Piaotian, Biquge, etc.) must implement these.
"""
@property
@abstractmethod
def root(self) -> str: ...
@property
@abstractmethod
def encoding(self) -> str: ...
@property
@abstractmethod
def chapter_list_selector(self) -> str: ...
# --------------------------
# Metadata extraction
# --------------------------
@abstractmethod
def parse_title(self, soup: BeautifulSoup) -> str: ...
@abstractmethod
def parse_author(self, soup: BeautifulSoup) -> str: ...
@abstractmethod
def parse_description(self, soup: BeautifulSoup) -> str: ...
@abstractmethod
def parse_cover(self, soup: BeautifulSoup, url: str) -> Optional[str]: ...
# --------------------------
# Chapter extraction
# --------------------------
@abstractmethod
def extract_chapter_page_url(self, soup: BeautifulSoup) -> str: ...
@abstractmethod
def parse_chapter_list(self, soup: BeautifulSoup) -> list: ...

121
bookscraper/scraper/sites/piaotian.py
Unescape View File

@ -1,121 +0,0 @@
# ============================================================
# File: scraper/sites/piaotian.py
# Purpose:
# Concrete SiteScraper implementation for ptwxz.com (Piaotian).
# Moves all parsing logic out of BookScraper.
# ============================================================
from scraper.sites.base import SiteScraper
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import re
from typing import Optional
class PiaotianScraper(SiteScraper):
root = "https://www.ptwxz.com"
encoding = "GB18030"
chapter_list_selector = "div.centent"
# ------------------------------------------------------------
# METADATA PARSING
# ------------------------------------------------------------
def parse_title(self, soup: BeautifulSoup) -> str:
h1 = soup.find("h1")
return h1.get_text(strip=True) if h1 else "UnknownBook"
def parse_author(self, soup: BeautifulSoup) -> str:
td = soup.find("td", string=lambda t: t and "作" in t)
raw = td.get_text(strip=True) if td else ""
return raw.split("：")[1] if "：" in raw else "UnknownAuthor"
def parse_description(self, soup: BeautifulSoup) -> str:
span = soup.find("span", string=lambda t: t and "内容简介" in t)
if not span:
return ""
parts = []
for sib in span.next_siblings:
# stop when next <span> reappears
if getattr(sib, "name", None) == "span":
break
text = (
sib.get_text(strip=True)
if hasattr(sib, "get_text")
else str(sib).strip()
)
if text:
parts.append(text)
return "\n".join(parts)
# ------------------------------------------------------------
# COVER PARSING
# (exactly your BookScraper._parse_cover logic)
# ------------------------------------------------------------
def parse_cover(self, soup: BeautifulSoup, url: str) -> Optional[str]:
# Extract book_id from URL
m = re.search(r"/(\d+)\.html$", url)
if not m:
return None
book_id = m.group(1)
# Extract vol (bookinfo/<vol>/<id>.html)
m2 = re.search(r"/bookinfo/(\d+)/", url)
volume = m2.group(1) if m2 else None
imgs = soup.find_all("img", src=True)
chosen = None
# Priority 1: match "/files/article/image/{vol}/{book_id}/"
if volume:
target_path = f"/files/article/image/{volume}/{book_id}/"
for img in imgs:
src = img["src"]
if target_path in src:
chosen = src
break
# Priority 2: endswith "/{book_id}s.jpg"
if not chosen:
target_suffix = f"/{book_id}s.jpg"
for img in imgs:
src = img["src"]
if src.endswith(target_suffix):
chosen = src
break
if not chosen:
return None
return urljoin(self.root, chosen)
# ------------------------------------------------------------
# CHAPTER EXTRACTION
# ------------------------------------------------------------
def extract_chapter_page_url(self, soup: BeautifulSoup) -> str:
node = soup.select_one(
"html > body > div:nth-of-type(6) > div:nth-of-type(2) > div > table"
)
href = node.select_one("a").get("href")
return urljoin(self.root, href)
def parse_chapter_list(self, soup: BeautifulSoup) -> list:
cont = soup.select_one(self.chapter_list_selector)
items = cont.select("ul li a[href]") if cont else []
chapters = []
idx = 1
for a in items:
href = a.get("href")
if not href.endswith(".html"):
continue
title = a.get_text(strip=True)
full_url = urljoin(self.root, href)
chapters.append({"num": idx, "title": title, "url": full_url})
idx += 1
return chapters

146
bookscraper/scraper/tasks/audio_tasks.py
Unescape View File

@ -1,8 +1,5 @@
# ============================================================
# File: scraper/tasks/audio_tasks.py
# Purpose: Convert chapter text files into audio using macOS
# “say”, with Redis-based slot control.
# Updated: now uses db.repository for audio counters.
# ============================================================
from celery_app import celery_app
@ -10,81 +7,61 @@ from logbus.publisher import log
import os
import subprocess
import time
import socket
import os
from scraper.progress import inc_audio_done, inc_audio_skipped
# from db.repository import inc_audio_done
from scraper.abort import abort_requested
from scraper.logger_decorators import logcall
from redis import Redis
from urllib.parse import urlparse
from scraper.services.audio_completion import trigger_audio_completion_check
# NEW — unified repository façade
from db.repository import (
inc_audio_done,
inc_audio_skipped,
)
# Kies lokale redis als aanwezig, anders standaard backend
redis_url = os.getenv("REDIS_BACKEND_LOCAL") or os.getenv("REDIS_BACKEND")
HOST = socket.gethostname()
parsed = urlparse(redis_url)
# ------------------------------------------------------------
# REDIS CLIENT SETUP
# REGULIER REDIS CLIENT (slots, file checks, state)
# ------------------------------------------------------------
redis_url = os.getenv("REDIS_BACKEND_LOCAL") or os.getenv("REDIS_BACKEND")
parsed = urlparse(redis_url)
# Slot locking Redis client
redis_client = Redis(
host=parsed.hostname,
port=parsed.port,
db=parsed.path.strip("/"),
)
# Abort + global progress flags always live in DB 0
# ------------------------------------------------------------
# BACKEND CLIENT (abort flags, progress counters) - altijd DB 0
# ------------------------------------------------------------
backend_client = Redis(
host=parsed.hostname,
port=parsed.port,
db=0,
)
# ------------------------------------------------------------
# CONFIG
# ------------------------------------------------------------
AUDIO_TIMEOUT = int(os.getenv("AUDIO_TIMEOUT_SECONDS", "300"))
AUDIO_VOICE = os.getenv("AUDIO_VOICE", "SinJi")
AUDIO_RATE = int(os.getenv("AUDIO_RATE", "200"))
HOST_PATH = os.getenv("HOST_PATH", "/app/output")
CONTAINER_PREFIX = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "/app/output")
AUDIO_SLOTS = int(os.getenv("AUDIO_SLOTS", "1"))
CONTAINER_PREFIX = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "/app/output")
# ============================================================
# CELERY TASK
# ============================================================
@celery_app.task(bind=True, queue="audio", ignore_result=True)
@logcall
def generate_audio(
self, book_id, volume_name, chapter_number, chapter_title, chapter_path
self, book_id, volume_name, chapter_number, chapter_title, chapter_text
):
"""
chapter_path: absolute container path to chapter text file.
"""
log(f"[AUDIO]({HOST}) CH{chapter_number}: START → {chapter_title}")
log(f"[AUDIO] CH{chapter_number}: START task → raw_input={chapter_text}")
# ------------------------------------------------------------
# ABORT CHECK
# ------------------------------------------------------------
# Abort early
if abort_requested(book_id, backend_client):
inc_audio_skipped(book_id)
log(f"[AUDIO]({HOST}) ABORT detected → skip CH{chapter_number}")
log(f"[AUDIO] ABORT detected → skip CH{chapter_number}")
return
# ------------------------------------------------------------
# ACQUIRE SLOT
# ------------------------------------------------------------
# ============================================================
# ACQUIRE AUDIO SLOT
# ============================================================
slot_key = None
ttl = AUDIO_TIMEOUT + 15
@ -95,13 +72,11 @@ def generate_audio(
log(f"[AUDIO] CH{chapter_number}: Acquired slot {i}/{AUDIO_SLOTS}")
break
# Need to wait
if slot_key is None:
log(f"[AUDIO] CH{chapter_number}: All slots busy → waiting…")
log(f"[AUDIO] CH{chapter_number}: All slots busy → waiting...")
start_wait = time.time()
while slot_key is None:
# Try all slots again
for i in range(1, AUDIO_SLOTS + 1):
key = f"audio_slot:{i}"
if redis_client.set(key, "1", nx=True, ex=ttl):
@ -109,32 +84,32 @@ def generate_audio(
log(f"[AUDIO] CH{chapter_number}: Slot acquired after wait")
break
# If still no slot
if not slot_key:
if abort_requested(book_id, backend_client):
log(f"[AUDIO] ABORT while waiting → skip CH{chapter_number}")
inc_audio_skipped(book_id)
return
if slot_key:
break
if time.time() - start_wait > ttl:
log(f"[AUDIO] CH{chapter_number}: Wait timeout → abort audio")
inc_audio_skipped(book_id)
return
if abort_requested(book_id, backend_client):
log(f"[AUDIO] ABORT while waiting → skip CH{chapter_number}")
return
time.sleep(0.25)
if time.time() - start_wait > ttl:
log(f"[AUDIO] CH{chapter_number}: Slot wait timeout → aborting audio")
return
# ------------------------------------------------------------
time.sleep(0.25)
# ============================================================
# PATH NORMALISATION
# ------------------------------------------------------------
container_path = chapter_path
# ============================================================
container_path = chapter_text
# Fix 1 — container_path kan None zijn → abort zonder crash
if not container_path:
log(f"[AUDIO] CH{chapter_number}: ERROR — no input file path provided")
log(f"[AUDIO] CH{chapter_number}: FATAL — no input path provided")
redis_client.delete(slot_key)
inc_audio_skipped(book_id)
return
# Strip container prefix so that host path is resolvable
# Fix 2 — veilige startswith
if CONTAINER_PREFIX and container_path.startswith(CONTAINER_PREFIX):
relative_path = container_path[len(CONTAINER_PREFIX) :].lstrip("/")
else:
@ -143,36 +118,35 @@ def generate_audio(
parts = relative_path.split("/")
if len(parts) < 3:
log(
f"[AUDIO] CH{chapter_number}: ERROR — cannot parse book/volume from {relative_path}"
f"[AUDIO] CH{chapter_number}: FATAL — cannot parse book/volume from {relative_path}"
)
redis_client.delete(slot_key)
inc_audio_skipped(book_id)
return
# book_from_path = parts[0] # volume_name passed explicitly anyway
# volume_from_path = parts[1]
book_from_path = parts[0]
volume_from_path = parts[1]
host_path = os.path.join(HOST_PATH, relative_path)
# ------------------------------------------------------------
# OUTPUT DIRECTORY
# ------------------------------------------------------------
base_dir = os.path.join(HOST_PATH, parts[0], parts[1], "Audio")
# ============================================================
# OUTPUT PREP
# ============================================================
base_dir = os.path.join(HOST_PATH, book_from_path, volume_from_path, "Audio")
os.makedirs(base_dir, exist_ok=True)
safe_num = f"{chapter_number:04d}"
audio_file = os.path.join(base_dir, f"{safe_num}.m4b")
# Skip if audio already exists
if os.path.exists(audio_file):
log(f"[AUDIO] CH{chapter_number}: Already exists → skip")
log(f"[AUDIO] Skip CH{chapter_number} → already exists")
redis_client.delete(slot_key)
inc_audio_skipped(book_id)
trigger_audio_completion_check(book_id)
return
# ------------------------------------------------------------
# BUILD TTS COMMAND
# ------------------------------------------------------------
# ============================================================
# BUILD CMD
# ============================================================
cmd = (
f"say --voice={AUDIO_VOICE} "
f"--input-file='{host_path}' "
@ -183,36 +157,30 @@ def generate_audio(
f"--data-format=aac"
)
log(f"[AUDIO]({HOST}) CH{chapter_number} → output: {audio_file}")
log(f"[AUDIO] CH{chapter_number}: CMD = {cmd}")
# ------------------------------------------------------------
# EXECUTE
# ------------------------------------------------------------
# ============================================================
# RUN TTS
# ============================================================
try:
subprocess.run(cmd, shell=True, check=True, timeout=AUDIO_TIMEOUT)
# NEW — repository façade
inc_audio_done(book_id)
trigger_audio_completion_check(book_id)
log(f"trigger_audio_completion_check ")
log(f"[AUDIO]({HOST}) CH{chapter_number}: Completed")
log(f"[AUDIO] CH{chapter_number}: Completed")
except subprocess.TimeoutExpired:
log(f"[AUDIO]({HOST}) CH{chapter_number}: TIMEOUT → removing file")
log(f"[AUDIO] CH{chapter_number}: TIMEOUT → remove incomplete file")
if os.path.exists(audio_file):
try:
os.remove(audio_file)
except Exception:
pass
inc_audio_skipped(book_id)
except subprocess.CalledProcessError as e:
log(f"[AUDIO] CH{chapter_number}: ERROR during say → {e}")
inc_audio_skipped(book_id)
except Exception as e:
log(f"[AUDIO] CH{chapter_number}: UNEXPECTED ERROR → {e}")
inc_audio_skipped(book_id)
finally:
if slot_key:

173
bookscraper/scraper/tasks/controller_tasks.py
Unescape View File

@ -1,167 +1,106 @@
# ============================================================
# File: scraper/tasks/controller_tasks.py
# Purpose:
# FULL scrape entrypoint + launching download/parse/save pipelines.
# NO result.get() anywhere. Scraping is done inline.
# Start the download → parse → save pipeline for a scraped book,
# including progress/abort tracking via book_id.
# ONLY THE CONTROLLER UPDATES PROGRESS (initial total).
# ============================================================
from celery_app import celery_app
from logbus.publisher import log
import os
import time
import redis
from scraper.download_controller import DownloadController
from scraper.progress import (
set_total,
)
from urllib.parse import urlparse
from scraper.logger_decorators import logcall
import redis
import os
from scraper.abort import abort_requested
from scraper.services.scrape_engine import ScrapeEngine
from scraper.services.site_resolver import SiteResolver
from db.repository import fetch_book, set_chapters_total
from scraper.download_controller import DownloadController
print(">>> [IMPORT] controller_tasks.py loaded")
# =============================================================
# 1) PUBLIC ENTRYPOINT — CALLED FROM /start
# =============================================================
@celery_app.task(
bind=True,
queue="controller",
ignore_result=False,
name="scraper.tasks.controller_tasks.start_full_scrape",
)
@logcall
def start_full_scrape(self, book_idx: str):
@celery_app.task(bind=True, queue="controller", ignore_result=False)
def launch_downloads(self, book_id: str, scrape_result: dict):
"""
FULL SCRAPE ENTRYPOINT.
Scraping is done inline → no Celery .get() needed.
"""
log(f"[CTRL] start_full_scrape(book_idx={book_idx})")
# Abort before doing anything
if abort_requested(book_idx):
log(f"[CTRL] PRE-ABORT flag detected for {book_idx}")
return {"book_idx": book_idx, "aborted": True, "reason": "pre-abort"}
# --------------------------------------------------------
# 1) Load book metadata from SQLite
# --------------------------------------------------------
book = fetch_book(book_idx)
if not book:
msg = f"[CTRL] Book '{book_idx}' not found in DB"
log(msg)
raise ValueError(msg)
url = book.get("book_url")
if not url:
msg = f"[CTRL] No book_url stored for {book_idx}"
log(msg)
raise ValueError(msg)
# --------------------------------------------------------
# 2) INLINE SCRAPE (fast, no Celery wait)
# --------------------------------------------------------
site = SiteResolver.resolve(url)
try:
scrape_result = ScrapeEngine.fetch_metadata_and_chapters(site, url)
log(f"[CTRL] Scrape OK for {book_idx}: {scrape_result.get('title')}")
except Exception as e:
log(f"[CTRL] ERROR during scrape of {book_idx}: {e}")
raise
# --------------------------------------------------------
# 3) Continue → dispatch pipelines
# --------------------------------------------------------
return launch_downloads(book_idx, scrape_result)
Launch the entire pipeline (download → parse → save),
AND initialize progress counters.
# =============================================================
# 2) PIPELINE DISPATCH (NOT a Celery task)
# =============================================================
@logcall
def launch_downloads(book_idx: str, scrape_result: dict):
"""
Launches the entire processing pipeline:
- initialize Redis UI state
- initialize SQLite totals
- dispatch per-chapter pipelines via DownloadController
Chapter-level progress is updated INSIDE the download/parse/save tasks.
This task MUST NOT call .get() on async subtasks (Celery restriction).
"""
title = scrape_result.get("title", "UnknownBook")
chapters = scrape_result.get("chapters", []) or []
total = len(chapters)
# ------------------------------------------------------------
# INIT REDIS STATE
# INIT BOOK STATE MODEL (required for Active Books dashboard)
# ------------------------------------------------------------
broker_url = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
parsed = urlparse(broker_url)
r = redis.Redis(
state = redis.Redis(
host=parsed.hostname,
port=parsed.port,
db=int(parsed.path.strip("/")),
decode_responses=True,
)
base = f"book:{book_idx}:state"
# Book metadata
state.set(f"book:{book_id}:title", title)
state.set(f"book:{book_id}:status", "starting")
r.hset(base, "title", title)
r.hset(base, "status", "starting")
r.hset(base, "chapters_total", total)
r.hset(base, "chapters_download_done", 0)
r.hset(base, "chapters_download_skipped", 0)
r.hset(base, "chapters_parsed_done", 0)
r.hset(base, "audio_done", 0)
r.hset(base, "audio_skipped", 0)
r.hset(base, "last_update", int(time.time()))
# Download counters
state.set(f"book:{book_id}:download:total", total)
state.set(f"book:{book_id}:download:done", 0)
# Audio counters (start at zero)
state.set(f"book:{book_id}:audio:done", 0)
# ------------------------------------------------------------
# INIT SQLITE SNAPSHOT
# INIT PROGRESS
# ------------------------------------------------------------
try:
set_chapters_total(book_idx, total)
except Exception as e:
log(f"[CTRL] ERROR updating SQLite totals: {e}")
raise
log(f"[CTRL] Initialized totals for {book_idx}: {total}")
set_total(book_id, total)
log(f"[CTRL] Progress initialized for {book_id}: total={total}")
# ------------------------------------------------------------
# ABORT CHECK BEFORE LAUNCHING JOBS
# BUILD CONTROLLER
# ------------------------------------------------------------
if abort_requested(book_idx):
log(f"[CTRL] ABORT flag detected — stopping BEFORE dispatch for {book_idx}")
r.hset(base, "status", "aborted")
return {"book_idx": book_idx, "aborted": True, "reason": "abort-before-start"}
ctl = DownloadController(book_id, scrape_result)
# ------------------------------------------------------------
# BUILD + DISPATCH PER-CHAPTER PIPELINES
# START PIPELINES (ASYNC)
# Returns a celery group AsyncResult. We DO NOT iterate or get().
# Progress & failures are handled by the worker subtasks.
# ------------------------------------------------------------
controller = DownloadController(book_idx, scrape_result)
try:
group_result = controller.start()
gid = getattr(group_result, "id", None)
log(f"[CTRL] Pipelines dispatched for {book_idx} (group_id={gid})")
except Exception as e:
log(f"[CTRL] ERROR dispatching pipelines for {book_idx}: {e}")
group_result = ctl.start()
log(
f"[CTRL] Pipelines dispatched for '{title}' "
f"(book_id={book_id}, group_id={group_result.id})"
)
# Abort flag set BEFORE tasks start?
if abort_requested(book_id):
log(f"[CTRL] ABORT requested before tasks start")
return {"book_id": book_id, "aborted": True}
except Exception as exc:
log(f"[CTRL] ERROR while dispatching pipelines: {exc}")
raise
# Update UI state to "downloading"
r.hset(base, "status", "downloading")
r.hset(base, "last_update", int(time.time()))
# ------------------------------------------------------------
# CONTROLLER DOES NOT WAIT FOR SUBTASK RESULTS
# (Download/parse/save tasks update progress themselves)
# ------------------------------------------------------------
log(f"[CTRL] Controller finished dispatch for book_id={book_id}")
return {
"book_idx": book_idx,
"book_id": book_id,
"total": total,
"started": True,
"group_id": gid,
"group_id": group_result.id,
}

158
bookscraper/scraper/tasks/download_tasks.py
Unescape View File

@ -1,24 +1,26 @@
# ============================================================
# File: scraper/tasks/download_tasks.py
# Purpose:
# Download chapter HTML into payload["html"].
# Updated for book_idx unified ID model.
# Purpose: Download chapter HTML with global concurrency,
# retry/backoff logic, 429 support, and abort-awareness.
#
# Logging:
# - timestamp + book_id in message
# - logbus.publisher → console
# - ui_log.push_ui → Redis GUI
# ============================================================
from celery_app import celery_app
from scraper.utils.utils import get_save_path
from scraper.utils import get_save_path
from scraper.abort import abort_requested, chapter_started, mark_chapter_started
# Unified repository façade
from db.repository import (
set_status,
inc_download_done,
inc_download_skipped,
from scraper.progress import (
inc_completed,
inc_chapter_done,
inc_chapter_download_skipped,
)
from db.repository import inc_downloaded, set_status
from logbus.publisher import log
from scraper.ui_log import push_ui
from scraper.logger_decorators import logcall
import requests
import redis
@ -33,9 +35,9 @@ print(">>> [IMPORT] download_tasks.py loaded")
# -----------------------------------------------------------
# TIMESTAMPED LOG WRAPPER
# -----------------------------------------------------------
def log_msg(book_idx: str, message: str):
def log_msg(book_id: str, message: str):
ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
full = f"{ts} [{book_idx}] {message}"
full = f"{ts} [{book_id}] {message}"
log(full)
push_ui(full)
@ -88,63 +90,46 @@ def release_global_slot():
# ============================================================
# CELERY TASK — Payload v3 (book_idx model)
# CELERY TASK — NEW SIGNATURE WITH chapter_dict + book_meta
# ============================================================
@celery_app.task(bind=True, queue="download", ignore_result=False)
@logcall
def download_chapter(self, payload: dict):
def download_chapter(self, book_id: str, chapter_dict: dict, book_meta: dict):
"""
Payload format:
{
"book_idx": str,
"chapter": {
New unified chapter model:
chapter_dict = {
"num": int,
"title": str,
"url": str,
"title": str,
"volume_path": str
},
"book_meta": dict,
# fields filled during pipeline:
"html": None | str,
"parsed": None | str,
"skipped": bool,
"path": None | str
}
"""
if not payload:
raise ValueError("download_chapter received empty payload")
}
book_idx = payload["book_idx"]
chapter = payload["chapter"]
book_meta = payload.get("book_meta") or {}
chapter_num = chapter["num"]
chapter_url = chapter["url"]
chapter_title = chapter.get("title") or f"Chapter {chapter_num}"
volume_path = chapter["volume_path"]
book_meta is propagated through the pipeline for parse/save.
"""
# -----------------------------------------------------------
# STATUS UPDATE (book is now in 'downloading')
# -----------------------------------------------------------
set_status(book_idx, "downloading")
chapter_num = chapter_dict.get("num")
chapter_url = chapter_dict.get("url")
chapter_title = chapter_dict.get("title") or f"Chapter {chapter_num}"
volume_path = chapter_dict.get("volume_path")
# -----------------------------------------------------------
# ABORT CHECK (skip if not yet started)
# ABORT BEFORE START
# -----------------------------------------------------------
if abort_requested(book_idx) and not chapter_started(book_idx, chapter_num):
log_msg(book_idx, f"[ABORT] Skip chapter {chapter_num}")
inc_download_skipped(book_idx)
payload["html"] = None
payload["skipped"] = True
payload["path"] = None
return payload
mark_chapter_started(book_idx, chapter_num)
if abort_requested(book_id) and not chapter_started(book_id, chapter_num):
msg = f"[ABORT] Skip chapter {chapter_num} (abort active, not started)"
log_msg(book_id, msg)
inc_chapter_download_skipped(book_id)
return {
"book_id": book_id,
"chapter": chapter_dict,
"html": None,
"skipped": True,
"path": None,
"abort": True,
"book_meta": book_meta,
}
# Mark chapter as started
mark_chapter_started(book_id, chapter_num)
# -----------------------------------------------------------
# SKIP IF FILE ALREADY EXISTS
@ -152,28 +137,34 @@ def download_chapter(self, payload: dict):
save_path = get_save_path(chapter_num, volume_path)
if os.path.exists(save_path):
log_msg(book_idx, f"[DL] SKIP {chapter_num} → {save_path}")
inc_download_skipped(book_idx)
payload["html"] = None
payload["skipped"] = True
payload["path"] = save_path
return payload
log_msg(book_id, f"[DL] SKIP {chapter_num} ({chapter_title}) → {save_path}")
return {
"book_id": book_id,
"chapter": chapter_dict,
"html": None,
"skipped": True,
"path": save_path,
"book_meta": book_meta,
}
# -----------------------------------------------------------
# GLOBAL DELAY + CONCURRENCY
# GLOBAL + SYNC DELAY
# -----------------------------------------------------------
if GLOBAL_DELAY > 0:
time.sleep(GLOBAL_DELAY)
wait_for_global_delay()
acquire_global_slot(MAX_CONCURRENCY)
# log_msg(book_id, f"[DL] ACQUIRED SLOT for chapter {chapter_num}")
# -----------------------------------------------------------
# HTTP DOWNLOAD
# -----------------------------------------------------------
try:
log_msg(book_idx, f"[DL] Downloading {chapter_num} ({chapter_title})")
log_msg(
book_id,
f"[DL] Downloading {chapter_num} ({chapter_title}): {chapter_url}",
)
resp = requests.get(
chapter_url,
@ -185,28 +176,41 @@ def download_chapter(self, payload: dict):
resp.encoding = resp.apparent_encoding or "gb2312"
html = resp.text
log_msg(book_idx, f"[DL] OK {chapter_num}: {len(html)} bytes")
log_msg(book_id, f"[DL] OK {chapter_num}: {len(html)} bytes")
payload["html"] = html
payload["skipped"] = False
payload["path"] = save_path
return payload
return {
"book_id": book_id,
"chapter": chapter_dict,
"html": html,
"skipped": False,
"path": save_path,
"book_meta": book_meta,
}
except Exception as exc:
attempt = self.request.retries
delay = BASE_DELAY * (BACKOFF**attempt)
# Handle 429
# Specific 429 handler
if getattr(getattr(exc, "response", None), "status_code", None) == 429:
log_msg(book_idx, f"[DL] 429 → WAIT {DELAY_429}s")
log_msg(
book_id,
f"[DL] 429 {chapter_num} → WAIT {DELAY_429}s "
f"(attempt {attempt}/{MAX_RETRIES})",
)
time.sleep(DELAY_429)
set_global_delay()
raise self.retry(exc=exc, countdown=0, max_retries=MAX_RETRIES)
# General retry with backoff
log_msg(book_idx, f"[DL] ERROR {chapter_num}: {exc} → retry {delay}s")
# Normal retry
log_msg(
book_id,
f"[DL] ERROR {chapter_num}: {exc} → retry in {delay}s "
f"(attempt {attempt}/{MAX_RETRIES})",
)
raise self.retry(exc=exc, countdown=delay, max_retries=MAX_RETRIES)
finally:
set_global_delay()
release_global_slot()
# log_msg(book_id, f"[DL] RELEASED SLOT for chapter {chapter_num}")

132
bookscraper/scraper/tasks/m4b_tasks.py
Unescape View File

@ -1,132 +0,0 @@
# ============================================================
# File: scraper/tasks/m4b_tasks.py
# ============================================================
import os
import subprocess
from typing import List
from celery_app import celery_app
from logbus.publisher import log
from scraper.logger_decorators import logcall
from db.repository import fetch_book, store_m4b_error
from scraper.scriptgen import build_merge_block
# ------------------------------------------------------------
# Helper: detect volumes (UNCHANGED)
# ------------------------------------------------------------
def detect_volumes(book_base: str) -> List[str]:
volumes = []
for name in os.listdir(book_base):
if name.lower().startswith("volume_"):
full = os.path.join(book_base, name)
if os.path.isdir(full):
volumes.append(name)
volumes.sort()
return volumes
# ------------------------------------------------------------
# Celery task
# ------------------------------------------------------------
@celery_app.task(bind=True, queue="m4b", ignore_result=True)
@logcall
def run_m4btool(self, book_idx: str):
log(f"[M4B] START book_idx={book_idx}")
book = fetch_book(book_idx)
if not book:
log(f"[M4B] Book not found in SQL: book_idx={book_idx}")
return
title = book.get("title", book_idx)
author = book.get("author", "Unknown")
output_root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
book_base = os.path.join(output_root, title)
log(f"[M4B] Book base directory: {book_base}")
if not os.path.isdir(book_base):
log(f"[M4B] Book directory missing: {book_base}")
return
volumes = detect_volumes(book_base)
if not volumes:
log(f"[M4B] No volumes found for book_idx={book_idx}")
return
log(f"[M4B] Volumes detected: {volumes}")
# --------------------------------------------------------
# Build canonical commands via scriptgen
# --------------------------------------------------------
merge_block = build_merge_block(
title, author, [(i + 1, v) for i, v in enumerate(volumes)]
)
commands = [c.strip() for c in merge_block.split("&&") if c.strip()]
for volume, cmd in zip(volumes, commands):
audio_dir = os.path.join(book_base, volume, "Audio")
if not os.path.isdir(audio_dir):
log(f"[M4B] SKIP {volume}: no Audio directory")
continue
log(f"[M4B] Running for volume={volume}")
log(f"[M4B] CMD: {cmd}")
try:
result = subprocess.run(
cmd,
cwd=book_base,
shell=True,
capture_output=True,
text=True,
check=True,
)
if result.stdout:
log(f"[M4B][STDOUT] {result.stdout}")
except subprocess.CalledProcessError as exc:
log(f"[M4B][FAILED] volume={volume}")
if exc.stdout:
log(f"[M4B][STDOUT] {exc.stdout}")
if exc.stderr:
log(f"[M4B][STDERR] {exc.stderr}")
store_m4b_error(
book_idx=book_idx,
volume=volume,
error_text=exc.stderr or str(exc),
)
continue
except Exception as exc:
log(f"[M4B][UNEXPECTED ERROR] volume={volume}: {exc}")
store_m4b_error(
book_idx=book_idx,
volume=volume,
error_text=str(exc),
)
continue
log(f"[M4B] FINISHED book_idx={book_idx}")
# ------------------------------------------------------------
# Orchestration helper (UNCHANGED)
# ------------------------------------------------------------
@logcall
def queue_m4b_for_book(book_idx: str):
log(f"[M4B] Queuing m4b-tool for book_idx={book_idx}")
celery_app.send_task(
"scraper.tasks.m4b_tasks.run_m4btool",
args=[book_idx],
queue="m4b",
)

178
bookscraper/scraper/tasks/parse_tasks.py
Unescape View File

@ -1,31 +1,33 @@
# ============================================================
# =========================================================
# File: scraper/tasks/parse_tasks.py
# Purpose: Parse downloaded HTML into clean chapter text.
# Enhanced Piaotia extractor + selector fallback + clean pipeline.
# Compatible with payload pipeline v3 + book_idx refactor.
# ============================================================
# Enhanced version: Piaotia H1→content extractor + clean pipeline
# NO HARDCODED REPLACEMENTS — everything comes from replacement files
# =========================================================
from celery_app import celery_app
from bs4 import BeautifulSoup, NavigableString, Comment
from bs4 import BeautifulSoup
from scraper.tasks.download_tasks import log_msg
from scraper.utils.utils import clean_text, load_all_replacements
from scraper.logger_decorators import logcall
from db.repository import inc_parsed_done
from scraper.utils import clean_text, load_all_replacements
from scraper.tasks.download_tasks import log_msg # unified logger
from bs4 import NavigableString, Comment
print(">>> [IMPORT] parse_tasks.py loaded (book_idx + payload v3)")
print(">>> [IMPORT] parse_tasks.py loaded (enhanced parser)")
# ============================================================
# PIAOTIA ADVANCED CONTENT EXTRACTOR
# ============================================================
def extract_piaotia_content(soup):
"""
Extract clean chapter content from Piaotia pages.
Start after the table following <H1>.
End before nav/ads/footer/copyright.
"""
h1 = soup.find("h1")
if not h1:
return None
# Find first table after <h1>
# -------- Find first table after <h1> --------
table = None
for sib in h1.next_siblings:
if getattr(sib, "name", None) == "table":
@ -37,39 +39,49 @@ def extract_piaotia_content(soup):
parts = []
# -------- Iterate after table --------
for sib in table.next_siblings:
name = getattr(sib, "name", None)
text = None
if hasattr(sib, "get_text"):
text = sib.get_text(strip=True)
# Stop conditions
# === STOP CONDITIONS ===
# Comments like <!-- 翻页上AD开始 -->
if isinstance(sib, Comment) and ("翻页" in sib):
break
# Explicit footer blocks
if name == "div":
sid = sib.get("id", "")
cls = sib.get("class", [])
if sid in ("thumb", "tags", "tips", "Commenddiv", "feit2"):
break
# Copyright block — strongest indicator
if text and ("重要声明" in text or "Copyright" in text):
break
# Navigation or 推荐阅读
if text and (text.startswith(("推荐阅读", "目录", "目 录"))):
break
# Skip scripts, ads, centers
if name in ("script", "style"):
continue
# Skip JS containers like <center><script>...</script></center>
if name == "center":
continue
# Accumulate
# === ACCUMULATE TEXT ===
if isinstance(sib, NavigableString):
s = sib.strip()
if s:
parts.append(s)
elif hasattr(sib, "get_text"):
t = sib.get_text(separator="\n").strip()
if t:
@ -78,44 +90,35 @@ def extract_piaotia_content(soup):
return "\n".join(parts).strip()
# ============================================================
# PARSE TASK — PAYLOAD PIPELINE v3 (book_idx)
# ============================================================
@celery_app.task(bind=True, queue="parse", ignore_result=False)
@logcall
def parse_chapter(self, payload: dict):
if not payload:
return {"skipped": True, "reason": "empty_payload"}
# NEW MODEL
book_idx = payload["book_idx"]
chapter = payload["chapter"]
book_meta = payload.get("book_meta") or {}
num = chapter.get("num")
title = chapter.get("title") or f"Chapter {num}"
html = payload.get("html")
def parse_chapter(self, download_result: dict):
"""
New signature under chapter_dict pipeline:
- receives ONLY the output dict from download_chapter
- book_meta is inside download_result["book_meta"]
- chapter_dict is inside download_result["chapter"]
"""
book_id = download_result.get("book_id", "NOBOOK")
chapter_dict = download_result.get("chapter") or {}
book_meta = download_result.get("book_meta") or {}
chapter_title = chapter_dict.get("title")
chapter_num = chapter_dict.get("num")
chapter_url = chapter_dict.get("url")
html = download_result.get("html")
# ------------------------------------------------------------
# DOWNLOAD SKIPPED → PARSE SKIP
# SKIPPED DOWNLOAD → SKIP PARSE
# ------------------------------------------------------------
if payload.get("skipped"):
log_msg(book_idx, f"[PARSE] SKIP chapter {num} (download skipped)")
return payload
if not html:
log_msg(book_idx, f"[PARSE] Missing HTML for chapter {num} → SKIP")
payload["parsed"] = None
payload["skipped"] = True
return payload
if download_result.get("skipped"):
log_msg(book_id, f"[PARSE] SKIP chapter {chapter_num} (download skipped)")
return download_result # already has chapter + book_meta + skipped
log_msg(book_idx, f"[PARSE] Parsing chapter {num}")
log_msg(book_id, f"[PARSE] Parsing chapter {chapter_num}")
soup = BeautifulSoup(html, "lxml")
# ------------------------------------------------------------
# STRICT SELECTORS
# STRICT SELECTORS (direct content blocks)
# ------------------------------------------------------------
selectors = [
"#content",
@ -138,20 +141,50 @@ def parse_chapter(self, payload: dict):
raw = None
# strict selectors failed → piaotia extractor
# --- STRICT SELECTOR FAILED → Try Piaotia extractor ---
if node is None:
raw = extract_piaotia_content(soup)
else:
raw = node.get_text(separator="\n")
# # ------------------------------------------------------------
# # PIAOTIA FALLBACK:
# # Extract content between <H1> and the "bottomlink" block.
# # ------------------------------------------------------------
# raw = None
# if node is None:
# h1 = soup.find("h1")
# if h1:
# content_parts = []
# for sib in h1.next_siblings:
# sib_class = getattr(sib, "get", lambda *_: None)("class")
# if sib_class and (
# "bottomlink" in sib_class or sib_class == "bottomlink"
# ):
# break
# if getattr(sib, "name", None) in ["script", "style", "center"]:
# continue
# if hasattr(sib, "get_text"):
# content_parts.append(sib.get_text(separator="\n"))
# else:
# content_parts.append(str(sib))
# raw = "\n".join(content_parts)
# ------------------------------------------------------------
# FINAL FALLBACK
# ------------------------------------------------------------
if raw is None:
for tag in soup(["script", "style", "noscript"]):
tag.decompose()
raw = soup.get_text(separator="\n")
if node:
raw = node.get_text(separator="\n")
else:
for tag in soup(["script", "style", "noscript"]):
tag.decompose()
raw = soup.get_text(separator="\n")
# ------------------------------------------------------------
# MULTIPASS CLEANING VIA replacement-block files
# MULTIPASS CLEANING via replacement files ONLY
# ------------------------------------------------------------
REPL = load_all_replacements()
@ -160,30 +193,28 @@ def parse_chapter(self, payload: dict):
text = clean_text(text, REPL)
# ------------------------------------------------------------
# Collapse double blank lines
# Collapse excessive empty lines
# ------------------------------------------------------------
cleaned = []
prev_blank = False
for line in text.split("\n"):
s = line.rstrip()
if s == "":
stripped = line.rstrip()
if stripped == "":
if prev_blank:
continue
prev_blank = True
cleaned.append("")
else:
prev_blank = False
cleaned.append(s)
cleaned.append(stripped)
text = "\n".join(cleaned)
text = f"{title}\n{text}"
text = chapter_title + "\n" + text
# ------------------------------------------------------------
# Header on chapter 1
# Add header to chapter 1
# ------------------------------------------------------------
if num == 1:
book_url = book_meta.get("book_url") or "UNKNOWN"
if chapter_num == 1:
book_url = book_meta.get("book_url") or book_meta.get("url") or "UNKNOWN"
header = (
f"{book_meta.get('title','')}\n"
f"Author: {book_meta.get('author','')}\n"
@ -192,14 +223,13 @@ def parse_chapter(self, payload: dict):
)
text = header + text
log_msg(book_idx, f"[PARSE] Parsed chapter {num}: {len(text)} chars")
# ------------------------------------------------------------
# OUTPUT PAYLOAD
# ------------------------------------------------------------
payload["parsed"] = text
payload["skipped"] = False
inc_parsed_done(book_idx)
log_msg(book_id, f"[PARSE] Parsed chapter {chapter_num}: {len(text)} chars")
return payload
# NEW RETURN FORMAT: chapter_dict stays intact
return {
"book_id": book_id,
"chapter": chapter_dict,
"text": text,
"length": len(text),
"book_meta": book_meta,
}

51
bookscraper/scraper/tasks/pipeline.py
Unescape View File

@ -1,16 +1,16 @@
# =========================================================
# File: scraper/tasks/pipeline.py
# Purpose:
# Build Celery chains for chapter processing using payload dict.
# Build Celery chains for chapter processing using chapter_dict.
#
# Pipeline v3:
# download_chapter(payload)
# → parse_chapter(payload)
# → save_chapter(payload)
# New Chain:
# download_chapter(book_id, chapter_dict, book_meta)
# → parse_chapter(download_result)
# → save_chapter(parsed_result)
# → update_progress(final_result, book_id)
#
# NOTE:
# - book_idx is the single authoritative key for all tasks
# - payload travels unchanged through the entire pipeline
# All subtasks pass through result dicts unchanged so the
# next stage receives the correct fields.
# =========================================================
from celery import chain
@ -18,33 +18,26 @@ from celery import chain
from scraper.tasks.download_tasks import download_chapter
from scraper.tasks.parse_tasks import parse_chapter
from scraper.tasks.save_tasks import save_chapter
from scraper.tasks.progress_tasks import update_progress
from scraper.logger_decorators import logcall
@logcall
def build_chapter_pipeline(book_idx: str, chapter_dict: dict, book_meta: dict):
"""
Create a payload object passed through the pipeline.
Consistent with the chapter_dict-based task signature.
def build_chapter_pipeline(
book_id: str,
chapter_dict: dict,
book_meta: dict,
):
"""
Build a Celery chain for one chapter using chapter_dict.
payload = {
"book_idx": book_idx,
"chapter": chapter_dict,
"book_meta": book_meta,
# Will be filled by download_chapter
"html": None,
# Will be filled by parse_chapter
"parsed": None,
# Set by download or parse on skip/404/etc
"skipped": False,
# Final path written by save_chapter
"path": None,
}
download_chapter(book_id, chapter_dict, book_meta)
→ parse_chapter(download_result)
→ save_chapter(parsed_result)
→ update_progress(result, book_id)
"""
return chain(
download_chapter.s(payload),
download_chapter.s(book_id, chapter_dict, book_meta),
parse_chapter.s(),
save_chapter.s(),
update_progress.s(book_id),
)

57
bookscraper/scraper/tasks/progress_tasks.py
Unescape View File

@ -0,0 +1,57 @@
# ============================================================
# File: scraper/tasks/progress_tasks.py
# Purpose: Central progress updater for chapter pipelines.
# Updated for chapter_dict pipeline model.
# ============================================================
from celery_app import celery_app
from scraper.progress import inc_completed, inc_skipped, inc_failed
from logbus.publisher import log
print(">>> [IMPORT] progress_tasks.py loaded")
@celery_app.task(bind=False, name="progress.update", queue="controller")
def update_progress(result: dict, book_id: str):
"""
Central progress logic:
- result: output of save_chapter
- book_id: explicitly passed by pipeline
IMPORTANT:
- save_chapter already updates counters for skipped & normal chapters
- progress.update MUST NOT double-increment
"""
ch = result.get("chapter") or {}
chapter_num = ch.get("num")
skipped = result.get("skipped", False)
failed = result.get("failed", False)
# ------------------------------------------------------------
# FAILED CASE
# ------------------------------------------------------------
if failed:
inc_failed(book_id)
log(f"[PROG] FAILED chapter {chapter_num}")
return result
# ------------------------------------------------------------
# SKIPPED CASE
# ------------------------------------------------------------
if skipped:
# save_chapter already did:
# inc_skipped(book_id)
log(f"[PROG] SKIPPED chapter {chapter_num}")
return result
# ------------------------------------------------------------
# NORMAL COMPLETION
# ------------------------------------------------------------
# save_chapter did NOT increment completed for skipped cases
# but DID inc_completed(book_id) for normal cases.
# update_progress should NOT double increment, so only log here.
log(f"[PROG] DONE chapter {chapter_num}")
return result

177
bookscraper/scraper/tasks/save_tasks.py
Unescape View File

@ -1,84 +1,139 @@
# ============================================================
# File: scraper/tasks/save_tasks.py (RESTORED AUDIO LOGIC + book_idx)
# File: scraper/tasks/save_tasks.py
# Purpose: Save parsed chapter text to disk + trigger audio.
# Updated for chapter_dict + book_meta pipeline model.
# ============================================================
print(">>> [IMPORT] save_tasks.py loaded")
from celery import shared_task
import os
from logbus.publisher import log
from scraper.logger_decorators import logcall
from scraper.utils.utils import get_save_path
from scraper.tasks.download_tasks import log_msg
from scraper.utils import get_save_path
from scraper.tasks.download_tasks import log_msg # unified logger
from scraper.progress import (
inc_completed,
inc_chapter_done,
inc_chapter_download_skipped,
)
from scraper.tasks.audio_tasks import generate_audio
from db.repository import inc_download_done, inc_download_skipped
@shared_task(bind=True, queue="save", ignore_result=False)
@logcall
def save_chapter(self, payload: dict):
if not payload:
log("[SAVE] ERROR: payload is None")
return {"error": True}
# NEW unified ID
book_idx = payload["book_idx"]
chapter = payload["chapter"]
parsed = payload.get("parsed")
path = payload.get("path")
skipped = payload.get("skipped")
num = chapter["num"]
title = chapter.get("title") or f"Chapter {num}"
volume = chapter.get("volume_path")
volume_name = os.path.basename(volume.rstrip("/"))
# ============================================================
# SKIPPED CASE (old behavior restored)
# ============================================================
if skipped or not parsed:
log_msg(book_idx, f"[SAVE] SKIP chapter {num}")
inc_download_skipped(book_idx)
# OLD behavior: even skipped chapters still queue audio
def save_chapter(self, parsed: dict):
"""
New pipeline model:
parsed = {
"book_id": str,
"chapter": chapter_dict,
"text": str,
"length": int,
"book_meta": dict,
"skipped": bool,
"path": optional str (if skipped)
}
"""
book_id = parsed.get("book_id", "NOBOOK")
chapter_dict = parsed.get("chapter") or {}
book_meta = parsed.get("book_meta") or {}
chapter_num = chapter_dict.get("num")
chapter_title = chapter_dict.get("title") or f"Chapter {chapter_num}"
volume_path = chapter_dict.get("volume_path")
# ------------------------------------------------------------
# VALIDATION
# ------------------------------------------------------------
if chapter_num is None or volume_path is None:
raise ValueError("Invalid parsed payload: chapter_dict missing fields.")
# ------------------------------------------------------------
# SKIPPED CASE
# ------------------------------------------------------------
if parsed.get("skipped"):
path = parsed.get("path", None)
log_msg(book_id, f"[SAVE] SKIP chapter {chapter_num} → {path}")
inc_chapter_download_skipped(book_id)
volume_name = os.path.basename(volume_path.rstrip("/"))
# Queue audio only if a valid file exists
if path and os.path.exists(path):
log_msg(book_idx, f"[AUDIO] Queueing audio for SKIPPED chapter {num}")
try:
generate_audio.delay(book_idx, volume_name, num, title, path)
except Exception as exc:
log_msg(book_idx, f"[AUDIO] ERROR queueing skipped audio: {exc}")
return payload
# ============================================================
generate_audio.delay(
book_id,
volume_name,
chapter_num,
chapter_title,
path,
)
log_msg(
book_id,
f"[AUDIO] Task queued (SKIPPED) for chapter {chapter_num} in {volume_name}",
)
except Exception as audio_exc:
log_msg(
book_id,
f"[AUDIO] ERROR queueing (SKIPPED) chapter {chapter_num}: {audio_exc}",
)
return {
"book_id": book_id,
"chapter": chapter_dict,
"path": path,
"skipped": True,
"book_meta": book_meta,
}
# ------------------------------------------------------------
# NORMAL SAVE CASE
# ============================================================
# ------------------------------------------------------------
try:
os.makedirs(volume, exist_ok=True)
save_path = get_save_path(num, volume)
text = parsed.get("text", "")
with open(save_path, "w", encoding="utf-8") as f:
f.write(parsed)
# Ensure volume folder exists
os.makedirs(volume_path, exist_ok=True)
log_msg(book_idx, f"[SAVE] Saved chapter {num} → {save_path}")
# Build final chapter file path
path = get_save_path(chapter_num, volume_path)
inc_download_done(book_idx)
# Write chapter text to file
with open(path, "w", encoding="utf-8") as f:
f.write(text)
# OLD behavior: ALWAYS queue audio
try:
generate_audio.delay(book_idx, volume_name, num, title, save_path)
log_msg(book_idx, f"[AUDIO] Task queued for chapter {num}")
except Exception as exc:
log_msg(book_idx, f"[AUDIO] ERROR queueing chapter {num}: {exc}")
log_msg(book_id, f"[SAVE] Saved chapter {chapter_num} → {path}")
inc_chapter_done(book_id)
inc_completed(book_id)
# Determine volume name
volume_name = os.path.basename(volume_path.rstrip("/"))
payload["path"] = save_path
payload["skipped"] = False
return payload
# Queue audio task
try:
generate_audio.delay(
book_id,
volume_name,
chapter_num,
chapter_title,
path,
)
log_msg(
book_id,
f"[AUDIO] Task queued for chapter {chapter_num} in {volume_name}",
)
except Exception as audio_exc:
log_msg(
book_id, f"[AUDIO] ERROR queueing chapter {chapter_num}: {audio_exc}"
)
return {
"book_id": book_id,
"chapter": chapter_dict,
"path": path,
"book_meta": book_meta,
}
except Exception as exc:
log_msg(book_idx, f"[SAVE] ERROR saving chapter {num}: {exc}")
log_msg(book_id, f"[SAVE] ERROR saving chapter {chapter_num}: {exc}")
raise

97
bookscraper/scraper/tasks/scraping.py
Unescape View File

@ -1,9 +1,7 @@
# ============================================================
# File: scraper/tasks/scraping.py
# Purpose:
# Scrape ONLY metadata + chapter list.
# Does NOT launch download controller anymore.
# Controller decides when pipelines start.
# Purpose: Scrape metadata + chapter list and initialise
# Redis progress tracking + launch download controller
# ============================================================
from celery_app import celery_app
@ -11,91 +9,88 @@ from logbus.publisher import log
import os
import redis
from scraper.logger_decorators import logcall
from scraper.sites import BookSite
from scraper.book_scraper import BookScraper
from scraper.abort import clear_abort
from scraper.ui_log import reset_ui_logs
from scraper.services.init_service import InitService
from scraper.abort import clear_abort # no circular deps
from scraper.ui_log import reset_ui_logs # <-- NEW IMPORT
print(">>> [IMPORT] scraping.py loaded")
# Redis connection (same DB as Celery broker)
# Redis connection (same as Celery broker)
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
@celery_app.task(
bind=True,
queue="scraping",
ignore_result=False,
name="scraper.tasks.scraping.start_scrape_book",
)
@logcall
@celery_app.task(bind=True, queue="scraping", ignore_result=False)
def start_scrape_book(self, url: str):
"""
Scrapes metadata + chapters.
DOES NOT START download / pipeline controller.
The controller_tasks.start_full_scrape() task will call this one.
"""
"""Scrapes metadata + chapters and prepares download tracking."""
# ------------------------------------------------------------
# CLEAR UI LOG BUFFER
# NEW: clear UI log buffer at start of new run
# ------------------------------------------------------------
reset_ui_logs()
log(f"[SCRAPING] Start scraping for: {url}")
# ------------------------------------------------------------
# SCRAPE (old engine)
# Book scrape
# ------------------------------------------------------------
site = BookSite()
scraper = BookScraper(site, url)
result = scraper.execute() # → { title, author, chapters, cover_url, ... }
result = scraper.execute() # returns dict with metadata + chapters
chapters = result.get("chapters", [])
full_count = len(chapters)
# ------------------------------------------------------------
# Compute unified book_idx
# ------------------------------------------------------------
book_idx = InitService.derive_book_id(url)
result["book_idx"] = book_idx
log(f"[SCRAPING] Assigned book_idx = {book_idx}")
# ------------------------------------------------------------
# DRY RUN TEST LIMIT
# DRY RUN
# ------------------------------------------------------------
DRY_RUN = os.getenv("DRY_RUN", "0") == "1"
TEST_LIMIT = int(os.getenv("TEST_LIMIT", "5"))
if DRY_RUN:
log(f"[SCRAPING] DRY RUN active → limiting chapters to {TEST_LIMIT}")
result["chapters"] = chapters[:TEST_LIMIT]
log(f"[SCRAPING] DRY_RUN: limiting chapters to {TEST_LIMIT}")
chapters = chapters[:TEST_LIMIT]
result["chapters"] = chapters
log(f"[SCRAPING] Completed scrape: {len(chapters)}/{full_count} chapters")
# ------------------------------------------------------------
# LOG RESULTS
# BOOK RUN ID (using title as ID)
# ------------------------------------------------------------
log(
f"[SCRAPING] Completed scrape: "
f"{len(result['chapters'])}/{full_count} chapters"
)
title = result.get("title") or "UnknownBook"
book_id = title # user requirement
result["book_id"] = book_id
log(f"[SCRAPING] Assigned book_id = '{book_id}'")
# ------------------------------------------------------------
# RESET ABORT + INITIALIZE LEGACY PROGRESS
# RESET ABORT + INITIALISE PROGRESS
# ------------------------------------------------------------
clear_abort(book_idx)
clear_abort(book_id)
r.set(f"progress:{book_idx}:total", len(result["chapters"]))
r.set(f"progress:{book_idx}:done", 0)
r.set(f"progress:{book_id}:total", len(chapters))
r.set(f"progress:{book_id}:done", 0)
r.delete(f"logs:{book_id}") # clear old logs if any
r.delete(f"logs:{book_idx}")
r.rpush(f"logs:{book_idx}", f":: SCRAPING STARTED for {url}")
r.rpush(f"logs:{book_idx}", f":: Found {len(result['chapters'])} chapters")
r.rpush(f"logs:{book_id}", f":: SCRAPING STARTED for {url}")
r.rpush(f"logs:{book_id}", f":: Found {len(chapters)} chapters")
# ------------------------------------------------------------
# IMPORTANT: DO NOT DISPATCH any pipelines here
# Controller will receive scrape_result and continue.
# DISPATCH DOWNLOAD CONTROLLER
# ------------------------------------------------------------
return result
celery_app.send_task(
"scraper.tasks.controller_tasks.launch_downloads",
args=[book_id, result],
queue="controller",
)
log(f"[SCRAPING] Dispatched download controller for '{book_id}'")
return {
"book_id": book_id,
"title": result.get("title"),
"author": result.get("author"),
"chapters": len(chapters),
}

149
bookscraper/scraper/tasks/statuscheck.py
Unescape View File

@ -1,149 +0,0 @@
# ============================================================
# File: scraper/tasks/statuscheck.py
# Purpose:
# Final status check after audio completion.
#
# Responsibilities:
# - Verify Redis counters (sanity check)
# - Verify filesystem (Audio files present)
# - Queue m4btool task
#
# Design rules:
# - Book-scope ONLY
# - No direct Redis usage
# - Repository is the single source of truth
# - Idempotent, defensive, non-blocking
# ============================================================
import os
from celery_app import celery_app
from logbus.publisher import log
from scraper.logger_decorators import logcall
from db.repository import (
get_audio_done,
get_chapters_total,
set_status,
fetch_book,
)
from scraper.tasks.m4b_tasks import run_m4btool
# ------------------------------------------------------------
# Helpers
# ------------------------------------------------------------
@log
def _detect_volumes(book_base: str):
"""
Return sorted list of Volume_XXX directories.
"""
vols = []
for name in os.listdir(book_base):
if name.lower().startswith("volume_"):
full = os.path.join(book_base, name)
if os.path.isdir(full):
vols.append(name)
vols.sort()
return vols
@logcall
def _count_audio_files(audio_dir: str) -> int:
"""
Count .m4b files in an Audio directory.
"""
if not os.path.isdir(audio_dir):
return 0
return len([f for f in os.listdir(audio_dir) if f.lower().endswith(".m4b")])
# ------------------------------------------------------------
# Celery task
# ------------------------------------------------------------
@celery_app.task(bind=True, queue="controller", ignore_result=True)
@logcall
def run_statuscheck(self, book_idx: str):
"""
Final statuscheck before m4btool execution.
Triggered exactly once by audio_completion quickcheck.
"""
log(f"[STATUSCHECK] START book={book_idx}")
# --------------------------------------------------------
# 1. Redis sanity check (via repository)
# --------------------------------------------------------
audio_done = get_audio_done(book_idx)
chapters_total = get_chapters_total(book_idx)
log(
f"[STATUSCHECK] Counters book={book_idx} "
f"audio_done={audio_done} chapters_total={chapters_total}"
)
if chapters_total <= 0:
log(f"[STATUSCHECK] No chapters_total → abort")
return
if audio_done < chapters_total:
# Defensive: should not happen, but never assume
log(
f"[STATUSCHECK] Audio not complete yet "
f"({audio_done}/{chapters_total}) → abort"
)
return
# --------------------------------------------------------
# 2. Fetch book metadata (for paths & m4b meta)
# --------------------------------------------------------
book = fetch_book(book_idx)
if not book:
log(f"[STATUSCHECK] Book not found in DB: {book_idx}")
return
title = book.get("title") or book_idx
author = book.get("author") or "Unknown"
# Base output directory
root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
book_base = os.path.join(root, title)
if not os.path.isdir(book_base):
log(f"[STATUSCHECK] Book directory missing: {book_base}")
return
# --------------------------------------------------------
# 3. Filesystem validation (light, non-blocking)
# --------------------------------------------------------
volumes = _detect_volumes(book_base)
if not volumes:
log(f"[STATUSCHECK] No volumes found for {book_idx}")
# Still allow m4btool to decide (it will no-op)
else:
for vol in volumes:
audio_dir = os.path.join(book_base, vol, "Audio")
count = _count_audio_files(audio_dir)
log(f"[STATUSCHECK] {vol}: " f"{count} audio files detected")
# --------------------------------------------------------
# 4. Queue m4btool (final pipeline step)
# --------------------------------------------------------
log(f"[STATUSCHECK] Queue m4btool for book={book_idx}")
set_status(book_idx, "m4b_running")
run_m4btool.delay(
book_idx=book_idx,
book_base=book_base,
meta={
"title": title,
"author": author,
},
)
log(f"[STATUSCHECK] DONE book={book_idx}")

0
bookscraper/scraper/utils/utils.py → bookscraper/scraper/utils.py
Unescape View File

0
bookscraper/scraper/utils/__init__.py
Unescape View File

272
bookscraper/scraper/utils/state_sync.py
Unescape View File

@ -1,272 +0,0 @@
# ============================================================
# File: scraper/utils/state_sync.py
# Purpose:
# State inspection + optional sync logic for unified book_idx model.
# Generates full book-card compatible dicts for debug UI.
# ============================================================
import os
import redis
from db.db import get_db
def _build_card(sqlite_row, redis_state, merged):
"""
Creates a dict that matches the fields required by components/bookcard.html:
b.book_idx
b.title
b.author
b.cover_path
b.status
b.created_at
b.download_done
b.download_total
b.audio_done
b.audio_total
"""
return {
"book_idx": sqlite_row.get("book_idx"),
"title": sqlite_row.get("title") or "Unknown",
"author": sqlite_row.get("author"),
"cover_path": sqlite_row.get("cover_path"),
# Use merged status (Redis > SQLite)
"status": merged.get("status") or sqlite_row.get("status") or "unknown",
# Meta
"created_at": sqlite_row.get("created_at"),
# Download counters
"download_done": merged.get("downloaded", 0),
"download_total": merged.get("chapters_total", 0),
# Audio counters
"audio_done": merged.get("audio_done", 0),
"audio_total": merged.get("chapters_total", 0),
}
# ============================================================
# INSPECT ONLY — NO WRITES
# ============================================================
def inspect_books_state_depecrated():
"""
Reads all books from SQLite and fetches Redis progress.
Builds:
• entry.sqlite
• entry.redis
• entry.would_merge_to
• entry.card (book-card compatible)
"""
r = redis.Redis.from_url(os.getenv("REDIS_BROKER"), decode_responses=True)
db = get_db()
cur = db.cursor()
cur.execute("SELECT * FROM books")
rows = cur.fetchall()
results = []
for row in rows:
sqlite_row = dict(row)
book_idx = sqlite_row["book_idx"]
redis_key = f"book:{book_idx}:state"
redis_state = r.hgetall(redis_key) or {}
# ================================
# DRY-RUN MERGE LOGIC
# ================================
merged = sqlite_row.copy()
if redis_state:
merged["downloaded"] = int(
redis_state.get("chapters_download_done", merged.get("downloaded", 0))
)
merged["parsed"] = int(
redis_state.get("chapters_parsed_done", merged.get("parsed", 0))
)
merged["audio_done"] = int(
redis_state.get("audio_done", merged.get("audio_done", 0))
)
merged["chapters_total"] = int(
redis_state.get("chapters_total", merged.get("chapters_total", 0))
)
merged["status"] = redis_state.get(
"status", merged.get("status", "unknown")
)
# ================================
# Build book-card data
# ================================
card = _build_card(sqlite_row, redis_state, merged)
# ================================
# Append final result entry
# ================================
results.append(
{
"book_idx": book_idx,
"title": sqlite_row.get("title"),
"sqlite": sqlite_row,
"redis": redis_state,
"would_merge_to": merged,
"card": card,
}
)
return results
# ============================================================
# INSPECT ONLY — NO WRITES
# ============================================================
def inspect_books_state():
"""
Reads canonical book state from repository.
Builds:
• entry.sqlite
• entry.redis
• entry.would_merge_to
• entry.card (book-card compatible)
"""
from db.repository import get_book_state
from db.db import get_db
db = get_db()
cur = db.cursor()
# Alleen nodig om te weten *welke* books er zijn
cur.execute("SELECT book_idx FROM books")
rows = cur.fetchall()
results = []
for row in rows:
book_idx = row["book_idx"]
# --------------------------------
# Canonical state (ENIGE waarheid)
# --------------------------------
state = get_book_state(book_idx)
# SQLite-view = alleen SQLite-kolommen
sqlite_view = {
k: v
for k, v in state.items()
if k
in (
"book_idx",
"title",
"author",
"description",
"cover_path",
"book_url",
"chapters_total",
"status",
"downloaded",
"parsed",
"audio_done",
"created_at",
"processdate",
"last_update",
)
}
# Redis-view = alleen Redis counters/status
redis_view = {
k: v
for k, v in state.items()
if k.startswith("chapters_")
or k in ("status", "audio_done", "audio_skipped")
}
merged = state # letterlijk de canonieke state
card = _build_card(sqlite_view, redis_view, merged)
results.append(
{
"book_idx": book_idx,
"title": state.get("title"),
"sqlite": sqlite_view,
"redis": redis_view,
"would_merge_to": merged,
"card": card,
}
)
return results
# ============================================================
# SYNC REDIS → SQLITE (writes)
# ============================================================
def sync_books_from_redis():
"""
Writes Redis progress values back into SQLite.
Uses unified book_idx as identifier.
"""
r = redis.Redis.from_url(os.getenv("REDIS_BROKER"), decode_responses=True)
db = get_db()
cur = db.cursor()
cur.execute("SELECT * FROM books")
rows = cur.fetchall()
results = []
for row in rows:
before = dict(row)
book_idx = before["book_idx"]
redis_key = f"book:{book_idx}:state"
redis_state = r.hgetall(redis_key)
if not redis_state:
results.append(
{
"book_idx": book_idx,
"before": before,
"redis": {},
"after": before,
}
)
continue
# Extract progress from Redis
downloaded = int(redis_state.get("chapters_download_done", 0))
parsed = int(redis_state.get("chapters_parsed_done", 0))
audio_done = int(redis_state.get("audio_done", 0))
total = int(redis_state.get("chapters_total", 0))
status = redis_state.get("status", before.get("status"))
# Update SQLite
cur.execute(
"""
UPDATE books
SET downloaded = ?, parsed = ?, audio_done = ?, chapters_total = ?, status = ?, last_update = datetime('now')
WHERE book_idx = ?
""",
(downloaded, parsed, audio_done, total, status, book_idx),
)
db.commit()
cur.execute("SELECT * FROM books WHERE book_idx = ?", (book_idx,))
after = dict(cur.fetchone())
results.append(
{
"book_idx": book_idx,
"before": before,
"redis": redis_state,
"after": after,
}
)
return results

BIN
bookscraper/static/covers/一剑朝天.jpg
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.1 KiB

BIN
bookscraper/static/covers/从吞噬开始.jpg
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.4 KiB

BIN
bookscraper/static/covers/流氓高手.jpg
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.5 KiB

BIN
bookscraper/static/covers/流氓高手II.jpg
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

310
bookscraper/static/css/bookcard.css
Unescape View File

@ -1,310 +0,0 @@
/* =======================================================================
File: static/css/bookcard.css
Purpose:
Styling voor registered book cards:
- status kleuren
- badges
- start/abort/statuscheck
- progress bars
======================================================================= */
/* -----------------------------------------------------------------------
GRID WRAPPER
----------------------------------------------------------------------- */
.registered-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(340px, 1fr));
gap: 20px;
margin-top: 15px;
}
/* -----------------------------------------------------------------------
BOOK CARD BASE
----------------------------------------------------------------------- */
.book-card {
position: relative;
display: grid;
grid-template-columns: 90px auto;
gap: 15px;
padding: 15px;
background: #fff;
border-radius: 10px;
border: 1px solid #e5e5e5;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05);
transition: border-color 0.25s ease, box-shadow 0.25s ease;
}
/* -----------------------------------------------------------------------
STATUS COLORS (BOOK CARD BORDER)
----------------------------------------------------------------------- */
/* Downloading / actief bezig */
.book-card.downloading {
border-color: #ff9500;
box-shadow: 0 0 6px rgba(255, 149, 0, 0.35);
}
/* Audio fase */
.book-card.audio {
border-color: #ffca28;
box-shadow: 0 0 6px rgba(255, 202, 40, 0.35);
}
/* Volledig klaar */
.book-card.done {
border: 2px solid #4caf50;
box-shadow: 0 0 6px rgba(76, 175, 80, 0.35);
}
/* Afgebroken */
.book-card.aborted {
border-color: #ff3b30;
box-shadow: 0 0 6px rgba(255, 59, 48, 0.35);
}
/* -----------------------------------------------------------------------
COVER
----------------------------------------------------------------------- */
.book-cover {
width: 90px;
}
.book-img {
width: 90px;
height: 130px;
object-fit: cover;
border-radius: 4px;
background: #f4f4f4;
}
.placeholder {
display: flex;
justify-content: center;
align-items: center;
color: #777;
font-size: 12px;
}
/* -----------------------------------------------------------------------
META
----------------------------------------------------------------------- */
.book-meta {
display: flex;
flex-direction: column;
justify-content: space-between;
}
.book-title {
font-size: 16px;
font-weight: bold;
}
.book-author {
font-size: 14px;
color: #444;
margin-bottom: 6px;
}
.book-created {
font-size: 12px;
color: #666;
}
/* -----------------------------------------------------------------------
ACTION BUTTONS
----------------------------------------------------------------------- */
.book-actions {
display: flex;
justify-content: flex-end;
gap: 10px;
margin-top: 10px;
}
.icon-btn {
width: 34px;
height: 34px;
border: none;
border-radius: 8px;
display: flex;
justify-content: center;
align-items: center;
font-size: 16px;
color: #fff;
cursor: pointer;
transition: background 0.15s ease, transform 0.1s ease;
}
/* Start */
.icon-start {
background: #2d8a3d;
}
.icon-start:hover {
background: #226c30;
transform: scale(1.05);
}
.icon-start:disabled {
background: #9bbb9f;
cursor: not-allowed;
opacity: 0.5;
}
/* Abort */
.icon-abort {
background: #c62828;
}
.icon-abort:hover {
background: #a31f1f;
transform: scale(1.05);
}
.icon-abort:disabled {
background: #d8a0a0;
cursor: not-allowed;
opacity: 0.5;
}
/* Hide */
.hide-form {
position: absolute;
top: 6px;
right: 6px;
}
.icon-hide {
background: #777;
}
.icon-hide:hover {
background: #555;
}
/* Statuscheck */
.statuscheck-btn {
background-color: #444;
color: #fff;
border: 1px solid #666;
margin-left: 4px;
padding: 4px 8px;
border-radius: 6px;
font-size: 12px;
cursor: pointer;
}
.statuscheck-btn:hover {
background-color: #333;
}
/* -----------------------------------------------------------------------
PROGRESS (FULL WIDTH)
----------------------------------------------------------------------- */
.book-progress {
grid-column: 1 / -1;
margin-top: 12px;
padding: 10px 12px;
background: #f6f6f6;
border-radius: 8px;
}
.progress-row {
margin-bottom: 4px;
}
.progress-label {
font-size: 12px;
margin-bottom: 4px;
color: #444;
}
/* BAR */
.progressbar {
position: relative;
width: 100%;
height: 14px;
background: #ddd;
border-radius: 7px;
overflow: hidden;
}
.progressbar-fill {
height: 100%;
transition: width 0.4s ease;
}
/* Download */
.progressbar-fill.download {
background: #2196f3;
}
/* Audio */
.progressbar-fill.audio {
background: #4caf50;
}
/* TEXT IN BAR */
.progressbar-text {
position: absolute;
inset: 0;
display: flex;
align-items: center;
justify-content: center;
font-size: 11px;
font-weight: 600;
color: #fff;
text-shadow: 0 1px 2px rgba(0, 0, 0, 0.6);
pointer-events: none;
}
/* -----------------------------------------------------------------------
STATUS BADGE
----------------------------------------------------------------------- */
.status-badge {
display: inline-block;
margin-bottom: 6px;
padding: 2px 8px;
font-size: 11px;
font-weight: 600;
border-radius: 10px;
text-transform: uppercase;
letter-spacing: 0.5px;
cursor: default;
}
/* DONE */
.status-badge.status-done {
background-color: #e6f4ea;
color: #2e7d32;
border: 1px solid #4caf50;
}
/* AUDIO */
.status-badge.status-audio {
background-color: #fff8e1;
color: #8d6e00;
border: 1px solid #ffca28;
}
/* DOWNLOADING */
.status-badge.status-downloading {
background-color: #e3f2fd;
color: #1565c0;
border: 1px solid #42a5f5;
}
/* Statuscheck */
.icon-statuscheck {
background: #444;
}
.icon-statuscheck:hover {
background: #333;
transform: scale(1.05);
}

190
bookscraper/static/css/dashboard.css
Unescape View File

@ -2,28 +2,33 @@
File: static/css/dashboard.css
Purpose:
Clean full-width vertical dashboard layout with large log viewer.
Book-card CSS is now moved to bookcard.css
======================================================================= */
/* -----------------------------------------------------------------------
1) GENERAL PAGE LAYOUT
----------------------------------------------------------------------- */
/* ------------------------------
GENERAL PAGE LAYOUT
------------------------------ */
/* Dashboard content should use full width */
.dashboard-container {
display: flex;
flex-direction: column;
width: 100%;
max-width: 1200px;
max-width: 1200px; /* voorkomt overflow rechts */
margin: 20px auto;
padding: 0 20px;
gap: 18px;
gap: 18px; /* kleiner dan 30px */
}
/* ------------------------------
SECTIONS (input, progress, logs)
------------------------------ */
.dashboard-section {
background: #ffffff;
padding: 16px;
padding: 16px; /* kleiner */
border-radius: 6px;
border: 1px solid #ddd;
margin: 0; /* weg extra witruimte */
}
.page-title {
@ -31,9 +36,9 @@
margin-bottom: 15px;
}
/* -----------------------------------------------------------------------
2) ACTIVE BOOK LIST (dashboard left panel)
----------------------------------------------------------------------- */
/* ------------------------------
BOOK LIST (optional)
------------------------------ */
.book-list {
display: flex;
@ -47,6 +52,7 @@
color: #777;
}
/* List item */
.book-list-item {
padding: 12px 16px;
background: #f7f7f7;
@ -67,6 +73,7 @@
border-color: #1e88e5;
}
/* Title + metadata */
.book-title {
font-size: 16px;
font-weight: 600;
@ -77,15 +84,20 @@
color: #555;
}
/* -----------------------------------------------------------------------
3) PROGRESS BOX
----------------------------------------------------------------------- */
.meta-label {
font-weight: 600;
}
/* ------------------------------
PROGRESS BOX
------------------------------ */
.progress-box {
background: #fafafa;
border: 1px solid #ddd;
padding: 8px;
padding: 18px;
border-radius: 6px;
width: 100%;
}
.progress-header h2 {
@ -129,35 +141,14 @@
margin-top: 4px;
}
.book-abort-area {
margin-top: 10px;
text-align: right;
}
.abort-btn {
padding: 6px 12px;
border-radius: 4px;
border: 1px solid #cc0000;
background: #ff4444;
color: white;
font-size: 12px;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.abort-btn:hover {
background: #ff2222;
border-color: #aa0000;
}
/* -----------------------------------------------------------------------
4) LOG VIEWER
----------------------------------------------------------------------- */
/* ------------------------------
LOG VIEWER — LARGE FULL-WIDTH
------------------------------ */
.log-viewer {
width: 100%;
max-width: 100%;
overflow: hidden;
overflow: hidden; /* voorkom horizontaal uitsteken */
}
.log-header {
@ -180,11 +171,11 @@
max-height: 75vh;
overflow-y: auto;
overflow-x: hidden;
overflow-x: hidden; /* voorkom dat de log naar rechts uitsteekt */
background: #000;
color: #00ff66;
border: 1px solid #0f0;
background: #000000; /* Pure terminal black */
color: #00ff66; /* Matrix / retro green */
border: 1px solid #0f0; /* neon green frame */
border-radius: 6px;
padding: 12px;
@ -192,39 +183,48 @@
font-size: 13px;
line-height: 1.35;
white-space: pre-wrap;
word-break: break-word;
white-space: pre-wrap; /* wraps text */
word-break: break-word; /* lange links breken */
}
/* Basestijl voor alle logregels */
.log-line {
white-space: pre-wrap;
padding: 2px 0;
font-family: "SF Mono", "Consolas", "Courier New", monospace;
}
/* Subklassen per logtype */
.log-line.default {
color: #00ff66;
color: #00ff66; /* groen */
}
.log-line.dl {
color: #00ccff;
color: #00ccff; /* cyan */
}
.log-line.parse {
color: #ffaa00;
color: #ffaa00; /* oranje */
}
.log-line.save {
color: #ffdd33;
color: #ffdd33; /* geel */
}
.log-line.audio {
color: #ff66ff;
color: #ff66ff; /* paars */
}
.log-line.ctrl {
color: #66aaff;
color: #66aaff; /* lichtblauw */
}
.log-line.error {
color: #ff3333;
color: #ff3333; /* rood */
}
/* -----------------------------------------------------------------------
5) PLACEHOLDER / FOOTER
----------------------------------------------------------------------- */
/* ------------------------------
PLACEHOLDER
------------------------------ */
.dashboard-placeholder {
font-size: 15px;
@ -232,7 +232,6 @@
text-align: center;
color: #777;
}
.footer {
text-align: center;
padding: 12px;
@ -241,72 +240,23 @@
font-size: 12px;
border-top: 1px solid #ddd;
}
/* -----------------------------
DROPDOWN NAVIGATION
------------------------------ */
/* Container for dropdown */
.nav-dropdown {
position: relative;
}
/* The clickable label ("Tools ▾") */
.nav-dropdown > .nav-item {
cursor: pointer;
.book-abort-area {
margin-top: 10px;
text-align: right;
}
/* Hide dropdown by default */
.dropdown-menu {
display: none;
position: absolute;
top: 100%;
right: 0;
background: #fff; /* zelfde achtergrond als navbar */
border: 1px solid #ddd;
padding: 8px 0;
margin: 0;
list-style: none; /* verwijder bolletjes */
.abort-btn {
padding: 6px 12px;
border-radius: 4px;
min-width: 160px;
z-index: 1000;
}
/* Show dropdown when hovering over parent */
.nav-dropdown:hover .dropdown-menu {
display: block;
}
/* Menu item styling */
.dropdown-menu li {
padding: 0;
margin: 0;
}
.dropdown-menu li a {
display: block;
padding: 8px 16px;
white-space: nowrap;
color: #333;
text-decoration: none;
}
/* Hover state */
.dropdown-menu li a:hover {
background: #f0f0f0;
}
table.kv {
border-collapse: collapse;
margin-bottom: 16px;
}
table.kv th {
text-align: left;
padding-right: 12px;
color: #777;
font-weight: normal;
border: 1px solid #cc0000;
background: #ff4444;
color: white;
font-size: 12px;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
table.kv td {
font-weight: 500;
.abort-btn:hover {
background: #ff2222;
border-color: #aa0000;
}

145
bookscraper/static/js/bookcard_controller.js
Unescape View File

@ -1,145 +0,0 @@
/* ============================================================
File: static/js/bookcard_controller.js
Purpose:
Single owner for updating book-card DOM from merged state
(would_merge_to)
============================================================ */
console.log("[BOOKCARD] controller loaded");
/* ============================================================
ENTRY POINT (called by state_updater.js)
============================================================ */
function updateBookCardsFromState(stateList) {
console.log("[BOOKCARD] updateBookCardsFromState called");
if (!Array.isArray(stateList)) {
console.warn("[BOOKCARD] Invalid stateList", stateList);
return;
}
const stateById = {};
stateList.forEach((entry) => {
const merged = entry.would_merge_to;
if (!merged || merged.book_idx == null) {
console.warn("[BOOKCARD] entry without merged/book_idx", entry);
return;
}
stateById[String(merged.book_idx)] = merged;
});
document.querySelectorAll(".book-card").forEach((card) => {
const bookIdx = card.dataset.bookIdx;
const state = stateById[bookIdx];
if (!state) {
console.debug("[BOOKCARD] No state for book_idx", bookIdx);
return;
}
console.log("[BOOKCARD] Updating card", bookIdx, state.status);
updateSingleBookCard(card, state);
});
}
/* ============================================================
SINGLE CARD UPDATE
============================================================ */
function updateSingleBookCard(card, state) {
console.log("[BOOKCARD] updateSingleBookCard", state.book_idx);
updateStatus(card, state);
updateStatusBadge(card, state);
updateButtons(card, state);
updateProgress(card, state);
}
/* ============================================================
STATUS
============================================================ */
function updateStatus(card, state) {
console.log("[BOOKCARD][STATUS]", state.book_idx, "→", state.status);
card.className = `book-card ${state.status || ""}`;
}
function updateStatusBadge(card, state) {
const badge = card.querySelector(".status-badge");
if (!badge) return;
const status = (state.status || "").toLowerCase();
badge.textContent = status.toUpperCase();
badge.className = `status-badge status-${status}`;
badge.title =
{
downloading: "Bezig met downloaden",
audio: "Downloads compleet, audio wordt gegenereerd",
done: "Alle chapters en audio zijn compleet",
}[status] || "";
}
/* ============================================================
BUTTONS
============================================================ */
function updateButtons(card, state) {
const startBtn = card.querySelector(".icon-start");
const abortBtn = card.querySelector(".icon-abort");
const busy = ["starting", "downloading", "parsing", "audio"];
console.log("[BOOKCARD][BUTTONS]", state.book_idx, "status:", state.status);
if (startBtn) {
// startBtn.disabled = busy.includes(state.status);
}
if (abortBtn) {
abortBtn.disabled = !busy.includes(state.status);
}
}
/* ============================================================
PROGRESS (DOWNLOAD + AUDIO)
============================================================ */
function updateProgress(card, s) {
const total = Number(s.chapters_total || 0);
// const downloadDone =
// Number(s.chapters_download_done || 0) +
// Number(s.chapters_download_skipped || 0);
const downloadDone = Number(s.downloaded || 0);
const audioDone = Number(s.audio_done || 0) + Number(s.audio_skipped || 0);
const downloadPct =
total > 0 ? Math.min((downloadDone / total) * 100, 100) : 0;
const audioPct = total > 0 ? Math.min((audioDone / total) * 100, 100) : 0;
console.log("[BOOKCARD][PROGRESS]", s.book_idx, {
total,
downloadDone,
audioDone,
downloadPct,
audioPct,
});
/* ---- DOWNLOAD ---- */
const dlBar = card.querySelector('[data-field="download_pct"]');
const dlText = card.querySelector('[data-field="download_text"]');
if (dlBar) dlBar.style.width = `${downloadPct}%`;
if (dlText) dlText.textContent = `${downloadDone} / ${total}`;
/* ---- AUDIO ---- */
const auBar = card.querySelector('[data-field="audio_pct"]');
const auText = card.querySelector('[data-field="audio_text"]');
if (auBar) auBar.style.width = `${audioPct}%`;
if (auText) auText.textContent = `${audioDone} / ${total}`;
}

284
bookscraper/static/js/dashboard.js
Unescape View File

@ -1,178 +1,200 @@
/* =======================================================================
File: static/js/dashboard.js
Purpose:
- Sidebar selectie
- Start / Abort acties
- UI status updates
NOTE:
- GEEN polling
- state_updater.js is leidend
Dashboard interactions:
- select book
- refresh logs
- refresh progress
NOTE:
$ / $$ / autoScroll komen uit helpers.js
======================================================================= */
console.log("[DASHBOARD] loaded");
/* ---------------------------------------------------------
Helpers
--------------------------------------------------------- */
Simple fetch wrapper
--------------------------------------------------------- */
async function apiGet(url) {
console.log("[DASHBOARD][API] GET", url);
try {
const r = await fetch(url, { cache: "no-store" });
if (!r.ok) {
console.warn("[DASHBOARD][API] GET failed", url, r.status);
return null;
}
const r = await fetch(url);
if (!r.ok) return null;
return await r.json();
} catch (e) {
console.error("[DASHBOARD][API] GET error", url, e);
console.error("API GET failed:", url, e);
return null;
}
}
function safeUpdateLogs(data) {
if (typeof window.updateLogs === "function") {
console.log("[DASHBOARD] updateLogs()");
window.updateLogs(data);
}
}
/* ---------------------------------------------------------
State
--------------------------------------------------------- */
let ACTIVE_BOOK_IDX = null;
Dashboard state
--------------------------------------------------------- */
let ACTIVE_BOOK = null;
let REFRESH_INTERVAL = null;
console.log(">>> dashboard.js LOADED");
/* ---------------------------------------------------------
DOM READY
--------------------------------------------------------- */
DOM Ready → setup
--------------------------------------------------------- */
document.addEventListener("DOMContentLoaded", () => {
console.log("[DASHBOARD] DOMContentLoaded");
bindSidebar();
bindBookCardButtons();
console.log(">>> dashboard.js DOMContentLoaded");
// =====================================================
// GLOBAL FALLBACK POLLING — ALWAYS FETCH LOGS
// Runs when no books exist or no selection has been made
// =====================================================
console.log(">>> dashboard.js: enabling global fallback polling");
setInterval(() => {
// if no active book → fetch global logs
if (!ACTIVE_BOOK) {
refreshBook(null); // triggers /logs
}
}, 2000);
const items = $$(".book-list-item");
console.log(">>> dashboard.js found book-list items:", items.length);
// Geen boeken → geen polling starten
// if (!items || items.length === 0) {
// console.log(">>> dashboard.js: geen boeken aanwezig, polling uit.");
// return;
// }
// Book selection listener
items.forEach((item) => {
item.addEventListener("click", () => {
console.log(">>> dashboard.js: user clicked book:", item.dataset.bookId);
selectBook(item.dataset.bookId);
});
});
const first = document.querySelector(".book-list-item");
if (first) {
console.log("[DASHBOARD] auto-select", first.dataset.bookIdx);
selectBook(first.dataset.bookIdx);
// Auto-select first book
if (!ACTIVE_BOOK && items[0]) {
console.log(
">>> dashboard.js: auto-select first book:",
items[0].dataset.bookId
);
selectBook(items[0].dataset.bookId);
}
});
/* ---------------------------------------------------------
Sidebar
--------------------------------------------------------- */
function bindSidebar() {
console.log("[DASHBOARD] bindSidebar()");
document.querySelectorAll(".book-list-item").forEach((item) => {
item.onclick = () => selectBook(item.dataset.bookIdx);
Select a book (updates UI + starts polling)
--------------------------------------------------------- */
function selectBook(bookId) {
console.log(">>> selectBook(", bookId, ")");
ACTIVE_BOOK = bookId;
// Highlight
$$(".book-list-item").forEach((el) => {
el.classList.toggle("active", el.dataset.bookId === bookId);
});
// Reset previous polling
if (REFRESH_INTERVAL) {
console.log(">>> dashboard.js: clearing previous polling interval");
clearInterval(REFRESH_INTERVAL);
}
// Start new polling
console.log(">>> dashboard.js: starting polling for bookId =", bookId);
REFRESH_INTERVAL = setInterval(() => {
refreshBook(ACTIVE_BOOK);
}, 2000);
// Immediate refresh
refreshBook(ACTIVE_BOOK);
}
setInterval(refreshActiveBooks, 2000);
async function refreshActiveBooks() {
const books = await apiGet("/api/books");
if (!books) return;
const container = $("#book-list");
if (!container) return;
function selectBook(bookIdx) {
if (!bookIdx || bookIdx === ACTIVE_BOOK_IDX) return;
// Herbouw de lijst
container.innerHTML = "";
books.forEach((b) => {
const div = document.createElement("div");
div.className = "book-list-item";
div.dataset.bookId = b.book_id;
ACTIVE_BOOK_IDX = bookIdx;
console.log("[DASHBOARD] selectBook", bookIdx);
div.innerHTML = `
<div class="book-title">${b.title}</div>
<div class="book-status">${b.status}</div>
<div class="book-progress">
${b.download_done}/${b.download_total} downloaded,
${b.audio_done}/${b.audio_total} audio
</div>
document.querySelectorAll(".book-list-item").forEach((el) => {
el.classList.toggle("active", el.dataset.bookIdx === bookIdx);
<button class="abort-btn" onclick="abortBook('${b.book_id}')">Abort</button>
`;
// Event listener opnieuw koppelen
div.addEventListener("click", () => selectBook(b.book_id));
container.appendChild(div);
});
refreshBook(bookIdx);
// Als ACTIVE_BOOK nog niet bekend → auto-selecteer eerste boek
if (!ACTIVE_BOOK && books.length > 0) {
selectBook(books[0].book_id);
}
}
/* ---------------------------------------------------------
Book refresh (NO POLLING)
--------------------------------------------------------- */
async function refreshBook(bookIdx) {
console.log("[DASHBOARD] refreshBook", bookIdx);
Fetch logs + progress from API
--------------------------------------------------------- */
async function refreshBook(bookId) {
console.log(">>> refreshBook(", bookId, ")");
const logs = await apiGet(`/api/book/${bookIdx}/logs`);
if (logs) safeUpdateLogs(logs);
// 1) Als er GEEN bookId is → haal alleen globale logs op
if (!bookId) {
console.log(">>> refreshBook: no active book → fetch /logs");
refreshBookCards();
}
const data = await apiGet("/logs");
if (data && data.logs) updateLogs(data.logs);
/* ---------------------------------------------------------
Bookcard buttons
--------------------------------------------------------- */
function bindBookCardButtons() {
console.log("[DASHBOARD] bindBookCardButtons()");
document.querySelectorAll(".icon-start").forEach((btn) => {
if (btn.dataset.bound) return;
btn.dataset.bound = "1";
btn.onclick = (e) => {
e.preventDefault();
const card = btn.closest(".book-card");
if (!card) return;
startBook(card.dataset.bookIdx);
};
});
return; // klaar
}
document.querySelectorAll(".icon-abort").forEach((btn) => {
if (btn.dataset.bound) return;
btn.dataset.bound = "1";
// 2) Als er WEL een boek is → haal book status + logs op
const state = await apiGet(`/api/book/${bookId}/status`);
const logs = await apiGet(`/api/book/${bookId}/logs`);
btn.onclick = (e) => {
e.preventDefault();
const card = btn.closest(".book-card");
if (!card) return;
abortBook(card.dataset.bookIdx);
};
});
}
console.log(">>> refreshBook state =", state);
console.log(">>> refreshBook logs =", logs);
/* ---------------------------------------------------------
START
--------------------------------------------------------- */
function startBook(bookIdx) {
console.log("[DASHBOARD] START", bookIdx);
fetch("/start", {
method: "POST",
headers: { "Content-Type": "application/x-www-form-urlencoded" },
body: `book_idx=${bookIdx}`,
}).then(() => refreshBook(bookIdx));
if (state) updateProgressBars(state);
if (logs) updateLogs(logs);
}
/* ---------------------------------------------------------
ABORT
--------------------------------------------------------- */
function abortBook(bookIdx) {
if (!confirm(`Abort book ${bookIdx}?`)) return;
Update LOG VIEW panel
--------------------------------------------------------- */
function updateLogs(logList) {
const output = $("#log-output");
if (!output) {
console.warn(">>> updateLogs: no #log-output element found");
return;
}
console.log("[DASHBOARD] ABORT", bookIdx);
output.innerHTML = "";
fetch(`/abort/${bookIdx}`, { method: "POST" }).then(() =>
refreshBook(bookIdx)
);
}
logList.forEach((line) => logAppend(line));
/* ---------------------------------------------------------
Bookcard UI refresh (non-progress)
--------------------------------------------------------- */
async function refreshBookCards() {
console.log("[DASHBOARD] refreshBookCards()");
const books = await apiGet("/api/books");
if (!books) return;
autoScroll(output);
}
document.querySelectorAll(".book-card").forEach((card) => {
const idx = card.dataset.bookIdx;
const info = books.find((b) => b.book_idx === idx);
if (!info) return;
console.log("[DASHBOARD] card status", idx, info.status);
card.className = `book-card ${info.status}`;
const abortBtn = card.querySelector(".icon-abort");
if (abortBtn) {
abortBtn.disabled = ![
"processing",
"downloading",
"parsing",
"audio",
].includes(info.status);
}
});
function abortBook(book_id) {
if (!confirm(`Abort tasks for book ${book_id}?`)) return;
fetch(`/abort/${book_id}`, { method: "POST" })
.then((r) => r.json())
.then((data) => {
console.log("Abort:", data);
})
.catch((err) => {
console.error("Abort failed:", err);
});
}

0
bookscraper/static/js/init_book.js
Unescape View File

101
bookscraper/static/js/inspect_state.js
Unescape View File

@ -1,101 +0,0 @@
/* ============================================================
File: static/js/inspect_state.js
Purpose:
- Receive merged state via state_updater.js
- Update ONLY the right-side state tables
- NO polling, NO fetch
============================================================ */
console.log("[inspect_state] JS loaded (subscriber mode)");
/* ------------------------------------------------------------
State subscription
------------------------------------------------------------ */
window.addEventListener("state:update", (e) => {
const entries = e.detail;
if (!Array.isArray(entries)) {
console.warn("[inspect_state] state:update payload is not array", entries);
return;
}
console.log("[inspect_state] state:update received entries:", entries.length);
updateInspectTables(entries);
});
/* ------------------------------------------------------------
Update tables
------------------------------------------------------------ */
function updateInspectTables(entries) {
console.log("[inspect_state] updating tables");
entries.forEach((entry) => {
const bookIdx = entry.book_idx;
if (bookIdx == null) {
console.warn("[inspect_state] entry without book_idx", entry);
return;
}
const block = document.querySelector(
`.state-block[data-book-idx="${bookIdx}"]`
);
if (!block) {
console.warn("[inspect_state] no state-block for book_idx", bookIdx);
return;
}
const table = block.querySelector(".state-table");
if (!table) {
console.warn("[inspect_state] no state-table for book_idx", bookIdx);
return;
}
console.log("[inspect_state] updating table for book_idx", bookIdx);
const sql = entry.sqlite || {};
const redis = entry.redis || {};
const merged = entry.would_merge_to || {};
table.innerHTML = `
<tr>
<th>Field</th>
<th>SQLite</th>
<th>Redis</th>
<th>Merged</th>
</tr>
${row("status", sql, redis, merged)}
${row("chapters_total", sql, redis, merged)}
${row("downloaded", sql, redis, merged)}
${row("chapters_download_done", sql, redis, merged)}
${row("chapters_download_skipped", sql, redis, merged)}
${row("parsed", sql, redis, merged)}
${row("chapters_parsed_done", sql, redis, merged)}
${row("audio_done", sql, redis, merged)}
${row("audio_skipped", sql, redis, merged)}
${row("last_update", sql, redis, merged)}
`;
});
}
/* ------------------------------------------------------------
Row helper
------------------------------------------------------------ */
function row(field, sql, redis, merged) {
const s = sql[field] ?? "";
const r = redis[field] ?? "";
const m = merged[field] ?? "";
const cls = String(s) === String(r) ? "same" : "diff";
return `
<tr>
<th>${field}</th>
<td class="${cls}">${s}</td>
<td class="${cls}">${r}</td>
<td>${m}</td>
</tr>
`;
}

122
bookscraper/static/js/log_view.js
Unescape View File

@ -1,105 +1,107 @@
/* =======================================================================
File: static/js/log_view.js
Purpose:
High-performance rolling log viewer
- efficient delta polling
- append-only mode (no DOM reset)
- rolling limit (prevents memory freeze)
- supports both global logs and per-book logs
Log viewer functionality:
- filtering
- clearing
- auto-scroll
- delta polling (efficient)
- rolling limit (prevent GUI freeze)
======================================================================= */
console.log(">>> log_view.js LOADING…");
/* ---------------------------------------------------------
Global log viewer state
Log filtering
--------------------------------------------------------- */
let LOG_FILTER = "ALL";
let LAST_LOG_INDEX = -1; // delta offset
const MAX_LOG_LINES = 600;
let LAST_LOG_INDEX = -1; // For delta polling
const MAX_LOG_LINES = 1000; // Rolling cap to prevent freezing
/* ---------------------------------------------------------
Apply filter on existing log lines
--------------------------------------------------------- */
function applyLogFilter() {
console.log(">>> log_view.js applyLogFilter(), filter =", LOG_FILTER);
const lines = $$(".log-line");
console.log(">>> log_view.js number of log-line elements:", lines.length);
lines.forEach((line) => {
const text = line.innerText;
const show = LOG_FILTER === "ALL" || (text && text.includes(LOG_FILTER));
line.style.display = show ? "block" : "none";
line.style.display =
LOG_FILTER === "ALL" || text.includes(LOG_FILTER) ? "block" : "none";
});
}
/* ---------------------------------------------------------
DOM Ready — bind clear/filter
UI bindings
--------------------------------------------------------- */
document.addEventListener("DOMContentLoaded", () => {
console.log(">>> log_view.js DOMContentLoaded");
const filterSel = $("#log-filter");
const clearBtn = $("#log-clear");
const output = $("#log-output");
if (!output) {
console.log(">>> log_view.js: No #log-output → viewer disabled");
if (!filterSel) {
console.log(">>> log_view.js: No log viewer found on this page.");
return;
}
console.log(">>> log_view.js: log viewer detected.");
// Filter dropdown
// filterSel.addEventListener("change", () => {
// LOG_FILTER = filterSel.value;
// console.log(">>> log_view.js filter changed to:", LOG_FILTER);
// applyLogFilter();
// });
// Clear log window
if (clearBtn) {
clearBtn.addEventListener("click", () => {
console.log(">>> log_view.js: Clear log viewer");
output.innerHTML = "";
LAST_LOG_INDEX = -1;
console.log(">>> log_view.js log-clear clicked → clearing output");
if (output) {
output.innerHTML = "";
LAST_LOG_INDEX = -1; // reset delta polling
}
});
}
});
/* ---------------------------------------------------------
Append ONE line
Append + Rolling buffer
--------------------------------------------------------- */
function rollingAppend(lineText) {
function logAppend(lineText) {
const output = $("#log-output");
if (!output) return;
const div = document.createElement("div");
div.classList.add("log-line");
// Type detection
if (lineText.includes("[DL]") || lineText.includes("[DOWNLOAD]"))
// -----------------------------------------------------
// Assign subtype classes
// -----------------------------------------------------
if (lineText.includes("[DL]") || lineText.includes("[DOWNLOAD]")) {
div.classList.add("dl");
else if (lineText.includes("[PARSE]")) div.classList.add("parse");
else if (lineText.includes("[SAVE]")) div.classList.add("save");
else if (lineText.includes("[AUDIO]")) div.classList.add("audio");
else if (lineText.includes("[CTRL]")) div.classList.add("ctrl");
else if (lineText.includes("[ERROR]")) div.classList.add("error");
else div.classList.add("default");
div.textContent = lineText;
output.appendChild(div);
// Rolling limit
while (output.childNodes.length > MAX_LOG_LINES) {
output.removeChild(output.firstChild);
} else if (lineText.includes("[PARSE]")) {
div.classList.add("parse");
} else if (lineText.includes("[SAVE]")) {
div.classList.add("save");
} else if (lineText.includes("[AUDIO]")) {
div.classList.add("audio");
} else if (lineText.includes("[CTRL]")) {
div.classList.add("ctrl");
} else if (lineText.includes("[ERROR]")) {
div.classList.add("error");
} else {
div.classList.add("default");
}
}
/* ---------------------------------------------------------
Primary entry: updateLogs()
Accepts:
{ logs:[...], last:N }
OR legacy:
{ lines:[...], last:N }
--------------------------------------------------------- */
function updateLogs(packet) {
const output = $("#log-output");
if (!output || !packet) return;
let lines = packet.logs || packet.lines || [];
if (!Array.isArray(lines)) return;
lines.forEach((line) => rollingAppend(line));
div.innerText = lineText;
output.appendChild(div);
// Correct unified delta index handling
if (packet.last !== undefined) {
LAST_LOG_INDEX = packet.last;
// Rolling buffer
while (output.children.length > MAX_LOG_LINES) {
output.removeChild(output.firstChild);
}
applyLogFilter();
@ -107,8 +109,7 @@ function updateLogs(packet) {
}
/* ---------------------------------------------------------
Delta polling — global logs ONLY
(dashboard.js overrides logs per-book)
Delta-based log polling
--------------------------------------------------------- */
function pollLogs() {
fetch(`/logs?last_index=${LAST_LOG_INDEX}`)
@ -116,8 +117,8 @@ function pollLogs() {
.then((data) => {
const lines = data.lines || [];
if (lines.length > 0) {
lines.forEach((line) => rollingAppend(line));
LAST_LOG_INDEX = data.last;
lines.forEach((line) => logAppend(line));
LAST_LOG_INDEX = data.total - 1;
}
})
.catch((err) => {
@ -125,6 +126,7 @@ function pollLogs() {
});
}
setInterval(pollLogs, 2800);
// Poll every 800 ms
setInterval(pollLogs, 1800);
console.log(">>> log_view.js LOADED");

72
bookscraper/static/js/progress.js
Unescape View File

@ -0,0 +1,72 @@
/* =======================================================================
File: static/js/progress.js
Purpose:
Update progress bars dynamically for the current book.
Expects data from API endpoints via dashboard.js or start.js.
======================================================================= */
console.log(">>> progress.js LOADED");
function updateProgressBars(data) {
console.log(">>> progress.js updateProgressBars() CALLED with:", data);
if (!data) {
console.warn(">>> progress.js: NO DATA RECEIVED");
return;
}
// Data format expected:
// {
// download_done,
// download_total,
// audio_done,
// audio_total
// }
const barDL = $(".progress-bar-fill");
const barAU = $(".progress-bar-fill.audio-fill");
console.log(">>> progress.js barDL =", barDL);
console.log(">>> progress.js barAU =", barAU);
const pctDL =
data.download_total > 0
? (100 * data.download_done) / data.download_total
: 0;
const pctAU =
data.audio_total > 0 ? (100 * data.audio_done) / data.audio_total : 0;
if (barDL) {
barDL.style.width = pctDL.toFixed(1) + "%";
console.log(">>> progress.js updated DL bar to", pctDL.toFixed(1) + "%");
} else {
console.warn(">>> progress.js: barDL NOT FOUND");
}
if (barAU) {
barAU.style.width = pctAU.toFixed(1) + "%";
console.log(">>> progress.js updated AU bar to", pctAU.toFixed(1) + "%");
} else {
console.warn(">>> progress.js: barAU NOT FOUND");
}
// Update textual stats
const stats = $$(".progress-stats span");
console.log(">>> progress.js stats elements found:", stats.length);
// Expected structure: [DL "x/y", DL "pct", AU "x/y", AU "pct"]
if (stats.length >= 4) {
stats[0].innerText = `${data.download_done} / ${data.download_total}`;
stats[1].innerText = pctDL.toFixed(1) + "%";
stats[2].innerText = `${data.audio_done} / ${data.audio_total}`;
stats[3].innerText = pctAU.toFixed(1) + "%";
console.log(">>> progress.js stats updated");
} else {
console.warn(
">>> progress.js: not enough stats spans, found",
stats.length
);
}
}

98
bookscraper/static/js/state_updater.js
Unescape View File

@ -1,98 +0,0 @@
/* ========================================================
File: static/js/state_updater.js
Purpose:
- Poll /api/state/all
- Dispatch merged state to subscribers
(bookcard_controller, inspect_state, others)
- Pause polling when tab inactive
======================================================== */
console.log("[STATE-UPDATER] loaded");
const STATE_POLL_INTERVAL_MS = 2500;
const STATE_ENDPOINT = "/api/state/all";
let STATE_TIMER = null;
/* ========================================================
INIT
======================================================== */
document.addEventListener("DOMContentLoaded", () => {
initStateUpdater();
});
function initStateUpdater() {
const cards = document.querySelectorAll(".book-card");
if (cards.length === 0) {
console.log("[STATE-UPDATER] No bookcards found — skipping");
return;
}
console.log(`[STATE-UPDATER] Starting updater for ${cards.length} bookcards`);
startPolling(true);
document.addEventListener("visibilitychange", () => {
document.hidden ? stopPolling() : startPolling(true);
});
}
/* ========================================================
DISPATCH
======================================================== */
function dispatchState(entries) {
console.debug("[STATE] dispatch", entries.length);
// 1. Bookcards
if (typeof window.updateBookCardsFromState === "function") {
window.updateBookCardsFromState(entries);
}
// 2. Inspect state tables / other subscribers
window.dispatchEvent(new CustomEvent("state:update", { detail: entries }));
}
/* ========================================================
POLLING CONTROL
======================================================== */
function startPolling(immediate = false) {
if (STATE_TIMER) return;
console.log("[STATE-UPDATER] Start polling");
if (immediate) pollState();
STATE_TIMER = setInterval(pollState, STATE_POLL_INTERVAL_MS);
}
function stopPolling() {
if (!STATE_TIMER) return;
console.log("[STATE-UPDATER] Stop polling (tab inactive)");
clearInterval(STATE_TIMER);
STATE_TIMER = null;
}
/* ========================================================
POLL API
======================================================== */
async function pollState() {
if (document.hidden) return;
try {
const resp = await fetch(STATE_ENDPOINT, { cache: "no-store" });
if (!resp.ok) return;
const entries = await resp.json();
if (!Array.isArray(entries)) return;
dispatchState(entries);
} catch (e) {
console.error("[STATE-UPDATER] poll error", e);
}
}

11
bookscraper/templates/base.html
Unescape View File

@ -20,16 +20,7 @@
<!-- JS -->
<script src="/static/js/app.js"></script>
<script src="/static/js/log_view.js"></script>
<script src="/static/js/progress.js"></script>
<script src="/static/js/dashboard.js"></script>
<!-- GLOBAL STATE UPDATER -->
<script src="/static/js/state_updater.js"></script>
<script>
document.addEventListener("DOMContentLoaded", () => {
if (typeof initStateUpdater === "function") {
initStateUpdater();
}
});
</script>
</body>
</html>

0
bookscraper/app/routes/__init__.py → bookscraper/templates/components/book_card.html
Unescape View File

12
bookscraper/templates/components/___book_list_item.html → bookscraper/templates/components/book_list_item.html
Unescape View File

@ -3,17 +3,17 @@
Purpose:
Dashboard weergave van één boek in de lijst.
Variabelen komen binnen via:
book.<veld>
→ Boek gebruikt nu uitsluitend book_idx als primaire sleutel
→ Dus alle velden moeten via "book.<veld>" aangesproken worden.
======================================================================= -->
<div class="book-list-item" data-book-idx="{{ book.book_idx }}">
<div class="book-list-item" data-book-id="{{ book.book_id }}">
<!-- Left area: title + metadata -->
<div class="book-info">
<div class="book-title">{{ book.title }}</div>
<div class="book-meta">
<span class="meta-label">IDX:</span> {{ book.book_idx }} {% if
<span class="meta-label">ID:</span> {{ book.book_id }} {% if
book.last_update %}
<span class="meta-separator">•</span>
<span class="meta-label">Updated:</span> {{ book.last_update }} {% endif
@ -56,10 +56,8 @@
<span class="mini-value">{{ pct_au }}%</span>
</div>
</div>
<!-- Abort button -->
<div class="book-abort-area">
<button class="abort-btn" onclick="abortBookAjax('{{ book.book_idx }}')">
<button class="abort-btn" onclick="abortBook('{{ book.book_id }}')">
Abort
</button>
</div>

90
bookscraper/templates/components/bookcard.html
Unescape View File

@ -1,90 +0,0 @@
{# ============================================================ File:
templates/components/bookcard.html Purpose: Eén enkele boekkaart (dumb
component) ============================================================ #}
<div class="book-card {{ b.status }}" data-book-idx="{{ b.book_idx }}">
<!-- HIDE -->
<form
action="/hide/{{ b.book_idx }}"
method="POST"
class="hide-form"
onsubmit="return confirm('Dit boek verbergen?')"
>
<button class="icon-btn icon-hide" title="Verbergen">
<i class="fa-solid fa-xmark"></i>
</button>
</form>
<!-- COVER -->
<div class="book-cover">
{% if b.cover_path %}
<img
src="/{{ b.cover_path }}"
class="book-img"
data-field="cover"
alt="cover"
/>
{% else %}
<div class="book-img placeholder" data-field="cover">?</div>
{% endif %}
</div>
<!-- META -->
<div class="book-meta">
<!-- STATUS BADGE -->
{% if b.status %}
<span
class="status-badge status-{{ b.status }}"
title="
{% if b.status == 'done' %}Alle chapters en audio zijn compleet{% endif %}
{% if b.status == 'audio' %}Downloads compleet, audio wordt nog gegenereerd{% endif %}
{% if b.status == 'downloading' %}Bezig met downloaden{% endif %}
"
>
{{ b.status | upper }}
</span>
{% endif %}
<div class="book-title" data-field="title">{{ b.title }}</div>
<div class="book-author" data-field="author">{{ b.author }}</div>
<div class="book-created">
Geregistreerd: <span data-field="created_at">{{ b.created_at }}</span>
</div>
<!-- ACTIONS -->
<div class="book-actions">
<!-- START -->
<form action="/start" method="POST">
<input type="hidden" name="book_idx" value="{{ b.book_idx }}" />
<button class="icon-btn icon-start" title="Start" data-action="start">
<i class="fa-solid fa-play"></i>
</button>
</form>
<!-- ABORT -->
<form action="/abort/{{ b.book_idx }}" method="POST">
<input type="hidden" name="book_idx" value="{{ b.book_idx }}" />
<button class="icon-btn icon-abort" title="Abort" data-action="abort">
<i class="fa-solid fa-stop"></i>
</button>
</form>
<form
method="post"
action="/inspect/statuscheck/{{ b.book_idx }}"
style="display: inline-block"
>
<button
type="submit"
class="icon-btn icon-statuscheck"
title="Herbereken status op basis van bestanden"
>
<i class="fa-solid fa-magnifying-glass-chart"></i>
</button>
</form>
</div>
</div>
<!-- PROGRESS -->
<div class="book-progress">{% include "components/progress_box.html" %}</div>
</div>

0
bookscraper/templates/components/init_book_form.html
Unescape View File

20
bookscraper/templates/components/nav.html
Unescape View File

@ -1,16 +1,16 @@
<!-- =======================================================================
File: templates/components/nav.html
Purpose: Global navigation bar for BookScraper UI (improved version)
Purpose: Global navigation bar for BookScraper UI
======================================================================= -->
<nav class="navbar">
<div class="nav-inner">
<!-- Branding / Home -->
<!-- Left side: Branding -->
<div class="nav-brand">
<a href="/">BookScraper</a>
</div>
<!-- Main navigation -->
<!-- Right side: Navigation Links -->
<ul class="nav-links">
<li>
<a href="/dashboard" class="nav-item"> Dashboard </a>
@ -21,19 +21,7 @@
</li>
<li>
<a href="/debug/inspect_state" class="nav-item"> State overview </a>
</li>
<!-- Tools dropdown -->
<li class="nav-dropdown">
<span class="nav-item">Tools ▾</span>
<ul class="dropdown-menu">
<li><a href="/api/db/books">DB Viewer</a></li>
<li><a href="/debug/inspect_state">Inspect State</a></li>
<li><a href="/debug/sync_state">Sync State</a></li>
<li><a href="/debug/redis-keys">Redis Keys</a></li>
<li><a href="/debug/queues">queues</a></li>
</ul>
<a href="/logs" class="nav-item"> Logs </a>
</li>
</ul>
</div>

71
bookscraper/templates/components/progress_box.html
Unescape View File

@ -1,34 +1,61 @@
<!-- =======================================================================
File: templates/components/progress_box.html
Purpose:
Dumb progress UI for a book card.
Initial values via Jinja, live updates via state_updater.js
======================================================================= -->
Purpose: Reusable progress overview (download + audio) for any book.
Notes:
- Expects the following variables from Flask:
book_id: str
title: str
download_total: int
download_done: int
audio_total: int
audio_done: int
- Pure HTML; JS for live updates will be added later.
======================================================================= -->
<div class="progress-box">
<!-- DOWNLOAD -->
<div class="progress-row">
<div class="progress-label">Download</div>
<div class="progressbar">
<div
class="progressbar-fill download"
data-field="download_pct"
style="width: 0%"
></div>
<div class="progressbar-text" data-field="download_text">0 / 0</div>
<!-- Header -->
<div class="progress-header">
<h2>Progress</h2>
{% if title %}
<div class="progress-subtitle">{{ title }}</div>
{% endif %} {% if book_id %}
<div class="progress-bookid">Book ID: <span>{{ book_id }}</span></div>
{% endif %}
</div>
<!-- DOWNLOAD SECTION -->
<div class="progress-section">
<h3>Download Progress</h3>
<div class="progress-bar">
{% set pct = 0 %} {% if download_total > 0 %} {% set pct = (100 *
download_done / download_total) | round(1) %} {% endif %}
<div class="progress-bar-fill" style="width: {{ pct }}%;"></div>
</div>
<div class="progress-stats">
<span>{{ download_done }} / {{ download_total }}</span>
<span>{{ pct }}%</span>
</div>
</div>
<!-- AUDIO -->
<div class="progress-row">
<div class="progress-label">Audio</div>
<div class="progressbar">
<!-- AUDIO SECTION -->
<div class="progress-section">
<h3>Audio Progress</h3>
<div class="progress-bar audio">
{% set pct2 = 0 %} {% if audio_total > 0 %} {% set pct2 = (100 *
audio_done / audio_total) | round(1) %} {% endif %}
<div
class="progressbar-fill audio"
data-field="audio_pct"
style="width: 0%"
class="progress-bar-fill audio-fill"
style="width: {{ pct2 }}%;"
></div>
<div class="progressbar-text" data-field="audio_text">0 / 0</div>
</div>
<div class="progress-stats">
<span>{{ audio_done }} / {{ audio_total }}</span>
<span>{{ pct2 }}%</span>
</div>
</div>
<script src="/static/js/progress.js"></script>
</div>

21
bookscraper/templates/components/registered_books.html
Unescape View File

@ -1,21 +0,0 @@
{# ============================================================ File:
templates/components/registered_books.html Purpose: Toon een grid van
geregistreerde boeken. Elke kaart wordt gerenderd via bookcard.html.
============================================================ #}
<section class="dashboard-section">
<h2>Geregistreerde boeken</h2>
{% if registered and registered|length > 0 %}
<div class="registered-grid">
{% for b in registered %} {% include "components/bookcard.html" %} {% endfor
%}
</div>
{% else %}
<p>Geen geregistreerde boeken.</p>
{% endif %}
</section>

19
bookscraper/templates/components/url_input.html
Unescape View File

@ -5,18 +5,17 @@
Used on landing pages or detail pages.
======================================================================= -->
<form method="POST" action="/init" class="url-form">
<label for="urls" class="url-label"> Book URL(s) – one per line: </label>
<form method="POST" action="/start" class="url-form">
<label for="url" class="url-label">Book URL:</label>
<textarea
id="urls"
name="urls"
<input
type="text"
id="url"
name="url"
class="url-input"
rows="5"
placeholder="https://www.piaotia.com/bookinfo/6/6072.html
https://www.piaotia.com/bookinfo/3/3785.html"
placeholder="https://www.piaotia.com/bookinfo/6/6072.html"
required
></textarea>
/>
<button type="submit" class="btn-primary url-submit">Register book(s)</button>
<button type="submit" class="btn-primary url-submit">Start Scraping</button>
</form>

13
bookscraper/templates/dashboard/book_detail.html
Unescape View File

@ -1,7 +1,7 @@
<!-- =======================================================================
File: templates/dashboard/book_detail.html
Purpose:
Detailpagina voor één book_idx.
Detailpagina voor één book_id.
Toont progress (download/audio) + filters + live logs.
======================================================================= -->
@ -15,9 +15,7 @@
<!-- Progress box -->
<section id="progressSection">
{% include "components/progress_box.html" with book_idx=book_idx,
title=title, download_total=download_total, download_done=download_done,
audio_total=audio_total, audio_done=audio_done %}
{% include "components/progress_box.html" %}
</section>
<!-- Log view -->
@ -29,10 +27,13 @@
<!-- PAGE-SPECIFIC JS -->
<script>
const BOOK_IDX = "{{ book_idx }}";
const BOOK_ID = "{{ book_id }}";
</script>
<!-- Shared log viewer -->
<script src="/static/js/log_view.js"></script>
<!-- Dashboard behaviour (only does something if dashboard HTML is present) -->
<script src="/static/js/dashboard.js"></script>
<!-- Existing global app logic -->
{% endblock %}

13
bookscraper/templates/dashboard/dashboard.html
Unescape View File

@ -25,8 +25,19 @@
<!-- ===========================================================
BOOK LIST
=========================================================== -->
<section class="dashboard-section">
<h2>Actieve boeken</h2>
{% if books and books|length > 0 %}
<div id="book-list" class="book-list">
{% for book in books %} {% include "components/book_list_item.html" %} {%
endfor %}
</div>
{% else %}
<div id="book-list" class="book-list-empty">Geen actieve boeken.</div>
{% endif %}
</section>
{% include "components/registered_books.html" %}
<hr />
<!-- ===========================================================

95
bookscraper/templates/debug/inspect_state.html
Unescape View File

@ -1,95 +0,0 @@
{# ============================================================ File:
templates/debug/inspect_state.html Purpose: Inspect SQLite vs Redis state per
book_idx - Initial render via Jinja - Live updates via inspect_state.js -
BookCard is server-rendered and NEVER replaced - Only the right-side state table
is updated dynamically
============================================================ #} {% extends
"layout.html" %} {% block content %}
<h1>State Inspection (SQL vs Redis)</h1>
<style>
.state-block {
display: grid;
grid-template-columns: 380px 1fr;
gap: 20px;
margin-bottom: 35px;
padding: 18px;
border: 1px solid #444;
background: #222;
border-radius: 8px;
}
.state-table {
width: 100%;
border-collapse: collapse;
}
.state-table th,
.state-table td {
border: 1px solid #555;
padding: 6px 10px;
}
.state-table th {
background: #333;
color: #fff;
}
.state-table td {
background: #2a2a2a;
color: #ddd;
}
.same {
color: #9f9 !important;
}
.diff {
color: #ff7b7b !important;
font-weight: bold;
}
</style>
<div id="state-container">
{% for entry in results %}
<div class="state-block" data-book-idx="{{ entry.book_idx }}">
<!-- LEFT: BookCard (server-rendered, NEVER replaced) -->
<div>
{% if entry.card %} {% with b = entry.card %} {% include
"components/bookcard.html" %} {% endwith %} {% else %}
<strong>{{ entry.book_idx }}</strong>
{% endif %}
</div>
<!-- RIGHT: State table (updated by JS) -->
<div>
<table class="state-table">
<tr>
<th>Field</th>
<th>SQLite</th>
<th>Redis</th>
<th>Merged</th>
</tr>
{% set sql = entry.sqlite %} {% set redis = entry.redis %} {% set merged
= entry.would_merge_to %} {% for field in [ "status", "chapters_total",
"downloaded", "chapters_download_done", "chapters_download_skipped",
"parsed", "chapters_parsed_done", "audio_done", "audio_skipped",
"last_update" ] %}
<tr>
<th>{{ field }}</th>
<td>{{ sql.get(field, "") }}</td>
<td>{{ redis.get(field, "") }}</td>
<td>{{ merged.get(field, "") }}</td>
</tr>
{% endfor %}
</table>
</div>
</div>
{% endfor %}
</div>
{% endblock %} {% block scripts %}
<script src="/static/js/inspect_state.js"></script>
{% endblock %}

91
bookscraper/templates/debug/queues.html
Unescape View File

@ -1,91 +0,0 @@
{% extends "layout.html" %} {% block content %}
<h1>Celery Queue Debug</h1>
<style>
.debug-section {
margin-bottom: 40px;
}
.debug-table {
width: 100%;
border-collapse: collapse;
margin-bottom: 20px;
}
.debug-table th,
.debug-table td {
border: 1px solid #444;
padding: 6px 10px;
}
.debug-table th {
background: #333;
color: #fff;
}
pre {
background: #1e1e1e;
color: #ddd;
padding: 10px;
overflow-x: auto;
}
code {
color: #9cf;
}
</style>
<div class="debug-section">
<h2>Workers</h2>
<h3>Active Tasks</h3>
<pre>{{ workers_active | tojson(indent=2) }}</pre>
<h3>Reserved</h3>
<pre>{{ workers_reserved | tojson(indent=2) }}</pre>
<h3>Scheduled</h3>
<pre>{{ workers_scheduled | tojson(indent=2) }}</pre>
</div>
<hr />
<div class="debug-section">
<h2>Queues</h2>
{% for q in queues %}
<div class="debug-queue">
<h3>{{ q.name }} ({{ q.length }} items)</h3>
<table class="debug-table">
<tr>
<th>Redis Key</th>
<td>{{ q.redis_key }}</td>
</tr>
<tr>
<th>Length</th>
<td>{{ q.length }}</td>
</tr>
<tr>
<th>Items (first 30)</th>
<td>
{% if q["items"] %}
<ul style="margin: 0; padding-left: 20px">
{% for item in q["items"] %}
<li><code>{{ item | e }}</code></li>
{% endfor %}
</ul>
{% else %}
<i>No items</i>
{% endif %}
</td>
</tr>
</table>
</div>
{% endfor %}
</div>
<script>
setInterval(() => {
window.location.reload();
}, 5000);
</script>
{% endblock %}

73
bookscraper/templates/index.html
Unescape View File

@ -1,53 +1,34 @@
<!DOCTYPE html>
<html lang="nl">
<head>
<meta charset="UTF-8" />
<head>
<meta charset="UTF-8">
<title>BookScraper</title>
<style>
body {
font-family: Arial, sans-serif;
padding: 40px;
max-width: 600px;
margin: auto;
}
h1 {
margin-bottom: 20px;
}
input[type="text"] {
width: 100%;
padding: 12px;
font-size: 16px;
border: 1px solid #ccc;
border-radius: 6px;
}
button {
margin-top: 20px;
padding: 12px 20px;
background: #007bff;
color: white;
border: none;
border-radius: 6px;
font-size: 16px;
cursor: pointer;
}
button:hover {
background: #0056b3;
}
body { font-family: Arial, sans-serif; padding: 40px; max-width: 600px; margin: auto; }
h1 { margin-bottom: 20px; }
input[type="text"] {
width: 100%; padding: 12px; font-size: 16px;
border: 1px solid #ccc; border-radius: 6px;
}
button {
margin-top: 20px;
padding: 12px 20px;
background: #007bff; color: white;
border: none; border-radius: 6px;
font-size: 16px; cursor: pointer;
}
button:hover { background: #0056b3; }
</style>
</head>
<body>
<h1>BookScraper WebGUI</h1>
</head>
<body>
<form action="/init" method="POST">
<label for="url">Geef een boek-URL op:</label><br /><br />
<input
type="text"
id="url"
name="url"
placeholder="https://example.com/book/12345"
required
/>
<button type="submit">Start Scraping</button>
</form>
</body>
<h1>BookScraper WebGUI</h1>
<form action="/start" method="POST">
<label for="url">Geef een boek-URL op:</label><br><br>
<input type="text" id="url" name="url" placeholder="https://example.com/book/12345" required>
<button type="submit">Start Scraping</button>
</form>
</body>
</html>

115
bookscraper/templates/inspect/statuscheck_result.html
Unescape View File

@ -1,115 +0,0 @@
{% extends "layout.html" %} {% block content %}
<h2>Statuscheck – Inspect</h2>
{% if error %}
<div class="error"><strong>Fout:</strong> {{ error }}</div>
{% else %}
<!-- ===================================================== -->
<!-- BOEK -->
<!-- ===================================================== -->
<h3>Boek</h3>
<table class="kv">
<tr>
<th>Book idx</th>
<td>{{ result.book_idx }}</td>
</tr>
<tr>
<th>Pad</th>
<td>{{ result.filesystem.book_dir }}</td>
</tr>
<tr>
<th>Bestaat</th>
<td>{{ result.filesystem.exists }}</td>
</tr>
</table>
<!-- ===================================================== -->
<!-- FILESYSTEM -->
<!-- ===================================================== -->
<h3>Filesystem (source of truth)</h3>
<table class="kv">
<tr>
<th>Volumes</th>
<td>{{ result.filesystem.volumes }}</td>
</tr>
<tr>
<th>Chapters (.txt)</th>
<td>{{ result.filesystem.chapters_txt }}</td>
</tr>
<tr>
<th>Audio (.m4b)</th>
<td>{{ result.filesystem.audio_files }}</td>
</tr>
</table>
<!-- ===================================================== -->
<!-- SQL -->
<!-- ===================================================== -->
<h3>SQL snapshot</h3>
<h4>Voor</h4>
<table class="kv">
<tr>
<th>Downloaded</th>
<td>{{ result.sql_before.downloaded }}</td>
</tr>
<tr>
<th>Audio done</th>
<td>{{ result.sql_before.audio_done }}</td>
</tr>
<tr>
<th>Status</th>
<td>{{ result.sql_before.status }}</td>
</tr>
</table>
<h4>Na</h4>
<table class="kv">
<tr>
<th>Downloaded</th>
<td>{{ result.sql_after.downloaded }}</td>
</tr>
<tr>
<th>Audio done</th>
<td>{{ result.sql_after.audio_done }}</td>
</tr>
<tr>
<th>Last update</th>
<td>{{ result.sql_after.last_update }}</td>
</tr>
</table>
<!-- ===================================================== -->
<!-- REPOSITORY -->
<!-- ===================================================== -->
<h3>Repository merged state (UI input)</h3>
<table class="kv">
<tr>
<th>Downloaded</th>
<td>{{ repo_state.downloaded }}</td>
</tr>
<tr>
<th>Audio done</th>
<td>{{ repo_state.audio_done }}</td>
</tr>
<tr>
<th>Chapters total</th>
<td>{{ repo_state.chapters_total }}</td>
</tr>
</table>
<details>
<summary>Raw repository state</summary>
<pre>{{ repo_state | tojson(indent=2) }}</pre>
</details>
{% endif %}
<hr />
<a href="/dashboard">← Terug naar dashboard</a>
{% endblock %}

11
bookscraper/templates/layout.html
Unescape View File

@ -14,12 +14,6 @@
<!-- CSS -->
<link rel="stylesheet" href="/static/css/style.css" />
<link rel="stylesheet" href="/static/css/dashboard.css" />
<link rel="stylesheet" href="/static/css/bookcard.css" />
<link
rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.0/css/all.min.css"
/>
<!-- GLOBAL HELPERS (moet ALTIJD boven alles geladen worden) -->
<script src="/static/js/helpers.js"></script>
@ -32,11 +26,6 @@
<footer class="footer">
BookScraper © 2025 — Powered by Celery + Redis
</footer>
{% block scripts %}{% endblock %}
<script src="/static/js/bookcard_controller.js"></script>
<script src="/static/js/state_updater.js"></script>
<script src="/static/js/dashboard.js"></script>
<!-- GLOBAL APP LOGIC (altijd als laatste) -->
<script src="/static/js/app.js"></script>

1
bookscraper/templates/result.html
Unescape View File

@ -66,7 +66,6 @@
display: none;
}
</style>
s
</head>
<body>

13
bookscraper/tools/mp4info
Unescape View File

@ -1,13 +0,0 @@
#!/bin/sh
# mp4info shim for m4b-tool (ffprobe-based)
if [ -z "$1" ]; then
echo "Usage: mp4info <file>" >&2
exit 1
fi
# ffprobe outputs float seconds; m4b-tool expects an integer
ffprobe -v error \
-show_entries format=duration \
-of default=noprint_wrappers=1:nokey=1 \
"$1" | awk '{ printf "%d\n", ($1 + 0.5) }'

10
bookscraper/worker/downloader.py
Unescape View File

@ -5,7 +5,7 @@ import requests
from io import BytesIO
from bs4 import BeautifulSoup
from scraper.logger import log_debug
from scraper.utils.utils import clean_text
from scraper.utils import clean_text
from urllib.parse import urljoin
@ -103,11 +103,8 @@ class ChapterDownloader:
collecting = True
continue
text = (
sib.get_text("\n", strip=True)
if hasattr(sib, "get_text")
else str(sib).strip()
)
text = sib.get_text("\n", strip=True) if hasattr(
sib, "get_text") else str(sib).strip()
if text:
parts.append(text)
@ -124,7 +121,6 @@ class ChapterDownloader:
vdir = f"{output_base}/v{volume}"
import os
os.makedirs(vdir, exist_ok=True)
fname = f"{number:05d}_{title}.txt"

Write Preview
Loading…
Cancel
Save