From fa2f212e0373a2b89d1fc961ef3342eebee21db1 Mon Sep 17 00:00:00 2001 From: "peter.fong" Date: Fri, 12 Dec 2025 22:24:19 +0100 Subject: [PATCH] inspect_state --- bookscraper/README.md | 2 +- bookscraper/app.py | 115 ++++++-- bookscraper/db/repository.py | 22 +- bookscraper/db/state_redis.py | 17 +- ...{_dep_progress.py => _dep_progress.py.bsk} | 0 bookscraper/scraper/abort.py | 3 +- bookscraper/scraper/scriptgen.py | 15 +- bookscraper/scraper/tasks/download_tasks.py | 3 - bookscraper/static/css/bookcard.css | 108 +++++-- bookscraper/static/js/bookcard_controller.js | 128 +++++++++ bookscraper/static/js/dashboard.js | 269 ++++++------------ bookscraper/static/js/inspect_state.js | 101 +++++++ bookscraper/static/js/progress.js | 72 ----- bookscraper/static/js/state_updater.js | 98 +++++++ bookscraper/templates/base.html | 11 +- .../templates/components/bookcard.html | 72 ++--- bookscraper/templates/components/nav.html | 2 +- .../templates/components/progress_box.html | 72 ++--- .../templates/components/url_input.html | 17 +- .../templates/debug/inspect_state.html | 84 +++--- bookscraper/templates/layout.html | 5 + bookscraper/templates/result.html | 124 +++++++- 22 files changed, 846 insertions(+), 494 deletions(-) rename bookscraper/scraper/{_dep_progress.py => _dep_progress.py.bsk} (100%) create mode 100644 bookscraper/static/js/bookcard_controller.js create mode 100644 bookscraper/static/js/inspect_state.js delete mode 100644 bookscraper/static/js/progress.js create mode 100644 bookscraper/static/js/state_updater.js diff --git a/bookscraper/README.md b/bookscraper/README.md index ab2775e..e53e997 100644 --- a/bookscraper/README.md +++ b/bookscraper/README.md @@ -137,7 +137,7 @@ docker compose up docker compose up -d -docker compose build web && docker compose up web +docker compose build --no-cache web && docker compose up web docker compose build worker_download && docker compose up worker_download diff --git a/bookscraper/app.py b/bookscraper/app.py index 07f8679..e0400d2 100644 --- a/bookscraper/app.py +++ b/bookscraper/app.py @@ -72,12 +72,18 @@ def dashboard(): logs_list = get_ui_logs() or [] registered_books = get_registered_books() log(f"[WEB] Registered books: {registered_books}") - reg = [b for b in get_registered_books() if b.get("status") != "hidden"] + from db.repository import fetch_all_books + from pprint import pprint + + pprint(fetch_all_books()) + pprint(get_registered_books()) + + # reg = [b for b in get_registered_books() if b.get("status") != "hidden"] return render_template( "dashboard/dashboard.html", books=list_active_books(), - registered=reg, + registered=registered_books, logs=logs_list, ) @@ -102,41 +108,86 @@ def book_detail(book_idx): @app.route("/init", methods=["POST"]) @logcall def init_book(): - url = request.form.get("url", "").strip() + # ------------------------------------------------- + # Accept single URL (legacy) OR multi-line URLs + # ------------------------------------------------- + raw_urls = request.form.get("urls") or request.form.get("url") or "" - if not url: + urls = [line.strip() for line in raw_urls.splitlines() if line.strip()] + + if not urls: return render_template( "dashboard/dashboard.html", - error="Geen URL opgegeven.", + error="Geen URL(s) opgegeven.", books=list_active_books(), registered=get_registered_books(), logs=get_ui_logs(), ) - try: - result = InitService.execute(url) - msg = f"Boek geregistreerd: {result.get('title')}" + # ------------------------------------------------- + # Duplicate check: existing book_ids + # ------------------------------------------------- + existing_books = {b["book_idx"] for b in fetch_all_books()} + results = [] - reg = [b for b in get_registered_books() if b.get("status") != "hidden"] + # ------------------------------------------------- + # Process each URL independently + # ------------------------------------------------- + for url in urls: + try: + book_id = InitService.derive_book_id(url) + + if book_id in existing_books: + results.append( + { + "url": url, + "status": "skipped", + "book_id": book_id, + "message": "Al geregistreerd", + } + ) + continue + + result = InitService.execute(url) + + results.append( + { + "url": url, + "status": "registered", + "book_id": result.get("book_id"), + "title": result.get("title"), + } + ) - return render_template( - "dashboard/dashboard.html", - message=msg, - books=list_active_books(), - registered=reg, - logs=get_ui_logs(), - ) + except Exception as e: + log_debug(f"[INIT] ERROR for url={url}: {e}") + results.append( + { + "url": url, + "status": "error", + "error": str(e), + } + ) - except Exception as e: - log_debug(f"[INIT] ERROR: {e}") - reg = [b for b in get_registered_books() if b.get("status") != "hidden"] - return render_template( - "dashboard/dashboard.html", - error=f"INIT mislukt: {e}", - books=list_active_books(), - registered=reg, - logs=get_ui_logs(), - ) + # ------------------------------------------------- + # Summary message + # ------------------------------------------------- + ok = sum(1 for r in results if r["status"] == "registered") + skipped = sum(1 for r in results if r["status"] == "skipped") + failed = sum(1 for r in results if r["status"] == "error") + + message = f"Geregistreerd: {ok}, overgeslagen: {skipped}, fouten: {failed}" + + reg = [b for b in get_registered_books() if b.get("status") != "hidden"] + + return render_template( + "dashboard/dashboard.html", + message=message, + init_results=results, # optioneel voor UI-weergave + books=list_active_books(), + registered=reg, + logs=get_ui_logs(), + ) @app.route("/hide/", methods=["POST"]) @@ -222,6 +273,18 @@ def abort_download(book_idx): # ===================================================== +@app.route("/api/state/all", methods=["GET"]) +@logcall +def api_state_all(): + """ + Returns the merged SQL + Redis state for all books + (same logic as /debug/inspect_state but JSON-only). + """ + from scraper.utils.state_sync import inspect_books_state + + return jsonify(inspect_books_state()) + + @app.route("/api/books") @logcall def api_books(): diff --git a/bookscraper/db/repository.py b/bookscraper/db/repository.py index bebc970..bd44d22 100644 --- a/bookscraper/db/repository.py +++ b/bookscraper/db/repository.py @@ -202,14 +202,18 @@ def update_book_after_full_scrape( @logcall def get_registered_books(): all_books = sql_fetch_all_books() - return [b for b in all_books if b.get("status") == "registered"] + HIDDEN_STATES = {"hidden"} + log(f"[DB] Fetched all books for registered filter, total={len(all_books)}") + return [b for b in all_books if b.get("status") not in HIDDEN_STATES] @logcall def get_active_books(): all_books = sql_fetch_all_books() + + HIDDEN_STATES = {"hidden", "done"} log(f"[DB] Fetched all books for active filter, total={len(all_books)}") - return [b for b in all_books if b.get("status") in ("active", "downloading")] + return [b for b in all_books if b.get("status") not in HIDDEN_STATES] # ============================================================ @@ -230,7 +234,7 @@ def set_chapters_total(book_idx, total): log(f"[DB] Setting chapter total for {book_idx} to {total}") redis_set_chapters_total(book_idx, total) sql_set_chapters_total(book_idx, total) - _legacy_set_total(book_idx, total) + # _legacy_set_total(book_idx, total) # ============================================================ @@ -240,15 +244,15 @@ def set_chapters_total(book_idx, total): def inc_download_done(book_idx, amount=1): log(f"[DB] Incrementing download done for {book_idx} by {amount}") redis_inc_download_done(book_idx, amount) - sql_inc_downloaded(book_idx, amount) - _legacy_inc_completed(book_idx) + # sql_inc_downloaded(book_idx, amount) + # _legacy_inc_completed(book_idx) @logcall def inc_download_skipped(book_idx, amount=1): log(f"[DB] Incrementing download skipped for {book_idx} by {amount}") redis_inc_download_skipped(book_idx, amount) - _legacy_inc_skipped(book_idx) + # _legacy_inc_skipped(book_idx) # ============================================================ @@ -258,7 +262,7 @@ def inc_download_skipped(book_idx, amount=1): def inc_parsed_done(book_idx, amount=1): log(f"[DB] Incrementing parsed done for {book_idx} by {amount}") redis_inc_parsed_done(book_idx, amount) - sql_inc_parsed(book_idx, amount) + # sql_inc_parsed(book_idx, amount) # ============================================================ @@ -267,7 +271,7 @@ def inc_parsed_done(book_idx, amount=1): @logcall def inc_audio_skipped(book_idx, amount=1): log(f"[DB] Incrementing audio skipped for {book_idx} by {amount}") - sql_inc_audio_skipped(book_idx, amount) + # sql_inc_audio_skipped(book_idx, amount) redis_inc_audio_skipped(book_idx, amount) @@ -275,7 +279,7 @@ def inc_audio_skipped(book_idx, amount=1): def inc_audio_done(book_idx, amount=1): log(f"[DB] Incrementing audio done for {book_idx} by {amount}") redis_inc_audio_done(book_idx, amount) - sql_inc_audio_done(book_idx, amount) + # sql_inc_audio_done(book_idx, amount) # ============================================================ diff --git a/bookscraper/db/state_redis.py b/bookscraper/db/state_redis.py index 2251f81..64aec84 100644 --- a/bookscraper/db/state_redis.py +++ b/bookscraper/db/state_redis.py @@ -26,6 +26,7 @@ def _key(book_idx: str) -> str: # STATUS # ------------------------------------------------------------ def redis_set_status(book_idx: str, status: str): + log(f"[DB-REDIS] Setting status for {book_idx} to {status}") key = _key(book_idx) r.hset(key, "status", status) r.hset(key, "last_update", int(time.time())) @@ -44,6 +45,7 @@ def redis_set_chapters_total(book_idx: str, total: int): # DOWNLOAD COUNTERS # ------------------------------------------------------------ def redis_inc_download_done(book_idx: str, amount: int = 1): + log(f"[DB-REDIS] Incrementing download done for {book_idx} by {amount}") key = _key(book_idx) r.hincrby(key, "chapters_download_done", amount) r.hset(key, "last_update", int(time.time())) @@ -60,6 +62,7 @@ def redis_inc_download_skipped(book_idx: str, amount: int = 1): # PARSE COUNTERS # ------------------------------------------------------------ def redis_inc_parsed_done(book_idx: str, amount: int = 1): + log(f"[DB-REDIS] Incrementing parsed done for {book_idx} by {amount}") key = _key(book_idx) r.hincrby(key, "chapters_parsed_done", amount) r.hset(key, "last_update", int(time.time())) @@ -77,10 +80,6 @@ def redis_inc_audio_done(book_idx: str, amount: int = 1): def redis_inc_audio_skipped(book_idx: str, amount: int = 1): log(f"[DB-REDIS] Incrementing audio skipped for {book_idx} by {amount}") - """ - New: Count skipped audio chapters (timeouts, pre-existing files, abort, etc.) - SQL does NOT track this; Redis-only metric. - """ key = _key(book_idx) r.hincrby(key, "audio_skipped", amount) r.hset(key, "last_update", int(time.time())) @@ -89,7 +88,7 @@ def redis_inc_audio_skipped(book_idx: str, amount: int = 1): # ------------------------------------------------------------ # INITIALISE BOOK STATE # ------------------------------------------------------------ -def init_book_state(book_id: str, title: str, url: str, chapters_total: int): +def init_book_state(book_idx: str, title: str, url: str, chapters_total: int): """ Initialiseert de complete Redis state voor een nieuw boek. LET OP: @@ -97,7 +96,7 @@ def init_book_state(book_id: str, title: str, url: str, chapters_total: int): - Alleen missende velden worden toegevoegd. """ - key = f"book:{book_id}:state" + key = f"book:{book_idx}:state" # Bestaat al? Dan vullen we alleen missende velden aan. exists = r.exists(key) @@ -105,7 +104,7 @@ def init_book_state(book_id: str, title: str, url: str, chapters_total: int): pipeline = r.pipeline() # Basis metadata - pipeline.hsetnx(key, "book_id", book_id) + pipeline.hsetnx(key, "book_id", book_idx) pipeline.hsetnx(key, "title", title or "") pipeline.hsetnx(key, "url", url or "") @@ -126,6 +125,6 @@ def init_book_state(book_id: str, title: str, url: str, chapters_total: int): pipeline.execute() if exists: - log(f"[DB-REDIS] init_book_state(): UPDATED existing state for {book_id}") + log(f"[DB-REDIS] init_book_state(): UPDATED existing state for {book_idx}") else: - log(f"[DB-REDIS] init_book_state(): CREATED new state for {book_id}") + log(f"[DB-REDIS] init_book_state(): CREATED new state for {book_idx}") diff --git a/bookscraper/scraper/_dep_progress.py b/bookscraper/scraper/_dep_progress.py.bsk similarity index 100% rename from bookscraper/scraper/_dep_progress.py rename to bookscraper/scraper/_dep_progress.py.bsk diff --git a/bookscraper/scraper/abort.py b/bookscraper/scraper/abort.py index 9753e53..b10c1a8 100644 --- a/bookscraper/scraper/abort.py +++ b/bookscraper/scraper/abort.py @@ -72,12 +72,11 @@ def abort_requested(book_idx: str, redis_client=None) -> bool: port = conn.get("port") db = conn.get("db") _debug( - f"[ABORT_DEBUG] first check book_idx={book_idx} " + # f"[ABORT_DEBUG] first check book_idx={book_idx} " f"redis={host}:{port} db={db}" ) except Exception: _debug(f"[ABORT_DEBUG] first check book_idx={book_idx}") - _seen_debug_keys.add(key) # Log ACTIVE state diff --git a/bookscraper/scraper/scriptgen.py b/bookscraper/scraper/scriptgen.py index 4f94d68..2b24395 100644 --- a/bookscraper/scraper/scriptgen.py +++ b/bookscraper/scraper/scriptgen.py @@ -45,11 +45,24 @@ def detect_volumes(book_base: str): # ------------------------------------------------------------ def build_merge_block(title: str, author: str, volumes): lines = [] + total_vols = len(volumes) + if total_vols >= 100: + pad = 3 + elif total_vols >= 10: + pad = 2 + else: + pad = 0 + for num, dirname in volumes: + if pad > 0: + vol_num = f"{num:0{pad}d}" + else: + vol_num = str(num) + line = ( f'm4b-tool merge --jobs=4 --writer="{author}" ' f'--albumartist="{author}" --album="{title}" ' - f'--name="{title}" --output-file="{title}-{num}.m4b" ' + f'--name="{title}" --output-file="{title}-{vol_num}.m4b" ' f'"{dirname}" -vvv' ) lines.append(line) diff --git a/bookscraper/scraper/tasks/download_tasks.py b/bookscraper/scraper/tasks/download_tasks.py index 9d14848..4a030d6 100644 --- a/bookscraper/scraper/tasks/download_tasks.py +++ b/bookscraper/scraper/tasks/download_tasks.py @@ -153,7 +153,6 @@ def download_chapter(self, payload: dict): if os.path.exists(save_path): log_msg(book_idx, f"[DL] SKIP {chapter_num} → {save_path}") - inc_download_skipped(book_idx) payload["html"] = None @@ -188,8 +187,6 @@ def download_chapter(self, payload: dict): log_msg(book_idx, f"[DL] OK {chapter_num}: {len(html)} bytes") - inc_download_done(book_idx) - payload["html"] = html payload["skipped"] = False payload["path"] = save_path diff --git a/bookscraper/static/css/bookcard.css b/bookscraper/static/css/bookcard.css index 588716e..78be639 100644 --- a/bookscraper/static/css/bookcard.css +++ b/bookscraper/static/css/bookcard.css @@ -2,11 +2,11 @@ File: static/css/bookcard.css Purpose: All styling for registered book cards (book-card) + - status colors + start/abort buttons. + status colors + start/abort buttons + progress bars ======================================================================= */ /* ----------------------------------------------------------------------- - GRID WRAPPER FOR REGISTERED BOOKS + GRID WRAPPER ----------------------------------------------------------------------- */ .registered-grid { @@ -17,7 +17,7 @@ } /* ----------------------------------------------------------------------- - MAIN BOOK CARD + BOOK CARD ----------------------------------------------------------------------- */ .book-card { @@ -36,7 +36,7 @@ } /* ----------------------------------------------------------------------- - BOOK STATUS COLORS + STATUS COLORS ----------------------------------------------------------------------- */ .book-card.processing { @@ -55,8 +55,8 @@ } .book-card.audio { - border-color: #e65100; - box-shadow: 0 0 6px rgba(230, 81, 0, 0.35); + border-color: #34c759; + box-shadow: 0 0 6px rgba(52, 199, 89, 0.35); } .book-card.completed { @@ -70,7 +70,7 @@ } /* ----------------------------------------------------------------------- - COVER IMAGE + COVER ----------------------------------------------------------------------- */ .book-cover { @@ -94,7 +94,7 @@ } /* ----------------------------------------------------------------------- - META INFORMATION + META ----------------------------------------------------------------------- */ .book-meta { @@ -106,25 +106,30 @@ .book-title { font-size: 16px; font-weight: bold; - margin-bottom: 4px; } .book-author { font-size: 14px; color: #444; - margin-bottom: 8px; + margin-bottom: 6px; } .book-created { font-size: 12px; color: #666; - margin-bottom: 10px; } /* ----------------------------------------------------------------------- - ICON BUTTONS + ACTION BUTTONS ----------------------------------------------------------------------- */ +.book-actions { + display: flex; + justify-content: flex-end; + gap: 10px; + margin-top: 10px; +} + .icon-btn { width: 34px; height: 34px; @@ -142,7 +147,7 @@ transition: background 0.15s ease, transform 0.1s ease; } -/* Start (green) */ +/* Start */ .icon-start { background: #2d8a3d; } @@ -150,15 +155,13 @@ background: #226c30; transform: scale(1.05); } - .icon-start:disabled { - background: #9bbb9f !important; + background: #9bbb9f; cursor: not-allowed; - transform: none; opacity: 0.5; } -/* Abort (red) */ +/* Abort */ .icon-abort { background: #c62828; } @@ -166,36 +169,83 @@ background: #a31f1f; transform: scale(1.05); } - .icon-abort:disabled { - background: #d8a0a0 !important; + background: #d8a0a0; cursor: not-allowed; - transform: none; opacity: 0.5; } -/* Hide button (gray) */ +/* Hide */ .hide-form { position: absolute; top: 6px; right: 6px; - margin: 0; } - .icon-hide { background: #777; } .icon-hide:hover { background: #555; - transform: scale(1.05); } + /* ----------------------------------------------------------------------- - BOOK ACTIONS (right aligned button row) + PROGRESS (FULL WIDTH) ----------------------------------------------------------------------- */ -.book-actions { - display: flex; - justify-content: flex-end; /* rechts uitlijnen */ - gap: 10px; /* ruimte tussen knoppen */ +.book-progress { + grid-column: 1 / -1; margin-top: 12px; + padding: 10px 12px; + background: #f6f6f6; + border-radius: 8px; +} + +.progress-row { + margin-bottom: 10px; +} + +.progress-label { + font-size: 12px; + margin-bottom: 4px; + color: #444; +} + +/* BAR */ +.progressbar { + position: relative; + width: 100%; + height: 14px; + background: #ddd; + border-radius: 7px; + overflow: hidden; +} + +.progressbar-fill { + height: 100%; + transition: width 0.4s ease; +} + +/* Download = blauw */ +.progressbar-fill.download { + background: #2196f3; +} + +/* Audio = groen */ +.progressbar-fill.audio { + background: #4caf50; +} + +/* TEXT IN BAR */ +.progressbar-text { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + + font-size: 11px; + font-weight: 600; + color: #fff; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.6); + pointer-events: none; } diff --git a/bookscraper/static/js/bookcard_controller.js b/bookscraper/static/js/bookcard_controller.js new file mode 100644 index 0000000..cfea644 --- /dev/null +++ b/bookscraper/static/js/bookcard_controller.js @@ -0,0 +1,128 @@ +/* ============================================================ + File: static/js/bookcard_controller.js + Purpose: + Single owner for updating book-card DOM from merged state + (would_merge_to) + ============================================================ */ + +console.log("[BOOKCARD] controller loaded"); + +/* ============================================================ + ENTRY POINT (called by state_updater.js) + ============================================================ */ + +function updateBookCardsFromState(stateList) { + console.log("[BOOKCARD] updateBookCardsFromState called"); + + if (!Array.isArray(stateList)) { + console.warn("[BOOKCARD] Invalid stateList", stateList); + return; + } + + const stateById = {}; + + stateList.forEach((entry) => { + const merged = entry.would_merge_to; + if (!merged || merged.book_idx == null) { + console.warn("[BOOKCARD] entry without merged/book_idx", entry); + return; + } + stateById[String(merged.book_idx)] = merged; + }); + + document.querySelectorAll(".book-card").forEach((card) => { + const bookIdx = card.dataset.bookIdx; + const state = stateById[bookIdx]; + + if (!state) { + console.debug("[BOOKCARD] No state for book_idx", bookIdx); + return; + } + + console.log("[BOOKCARD] Updating card", bookIdx, state.status); + updateSingleBookCard(card, state); + }); +} + +/* ============================================================ + SINGLE CARD UPDATE + ============================================================ */ + +function updateSingleBookCard(card, state) { + console.log("[BOOKCARD] updateSingleBookCard", state.book_idx); + + updateStatus(card, state); + updateButtons(card, state); + updateProgress(card, state); +} + +/* ============================================================ + STATUS + ============================================================ */ + +function updateStatus(card, state) { + console.log("[BOOKCARD][STATUS]", state.book_idx, "→", state.status); + card.className = `book-card ${state.status || ""}`; +} + +/* ============================================================ + BUTTONS + ============================================================ */ + +function updateButtons(card, state) { + const startBtn = card.querySelector(".icon-start"); + const abortBtn = card.querySelector(".icon-abort"); + + const busy = ["starting", "downloading", "parsing", "audio"]; + + console.log("[BOOKCARD][BUTTONS]", state.book_idx, "status:", state.status); + + if (startBtn) { + // startBtn.disabled = busy.includes(state.status); + } + + if (abortBtn) { + abortBtn.disabled = !busy.includes(state.status); + } +} + +/* ============================================================ + PROGRESS (DOWNLOAD + AUDIO) + ============================================================ */ + +function updateProgress(card, s) { + const total = Number(s.chapters_total || 0); + + const downloadDone = + Number(s.chapters_download_done || 0) + + Number(s.chapters_download_skipped || 0); + + const audioDone = Number(s.audio_done || 0) + Number(s.audio_skipped || 0); + + const downloadPct = + total > 0 ? Math.min((downloadDone / total) * 100, 100) : 0; + + const audioPct = total > 0 ? Math.min((audioDone / total) * 100, 100) : 0; + + console.log("[BOOKCARD][PROGRESS]", s.book_idx, { + total, + downloadDone, + audioDone, + downloadPct, + audioPct, + }); + + /* ---- DOWNLOAD ---- */ + const dlBar = card.querySelector('[data-field="download_pct"]'); + const dlText = card.querySelector('[data-field="download_text"]'); + + if (dlBar) dlBar.style.width = `${downloadPct}%`; + if (dlText) dlText.textContent = `${downloadDone} / ${total}`; + + /* ---- AUDIO ---- */ + const auBar = card.querySelector('[data-field="audio_pct"]'); + const auText = card.querySelector('[data-field="audio_text"]'); + + if (auBar) auBar.style.width = `${audioPct}%`; + if (auText) auText.textContent = `${audioDone} / ${total}`; +} diff --git a/bookscraper/static/js/dashboard.js b/bookscraper/static/js/dashboard.js index 2945d06..c11b0b4 100644 --- a/bookscraper/static/js/dashboard.js +++ b/bookscraper/static/js/dashboard.js @@ -1,247 +1,159 @@ /* ======================================================================= File: static/js/dashboard.js Purpose: - Dashboard interactions: - - Select active book_idx - - Live logs & progress - - Bookcard AJAX start/abort - NOTE: - updateLogs() is provided by log_view.js + - Sidebar selectie + - Start / Abort acties + - UI status updates + NOTE: + - GEEN polling + - state_updater.js is leidend ======================================================================= */ +console.log("[DASHBOARD] loaded"); + /* --------------------------------------------------------- - Utility: Safe fetch wrapper - --------------------------------------------------------- */ + Helpers +--------------------------------------------------------- */ async function apiGet(url) { + console.log("[DASHBOARD][API] GET", url); try { - const r = await fetch(url); - if (!r.ok) return null; + const r = await fetch(url, { cache: "no-store" }); + if (!r.ok) { + console.warn("[DASHBOARD][API] GET failed", url, r.status); + return null; + } return await r.json(); } catch (e) { - console.error("API GET failed:", url, e); + console.error("[DASHBOARD][API] GET error", url, e); return null; } } +function safeUpdateLogs(data) { + if (typeof window.updateLogs === "function") { + console.log("[DASHBOARD] updateLogs()"); + window.updateLogs(data); + } +} + /* --------------------------------------------------------- - Dashboard state - --------------------------------------------------------- */ + State +--------------------------------------------------------- */ let ACTIVE_BOOK_IDX = null; -let REFRESH_INTERVAL = null; - -console.log(">>> dashboard.js LOADED"); /* --------------------------------------------------------- DOM READY - --------------------------------------------------------- */ +--------------------------------------------------------- */ document.addEventListener("DOMContentLoaded", () => { - console.log(">>> dashboard.js DOMContentLoaded"); - - // Fallback: global logs when no active book_idx - setInterval(() => { - if (!ACTIVE_BOOK_IDX) refreshBook(null); - }, 2000); + console.log("[DASHBOARD] DOMContentLoaded"); - // Sidebar items - const items = $$(".book-list-item"); - items.forEach((item) => { - item.addEventListener("click", () => { - selectBook(item.dataset.bookIdx); - }); - }); - - // Auto-select first book - if (!ACTIVE_BOOK_IDX && items[0]) { - selectBook(items[0].dataset.bookIdx); - } - - // Bind start/abort buttons inside cards + bindSidebar(); bindBookCardButtons(); - // Refresh sidebar every few seconds - setInterval(refreshActiveBooks, 2800); + const first = document.querySelector(".book-list-item"); + if (first) { + console.log("[DASHBOARD] auto-select", first.dataset.bookIdx); + selectBook(first.dataset.bookIdx); + } }); /* --------------------------------------------------------- - Select a book_idx - --------------------------------------------------------- */ -function selectBook(bookIdx) { - ACTIVE_BOOK_IDX = bookIdx; - console.log(">>> Selecting book_idx", bookIdx); - - // Highlight sidebar - $$(".book-list-item").forEach((el) => { - el.classList.toggle("active", el.dataset.bookIdx === bookIdx); + Sidebar +--------------------------------------------------------- */ +function bindSidebar() { + console.log("[DASHBOARD] bindSidebar()"); + document.querySelectorAll(".book-list-item").forEach((item) => { + item.onclick = () => selectBook(item.dataset.bookIdx); }); - - // Reset polling - if (REFRESH_INTERVAL) clearInterval(REFRESH_INTERVAL); - - REFRESH_INTERVAL = setInterval(() => { - refreshBook(ACTIVE_BOOK_IDX); - }, 2000); - - refreshBook(ACTIVE_BOOK_IDX); } -/* --------------------------------------------------------- - Refresh sidebar list - --------------------------------------------------------- */ -async function refreshActiveBooks() { - const books = await apiGet("/api/books"); - if (!books) return; - - const container = $("#book-list"); - if (!container) return; - - container.innerHTML = ""; - - books.forEach((b) => { - const div = document.createElement("div"); - div.className = "book-list-item"; - div.dataset.bookIdx = b.book_idx; +function selectBook(bookIdx) { + if (!bookIdx || bookIdx === ACTIVE_BOOK_IDX) return; - div.innerHTML = ` -
${b.title}
-
${b.status}
-
- ${b.download_done}/${b.download_total} downloaded, - ${b.audio_done}/${b.audio_total} audio -
- `; + ACTIVE_BOOK_IDX = bookIdx; + console.log("[DASHBOARD] selectBook", bookIdx); - div.addEventListener("click", () => selectBook(b.book_idx)); - container.appendChild(div); + document.querySelectorAll(".book-list-item").forEach((el) => { + el.classList.toggle("active", el.dataset.bookIdx === bookIdx); }); - if (!ACTIVE_BOOK_IDX && books.length > 0) { - selectBook(books[0].book_idx); - } + refreshBook(bookIdx); } /* --------------------------------------------------------- - Fetch logs + progress - --------------------------------------------------------- */ + Book refresh (NO POLLING) +--------------------------------------------------------- */ async function refreshBook(bookIdx) { - if (!bookIdx) { - const data = await apiGet("/logs"); - if (data) updateLogs(data); - return; - } + console.log("[DASHBOARD] refreshBook", bookIdx); - const state = await apiGet(`/api/book/${bookIdx}/status`); const logs = await apiGet(`/api/book/${bookIdx}/logs`); + if (logs) safeUpdateLogs(logs); - if (state) { - updateProgressBars(state); - refreshBookCards(); - } - if (logs) updateLogs(logs); + refreshBookCards(); } /* --------------------------------------------------------- - BOOKCARD BUTTON BINDING (idempotent) - --------------------------------------------------------- */ + Bookcard buttons +--------------------------------------------------------- */ function bindBookCardButtons() { - console.log(">>> bindBookCardButtons() scanning…"); + console.log("[DASHBOARD] bindBookCardButtons()"); - // START BUTTONS - document.querySelectorAll(".book-card .icon-start").forEach((btn) => { - if (btn.dataset.bound === "1") return; + document.querySelectorAll(".icon-start").forEach((btn) => { + if (btn.dataset.bound) return; btn.dataset.bound = "1"; - btn.addEventListener("click", (ev) => { - ev.preventDefault(); - if (btn.disabled) return; - + btn.onclick = (e) => { + e.preventDefault(); const card = btn.closest(".book-card"); - const bookIdx = card?.dataset.bookIdx; - - console.log(">>> START clicked:", bookIdx); - if (!bookIdx) { - console.error(">>> ERROR: bookIdx missing on .book-card dataset"); - return; - } - - startBook(bookIdx); - }); + if (!card) return; + startBook(card.dataset.bookIdx); + }; }); - // ABORT BUTTONS - document.querySelectorAll(".book-card .icon-abort").forEach((btn) => { - if (btn.dataset.bound === "1") return; + document.querySelectorAll(".icon-abort").forEach((btn) => { + if (btn.dataset.bound) return; btn.dataset.bound = "1"; - btn.addEventListener("click", (ev) => { - ev.preventDefault(); - if (btn.disabled) return; - + btn.onclick = (e) => { + e.preventDefault(); const card = btn.closest(".book-card"); - const bookIdx = card?.dataset.bookIdx; - - console.log(">>> ABORT clicked:", bookIdx); - if (!bookIdx) { - console.error(">>> ERROR: bookIdx missing on .book-card dataset"); - return; - } - - abortBookAjax(bookIdx); - }); + if (!card) return; + abortBook(card.dataset.bookIdx); + }; }); } /* --------------------------------------------------------- - AJAX START - --------------------------------------------------------- */ + START +--------------------------------------------------------- */ function startBook(bookIdx) { - console.log(">>> startBook():", bookIdx); + console.log("[DASHBOARD] START", bookIdx); fetch("/start", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: `book_idx=${bookIdx}`, // backend expects field name book_idx - }) - .then(async (r) => { - console.log(">>> /start status:", r.status); - - let data = null; - try { - data = await r.json(); - } catch (e) {} - - console.log(">>> /start response:", data); - refreshBookCards(); - refreshBook(bookIdx); - }) - .catch((err) => console.error("Start failed:", err)); + body: `book_idx=${bookIdx}`, + }).then(() => refreshBook(bookIdx)); } /* --------------------------------------------------------- - AJAX ABORT - --------------------------------------------------------- */ -function abortBookAjax(bookIdx) { - if (!confirm(`Abort tasks for book ${bookIdx}?`)) return; - - console.log(">>> abortBookAjax():", bookIdx); + ABORT +--------------------------------------------------------- */ +function abortBook(bookIdx) { + if (!confirm(`Abort book ${bookIdx}?`)) return; - fetch(`/abort/${bookIdx}`, { method: "POST" }) - .then(async (r) => { - let data = null; - try { - data = await r.json(); - } catch (e) {} - console.log(">>> /abort response:", data); + console.log("[DASHBOARD] ABORT", bookIdx); - refreshBookCards(); - refreshBook(bookIdx); - }) - .catch((err) => console.error("Abort failed:", err)); + fetch(`/abort/${bookIdx}`, { method: "POST" }).then(() => + refreshBook(bookIdx) + ); } /* --------------------------------------------------------- - Refresh all book-cards (status, classes, buttons) - --------------------------------------------------------- */ + Bookcard UI refresh (non-progress) +--------------------------------------------------------- */ async function refreshBookCards() { + console.log("[DASHBOARD] refreshBookCards()"); const books = await apiGet("/api/books"); if (!books) return; @@ -250,22 +162,17 @@ async function refreshBookCards() { const info = books.find((b) => b.book_idx === idx); if (!info) return; - // Status CSS + console.log("[DASHBOARD] card status", idx, info.status); card.className = `book-card ${info.status}`; - // Button states - const startBtn = card.querySelector(".icon-start"); const abortBtn = card.querySelector(".icon-abort"); - - if (startBtn) startBtn.disabled = info.status !== "registered"; - if (abortBtn) + if (abortBtn) { abortBtn.disabled = ![ "processing", "downloading", "parsing", "audio", ].includes(info.status); + } }); - - bindBookCardButtons(); } diff --git a/bookscraper/static/js/inspect_state.js b/bookscraper/static/js/inspect_state.js new file mode 100644 index 0000000..fed305e --- /dev/null +++ b/bookscraper/static/js/inspect_state.js @@ -0,0 +1,101 @@ +/* ============================================================ + File: static/js/inspect_state.js + Purpose: + - Receive merged state via state_updater.js + - Update ONLY the right-side state tables + - NO polling, NO fetch + ============================================================ */ + +console.log("[inspect_state] JS loaded (subscriber mode)"); + +/* ------------------------------------------------------------ + State subscription + ------------------------------------------------------------ */ + +window.addEventListener("state:update", (e) => { + const entries = e.detail; + + if (!Array.isArray(entries)) { + console.warn("[inspect_state] state:update payload is not array", entries); + return; + } + + console.log("[inspect_state] state:update received entries:", entries.length); + updateInspectTables(entries); +}); + +/* ------------------------------------------------------------ + Update tables + ------------------------------------------------------------ */ + +function updateInspectTables(entries) { + console.log("[inspect_state] updating tables"); + + entries.forEach((entry) => { + const bookIdx = entry.book_idx; + if (bookIdx == null) { + console.warn("[inspect_state] entry without book_idx", entry); + return; + } + + const block = document.querySelector( + `.state-block[data-book-idx="${bookIdx}"]` + ); + if (!block) { + console.warn("[inspect_state] no state-block for book_idx", bookIdx); + return; + } + + const table = block.querySelector(".state-table"); + if (!table) { + console.warn("[inspect_state] no state-table for book_idx", bookIdx); + return; + } + + console.log("[inspect_state] updating table for book_idx", bookIdx); + + const sql = entry.sqlite || {}; + const redis = entry.redis || {}; + const merged = entry.would_merge_to || {}; + + table.innerHTML = ` + + Field + SQLite + Redis + Merged + + ${row("status", sql, redis, merged)} + ${row("chapters_total", sql, redis, merged)} + ${row("downloaded", sql, redis, merged)} + ${row("chapters_download_done", sql, redis, merged)} + ${row("chapters_download_skipped", sql, redis, merged)} + ${row("parsed", sql, redis, merged)} + ${row("chapters_parsed_done", sql, redis, merged)} + ${row("audio_done", sql, redis, merged)} + ${row("audio_skipped", sql, redis, merged)} + ${row("last_update", sql, redis, merged)} + `; + }); +} + +/* ------------------------------------------------------------ + Row helper + ------------------------------------------------------------ */ + +function row(field, sql, redis, merged) { + const s = sql[field] ?? ""; + const r = redis[field] ?? ""; + const m = merged[field] ?? ""; + + const cls = String(s) === String(r) ? "same" : "diff"; + + return ` + + ${field} + ${s} + ${r} + ${m} + + `; +} diff --git a/bookscraper/static/js/progress.js b/bookscraper/static/js/progress.js deleted file mode 100644 index bcd834d..0000000 --- a/bookscraper/static/js/progress.js +++ /dev/null @@ -1,72 +0,0 @@ -/* ======================================================================= - File: static/js/progress.js - Purpose: - Update progress bars dynamically for the current book. - Only updates the main progress box (book_detail page). - ======================================================================= */ - -console.log(">>> progress.js LOADED"); - -function updateProgressBars(data) { - console.log(">>> progress.js updateProgressBars() CALLED with:", data); - - if (!data) { - console.warn(">>> progress.js: NO DATA RECEIVED"); - return; - } - - // We always update inside the main progress box: - const container = document.querySelector("#progressSection"); - if (!container) { - console.warn(">>> progress.js: #progressSection NOT FOUND"); - return; - } - - // Select bars ONLY inside the correct section - const barDL = container.querySelector( - ".progress-bar:not(.audio) .progress-bar-fill" - ); - const barAU = container.querySelector( - ".progress-bar.audio .progress-bar-fill" - ); - - const pctDL = - data.download_total > 0 - ? (100 * data.download_done) / data.download_total - : 0; - - const pctAU = - data.audio_total > 0 ? (100 * data.audio_done) / data.audio_total : 0; - - if (barDL) { - barDL.style.width = pctDL.toFixed(1) + "%"; - console.log(">>> progress.js DL bar =", pctDL.toFixed(1) + "%"); - } else { - console.warn(">>> progress.js: barDL NOT FOUND INSIDE #progressSection"); - } - - if (barAU) { - barAU.style.width = pctAU.toFixed(1) + "%"; - console.log(">>> progress.js AU bar =", pctAU.toFixed(1) + "%"); - } else { - console.warn(">>> progress.js: barAU NOT FOUND INSIDE #progressSection"); - } - - // Textual stats — only update inside progress box - const stats = container.querySelectorAll(".progress-stats span"); - - // Expected: [DL x/y, DL %, AU x/y, AU %] - if (stats.length >= 4) { - stats[0].innerText = `${data.download_done} / ${data.download_total}`; - stats[1].innerText = pctDL.toFixed(1) + "%"; - stats[2].innerText = `${data.audio_done} / ${data.audio_total}`; - stats[3].innerText = pctAU.toFixed(1) + "%"; - - console.log(">>> progress.js stats updated"); - } else { - console.warn( - ">>> progress.js: not enough stats spans in the container, found", - stats.length - ); - } -} diff --git a/bookscraper/static/js/state_updater.js b/bookscraper/static/js/state_updater.js new file mode 100644 index 0000000..dda88e8 --- /dev/null +++ b/bookscraper/static/js/state_updater.js @@ -0,0 +1,98 @@ +/* ======================================================== + File: static/js/state_updater.js + Purpose: + - Poll /api/state/all + - Dispatch merged state to subscribers + (bookcard_controller, inspect_state, others) + - Pause polling when tab inactive + ======================================================== */ + +console.log("[STATE-UPDATER] loaded"); + +const STATE_POLL_INTERVAL_MS = 2500; +const STATE_ENDPOINT = "/api/state/all"; + +let STATE_TIMER = null; + +/* ======================================================== + INIT + ======================================================== */ + +document.addEventListener("DOMContentLoaded", () => { + initStateUpdater(); +}); + +function initStateUpdater() { + const cards = document.querySelectorAll(".book-card"); + + if (cards.length === 0) { + console.log("[STATE-UPDATER] No bookcards found — skipping"); + return; + } + + console.log(`[STATE-UPDATER] Starting updater for ${cards.length} bookcards`); + + startPolling(true); + + document.addEventListener("visibilitychange", () => { + document.hidden ? stopPolling() : startPolling(true); + }); +} + +/* ======================================================== + DISPATCH + ======================================================== */ + +function dispatchState(entries) { + console.debug("[STATE] dispatch", entries.length); + + // 1. Bookcards + if (typeof window.updateBookCardsFromState === "function") { + window.updateBookCardsFromState(entries); + } + + // 2. Inspect state tables / other subscribers + window.dispatchEvent(new CustomEvent("state:update", { detail: entries })); +} + +/* ======================================================== + POLLING CONTROL + ======================================================== */ + +function startPolling(immediate = false) { + if (STATE_TIMER) return; + + console.log("[STATE-UPDATER] Start polling"); + + if (immediate) pollState(); + + STATE_TIMER = setInterval(pollState, STATE_POLL_INTERVAL_MS); +} + +function stopPolling() { + if (!STATE_TIMER) return; + + console.log("[STATE-UPDATER] Stop polling (tab inactive)"); + clearInterval(STATE_TIMER); + STATE_TIMER = null; +} + +/* ======================================================== + POLL API + ======================================================== */ + +async function pollState() { + if (document.hidden) return; + + try { + const resp = await fetch(STATE_ENDPOINT, { cache: "no-store" }); + if (!resp.ok) return; + + const entries = await resp.json(); + if (!Array.isArray(entries)) return; + + dispatchState(entries); + } catch (e) { + console.error("[STATE-UPDATER] poll error", e); + } +} diff --git a/bookscraper/templates/base.html b/bookscraper/templates/base.html index 57961e4..c9394a1 100644 --- a/bookscraper/templates/base.html +++ b/bookscraper/templates/base.html @@ -20,7 +20,16 @@ - + + + + diff --git a/bookscraper/templates/components/bookcard.html b/bookscraper/templates/components/bookcard.html index 4815b5a..023a6bb 100644 --- a/bookscraper/templates/components/bookcard.html +++ b/bookscraper/templates/components/bookcard.html @@ -1,63 +1,48 @@ -{# ============================================================ - File: templates/components/bookcard.html - Purpose: - Eén enkele boekkaart met: - - status styles - - cover - - metadata - - hide button - - start (play) - - abort (stop) - Requires: - variable "b" in context - ============================================================ #} +{# ============================================================ File: +templates/components/bookcard.html Purpose: Eén enkele boekkaart (dumb +component) ============================================================ #}
- - +
- +
{% if b.cover_path %} - cover + cover {% else %} -
?
+
?
{% endif %}
- +
-
{{ b.title }}
-
{{ b.author }}
-
Geregistreerd: {{ b.created_at }}
+
{{ b.title }}
+
{{ b.author }}
+
+ Geregistreerd: {{ b.created_at }} +
+
-
@@ -65,18 +50,13 @@
-
+
-
- - + +
{% include "components/progress_box.html" %}
+ diff --git a/bookscraper/templates/components/nav.html b/bookscraper/templates/components/nav.html index 10e6485..4a97acd 100644 --- a/bookscraper/templates/components/nav.html +++ b/bookscraper/templates/components/nav.html @@ -21,7 +21,7 @@
  • - Logs + State overview
    • diff --git a/bookscraper/templates/components/progress_box.html b/bookscraper/templates/components/progress_box.html index 8380c2a..3847505 100644 --- a/bookscraper/templates/components/progress_box.html +++ b/bookscraper/templates/components/progress_box.html @@ -1,62 +1,34 @@ + Purpose: + Dumb progress UI for a book card. + Initial values via Jinja, live updates via state_updater.js +======================================================================= -->
    - -
    -

    Progress

    - {% if title %} -
    {{ title }}
    - {% endif %} {% if book_idx %} -
    Book IDX: {{ book_idx }}
    - {% endif %} -
    - - -
    -

    Download Progress

    - -
    - {% set pct = 0 %} {% if download_total > 0 %} {% set pct = (100 * - download_done / download_total) | round(1) %} {% endif %} -
    -
    - -
    - {{ download_done }} / {{ download_total }} - {{ pct }}% + +
    +
    Download
    +
    +
    +
    0 / 0
    - -
    -

    Audio Progress

    - -
    - {% set pct2 = 0 %} {% if audio_total > 0 %} {% set pct2 = (100 * - audio_done / audio_total) | round(1) %} {% endif %} + +
    +
    Audio
    +
    -
    - -
    - {{ audio_done }} / {{ audio_total }} - {{ pct2 }}% +
    0 / 0
    - -
    diff --git a/bookscraper/templates/components/url_input.html b/bookscraper/templates/components/url_input.html index d69db52..882bc3f 100644 --- a/bookscraper/templates/components/url_input.html +++ b/bookscraper/templates/components/url_input.html @@ -6,16 +6,17 @@ ======================================================================= -->
    - + - + > - +
    diff --git a/bookscraper/templates/debug/inspect_state.html b/bookscraper/templates/debug/inspect_state.html index 752266b..6501dc1 100644 --- a/bookscraper/templates/debug/inspect_state.html +++ b/bookscraper/templates/debug/inspect_state.html @@ -1,7 +1,8 @@ {# ============================================================ File: templates/debug/inspect_state.html Purpose: Inspect SQLite vs Redis state per -book_idx. Left side: full book-card UI (same component as dashboard) Right side: -SQL / Redis / merged comparison table. +book_idx - Initial render via Jinja - Live updates via inspect_state.js - +BookCard is server-rendered and NEVER replaced - Only the right-side state table +is updated dynamically ============================================================ #} {% extends "layout.html" %} {% block content %} @@ -43,53 +44,52 @@ SQL / Redis / merged comparison table. .same { color: #9f9 !important; } + .diff { color: #ff7b7b !important; font-weight: bold; } - .empty { - color: #aaa !important; - font-style: italic; - } -{% macro cmp(sqlval, redisval) %} {% if (sqlval|string) == (redisval|string) %} -{{ sqlval }} -{{ redisval }} -{% else %} -{{ sqlval }} -{{ redisval }} -{% endif %} {% endmacro %} {% for entry in results %} -
    - -
    - {% with b = entry.card %} {% include "components/bookcard.html" %} {% - endwith %} -
    +
    + {% for entry in results %} +
    + +
    + {% if entry.card %} {% with b = entry.card %} {% include + "components/bookcard.html" %} {% endwith %} {% else %} + {{ entry.book_idx }} + {% endif %} +
    - -
    - - - - - - - + +
    +
    FieldSQLiteRedisMerged Result
    + + + + + + - {% set sql = entry.sqlite %} {% set redis = entry.redis %} {% set merged = - entry.would_merge_to %} {% for field in [ "status", "chapters_total", - "downloaded", "chapters_download_done", "chapters_download_skipped", - "parsed", "chapters_parsed_done", "audio_done", "audio_skipped", - "last_update" ] %} - - - - - - - {% endfor %} -
    FieldSQLiteRedisMerged
    {{ field }}{{ sql.get(field, '') }}{{ redis.get(field, '') }}{{ merged.get(field, '') }}
    + {% set sql = entry.sqlite %} {% set redis = entry.redis %} {% set merged + = entry.would_merge_to %} {% for field in [ "status", "chapters_total", + "downloaded", "chapters_download_done", "chapters_download_skipped", + "parsed", "chapters_parsed_done", "audio_done", "audio_skipped", + "last_update" ] %} + + {{ field }} + {{ sql.get(field, "") }} + {{ redis.get(field, "") }} + {{ merged.get(field, "") }} + + {% endfor %} + +
    + {% endfor %}
    -{% endfor %} {% endblock %} + +{% endblock %} {% block scripts %} + +{% endblock %} diff --git a/bookscraper/templates/layout.html b/bookscraper/templates/layout.html index 14d6788..510b793 100644 --- a/bookscraper/templates/layout.html +++ b/bookscraper/templates/layout.html @@ -32,6 +32,11 @@
    BookScraper © 2025 — Powered by Celery + Redis
    + {% block scripts %}{% endblock %} + + + + diff --git a/bookscraper/templates/result.html b/bookscraper/templates/result.html index 16ef2d5..cc7fdec 100644 --- a/bookscraper/templates/result.html +++ b/bookscraper/templates/result.html @@ -32,6 +32,7 @@ font-size: 13px; } + /* NEW: Clear button */ #clearLogBtn { margin-bottom: 10px; padding: 8px 16px; @@ -65,11 +66,12 @@ display: none; } + s ← Terug -

    Scrape Resultaat

    +

    Scrape Resultaat--

    {% if error %}
    Fout: {{ error }}
    - {% endif %} - - {% if message %} + {% endif %} {% if message %}
    {{ message }}
    {% endif %} @@ -114,29 +114,127 @@ class="box hidden" style="background: #ffefef; border-left: 5px solid #cc0000" > - Mislukte hoofdstukken: + Failed chapters:
      Live log:
      +
      + +