filesystem-based statuscheck, merged state fixes

bookscraper/app.py

@@ -349,6 +349,25 @@ def logs():
     return jsonify({"lines": new_lines, "last": new_last})
 
 
+from flask import render_template
+from scraper.services.status_check_service import StatusCheckService
+from logbus.publisher import log
+
+
+from db.repository import get_book_state
+
+
+@app.route("/inspect/statuscheck/<book_idx>", methods=["POST"])
+@logcall
+def inspect_statuscheck(book_idx):
+    try:
+        StatusCheckService.run(book_idx)
+        return ("", 204)  # background action, geen UI
+    except Exception as e:
+        log(f"[STATUSCHECK] ERROR book_idx={book_idx}: {e}")
+        return jsonify({"error": str(e)}), 500
+
+
 # =====================================================
 # SECTION 4 — DEBUG ROUTES
 # =====================================================

bookscraper/db/repository.py

@@ -304,53 +304,87 @@ def inc_audio_done_legacy(book_idx, amount=1):
 # ============================================================
 # READ — DERIVED BOOK STATE
 # ============================================================
-
-
 @logcall
 def get_book_state(book_idx):
     """
-    Canonical read-model for a single book.
+    Canonical merged read-model for a single book.
-
+
-    Responsibilities:
+    Gedrag:
-    - Read SQLite snapshot (static metadata)
+    - Leest SQL (snapshot)
-    - Read Redis live state (counters / status)
+    - Leest Redis (live counters)
-    - Compute derived fields (NO UI logic)
+    - Rekent naar merged
-
+    - GEEN writes
-    Invariants:
+    - GEEN side-effects
-    - downloaded = chapters_download_done + chapters_download_skipped
+
+    Merge-regels:
+    - merged = max(sql, redis)
+    - merged wordt gecapt op chapters_total
     """
 
-    # --- SQLite snapshot ---
+    # ----------------------------------------------------
+    # 1. Fetch bronnen
+    # ----------------------------------------------------
     sqlite_row = sql_fetch_book(book_idx) or {}
 
-    # --- Redis live state ---
     key = f"book:{book_idx}:state"
     redis_state = _r.hgetall(key) or {}
 
-    # Normalize numeric redis values
     def _int(v):
         try:
             return int(v)
         except Exception:
             return 0
 
-    # --- primary counters ---
+    # ----------------------------------------------------
-    chapters_done = _int(redis_state.get("chapters_download_done"))
+    # 2. SQL snapshot
-    chapters_skipped = _int(redis_state.get("chapters_download_skipped"))
+    # ----------------------------------------------------
-
+    chapters_total = _int(sqlite_row.get("chapters_total"))
-    # --- derived counters ---
+    sql_downloaded = _int(sqlite_row.get("downloaded"))
-    downloaded = chapters_done + chapters_skipped
+    sql_audio_done = _int(sqlite_row.get("audio_done"))
-
+
-    # --- build canonical state ---
+    # ----------------------------------------------------
+    # 3. Redis live counters
+    # ----------------------------------------------------
+    redis_downloaded = _int(redis_state.get("chapters_download_done")) + _int(
+        redis_state.get("chapters_download_skipped")
+    )
+
+    redis_audio_done = _int(redis_state.get("audio_done"))
+
+    # ----------------------------------------------------
+    # 4. Merge (SQL vs Redis)
+    # ----------------------------------------------------
+    merged_downloaded = max(sql_downloaded, redis_downloaded)
+    merged_audio_done = max(sql_audio_done, redis_audio_done)
+
+    if chapters_total > 0:
+        merged_downloaded = min(merged_downloaded, chapters_total)
+        merged_audio_done = min(merged_audio_done, chapters_total)
+
+    # ----------------------------------------------------
+    # 5. Bouw merged state (read-only)
+    # ----------------------------------------------------
     state = {}
 
-    # 1) start with SQLite snapshot
+    # Basis = SQL
     state.update(sqlite_row)
 
-    # 2) overlay Redis live fields
+    # Overschrijf alleen met merged conclusies
-    state.update(redis_state)
+    state["downloaded"] = merged_downloaded
-
+    state["audio_done"] = merged_audio_done
-    # 3) enforce derived invariants
+    state["chapters_total"] = chapters_total
-    state["downloaded"] = downloaded
+    # ----------------------------------------------------
+    # 4b. Derive status (READ-ONLY)
+    # ----------------------------------------------------
+    derived_status = sqlite_row.get("status") or "unknown"
+
+    if chapters_total > 0:
+        if merged_downloaded < chapters_total:
+            derived_status = "downloading"
+        elif merged_downloaded == chapters_total and merged_audio_done < chapters_total:
+            derived_status = "audio"
+        elif merged_audio_done == chapters_total:
+            derived_status = "done"
+    state["status"] = derived_status
 
     return state

bookscraper/scraper/scriptgen.py

@@ -5,7 +5,6 @@
 import os
 import stat
 from logbus.publisher import log
-
 from scraper.logger_decorators import logcall
 
 TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates")
@@ -45,26 +44,40 @@ def detect_volumes(book_base: str):
 # ------------------------------------------------------------
 def build_merge_block(title: str, author: str, volumes):
     lines = []
+
+    # --------------------------------------------------------
+    # Normalize input (defensive)
+    # --------------------------------------------------------
+    title = (title or "").strip()
+    author = (author or "").strip()
+
     total_vols = len(volumes)
+
+    # Padding-regel:
+    # - altijd minimaal 2 (01, 02)
+    # - 3 bij >=100
     if total_vols >= 100:
         pad = 3
-    elif total_vols >= 10:
-        pad = 2
     else:
-        pad = 0
+        pad = 2
 
     for num, dirname in volumes:
-        if pad > 0:
+        vol_num = f"{num:0{pad}d}"  # voor filename
-            vol_num = f"{num:0{pad}d}"
+        series_part = f"{num:0{pad}d}"  # voor series-part (string!)
-        else:
-            vol_num = str(num)
 
         line = (
-            f'm4b-tool merge --jobs=4 --writer="{author}" '
+            f"m4b-tool merge --jobs=4 "
-            f'--albumartist="{author}" --album="{title}" '
+            f'--writer="{author}" '
-            f'--name="{title}" --output-file="{title}-{vol_num}.m4b" '
+            f'--sortalbum="{title}" '
+            f'--albumartist="{author}" '
+            f'--album="{title}" '
+            f'--name="{title}" '
+            f'--series="{title}" '
+            f'--series-part="{series_part}" '
+            f'--output-file="{title}-{vol_num}.m4b" '
             f'"{dirname}" -vvv'
         )
+
         lines.append(line)
 
     if not lines:
@@ -76,7 +89,14 @@ def build_merge_block(title: str, author: str, volumes):
 # ------------------------------------------------------------
 # Main generator
 # ------------------------------------------------------------
+@logcall
 def generate_all_scripts(book_base: str, title: str, author: str):
+    # --------------------------------------------------------
+    # Defensive normalize
+    # --------------------------------------------------------
+    title = (title or "").strip()
+    author = (author or "").strip()
+
     log(f"[SCRIPTGEN] Generating scripts in {book_base}")
 
     # Load templates

bookscraper/scraper/services/status_check_service.py

@@ -0,0 +1,135 @@
+# ============================================================
+# File: scraper/services/status_check_service.py
+# Purpose:
+#   Handmatige, idempotente statuscheck per boek.
+#
+#   Bepaalt op basis van het filesystem:
+#     - aantal gedownloade chapters (.txt)
+#     - aantal gegenereerde audiofiles (.m4b)
+#
+#   En schrijft deze gevalideerde werkelijkheid naar SQL.
+#
+#   LET OP:
+#   - Geen Redis
+#   - Geen Celery
+#   - Geen status-transities
+#   - Geen pipeline-logica
+# ============================================================
+
+import os
+from datetime import datetime
+from typing import Dict, Any
+
+from logbus.publisher import log
+from scraper.logger_decorators import logcall
+
+from db.state_sql import sql_fetch_book, sql_update_book
+
+
+class StatusCheckService:
+    """
+    Statuscheck op basis van filesystem.
+    Single source of truth = disk.
+    """
+
+    @staticmethod
+    @logcall
+    def run(book_idx: str) -> Dict[str, Any]:
+        """
+        Voer statuscheck uit voor één boek.
+
+        Returns een inspecteerbaar dict met:
+          - filesystem tellingen
+          - SQL before / after snapshot
+        """
+
+        # ----------------------------------------------------
+        # 1. SQL fetch (bestaat het boek?)
+        # ----------------------------------------------------
+        sql_before = sql_fetch_book(book_idx)
+
+        if not sql_before:
+            raise ValueError(f"[STATUSCHECK] Book not found in SQL: {book_idx}")
+
+        # ----------------------------------------------------
+        # 2. Bepaal filesystem root
+        # ----------------------------------------------------
+        output_root = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
+        title = sql_before.get("title")
+        book_dir = os.path.join(output_root, title)
+
+        if not os.path.isdir(book_dir):
+            log(
+                f"[STATUSCHECK] No output directory for book_idx={book_idx} : title='{title}')"
+            )
+            chapters_txt = 0
+            audio_files = 0
+            volumes = 0
+        else:
+            chapters_txt = 0
+            audio_files = 0
+            volumes = 0
+
+            # ------------------------------------------------
+            # 3. Scan volumes
+            # ------------------------------------------------
+            for entry in os.listdir(book_dir):
+                if not entry.lower().startswith("volume_"):
+                    continue
+
+                volumes += 1
+                volume_path = os.path.join(book_dir, entry)
+
+                if not os.path.isdir(volume_path):
+                    continue
+
+                # ---- TXT chapters ----
+                for fname in os.listdir(volume_path):
+                    if fname.lower().endswith(".txt"):
+                        chapters_txt += 1
+
+                # ---- Audio ----
+                audio_dir = os.path.join(volume_path, "Audio")
+                if os.path.isdir(audio_dir):
+                    for fname in os.listdir(audio_dir):
+                        if fname.lower().endswith(".m4b"):
+                            audio_files += 1
+
+        # ----------------------------------------------------
+        # 4. SQL update (snapshot)
+        # ----------------------------------------------------
+        now = datetime.utcnow().isoformat(timespec="seconds")
+
+        update_fields = {
+            "downloaded": chapters_txt,
+            "audio_done": audio_files,
+            "last_update": now,
+        }
+
+        sql_update_book(book_idx, update_fields)
+
+        sql_after = sql_fetch_book(book_idx)
+
+        # ----------------------------------------------------
+        # 5. Resultaat voor inspect/debug
+        # ----------------------------------------------------
+        result = {
+            "book_idx": book_idx,
+            "filesystem": {
+                "book_dir": book_dir,
+                "exists": os.path.isdir(book_dir),
+                "volumes": volumes,
+                "chapters_txt": chapters_txt,
+                "audio_files": audio_files,
+            },
+            "sql_before": sql_before,
+            "sql_after": sql_after,
+            "notes": [],
+        }
+
+        log(
+            f"[STATUSCHECK] book_idx={book_idx} "
+            f"chapters={chapters_txt} audio={audio_files}"
+        )
+
+        return result

bookscraper/static/css/bookcard.css

@@ -1,8 +1,11 @@
 /* =======================================================================
    File: static/css/bookcard.css
    Purpose:
-      All styling for registered book cards (book-card) +
+      Styling voor registered book cards:
-      status colors + start/abort buttons + progress bars
+      - status kleuren
+      - badges
+      - start/abort/statuscheck
+      - progress bars
    ======================================================================= */
 
 /* -----------------------------------------------------------------------
@@ -17,7 +20,7 @@
 }
 
 /* -----------------------------------------------------------------------
-   BOOK CARD
+   BOOK CARD BASE
    ----------------------------------------------------------------------- */
 
 .book-card {
@@ -36,34 +39,28 @@
 }
 
 /* -----------------------------------------------------------------------
-   STATUS COLORS
+   STATUS COLORS (BOOK CARD BORDER)
    ----------------------------------------------------------------------- */
 
-.book-card.processing {
+/* Downloading / actief bezig */
-  border-color: #007aff;
-  box-shadow: 0 0 6px rgba(0, 122, 255, 0.35);
-}
-
 .book-card.downloading {
   border-color: #ff9500;
   box-shadow: 0 0 6px rgba(255, 149, 0, 0.35);
 }
 
-.book-card.parsing {
+/* Audio fase */
-  border-color: #ffcc00;
-  box-shadow: 0 0 6px rgba(255, 204, 0, 0.35);
-}
-
 .book-card.audio {
-  border-color: #34c759;
+  border-color: #ffca28;
-  box-shadow: 0 0 6px rgba(52, 199, 89, 0.35);
+  box-shadow: 0 0 6px rgba(255, 202, 40, 0.35);
 }
 
-.book-card.completed {
+/* Volledig klaar */
-  border-color: #34c759;
+.book-card.done {
-  box-shadow: 0 0 6px rgba(52, 199, 89, 0.35);
+  border: 2px solid #4caf50;
+  box-shadow: 0 0 6px rgba(76, 175, 80, 0.35);
 }
 
+/* Afgebroken */
 .book-card.aborted {
   border-color: #ff3b30;
   box-shadow: 0 0 6px rgba(255, 59, 48, 0.35);
@@ -188,6 +185,21 @@
   background: #555;
 }
 
+/* Statuscheck */
+.statuscheck-btn {
+  background-color: #444;
+  color: #fff;
+  border: 1px solid #666;
+  margin-left: 4px;
+  padding: 4px 8px;
+  border-radius: 6px;
+  font-size: 12px;
+  cursor: pointer;
+}
+.statuscheck-btn:hover {
+  background-color: #333;
+}
+
 /* -----------------------------------------------------------------------
    PROGRESS (FULL WIDTH)
    ----------------------------------------------------------------------- */
@@ -201,7 +213,7 @@
 }
 
 .progress-row {
-  margin-bottom: 10px;
+  margin-bottom: 4px;
 }
 
 .progress-label {
@@ -225,12 +237,12 @@
   transition: width 0.4s ease;
 }
 
-/* Download = blauw */
+/* Download */
 .progressbar-fill.download {
   background: #2196f3;
 }
 
-/* Audio = groen */
+/* Audio */
 .progressbar-fill.audio {
   background: #4caf50;
 }
@@ -249,3 +261,50 @@
   text-shadow: 0 1px 2px rgba(0, 0, 0, 0.6);
   pointer-events: none;
 }
+
+/* -----------------------------------------------------------------------
+   STATUS BADGE
+   ----------------------------------------------------------------------- */
+
+.status-badge {
+  display: inline-block;
+  margin-bottom: 6px;
+  padding: 2px 8px;
+  font-size: 11px;
+  font-weight: 600;
+  border-radius: 10px;
+  text-transform: uppercase;
+  letter-spacing: 0.5px;
+  cursor: default;
+}
+
+/* DONE */
+.status-badge.status-done {
+  background-color: #e6f4ea;
+  color: #2e7d32;
+  border: 1px solid #4caf50;
+}
+
+/* AUDIO */
+.status-badge.status-audio {
+  background-color: #fff8e1;
+  color: #8d6e00;
+  border: 1px solid #ffca28;
+}
+
+/* DOWNLOADING */
+.status-badge.status-downloading {
+  background-color: #e3f2fd;
+  color: #1565c0;
+  border: 1px solid #42a5f5;
+}
+
+/* Statuscheck */
+.icon-statuscheck {
+  background: #444;
+}
+
+.icon-statuscheck:hover {
+  background: #333;
+  transform: scale(1.05);
+}

bookscraper/static/css/dashboard.css

@@ -84,9 +84,8 @@
 .progress-box {
   background: #fafafa;
   border: 1px solid #ddd;
-  padding: 18px;
+  padding: 8px;
   border-radius: 6px;
-  width: 100%;
 }
 
 .progress-header h2 {
@@ -295,3 +294,19 @@
 .dropdown-menu li a:hover {
   background: #f0f0f0;
 }
+
+table.kv {
+  border-collapse: collapse;
+  margin-bottom: 16px;
+}
+
+table.kv th {
+  text-align: left;
+  padding-right: 12px;
+  color: #777;
+  font-weight: normal;
+}
+
+table.kv td {
+  font-weight: 500;
+}

bookscraper/static/js/bookcard_controller.js

@@ -52,6 +52,7 @@ function updateSingleBookCard(card, state) {
   console.log("[BOOKCARD] updateSingleBookCard", state.book_idx);
 
   updateStatus(card, state);
+  updateStatusBadge(card, state);
   updateButtons(card, state);
   updateProgress(card, state);
 }
@@ -64,6 +65,21 @@ function updateStatus(card, state) {
   console.log("[BOOKCARD][STATUS]", state.book_idx, "→", state.status);
   card.className = `book-card ${state.status || ""}`;
 }
+function updateStatusBadge(card, state) {
+  const badge = card.querySelector(".status-badge");
+  if (!badge) return;
+
+  const status = (state.status || "").toLowerCase();
+
+  badge.textContent = status.toUpperCase();
+  badge.className = `status-badge status-${status}`;
+  badge.title =
+    {
+      downloading: "Bezig met downloaden",
+      audio: "Downloads compleet, audio wordt gegenereerd",
+      done: "Alle chapters en audio zijn compleet",
+    }[status] || "";
+}
 
 /* ============================================================
    BUTTONS

bookscraper/templates/components/bookcard.html

@@ -31,6 +31,20 @@ component) ============================================================ #}
 
   <!-- META -->
   <div class="book-meta">
+    <!-- STATUS BADGE -->
+    {% if b.status %}
+    <span
+      class="status-badge status-{{ b.status }}"
+      title="
+{% if b.status == 'done' %}Alle chapters en audio zijn compleet{% endif %}
+{% if b.status == 'audio' %}Downloads compleet, audio wordt nog gegenereerd{% endif %}
+{% if b.status == 'downloading' %}Bezig met downloaden{% endif %}
+"
+    >
+      {{ b.status | upper }}
+    </span>
+    {% endif %}
+
     <div class="book-title" data-field="title">{{ b.title }}</div>
     <div class="book-author" data-field="author">{{ b.author }}</div>
     <div class="book-created">
@@ -54,6 +68,20 @@ component) ============================================================ #}
           <i class="fa-solid fa-stop"></i>
         </button>
       </form>
+
+      <form
+        method="post"
+        action="/inspect/statuscheck/{{ b.book_idx }}"
+        style="display: inline-block"
+      >
+        <button
+          type="submit"
+          class="icon-btn icon-statuscheck"
+          title="Herbereken status op basis van bestanden"
+        >
+          <i class="fa-solid fa-magnifying-glass-chart"></i>
+        </button>
+      </form>
     </div>
   </div>
 

bookscraper/templates/dashboard/dashboard.html

@@ -22,25 +22,11 @@
 
   <hr />
 
-  {% include "components/registered_books.html" %}
-
-  <hr />
   <!-- ===========================================================
        BOOK LIST
        =========================================================== -->
-  <section class="dashboard-section">
-    <h2>Actieve boeken</h2>
-
-    {% if books and books|length > 0 %}
-    <div id="book-list" class="book-list">
-      {% for book in books %} {% include "components/book_list_item.html" %} {%
-      endfor %}
-    </div>
-    {% else %}
-    <div id="book-list" class="book-list-empty">Geen actieve boeken.</div>
-    {% endif %}
-  </section>
 
+  {% include "components/registered_books.html" %}
   <hr />
 
   <!-- ===========================================================

bookscraper/templates/inspect/statuscheck_result.html

@@ -0,0 +1,115 @@
+{% extends "layout.html" %} {% block content %}
+
+<h2>Statuscheck – Inspect</h2>
+
+{% if error %}
+<div class="error"><strong>Fout:</strong> {{ error }}</div>
+{% else %}
+
+<!-- ===================================================== -->
+<!-- BOEK -->
+<!-- ===================================================== -->
+<h3>Boek</h3>
+<table class="kv">
+  <tr>
+    <th>Book idx</th>
+    <td>{{ result.book_idx }}</td>
+  </tr>
+  <tr>
+    <th>Pad</th>
+    <td>{{ result.filesystem.book_dir }}</td>
+  </tr>
+  <tr>
+    <th>Bestaat</th>
+    <td>{{ result.filesystem.exists }}</td>
+  </tr>
+</table>
+
+<!-- ===================================================== -->
+<!-- FILESYSTEM -->
+<!-- ===================================================== -->
+<h3>Filesystem (source of truth)</h3>
+<table class="kv">
+  <tr>
+    <th>Volumes</th>
+    <td>{{ result.filesystem.volumes }}</td>
+  </tr>
+  <tr>
+    <th>Chapters (.txt)</th>
+    <td>{{ result.filesystem.chapters_txt }}</td>
+  </tr>
+  <tr>
+    <th>Audio (.m4b)</th>
+    <td>{{ result.filesystem.audio_files }}</td>
+  </tr>
+</table>
+
+<!-- ===================================================== -->
+<!-- SQL -->
+<!-- ===================================================== -->
+<h3>SQL snapshot</h3>
+
+<h4>Voor</h4>
+<table class="kv">
+  <tr>
+    <th>Downloaded</th>
+    <td>{{ result.sql_before.downloaded }}</td>
+  </tr>
+  <tr>
+    <th>Audio done</th>
+    <td>{{ result.sql_before.audio_done }}</td>
+  </tr>
+  <tr>
+    <th>Status</th>
+    <td>{{ result.sql_before.status }}</td>
+  </tr>
+</table>
+
+<h4>Na</h4>
+<table class="kv">
+  <tr>
+    <th>Downloaded</th>
+    <td>{{ result.sql_after.downloaded }}</td>
+  </tr>
+  <tr>
+    <th>Audio done</th>
+    <td>{{ result.sql_after.audio_done }}</td>
+  </tr>
+  <tr>
+    <th>Last update</th>
+    <td>{{ result.sql_after.last_update }}</td>
+  </tr>
+</table>
+
+<!-- ===================================================== -->
+<!-- REPOSITORY -->
+<!-- ===================================================== -->
+<h3>Repository merged state (UI input)</h3>
+
+<table class="kv">
+  <tr>
+    <th>Downloaded</th>
+    <td>{{ repo_state.downloaded }}</td>
+  </tr>
+  <tr>
+    <th>Audio done</th>
+    <td>{{ repo_state.audio_done }}</td>
+  </tr>
+  <tr>
+    <th>Chapters total</th>
+    <td>{{ repo_state.chapters_total }}</td>
+  </tr>
+</table>
+
+<details>
+  <summary>Raw repository state</summary>
+  <pre>{{ repo_state | tojson(indent=2) }}</pre>
+</details>
+
+{% endif %}
+
+<hr />
+
+<a href="/dashboard">← Terug naar dashboard</a>
+
+{% endblock %}