parent
292c9246a1
commit
feb8ca60d7
@ -0,0 +1,79 @@
|
||||
# ============================================================
|
||||
# File: db/state_redis.py
|
||||
# Purpose:
|
||||
# Low-level Redis counters/state for BookScraper.
|
||||
# Used ONLY by db.repository façade.
|
||||
# ============================================================
|
||||
|
||||
import os
|
||||
import time
|
||||
import redis
|
||||
|
||||
from logbus.publisher import log
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
|
||||
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# STATUS
|
||||
# ------------------------------------------------------------
|
||||
def redis_set_status(book_id: str, status: str):
|
||||
key = f"book:{book_id}:state"
|
||||
r.hset(key, "status", status)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# TOTAL CHAPTERS
|
||||
# ------------------------------------------------------------
|
||||
def redis_set_chapters_total(book_id: str, total: int):
|
||||
key = f"book:{book_id}:state"
|
||||
r.hset(key, "chapters_total", total)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# DOWNLOAD COUNTERS
|
||||
# ------------------------------------------------------------
|
||||
def redis_inc_download_done(book_id: str, amount: int = 1):
|
||||
key = f"book:{book_id}:state"
|
||||
r.hincrby(key, "chapters_download_done", amount)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
def redis_inc_download_skipped(book_id: str, amount: int = 1):
|
||||
log(f"[DB-REDIS] Incrementing download skipped for {book_id} by {amount}")
|
||||
key = f"book:{book_id}:state"
|
||||
r.hincrby(key, "chapters_download_skipped", amount)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# PARSE COUNTERS
|
||||
# ------------------------------------------------------------
|
||||
def redis_inc_parsed_done(book_id: str, amount: int = 1):
|
||||
key = f"book:{book_id}:state"
|
||||
r.hincrby(key, "chapters_parsed_done", amount)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# AUDIO COUNTERS
|
||||
# ------------------------------------------------------------
|
||||
def redis_inc_audio_done(book_id: str, amount: int = 1):
|
||||
log(f"[DB-REDIS] Incrementing audio done for {book_id} by {amount}")
|
||||
key = f"book:{book_id}:state"
|
||||
r.hincrby(key, "audio_done", amount)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
|
||||
|
||||
def redis_inc_audio_skipped(book_id: str, amount: int = 1):
|
||||
log(f"[DB-REDIS] Incrementing audio skipped for {book_id} by {amount}")
|
||||
"""
|
||||
New: Count skipped audio chapters (timeouts, pre-existing files, abort, etc.)
|
||||
SQL does NOT track this; Redis-only metric.
|
||||
"""
|
||||
key = f"book:{book_id}:state"
|
||||
r.hincrby(key, "audio_skipped", amount)
|
||||
r.hset(key, "last_update", int(time.time()))
|
||||
@ -0,0 +1,165 @@
|
||||
# ============================================================
|
||||
# File: db/state_sql.py
|
||||
# Purpose:
|
||||
# Low-level SQLite snapshot layer for BookScraper metadata.
|
||||
# Used ONLY through db.repository façade.
|
||||
# ============================================================
|
||||
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
from logbus.publisher import log
|
||||
|
||||
DB_PATH = os.getenv("BOOKSCRAPER_DB", "/app/db/books.db")
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# INTERNAL HELPERS
|
||||
# ------------------------------------------------------------
|
||||
def _connect():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# FETCH
|
||||
# ------------------------------------------------------------
|
||||
def sql_fetch_book(book_id):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM books WHERE book_id = ?", (book_id,))
|
||||
row = cur.fetchone()
|
||||
conn.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def sql_fetch_all_books():
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT * FROM books ORDER BY rowid DESC")
|
||||
rows = cur.fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# REGISTER / UPDATE
|
||||
# ------------------------------------------------------------
|
||||
def sql_register_book(book_id, fields: dict):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
cols = ", ".join(["book_id"] + list(fields.keys()))
|
||||
placeholders = ", ".join(["?"] * (1 + len(fields)))
|
||||
values = [book_id] + list(fields.values())
|
||||
|
||||
cur.execute(
|
||||
f"INSERT OR REPLACE INTO books ({cols}) VALUES ({placeholders})", values
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def sql_update_book(book_id, fields: dict):
|
||||
if not fields:
|
||||
return
|
||||
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
|
||||
set_clause = ", ".join([f"{k} = ?" for k in fields])
|
||||
params = list(fields.values()) + [book_id]
|
||||
|
||||
cur.execute(f"UPDATE books SET {set_clause} WHERE book_id = ?", params)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# STATUS
|
||||
# ------------------------------------------------------------
|
||||
def sql_set_status(book_id, status: str):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute("UPDATE books SET status = ? WHERE book_id = ?", (status, book_id))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# CHAPTER TOTAL (snapshot)
|
||||
# ------------------------------------------------------------
|
||||
def sql_set_chapters_total(book_id, total: int):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"UPDATE books SET chapters_total = ? WHERE book_id = ?", (total, book_id)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# COUNTERS (SNAPSHOT-ONLY)
|
||||
# ------------------------------------------------------------
|
||||
def sql_inc_downloaded(book_id, amount=1):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE books
|
||||
SET downloaded = COALESCE(downloaded,0) + ?
|
||||
WHERE book_id = ?
|
||||
""",
|
||||
(amount, book_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def sql_inc_parsed(book_id, amount=1):
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE books
|
||||
SET parsed = COALESCE(parsed,0) + ?
|
||||
WHERE book_id = ?
|
||||
""",
|
||||
(amount, book_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def sql_inc_audio_done(book_id, amount=1):
|
||||
log(f"[DB-SQL] Incrementing audio done for {book_id} by {amount}")
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE books
|
||||
SET audio_done = COALESCE(audio_done,0) + ?
|
||||
WHERE book_id = ?
|
||||
""",
|
||||
(amount, book_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def sql_inc_audio_skipped(book_id, amount=1):
|
||||
log(f"[DB-SQL] Incrementing audio skipped for {book_id} by {amount}")
|
||||
conn = _connect()
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE books
|
||||
SET audio_skipped = COALESCE(audio_skipped,0) + ?
|
||||
WHERE book_id = ?
|
||||
""",
|
||||
(amount, book_id),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
@ -1,21 +1,66 @@
|
||||
# ============================================================
|
||||
# File: scraper/book_scraper.py
|
||||
# Purpose:
|
||||
# Backwards-compatible wrapper giving same API as before.
|
||||
# Uses the new engine under the hood.
|
||||
# Backwards-compatible wrapper giving the SAME public API
|
||||
# as the old BookScraper, but internally uses ScrapeEngine.
|
||||
#
|
||||
# execute() → full metadata + chapterlist
|
||||
#
|
||||
# (* Chapter downloading komt later in ScrapeEngine,
|
||||
# maar deze wrapper hoeft NIET aangepast te worden.)
|
||||
# ============================================================
|
||||
|
||||
from scraper.engine.parser import extract_metadata_full
|
||||
from scraper.logger_decorators import logcall
|
||||
from scraper.services.scrape_engine import ScrapeEngine
|
||||
|
||||
|
||||
class BookScraper:
|
||||
def __init__(self, site_scraper, url):
|
||||
"""
|
||||
Backwards-compatible BookScraper façade.
|
||||
|
||||
In het oude systeem deed BookScraper ALLES:
|
||||
- metadata ophalen
|
||||
- cover ophalen
|
||||
- hoofdstukkenlijst
|
||||
- hoofdstukken downloaden
|
||||
- volume folders
|
||||
- skip logic
|
||||
|
||||
In het nieuwe systeem is dát opgesplitst:
|
||||
|
||||
ScrapeEngine → metadata / chapterlist / download engine (in ontwikkeling)
|
||||
BookScraper → behoudt dezelfde API als voorheen
|
||||
|
||||
Daardoor kunnen Celery-tasks en oudere modules blijven werken
|
||||
zonder refactor-chaos.
|
||||
"""
|
||||
|
||||
@logcall
|
||||
def __init__(self, site_scraper, url: str):
|
||||
self.site = site_scraper
|
||||
self.url = url
|
||||
|
||||
@logcall
|
||||
def execute(self):
|
||||
"""
|
||||
Backwards compatible full scrape:
|
||||
returns {title, author, description, cover_url, chapters, book_url}
|
||||
Public legacy API.
|
||||
Retourneert metadata + chapters EXACT zoals de oude BookScraper
|
||||
vóór downloadfase.
|
||||
|
||||
Dit is belangrijk:
|
||||
- INIT-flow gebruikt metadata only
|
||||
- scraping tasks gebruiken chapterlist
|
||||
"""
|
||||
return extract_metadata_full(self.url, self.site)
|
||||
|
||||
data = ScrapeEngine.fetch_metadata_and_chapters(self.site, self.url)
|
||||
|
||||
# Legacy output structuur volledig repliceren:
|
||||
return {
|
||||
"title": data.get("title"),
|
||||
"author": data.get("author"),
|
||||
"description": data.get("description"),
|
||||
"cover_url": data.get("cover_url"),
|
||||
"chapters": data.get("chapters", []),
|
||||
"chapters_total": data.get("chapters_total", 0),
|
||||
"book_url": data.get("book_url"),
|
||||
}
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
# ============================================================
|
||||
# File: scraper/logger_decorators.py
|
||||
# Purpose: Function-call logging decorator
|
||||
# ============================================================
|
||||
|
||||
from functools import wraps
|
||||
from scraper.logger import log_debug
|
||||
|
||||
|
||||
def logcall(func):
|
||||
"""
|
||||
Decorator: log function name + arguments every time it's called.
|
||||
Usage: @logcall above any function.
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
# Naam van de functie
|
||||
name = func.__qualname__
|
||||
|
||||
# Eerste logregel vóór uitvoering
|
||||
# log_debug(f"[CALL] {name} args={args} kwargs={kwargs}")
|
||||
log_debug(f"[CALL] {name} args={args}")
|
||||
# log_debug(f"[CALL] {name}")
|
||||
|
||||
result = func(*args, **kwargs)
|
||||
|
||||
# Log ná uitvoering
|
||||
# log_debug(f"[RETURN] {name} → {result}")
|
||||
|
||||
return result
|
||||
|
||||
return wrapper
|
||||
@ -1,57 +0,0 @@
|
||||
# ============================================================
|
||||
# File: scraper/tasks/progress_tasks.py
|
||||
# Purpose: Central progress updater for chapter pipelines.
|
||||
# Updated for chapter_dict pipeline model.
|
||||
# ============================================================
|
||||
|
||||
from celery_app import celery_app
|
||||
from scraper.progress import inc_completed, inc_skipped, inc_failed
|
||||
from logbus.publisher import log
|
||||
|
||||
print(">>> [IMPORT] progress_tasks.py loaded")
|
||||
|
||||
|
||||
@celery_app.task(bind=False, name="progress.update", queue="controller")
|
||||
def update_progress(result: dict, book_id: str):
|
||||
"""
|
||||
Central progress logic:
|
||||
- result: output of save_chapter
|
||||
- book_id: explicitly passed by pipeline
|
||||
|
||||
IMPORTANT:
|
||||
- save_chapter already updates counters for skipped & normal chapters
|
||||
- progress.update MUST NOT double-increment
|
||||
"""
|
||||
|
||||
ch = result.get("chapter") or {}
|
||||
chapter_num = ch.get("num")
|
||||
|
||||
skipped = result.get("skipped", False)
|
||||
failed = result.get("failed", False)
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# FAILED CASE
|
||||
# ------------------------------------------------------------
|
||||
if failed:
|
||||
inc_failed(book_id)
|
||||
log(f"[PROG] FAILED chapter {chapter_num}")
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# SKIPPED CASE
|
||||
# ------------------------------------------------------------
|
||||
if skipped:
|
||||
# save_chapter already did:
|
||||
# inc_skipped(book_id)
|
||||
log(f"[PROG] SKIPPED chapter {chapter_num}")
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# NORMAL COMPLETION
|
||||
# ------------------------------------------------------------
|
||||
# save_chapter did NOT increment completed for skipped cases
|
||||
# but DID inc_completed(book_id) for normal cases.
|
||||
# update_progress should NOT double increment, so only log here.
|
||||
log(f"[PROG] DONE chapter {chapter_num}")
|
||||
|
||||
return result
|
||||
@ -1,139 +1,83 @@
|
||||
# ============================================================
|
||||
# File: scraper/tasks/save_tasks.py
|
||||
# Purpose: Save parsed chapter text to disk + trigger audio.
|
||||
# Updated for chapter_dict + book_meta pipeline model.
|
||||
# File: scraper/tasks/save_tasks.py (RESTORED AUDIO LOGIC)
|
||||
# ============================================================
|
||||
|
||||
print(">>> [IMPORT] save_tasks.py loaded")
|
||||
|
||||
from celery import shared_task
|
||||
import os
|
||||
from scraper.utils import get_save_path
|
||||
from scraper.tasks.download_tasks import log_msg # unified logger
|
||||
from scraper.progress import (
|
||||
inc_completed,
|
||||
inc_chapter_done,
|
||||
inc_chapter_download_skipped,
|
||||
)
|
||||
|
||||
from logbus.publisher import log
|
||||
from scraper.logger_decorators import logcall
|
||||
from scraper.utils.utils import get_save_path
|
||||
from scraper.tasks.download_tasks import log_msg
|
||||
from scraper.tasks.audio_tasks import generate_audio
|
||||
|
||||
from db.repository import inc_download_done, inc_download_skipped
|
||||
|
||||
|
||||
@shared_task(bind=True, queue="save", ignore_result=False)
|
||||
def save_chapter(self, parsed: dict):
|
||||
"""
|
||||
New pipeline model:
|
||||
parsed = {
|
||||
"book_id": str,
|
||||
"chapter": chapter_dict,
|
||||
"text": str,
|
||||
"length": int,
|
||||
"book_meta": dict,
|
||||
"skipped": bool,
|
||||
"path": optional str (if skipped)
|
||||
}
|
||||
"""
|
||||
|
||||
book_id = parsed.get("book_id", "NOBOOK")
|
||||
chapter_dict = parsed.get("chapter") or {}
|
||||
book_meta = parsed.get("book_meta") or {}
|
||||
|
||||
chapter_num = chapter_dict.get("num")
|
||||
chapter_title = chapter_dict.get("title") or f"Chapter {chapter_num}"
|
||||
volume_path = chapter_dict.get("volume_path")
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# VALIDATION
|
||||
# ------------------------------------------------------------
|
||||
if chapter_num is None or volume_path is None:
|
||||
raise ValueError("Invalid parsed payload: chapter_dict missing fields.")
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# SKIPPED CASE
|
||||
# ------------------------------------------------------------
|
||||
if parsed.get("skipped"):
|
||||
path = parsed.get("path", None)
|
||||
log_msg(book_id, f"[SAVE] SKIP chapter {chapter_num} → {path}")
|
||||
|
||||
inc_chapter_download_skipped(book_id)
|
||||
|
||||
volume_name = os.path.basename(volume_path.rstrip("/"))
|
||||
|
||||
# Queue audio only if a valid file exists
|
||||
@logcall
|
||||
def save_chapter(self, payload: dict):
|
||||
|
||||
if not payload:
|
||||
log("[SAVE] ERROR: payload is None")
|
||||
return {"error": True}
|
||||
|
||||
book_id = payload["book_id"]
|
||||
chapter = payload["chapter"]
|
||||
parsed = payload.get("parsed")
|
||||
path = payload.get("path")
|
||||
skipped = payload.get("skipped")
|
||||
|
||||
num = chapter["num"]
|
||||
title = chapter.get("title") or f"Chapter {num}"
|
||||
volume = chapter.get("volume_path")
|
||||
volume_name = os.path.basename(volume.rstrip("/"))
|
||||
|
||||
# ============================================================
|
||||
# SKIPPED CASE (restore old behavior)
|
||||
# ============================================================
|
||||
if skipped or not parsed:
|
||||
log_msg(book_id, f"[SAVE] SKIP chapter {num}")
|
||||
inc_download_skipped(book_id)
|
||||
|
||||
# Restore old behavior:
|
||||
# If file already exists, STILL trigger audio.
|
||||
if path and os.path.exists(path):
|
||||
log_msg(book_id, f"[AUDIO] Queueing audio for SKIPPED chapter {num}")
|
||||
try:
|
||||
generate_audio.delay(
|
||||
book_id,
|
||||
volume_name,
|
||||
chapter_num,
|
||||
chapter_title,
|
||||
path,
|
||||
)
|
||||
log_msg(
|
||||
book_id,
|
||||
f"[AUDIO] Task queued (SKIPPED) for chapter {chapter_num} in {volume_name}",
|
||||
)
|
||||
except Exception as audio_exc:
|
||||
log_msg(
|
||||
book_id,
|
||||
f"[AUDIO] ERROR queueing (SKIPPED) chapter {chapter_num}: {audio_exc}",
|
||||
)
|
||||
|
||||
return {
|
||||
"book_id": book_id,
|
||||
"chapter": chapter_dict,
|
||||
"path": path,
|
||||
"skipped": True,
|
||||
"book_meta": book_meta,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# NORMAL SAVE CASE
|
||||
# ------------------------------------------------------------
|
||||
try:
|
||||
text = parsed.get("text", "")
|
||||
generate_audio.delay(book_id, volume_name, num, title, path)
|
||||
except Exception as exc:
|
||||
log_msg(book_id, f"[AUDIO] ERROR queueing skipped audio: {exc}")
|
||||
|
||||
# Ensure volume folder exists
|
||||
os.makedirs(volume_path, exist_ok=True)
|
||||
return payload
|
||||
|
||||
# Build final chapter file path
|
||||
path = get_save_path(chapter_num, volume_path)
|
||||
# ============================================================
|
||||
# NORMAL SAVE CASE
|
||||
# ============================================================
|
||||
try:
|
||||
os.makedirs(volume, exist_ok=True)
|
||||
save_path = get_save_path(num, volume)
|
||||
|
||||
# Write chapter text to file
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
with open(save_path, "w", encoding="utf-8") as f:
|
||||
f.write(parsed)
|
||||
|
||||
log_msg(book_id, f"[SAVE] Saved chapter {chapter_num} → {path}")
|
||||
inc_chapter_done(book_id)
|
||||
inc_completed(book_id)
|
||||
log_msg(book_id, f"[SAVE] Saved chapter {num} → {save_path}")
|
||||
|
||||
# Determine volume name
|
||||
volume_name = os.path.basename(volume_path.rstrip("/"))
|
||||
inc_download_done(book_id)
|
||||
|
||||
# Queue audio task
|
||||
# Restore old behavior → ALWAYS queue audio
|
||||
try:
|
||||
generate_audio.delay(
|
||||
book_id,
|
||||
volume_name,
|
||||
chapter_num,
|
||||
chapter_title,
|
||||
path,
|
||||
)
|
||||
log_msg(
|
||||
book_id,
|
||||
f"[AUDIO] Task queued for chapter {chapter_num} in {volume_name}",
|
||||
)
|
||||
except Exception as audio_exc:
|
||||
log_msg(
|
||||
book_id, f"[AUDIO] ERROR queueing chapter {chapter_num}: {audio_exc}"
|
||||
)
|
||||
|
||||
return {
|
||||
"book_id": book_id,
|
||||
"chapter": chapter_dict,
|
||||
"path": path,
|
||||
"book_meta": book_meta,
|
||||
}
|
||||
generate_audio.delay(book_id, volume_name, num, title, save_path)
|
||||
log_msg(book_id, f"[AUDIO] Task queued for chapter {num}")
|
||||
except Exception as exc:
|
||||
log_msg(book_id, f"[AUDIO] ERROR queueing chapter {num}: {exc}")
|
||||
|
||||
payload["path"] = save_path
|
||||
payload["skipped"] = False
|
||||
return payload
|
||||
|
||||
except Exception as exc:
|
||||
log_msg(book_id, f"[SAVE] ERROR saving chapter {chapter_num}: {exc}")
|
||||
log_msg(book_id, f"[SAVE] ERROR saving chapter {num}: {exc}")
|
||||
raise
|
||||
|
||||
@ -0,0 +1,141 @@
|
||||
# ============================================================
|
||||
# File: scraper/utils/state_sync.py
|
||||
# Purpose:
|
||||
# State inspection + optional sync logic for book progress.
|
||||
# This version provides:
|
||||
# • inspect_books_state() → NO writes, just a dry-run
|
||||
# • sync_books_from_redis() → NOT USED YET (kept commented)
|
||||
# ============================================================
|
||||
|
||||
import os
|
||||
import redis
|
||||
from db.db import get_db
|
||||
|
||||
|
||||
def inspect_books_state():
|
||||
"""
|
||||
Reads all books from SQLite and fetches Redis progress,
|
||||
but performs NO writes. Only shows:
|
||||
- sqlite row
|
||||
- redis state
|
||||
- merged result (dry-run)
|
||||
|
||||
Returns a list of inspection dicts.
|
||||
"""
|
||||
r = redis.Redis.from_url(os.getenv("REDIS_BROKER"))
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
|
||||
cur.execute("SELECT * FROM books")
|
||||
rows = cur.fetchall()
|
||||
|
||||
results = []
|
||||
|
||||
for row in rows:
|
||||
book_id = row["book_id"]
|
||||
sqlite_row = dict(row)
|
||||
|
||||
# Read redis state
|
||||
redis_key = f"book:{book_id}:state"
|
||||
progress = r.hgetall(redis_key)
|
||||
|
||||
if progress:
|
||||
decoded = {k.decode(): v.decode() for k, v in progress.items()}
|
||||
else:
|
||||
decoded = {}
|
||||
|
||||
# Determine dry-run merged result
|
||||
merged = sqlite_row.copy()
|
||||
|
||||
if decoded:
|
||||
merged["downloaded"] = int(
|
||||
decoded.get("download_done", merged.get("downloaded", 0))
|
||||
)
|
||||
merged["parsed"] = int(decoded.get("parsed_done", merged.get("parsed", 0)))
|
||||
merged["audio_done"] = int(
|
||||
decoded.get("audio_done", merged.get("audio_done", 0))
|
||||
)
|
||||
merged["chapters_total"] = int(
|
||||
decoded.get("chapters_total", merged.get("chapters_total", 0))
|
||||
)
|
||||
merged["status"] = decoded.get("status", merged.get("status", "unknown"))
|
||||
|
||||
results.append(
|
||||
{
|
||||
"book_id": book_id,
|
||||
"sqlite": sqlite_row,
|
||||
"redis": decoded,
|
||||
"would_merge_to": merged,
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def sync_books_from_redis():
|
||||
"""
|
||||
Reads all books from SQLite, fetches Redis progress,
|
||||
and updates SQLite rows accordingly.
|
||||
|
||||
Returns a list of {
|
||||
"book_id": ...,
|
||||
"before": ...,
|
||||
"redis": ...,
|
||||
"after": ...
|
||||
}
|
||||
"""
|
||||
r = redis.Redis.from_url(os.getenv("REDIS_BROKER"))
|
||||
db = get_db()
|
||||
cur = db.cursor()
|
||||
|
||||
# Haal alle boeken op
|
||||
cur.execute("SELECT * FROM books")
|
||||
rows = cur.fetchall()
|
||||
|
||||
results = []
|
||||
|
||||
for row in rows:
|
||||
book_id = row["book_id"]
|
||||
before = dict(row)
|
||||
|
||||
redis_key = f"book:{book_id}:state"
|
||||
progress = r.hgetall(redis_key)
|
||||
|
||||
if not progress:
|
||||
results.append(
|
||||
{"book_id": book_id, "before": before, "redis": {}, "after": before}
|
||||
)
|
||||
continue
|
||||
|
||||
# Decode Redis bytes → string dictionary
|
||||
decoded = {k.decode(): v.decode() for k, v in progress.items()}
|
||||
|
||||
# Extract counters
|
||||
downloaded = int(decoded.get("download_done", 0))
|
||||
parsed = int(decoded.get("parsed_done", 0))
|
||||
audio_done = int(decoded.get("audio_done", 0))
|
||||
chapters_total = int(decoded.get("chapters_total", 0))
|
||||
|
||||
# Redis status wins
|
||||
status = decoded.get("status", before["status"])
|
||||
|
||||
# Write back to SQLite
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE books
|
||||
SET downloaded = ?, parsed = ?, audio_done = ?, chapters_total = ?, status = ?, last_update = datetime('now')
|
||||
WHERE book_id = ?
|
||||
""",
|
||||
(downloaded, parsed, audio_done, chapters_total, status, book_id),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# Fetch updated row
|
||||
cur.execute("SELECT * FROM books WHERE book_id = ?", (book_id,))
|
||||
after = dict(cur.fetchone())
|
||||
|
||||
results.append(
|
||||
{"book_id": book_id, "before": before, "redis": decoded, "after": after}
|
||||
)
|
||||
|
||||
return results
|
||||
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 6.4 KiB |
|
Before Width: | Height: | Size: 3.5 KiB |
|
Before Width: | Height: | Size: 12 KiB |
@ -0,0 +1,201 @@
|
||||
/* =======================================================================
|
||||
File: static/css/bookcard.css
|
||||
Purpose:
|
||||
All styling for registered book cards (book-card) +
|
||||
status colors + start/abort buttons.
|
||||
======================================================================= */
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
GRID WRAPPER FOR REGISTERED BOOKS
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.registered-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(340px, 1fr));
|
||||
gap: 20px;
|
||||
margin-top: 15px;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
MAIN BOOK CARD
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.book-card {
|
||||
position: relative;
|
||||
display: grid;
|
||||
grid-template-columns: 90px auto;
|
||||
gap: 15px;
|
||||
|
||||
padding: 15px;
|
||||
background: #fff;
|
||||
border-radius: 10px;
|
||||
border: 1px solid #e5e5e5;
|
||||
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05);
|
||||
|
||||
transition: border-color 0.25s ease, box-shadow 0.25s ease;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
BOOK STATUS COLORS
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.book-card.processing {
|
||||
border-color: #007aff;
|
||||
box-shadow: 0 0 6px rgba(0, 122, 255, 0.35);
|
||||
}
|
||||
|
||||
.book-card.downloading {
|
||||
border-color: #ff9500;
|
||||
box-shadow: 0 0 6px rgba(255, 149, 0, 0.35);
|
||||
}
|
||||
|
||||
.book-card.parsing {
|
||||
border-color: #ffcc00;
|
||||
box-shadow: 0 0 6px rgba(255, 204, 0, 0.35);
|
||||
}
|
||||
|
||||
.book-card.audio {
|
||||
border-color: #e65100;
|
||||
box-shadow: 0 0 6px rgba(230, 81, 0, 0.35);
|
||||
}
|
||||
|
||||
.book-card.completed {
|
||||
border-color: #34c759;
|
||||
box-shadow: 0 0 6px rgba(52, 199, 89, 0.35);
|
||||
}
|
||||
|
||||
.book-card.aborted {
|
||||
border-color: #ff3b30;
|
||||
box-shadow: 0 0 6px rgba(255, 59, 48, 0.35);
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
COVER IMAGE
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.book-cover {
|
||||
width: 90px;
|
||||
}
|
||||
|
||||
.book-img {
|
||||
width: 90px;
|
||||
height: 130px;
|
||||
object-fit: cover;
|
||||
border-radius: 4px;
|
||||
background: #f4f4f4;
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
color: #777;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
META INFORMATION
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.book-meta {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: space-between;
|
||||
}
|
||||
|
||||
.book-title {
|
||||
font-size: 16px;
|
||||
font-weight: bold;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.book-author {
|
||||
font-size: 14px;
|
||||
color: #444;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.book-created {
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
/* -----------------------------------------------------------------------
|
||||
ICON BUTTONS
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.icon-btn {
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
|
||||
font-size: 16px;
|
||||
color: #fff;
|
||||
cursor: pointer;
|
||||
|
||||
transition: background 0.15s ease, transform 0.1s ease;
|
||||
}
|
||||
|
||||
/* Start (green) */
|
||||
.icon-start {
|
||||
background: #2d8a3d;
|
||||
}
|
||||
.icon-start:hover {
|
||||
background: #226c30;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
.icon-start:disabled {
|
||||
background: #9bbb9f !important;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
/* Abort (red) */
|
||||
.icon-abort {
|
||||
background: #c62828;
|
||||
}
|
||||
.icon-abort:hover {
|
||||
background: #a31f1f;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
.icon-abort:disabled {
|
||||
background: #d8a0a0 !important;
|
||||
cursor: not-allowed;
|
||||
transform: none;
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
/* Hide button (gray) */
|
||||
.hide-form {
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
right: 6px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.icon-hide {
|
||||
background: #777;
|
||||
}
|
||||
.icon-hide:hover {
|
||||
background: #555;
|
||||
transform: scale(1.05);
|
||||
}
|
||||
/* -----------------------------------------------------------------------
|
||||
BOOK ACTIONS (right aligned button row)
|
||||
----------------------------------------------------------------------- */
|
||||
|
||||
.book-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end; /* rechts uitlijnen */
|
||||
gap: 10px; /* ruimte tussen knoppen */
|
||||
margin-top: 12px;
|
||||
}
|
||||
@ -0,0 +1,82 @@
|
||||
{# ============================================================
|
||||
File: templates/components/bookcard.html
|
||||
Purpose:
|
||||
Eén enkele boekkaart met:
|
||||
- status styles
|
||||
- cover
|
||||
- metadata
|
||||
- hide button
|
||||
- start (play)
|
||||
- abort (stop)
|
||||
Requires:
|
||||
variable "b" in context
|
||||
============================================================ #}
|
||||
|
||||
<div class="book-card {{ b.status }}" data-book-id="{{ b.book_id }}">
|
||||
|
||||
<!-- ======================================================
|
||||
HIDE BUTTON (icon-only)
|
||||
====================================================== -->
|
||||
<form
|
||||
action="/hide/{{ b.book_id }}"
|
||||
method="POST"
|
||||
onsubmit="return confirm('Dit boek verbergen?')"
|
||||
class="hide-form"
|
||||
>
|
||||
<button class="icon-btn icon-hide" title="Verbergen">
|
||||
<i class="fa-solid fa-xmark"></i>
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<!-- ======================================================
|
||||
COVER
|
||||
====================================================== -->
|
||||
<div class="book-cover">
|
||||
{% if b.cover_path %}
|
||||
<img src="/{{ b.cover_path }}" alt="cover" class="book-img" />
|
||||
{% else %}
|
||||
<div class="book-img placeholder">?</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<!-- ======================================================
|
||||
META + BUTTONS
|
||||
====================================================== -->
|
||||
<div class="book-meta">
|
||||
<div class="book-title">{{ b.title }}</div>
|
||||
<div class="book-author">{{ b.author }}</div>
|
||||
<div class="book-created">Geregistreerd: {{ b.created_at }}</div>
|
||||
|
||||
<div class="book-actions">
|
||||
<!-- START -->
|
||||
<form action="/start" method="POST">
|
||||
<input type="hidden" name="book_id" value="{{ b.book_id }}" />
|
||||
<button
|
||||
class="icon-btn icon-start"
|
||||
title="Start scraping"
|
||||
{% if b.status != "registered" %}
|
||||
disabled
|
||||
{% endif %}
|
||||
>
|
||||
<i class="fa-solid fa-play"></i>
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<!-- ABORT -->
|
||||
<form action="/abort/{{ b.book_id }}" method="POST">
|
||||
<input type="hidden" name="book_id" value="{{ b.book_id }}" />
|
||||
<button
|
||||
class="icon-btn icon-abort"
|
||||
title="Stoppen (abort)"
|
||||
{% if b.status not in ["processing","downloading","parsing","audio"] %}
|
||||
disabled
|
||||
{% endif %}
|
||||
>
|
||||
<i class="fa-solid fa-stop"></i>
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
</div> <!-- einde .book-meta -->
|
||||
|
||||
</div> <!-- einde .book-card -->
|
||||
@ -0,0 +1,21 @@
|
||||
{# ============================================================ File:
|
||||
templates/components/registered_books.html Purpose: Toon een grid van
|
||||
geregistreerde boeken. Elke kaart wordt gerenderd via bookcard.html.
|
||||
============================================================ #}
|
||||
|
||||
<section class="dashboard-section">
|
||||
<h2>Geregistreerde boeken</h2>
|
||||
|
||||
{% if registered and registered|length > 0 %}
|
||||
|
||||
<div class="registered-grid">
|
||||
{% for b in registered %} {% include "components/bookcard.html" %} {% endfor
|
||||
%}
|
||||
</div>
|
||||
|
||||
{% else %}
|
||||
|
||||
<p>Geen geregistreerde boeken.</p>
|
||||
|
||||
{% endif %}
|
||||
</section>
|
||||
@ -0,0 +1,88 @@
|
||||
{% extends "layout.html" %} {% block content %}
|
||||
|
||||
<h1>State Inspection (SQL vs Redis)</h1>
|
||||
|
||||
<style>
|
||||
.state-card {
|
||||
border: 1px solid #444;
|
||||
padding: 18px;
|
||||
margin-bottom: 30px;
|
||||
background: #222;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.state-title {
|
||||
font-size: 1.4em;
|
||||
margin-bottom: 14px;
|
||||
color: #9cf;
|
||||
}
|
||||
|
||||
table.state-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
|
||||
.state-table th,
|
||||
.state-table td {
|
||||
border: 1px solid #555;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
|
||||
.state-table th {
|
||||
background: #333;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.same {
|
||||
color: #9f9;
|
||||
}
|
||||
|
||||
.diff {
|
||||
color: #ff7b7b;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.empty {
|
||||
color: #aaa;
|
||||
font-style: italic;
|
||||
}
|
||||
</style>
|
||||
|
||||
{% macro cmp(sqlval, redisval) %} {% if (sqlval|string) == (redisval|string) %}
|
||||
<td class="same">{{ sqlval }}</td>
|
||||
<td class="same">{{ redisval }}</td>
|
||||
{% else %}
|
||||
<td class="diff">{{ sqlval }}</td>
|
||||
<td class="diff">{{ redisval }}</td>
|
||||
{% endif %} {% endmacro %} {% for entry in results %}
|
||||
<div class="state-card">
|
||||
<div class="state-title">📘 {{ entry.book_id }}</div>
|
||||
|
||||
{% set sql = entry.sqlite %} {% set redis = entry.redis %} {% set merged =
|
||||
entry.would_merge_to %}
|
||||
|
||||
<table class="state-table">
|
||||
<tr>
|
||||
<th>Field</th>
|
||||
<th>SQLite</th>
|
||||
<th>Redis</th>
|
||||
<th>Merged Result</th>
|
||||
</tr>
|
||||
|
||||
{% for field in [ "status", "chapters_total", "downloaded",
|
||||
"chapters_download_done", "chapters_download_skipped", "parsed",
|
||||
"chapters_parsed_done", "audio_done", "audio_skipped", "last_update" ] %}
|
||||
|
||||
<tr>
|
||||
<th>{{ field }}</th>
|
||||
|
||||
<td>{{ sql.get(field, '') }}</td>
|
||||
<td>{{ redis.get(field, '') }}</td>
|
||||
<td>{{ merged.get(field, '') }}</td>
|
||||
</tr>
|
||||
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
{% endfor %} {% endblock %}
|
||||
@ -0,0 +1,91 @@
|
||||
{% extends "layout.html" %} {% block content %}
|
||||
<h1>Celery Queue Debug</h1>
|
||||
|
||||
<style>
|
||||
.debug-section {
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
.debug-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
.debug-table th,
|
||||
.debug-table td {
|
||||
border: 1px solid #444;
|
||||
padding: 6px 10px;
|
||||
}
|
||||
.debug-table th {
|
||||
background: #333;
|
||||
color: #fff;
|
||||
}
|
||||
pre {
|
||||
background: #1e1e1e;
|
||||
color: #ddd;
|
||||
padding: 10px;
|
||||
overflow-x: auto;
|
||||
}
|
||||
code {
|
||||
color: #9cf;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="debug-section">
|
||||
<h2>Workers</h2>
|
||||
|
||||
<h3>Active Tasks</h3>
|
||||
<pre>{{ workers_active | tojson(indent=2) }}</pre>
|
||||
|
||||
<h3>Reserved</h3>
|
||||
<pre>{{ workers_reserved | tojson(indent=2) }}</pre>
|
||||
|
||||
<h3>Scheduled</h3>
|
||||
<pre>{{ workers_scheduled | tojson(indent=2) }}</pre>
|
||||
</div>
|
||||
|
||||
<hr />
|
||||
|
||||
<div class="debug-section">
|
||||
<h2>Queues</h2>
|
||||
|
||||
{% for q in queues %}
|
||||
<div class="debug-queue">
|
||||
<h3>{{ q.name }} ({{ q.length }} items)</h3>
|
||||
|
||||
<table class="debug-table">
|
||||
<tr>
|
||||
<th>Redis Key</th>
|
||||
<td>{{ q.redis_key }}</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Length</th>
|
||||
<td>{{ q.length }}</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<th>Items (first 30)</th>
|
||||
<td>
|
||||
{% if q["items"] %}
|
||||
<ul style="margin: 0; padding-left: 20px">
|
||||
{% for item in q["items"] %}
|
||||
<li><code>{{ item | e }}</code></li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
{% else %}
|
||||
<i>No items</i>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<script>
|
||||
setInterval(() => {
|
||||
window.location.reload();
|
||||
}, 5000);
|
||||
</script>
|
||||
|
||||
{% endblock %}
|
||||
Loading…
Reference in new issue