kmftools/bookscraper/app.py

# ============================================
# File: bookscraper/app.py  (ASYNC SCRAPING)
# ============================================

from dotenv import load_dotenv

load_dotenv()

print(">>> [WEB] Importing celery_app …")
from celery_app import celery_app
from db.db import init_db

init_db()  # ensure DB schema exists before Flask starts

from flask import Flask, render_template, request, jsonify
from scraper.logger import log_debug

# Abort + Progress (per book_id)
from scraper.abort import set_abort
from scraper.progress import get_progress

# UI LOGS (GLOBAL — no book_id)
from scraper.ui_log import get_ui_logs, reset_ui_logs

from celery.result import AsyncResult
from scraper.state import state as r

# Cover serving
from flask import send_from_directory
import os

import redis

# Flask
app = Flask(__name__)

# =====================================================
# STATIC FILE SERVING FOR OUTPUT
# =====================================================
OUTPUT_ROOT = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")


@app.route("/output/<path:filename>")
def serve_output(filename):
    return send_from_directory(OUTPUT_ROOT, filename, as_attachment=False)


# =====================================================
# HOME PAGE
# =====================================================
@app.route("/", methods=["GET"])
def index():
    return render_template("index.html")


# =====================================================
# START SCRAPING (async via Celery)
# =====================================================
@app.route("/start", methods=["POST"])
def start_scraping():
    url = request.form.get("url", "").strip()

    if not url:
        # ★ FIX: dashboard moet altijd books + logs meekrijgen
        return render_template(
            "dashboard/dashboard.html",
            error="Geen URL opgegeven.",
            books=list_active_books(),
            logs=get_ui_logs(),
        )

    reset_ui_logs()
    log_debug(f"[WEB] Scraping via Celery: {url}")

    async_result = celery_app.send_task(
        "scraper.tasks.scraping.start_scrape_book",
        args=[url],
        queue="scraping",
    )

    # ★ FIX: direct dashboard tonen met actuele data
    return render_template(
        "dashboard/dashboard.html",
        scraping_task_id=async_result.id,
        books=list_active_books(),
        logs=get_ui_logs(),
    )


# =====================================================
# CLEAR UI LOGS
# =====================================================
@app.route("/clear-logs", methods=["POST"])
def clear_logs():
    reset_ui_logs()
    return jsonify({"status": "ok", "message": "UI logs cleared"})


# =====================================================
# ABORT (per book_id)
# =====================================================
@app.route("/abort/<book_id>", methods=["POST"])
def abort_download(book_id):
    log_debug(f"[WEB] Abort requested for book: {book_id}")
    set_abort(book_id)
    return jsonify({"status": "ok", "aborted": book_id})


# =====================================================
# PROGRESS (per book_id)
# =====================================================
@app.route("/progress/<book_id>", methods=["GET"])
def progress(book_id):
    return jsonify(get_progress(book_id))


# =====================================================
# CELERY RESULT → return book_id
# =====================================================
@app.route("/celery-result/<task_id>", methods=["GET"])
def celery_result(task_id):
    result = AsyncResult(task_id, app=celery_app)

    if result.successful():
        return jsonify({"ready": True, "result": result.get()})
    if result.failed():
        return jsonify({"ready": True, "error": "failed"})
    return jsonify({"ready": False})


# =====================================================
# API: book status new model
# =====================================================
def getStatus(book_id):

    state = r.hgetall(f"book:{book_id}:state")
    status = state.get("status") or "unknown"
    dl_done = int(state.get("chapters_download_done", 0))
    dl_skipped = int(state.get("chapters_download_skipped", 0))
    dl_total = int(state.get("chapters_total", 0))
    au_done = int(state.get("audio_done") or 0)
    title = state.get("title") or book_id

    au_total = dl_total

    return {
        "book_id": book_id,
        "title": title,
        "status": status,
        "download_done": dl_done,
        "download_skipped": dl_skipped,
        "download_total": dl_total,
        "audio_done": au_done,
        "audio_total": au_total,
    }


# =====================================================
# REDIS BACKEND — BOOK STATE MODEL
# =====================================================
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)


def list_active_booksold():
    """Return list of active books from Redis Book State Model."""
    keys = r.keys("book:*:state")
    books = []

    for key in keys:
        book_id = key.split(":")[1]
        print(book_id)
        books.append(getStatus(book_id))

    return books


def list_active_books():
    books = []
    for key in r.scan_iter(match="book:*:state", count=1000):
        first = key.find(":")
        second = key.find(":", first + 1)
        book_id = key[first + 1 : second]
        books.append(getStatus(book_id))
    return books


# =====================================================
# API: list all active books
# =====================================================
@app.route("/api/books")
def api_books():
    return jsonify(list_active_books())


@app.route("/api/book/<book_id>/status")
def api_book_status(book_id):

    return jsonify(getStatus(book_id))


# =====================================================
# API: book logs
# =====================================================
@app.route("/api/book/<book_id>/logs")
def api_book_logs(book_id):
    logs = r.lrange(f"logs:{book_id}", 0, -1) or []
    return jsonify(logs)


# =====================================================
# VIEW: DASHBOARD
# =====================================================
@app.route("/dashboard")
def dashboard():
    logs_list = get_ui_logs() or []
    # ★ FIX: dashboard moet altijd books + logs krijgen
    return render_template(
        "dashboard/dashboard.html",
        books=list_active_books(),
        logs=logs_list,  # dashboard krijgt LIST, geen dict
    )


# =====================================================
# VIEW: BOOK DETAIL PAGE
# =====================================================
@app.route("/book/<book_id>")
def book_detail(book_id):
    title = r.get(f"book:{book_id}:title") or book_id
    return render_template(
        "dashboard/book_detail.html",
        book_id=book_id,
        title=title,
        logs=get_ui_logs(),
    )


@app.route("/debug/redis-keys")
def debug_redis_keys():
    cursor = 0
    results = {}

    while True:
        cursor, keys = r.scan(cursor, match="*", count=200)
        for k in keys:
            try:
                results[k] = r.get(k)
            except:
                results[k] = "<non-string value>"
        if cursor == 0:
            break

    return jsonify(results)


# ============================================================
# Rolling log endpoint (no new file)
# ============================================================

from flask import jsonify, request

# =====================================================
# ROLLING LOG ENDPOINT — DELTA POLLING VIA ui_log
# =====================================================
from scraper.ui_log import get_ui_logs_delta


@app.route("/logs", methods=["GET"])
def logs():
    """
    Delta log delivery for WebGUI.
    Browser sends ?last_index=N, we return only new lines.
    """
    try:
        last_index = int(request.args.get("last_index", -1))
    except:
        last_index = -1

    new_lines, total = get_ui_logs_delta(last_index)

    return jsonify({"lines": new_lines, "total": total})


# =====================================================
# RUN FLASK
# =====================================================
if __name__ == "__main__":
    debug = os.getenv("FLASK_DEBUG", "0") == "1"
    host = os.getenv("HOST", "0.0.0.0")
    port = int(os.getenv("PORT", "5000"))

    log_debug(f"[WEB] Starting Flask server on {host}:{port}, debug={debug}")
    app.run(host=host, port=port, debug=debug)