You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
292 lines
8.2 KiB
292 lines
8.2 KiB
# ============================================
|
|
# File: bookscraper/app.py (ASYNC SCRAPING)
|
|
# ============================================
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
print(">>> [WEB] Importing celery_app …")
|
|
from celery_app import celery_app
|
|
|
|
from flask import Flask, render_template, request, jsonify
|
|
from scraper.logger import log_debug
|
|
|
|
# Abort + Progress (per book_id)
|
|
from scraper.abort import set_abort
|
|
from scraper.progress import get_progress
|
|
|
|
# UI LOGS (GLOBAL — no book_id)
|
|
from scraper.ui_log import get_ui_logs, reset_ui_logs
|
|
|
|
from celery.result import AsyncResult
|
|
from scraper.state import state as r
|
|
|
|
# Cover serving
|
|
from flask import send_from_directory
|
|
import os
|
|
|
|
import redis
|
|
|
|
# Flask
|
|
app = Flask(__name__)
|
|
|
|
# =====================================================
|
|
# STATIC FILE SERVING FOR OUTPUT
|
|
# =====================================================
|
|
OUTPUT_ROOT = os.getenv("BOOKSCRAPER_OUTPUT_DIR", "output")
|
|
|
|
|
|
@app.route("/output/<path:filename>")
|
|
def serve_output(filename):
|
|
return send_from_directory(OUTPUT_ROOT, filename, as_attachment=False)
|
|
|
|
|
|
# =====================================================
|
|
# HOME PAGE
|
|
# =====================================================
|
|
@app.route("/", methods=["GET"])
|
|
def index():
|
|
return render_template("index.html")
|
|
|
|
|
|
# =====================================================
|
|
# START SCRAPING (async via Celery)
|
|
# =====================================================
|
|
@app.route("/start", methods=["POST"])
|
|
def start_scraping():
|
|
url = request.form.get("url", "").strip()
|
|
|
|
if not url:
|
|
# ★ FIX: dashboard moet altijd books + logs meekrijgen
|
|
return render_template(
|
|
"dashboard/dashboard.html",
|
|
error="Geen URL opgegeven.",
|
|
books=list_active_books(),
|
|
logs=get_ui_logs(),
|
|
)
|
|
|
|
reset_ui_logs()
|
|
log_debug(f"[WEB] Scraping via Celery: {url}")
|
|
|
|
async_result = celery_app.send_task(
|
|
"scraper.tasks.scraping.start_scrape_book",
|
|
args=[url],
|
|
queue="scraping",
|
|
)
|
|
|
|
# ★ FIX: direct dashboard tonen met actuele data
|
|
return render_template(
|
|
"dashboard/dashboard.html",
|
|
scraping_task_id=async_result.id,
|
|
books=list_active_books(),
|
|
logs=get_ui_logs(),
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# CLEAR UI LOGS
|
|
# =====================================================
|
|
@app.route("/clear-logs", methods=["POST"])
|
|
def clear_logs():
|
|
reset_ui_logs()
|
|
return jsonify({"status": "ok", "message": "UI logs cleared"})
|
|
|
|
|
|
# =====================================================
|
|
# ABORT (per book_id)
|
|
# =====================================================
|
|
@app.route("/abort/<book_id>", methods=["POST"])
|
|
def abort_download(book_id):
|
|
log_debug(f"[WEB] Abort requested for book: {book_id}")
|
|
set_abort(book_id)
|
|
return jsonify({"status": "ok", "aborted": book_id})
|
|
|
|
|
|
# =====================================================
|
|
# PROGRESS (per book_id)
|
|
# =====================================================
|
|
@app.route("/progress/<book_id>", methods=["GET"])
|
|
def progress(book_id):
|
|
return jsonify(get_progress(book_id))
|
|
|
|
|
|
# =====================================================
|
|
# CELERY RESULT → return book_id
|
|
# =====================================================
|
|
@app.route("/celery-result/<task_id>", methods=["GET"])
|
|
def celery_result(task_id):
|
|
result = AsyncResult(task_id, app=celery_app)
|
|
|
|
if result.successful():
|
|
return jsonify({"ready": True, "result": result.get()})
|
|
if result.failed():
|
|
return jsonify({"ready": True, "error": "failed"})
|
|
return jsonify({"ready": False})
|
|
|
|
|
|
# =====================================================
|
|
# API: book status new model
|
|
# =====================================================
|
|
def getStatus(book_id):
|
|
|
|
state = r.hgetall(f"book:{book_id}:state")
|
|
status = state.get("status") or "unknown"
|
|
dl_done = int(state.get("chapters_download_done", 0))
|
|
dl_skipped = int(state.get("chapters_download_skipped", 0))
|
|
dl_total = int(state.get("chapters_total", 0))
|
|
au_done = int(state.get("audio_done") or 0)
|
|
title = state.get("title") or book_id
|
|
|
|
au_total = dl_total
|
|
|
|
return {
|
|
"book_id": book_id,
|
|
"title": title,
|
|
"status": status,
|
|
"download_done": dl_done,
|
|
"download_skipped": dl_skipped,
|
|
"download_total": dl_total,
|
|
"audio_done": au_done,
|
|
"audio_total": au_total,
|
|
}
|
|
|
|
|
|
# =====================================================
|
|
# REDIS BACKEND — BOOK STATE MODEL
|
|
# =====================================================
|
|
REDIS_URL = os.getenv("REDIS_BROKER", "redis://redis:6379/0")
|
|
r = redis.Redis.from_url(REDIS_URL, decode_responses=True)
|
|
|
|
|
|
def list_active_booksold():
|
|
"""Return list of active books from Redis Book State Model."""
|
|
keys = r.keys("book:*:state")
|
|
books = []
|
|
|
|
for key in keys:
|
|
book_id = key.split(":")[1]
|
|
print(book_id)
|
|
books.append(getStatus(book_id))
|
|
|
|
return books
|
|
|
|
|
|
def list_active_books():
|
|
books = []
|
|
for key in r.scan_iter(match="book:*:state", count=1000):
|
|
first = key.find(":")
|
|
second = key.find(":", first + 1)
|
|
book_id = key[first + 1 : second]
|
|
books.append(getStatus(book_id))
|
|
return books
|
|
|
|
|
|
# =====================================================
|
|
# API: list all active books
|
|
# =====================================================
|
|
@app.route("/api/books")
|
|
def api_books():
|
|
return jsonify(list_active_books())
|
|
|
|
|
|
@app.route("/api/book/<book_id>/status")
|
|
def api_book_status(book_id):
|
|
|
|
return jsonify(getStatus(book_id))
|
|
|
|
|
|
# =====================================================
|
|
# API: book logs
|
|
# =====================================================
|
|
@app.route("/api/book/<book_id>/logs")
|
|
def api_book_logs(book_id):
|
|
logs = r.lrange(f"logs:{book_id}", 0, -1) or []
|
|
return jsonify(logs)
|
|
|
|
|
|
# =====================================================
|
|
# VIEW: DASHBOARD
|
|
# =====================================================
|
|
@app.route("/dashboard")
|
|
def dashboard():
|
|
logs_list = get_ui_logs() or []
|
|
# ★ FIX: dashboard moet altijd books + logs krijgen
|
|
return render_template(
|
|
"dashboard/dashboard.html",
|
|
books=list_active_books(),
|
|
logs=logs_list, # dashboard krijgt LIST, geen dict
|
|
)
|
|
|
|
|
|
# =====================================================
|
|
# VIEW: BOOK DETAIL PAGE
|
|
# =====================================================
|
|
@app.route("/book/<book_id>")
|
|
def book_detail(book_id):
|
|
title = r.get(f"book:{book_id}:title") or book_id
|
|
return render_template(
|
|
"dashboard/book_detail.html",
|
|
book_id=book_id,
|
|
title=title,
|
|
logs=get_ui_logs(),
|
|
)
|
|
|
|
|
|
@app.route("/debug/redis-keys")
|
|
def debug_redis_keys():
|
|
cursor = 0
|
|
results = {}
|
|
|
|
while True:
|
|
cursor, keys = r.scan(cursor, match="*", count=200)
|
|
for k in keys:
|
|
try:
|
|
results[k] = r.get(k)
|
|
except:
|
|
results[k] = "<non-string value>"
|
|
if cursor == 0:
|
|
break
|
|
|
|
return jsonify(results)
|
|
|
|
|
|
# ============================================================
|
|
# Rolling log endpoint (no new file)
|
|
# ============================================================
|
|
|
|
from flask import jsonify, request
|
|
|
|
# =====================================================
|
|
# ROLLING LOG ENDPOINT — DELTA POLLING VIA ui_log
|
|
# =====================================================
|
|
from scraper.ui_log import get_ui_logs_delta
|
|
|
|
|
|
@app.route("/logs", methods=["GET"])
|
|
def logs():
|
|
"""
|
|
Delta log delivery for WebGUI.
|
|
Browser sends ?last_index=N, we return only new lines.
|
|
"""
|
|
try:
|
|
last_index = int(request.args.get("last_index", -1))
|
|
except:
|
|
last_index = -1
|
|
|
|
new_lines, total = get_ui_logs_delta(last_index)
|
|
|
|
return jsonify({"lines": new_lines, "total": total})
|
|
|
|
|
|
# =====================================================
|
|
# RUN FLASK
|
|
# =====================================================
|
|
if __name__ == "__main__":
|
|
debug = os.getenv("FLASK_DEBUG", "0") == "1"
|
|
host = os.getenv("HOST", "0.0.0.0")
|
|
port = int(os.getenv("PORT", "5000"))
|
|
|
|
log_debug(f"[WEB] Starting Flask server on {host}:{port}, debug={debug}")
|
|
app.run(host=host, port=port, debug=debug)
|