You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
37 lines
1.1 KiB
37 lines
1.1 KiB
# ============================================================
|
|
# File: scraper/tasks/progress_tasks.py
|
|
# Purpose: Central progress updater for chapter pipelines.
|
|
# ============================================================
|
|
|
|
from celery_app import celery_app
|
|
from scraper.progress import inc_completed, inc_skipped, inc_failed
|
|
from logbus.publisher import log
|
|
|
|
print(">>> [IMPORT] progress_tasks.py loaded")
|
|
|
|
|
|
@celery_app.task(bind=False, name="progress.update", queue="controller")
|
|
def update_progress(result: dict, book_id: str):
|
|
"""
|
|
Central progress logic:
|
|
- result: output of save_chapter
|
|
- book_id: explicitly passed by pipeline
|
|
"""
|
|
|
|
ch = result.get("chapter")
|
|
skipped = result.get("skipped", False)
|
|
failed = result.get("failed", False)
|
|
|
|
if failed:
|
|
inc_failed(book_id)
|
|
log(f"[PROG] FAILED chapter {ch}")
|
|
elif skipped:
|
|
inc_skipped(book_id)
|
|
inc_completed(book_id)
|
|
log(f"[PROG] SKIPPED chapter {ch}")
|
|
else:
|
|
inc_completed(book_id)
|
|
log(f"[PROG] DONE chapter {ch}")
|
|
|
|
return result
|