You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
22 lines
638 B
22 lines
638 B
# scraper/tasks/controller_tasks.py
|
|
|
|
from celery_app import celery_app
|
|
from logbus.publisher import log
|
|
from scraper.download_controller import DownloadController
|
|
|
|
print(">>> [IMPORT] controller_tasks.py loaded")
|
|
|
|
|
|
@celery_app.task(bind=True, queue="controller", ignore_result=False)
|
|
def launch_downloads(self, scrape_result: dict):
|
|
"""Start complete download → parse → save pipeline."""
|
|
|
|
log("[CTRL] Launching DownloadController...")
|
|
|
|
ctl = DownloadController(scrape_result)
|
|
async_result = ctl.start()
|
|
|
|
log("[CTRL] Pipelines dispatched.")
|
|
|
|
return {"pipelines_started": len(scrape_result.get("chapters", []))}
|