You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
59 lines
1.9 KiB
59 lines
1.9 KiB
# celery_app.py
|
|
import os
|
|
from celery import Celery
|
|
from dotenv import load_dotenv
|
|
|
|
print(">>> [celery_app] Loading .env BEFORE initializing Celery...")
|
|
load_dotenv()
|
|
from db.db import init_db
|
|
|
|
init_db() # ensures DB exists for all workers
|
|
|
|
BROKER = os.getenv("REDIS_BROKER")
|
|
BACKEND = os.getenv("REDIS_BACKEND")
|
|
|
|
print(">>> [celery_app] BROKER =", BROKER)
|
|
print(">>> [celery_app] BACKEND =", BACKEND)
|
|
|
|
celery_app = Celery(
|
|
"bookscraper",
|
|
broker=BROKER,
|
|
backend=BACKEND,
|
|
include=[
|
|
"scraper.tasks.scraping",
|
|
"scraper.tasks.controller_tasks",
|
|
"scraper.tasks.download_tasks",
|
|
"scraper.tasks.parse_tasks",
|
|
"scraper.tasks.save_tasks",
|
|
# --------------------------------------------------------
|
|
# AUDIO TASKS (NEW)
|
|
# --------------------------------------------------------
|
|
"scraper.tasks.audio_tasks",
|
|
],
|
|
)
|
|
|
|
celery_app.conf.task_routes = {
|
|
"scraper.tasks.scraping.*": {"queue": "scraping"},
|
|
"scraper.tasks.controller_tasks.*": {"queue": "controller"},
|
|
"scraper.tasks.download_tasks.*": {"queue": "download"},
|
|
"scraper.tasks.parse_tasks.*": {"queue": "parse"},
|
|
"scraper.tasks.save_tasks.*": {"queue": "save"},
|
|
# ------------------------------------------------------------
|
|
# AUDIO ROUTING (NEW)
|
|
# ------------------------------------------------------------
|
|
"scraper.tasks.audio_tasks.*": {"queue": "audio"},
|
|
}
|
|
|
|
# ------------------------------------------------------------
|
|
# EXTRA DEBUG: test import of included modules
|
|
# ------------------------------------------------------------
|
|
print(">>> [celery_app] Testing imports for included task modules...")
|
|
for module in celery_app.conf.include:
|
|
try:
|
|
__import__(module)
|
|
print(f">>> [celery_app] OK import {module}")
|
|
except Exception as e:
|
|
print(f">>> [celery_app] FAILED import {module}: {e}")
|
|
|
|
print(">>> [celery_app] Celery initialization complete.")
|