You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
36 lines
1.0 KiB
36 lines
1.0 KiB
from celery import shared_task
|
|
from scraper.book_scraper import BookScraper
|
|
from scraper.sites import BookSite
|
|
from logbus.publisher import log
|
|
|
|
|
|
@shared_task(bind=True, queue="scraping")
|
|
def scrape_book(self, url):
|
|
"""
|
|
HIGH-LEVEL SCRAPER TASK
|
|
Roept synchronen BookScraper aan voor een volledige scrape.
|
|
"""
|
|
log(f"[SCRAPER] Start scrape: {url}")
|
|
|
|
scraper = BookScraper(BookSite(), url)
|
|
result = scraper.execute()
|
|
|
|
log(f"[SCRAPER] Finished scrape: {url}")
|
|
return {"title": result["title"]}
|
|
|
|
|
|
@shared_task(bind=True, queue="download", max_retries=5)
|
|
def download_chapter_task(self, number, title, url, output_base):
|
|
"""
|
|
Download alleen één chapter.
|
|
download_worker.py voert dit uiteindelijk uit.
|
|
"""
|
|
from worker.download_worker import download_single_chapter
|
|
|
|
try:
|
|
return download_single_chapter(number, title, url, output_base)
|
|
|
|
except Exception as e:
|
|
log(f"[DOWNLOAD] Error while downloading chapter {number}: {e}")
|
|
raise self.retry(countdown=3)
|