You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
156 lines
5.1 KiB
156 lines
5.1 KiB
# =========================================================
|
|
# File: scraper/tasks/parse_tasks.py
|
|
# Purpose: Parse downloaded HTML into clean chapter text.
|
|
# Enhanced version: Piaotia H1→content extractor + clean pipeline
|
|
# NO HARDCODED REPLACEMENTS — everything comes from replacement files
|
|
# =========================================================
|
|
|
|
from celery_app import celery_app
|
|
from bs4 import BeautifulSoup
|
|
|
|
from scraper.utils import clean_text, load_all_replacements
|
|
from scraper.tasks.download_tasks import log_msg # unified logger
|
|
|
|
print(">>> [IMPORT] parse_tasks.py loaded (enhanced parser)")
|
|
|
|
|
|
@celery_app.task(bind=True, queue="parse", ignore_result=False)
|
|
def parse_chapter(self, download_result: dict, meta: dict):
|
|
|
|
book_id = download_result.get("book_id", "NOBOOK")
|
|
|
|
# ------------------------------------------------------------
|
|
# SKIPPED DOWNLOAD → SKIP PARSE
|
|
# ------------------------------------------------------------
|
|
if download_result.get("skipped"):
|
|
chapter = download_result.get("chapter")
|
|
log_msg(book_id, f"[PARSE] SKIP chapter {chapter} (download skipped)")
|
|
download_result["book_id"] = book_id
|
|
return download_result
|
|
|
|
# ------------------------------------------------------------
|
|
# NORMAL PARSE
|
|
# ------------------------------------------------------------
|
|
chapter_num = download_result["chapter"]
|
|
chapter_url = download_result["url"]
|
|
html = download_result["html"]
|
|
|
|
log_msg(book_id, f"[PARSE] Parsing chapter {chapter_num}")
|
|
|
|
soup = BeautifulSoup(html, "lxml")
|
|
|
|
# ------------------------------------------------------------
|
|
# STRICT SELECTORS (direct content blocks)
|
|
# ------------------------------------------------------------
|
|
selectors = [
|
|
"#content",
|
|
"div#content",
|
|
".content",
|
|
"div.content",
|
|
"#chaptercontent",
|
|
"div#chaptercontent",
|
|
"#chapterContent",
|
|
".read-content",
|
|
"div.read-content",
|
|
]
|
|
|
|
node = None
|
|
for sel in selectors:
|
|
tmp = soup.select_one(sel)
|
|
if tmp:
|
|
node = tmp
|
|
break
|
|
|
|
# ------------------------------------------------------------
|
|
# PIAOTIA FALLBACK:
|
|
# Extract content between <H1> and the "bottomlink" block.
|
|
# ------------------------------------------------------------
|
|
raw = None
|
|
if node is None:
|
|
h1 = soup.find("h1")
|
|
if h1:
|
|
content_parts = []
|
|
for sib in h1.next_siblings:
|
|
|
|
# stop at bottom navigation/footer block
|
|
sib_class = getattr(sib, "get", lambda *_: None)("class")
|
|
if sib_class and (
|
|
"bottomlink" in sib_class or sib_class == "bottomlink"
|
|
):
|
|
break
|
|
|
|
# ignore typical noise containers
|
|
if getattr(sib, "name", None) in ["script", "style", "center"]:
|
|
continue
|
|
|
|
if hasattr(sib, "get_text"):
|
|
content_parts.append(sib.get_text(separator="\n"))
|
|
else:
|
|
content_parts.append(str(sib))
|
|
|
|
raw = "\n".join(content_parts)
|
|
|
|
# ------------------------------------------------------------
|
|
# FINAL FALLBACK
|
|
# ------------------------------------------------------------
|
|
if raw is None:
|
|
if node:
|
|
raw = node.get_text(separator="\n")
|
|
else:
|
|
# drop scripts & styles
|
|
for tag in soup(["script", "style", "noscript"]):
|
|
tag.decompose()
|
|
|
|
raw = soup.get_text(separator="\n")
|
|
|
|
# ------------------------------------------------------------
|
|
# MULTIPASS CLEANING via replacement files ONLY
|
|
# ------------------------------------------------------------
|
|
REPL = load_all_replacements()
|
|
|
|
text = raw
|
|
for _ in range(5): # like the C# CleanText loop
|
|
text = clean_text(text, REPL)
|
|
|
|
# ------------------------------------------------------------
|
|
# Collapse excessive empty lines
|
|
# ------------------------------------------------------------
|
|
cleaned = []
|
|
prev_blank = False
|
|
|
|
for line in text.split("\n"):
|
|
stripped = line.rstrip()
|
|
if stripped == "":
|
|
if prev_blank:
|
|
continue
|
|
prev_blank = True
|
|
cleaned.append("")
|
|
else:
|
|
prev_blank = False
|
|
cleaned.append(stripped)
|
|
|
|
text = "\n".join(cleaned)
|
|
|
|
# ------------------------------------------------------------
|
|
# Add header to chapter 1
|
|
# ------------------------------------------------------------
|
|
if chapter_num == 1:
|
|
book_url = meta.get("book_url") or meta.get("url") or "UNKNOWN"
|
|
header = (
|
|
f"{meta.get('title','')}\n"
|
|
f"Author: {meta.get('author','')}\n"
|
|
f"Description:\n{meta.get('description','')}\n"
|
|
f"Book URL: {book_url}\n" + "-" * 50 + "\n\n"
|
|
)
|
|
text = header + text
|
|
|
|
log_msg(book_id, f"[PARSE] Parsed chapter {chapter_num}: {len(text)} chars")
|
|
|
|
return {
|
|
"book_id": book_id,
|
|
"chapter": chapter_num,
|
|
"url": chapter_url,
|
|
"text": text,
|
|
"length": len(text),
|
|
}
|