mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 15:20:30 +08:00
Feat: support epub parsing (#13650)
Closes #1398 ### What problem does this PR solve? Adds native support for EPUB files. EPUB content is extracted in spine (reading) order and parsed using the existing HTML parser. No new dependencies required. ### Type of change - [x] New Feature (non-breaking change which adds functionality) To check this parser manually: ```python uv run --python 3.12 python -c " from deepdoc.parser import EpubParser with open('$HOME/some_epub_book.epub', 'rb') as f: data = f.read() sections = EpubParser()(None, binary=data, chunk_token_num=512) print(f'Got {len(sections)} sections') for i, s in enumerate(sections[:5]): print(f'\n--- Section {i} ---') print(s[:200]) " ```
This commit is contained in:
@@ -33,7 +33,7 @@ from common.constants import LLMType
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_by_type_and_name, get_tenant_default_model_by_type
|
||||
from rag.utils.file_utils import extract_embed_file, extract_links_from_pdf, extract_links_from_docx, extract_html
|
||||
from deepdoc.parser import DocxParser, ExcelParser, HtmlParser, JsonParser, MarkdownElementExtractor, MarkdownParser, PdfParser, TxtParser
|
||||
from deepdoc.parser import DocxParser, EpubParser, ExcelParser, HtmlParser, JsonParser, MarkdownElementExtractor, MarkdownParser, PdfParser, TxtParser
|
||||
from deepdoc.parser.figure_parser import VisionFigureParser, vision_figure_parser_docx_wrapper_naive, vision_figure_parser_pdf_wrapper
|
||||
from deepdoc.parser.pdf_parser import PlainParser, VisionParser
|
||||
from deepdoc.parser.docling_parser import DoclingParser
|
||||
@@ -953,6 +953,14 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, lang="Chinese", ca
|
||||
sections = _normalize_section_text_for_rtl_presentation_forms(sections)
|
||||
callback(0.8, "Finish parsing.")
|
||||
|
||||
elif re.search(r"\.epub$", filename, re.IGNORECASE):
|
||||
callback(0.1, "Start to parse.")
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 128))
|
||||
sections = EpubParser()(filename, binary, chunk_token_num)
|
||||
sections = [(_, "") for _ in sections if _]
|
||||
sections = _normalize_section_text_for_rtl_presentation_forms(sections)
|
||||
callback(0.8, "Finish parsing.")
|
||||
|
||||
elif re.search(r"\.(json|jsonl|ldjson)$", filename, re.IGNORECASE):
|
||||
callback(0.1, "Start to parse.")
|
||||
chunk_token_num = int(parser_config.get("chunk_token_num", 128))
|
||||
|
||||
@@ -43,10 +43,9 @@ from rag.nlp import BULLET_PATTERN, bullets_category, docx_question_level, not_b
|
||||
from rag.utils.base64_image import image2id
|
||||
|
||||
|
||||
|
||||
|
||||
from common.misc_utils import thread_pool_exec
|
||||
|
||||
|
||||
class ParserParam(ProcessParamBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -82,6 +81,10 @@ class ParserParam(ProcessParamBase):
|
||||
"json",
|
||||
],
|
||||
"video": [],
|
||||
"epub": [
|
||||
"text",
|
||||
"json",
|
||||
],
|
||||
}
|
||||
|
||||
self.setups = {
|
||||
@@ -166,6 +169,12 @@ class ParserParam(ProcessParamBase):
|
||||
"output_format": "text",
|
||||
"prompt": "",
|
||||
},
|
||||
"epub": {
|
||||
"suffix": [
|
||||
"epub",
|
||||
],
|
||||
"output_format": "json",
|
||||
},
|
||||
}
|
||||
|
||||
def check(self):
|
||||
@@ -219,6 +228,11 @@ class ParserParam(ProcessParamBase):
|
||||
email_output_format = email_config.get("output_format", "")
|
||||
self.check_valid_value(email_output_format, "Email output format abnormal.", self.allowed_output_format["email"])
|
||||
|
||||
epub_config = self.setups.get("epub", "")
|
||||
if epub_config:
|
||||
epub_output_format = epub_config.get("output_format", "")
|
||||
self.check_valid_value(epub_output_format, "EPUB output format abnormal.", self.allowed_output_format["epub"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {}
|
||||
|
||||
@@ -390,9 +404,7 @@ class Parser(ProcessBase):
|
||||
box = {
|
||||
"text": text,
|
||||
"image": pdf_parser.crop(poss, 1) if isinstance(poss, str) and poss else None,
|
||||
"positions": [[pos[0][-1], *pos[1:]] for pos in pdf_parser.extract_positions(poss)]
|
||||
if isinstance(poss, str) and poss
|
||||
else [],
|
||||
"positions": [[pos[0][-1], *pos[1:]] for pos in pdf_parser.extract_positions(poss)] if isinstance(poss, str) and poss else [],
|
||||
}
|
||||
bboxes.append(box)
|
||||
elif parse_method.lower() == "tcadp parser":
|
||||
@@ -698,7 +710,6 @@ class Parser(ProcessBase):
|
||||
markdown_text = docx_parser.to_markdown(name, binary=blob)
|
||||
self.set_output("markdown", markdown_text)
|
||||
|
||||
|
||||
def _slides(self, name, blob, **kwargs):
|
||||
self.callback(random.randint(1, 5) / 100.0, "Start to work on a PowerPoint Document")
|
||||
|
||||
@@ -839,11 +850,13 @@ class Parser(ProcessBase):
|
||||
else:
|
||||
txt = cv_model.describe(img_binary.read())
|
||||
|
||||
json_result = [{
|
||||
"text": txt,
|
||||
"image": img,
|
||||
"doc_type_kwd": "image",
|
||||
}]
|
||||
json_result = [
|
||||
{
|
||||
"text": txt,
|
||||
"image": img,
|
||||
"doc_type_kwd": "image",
|
||||
}
|
||||
]
|
||||
self.set_output("json", json_result)
|
||||
|
||||
def _audio(self, name, blob, **kwargs):
|
||||
@@ -1013,6 +1026,22 @@ class Parser(ProcessBase):
|
||||
content_txt += fb
|
||||
self.set_output("text", content_txt)
|
||||
|
||||
def _epub(self, name, blob, **kwargs):
|
||||
from deepdoc.parser import EpubParser
|
||||
|
||||
self.callback(random.randint(1, 5) / 100.0, "Start to work on an EPUB.")
|
||||
conf = self._param.setups["epub"]
|
||||
self.set_output("output_format", conf["output_format"])
|
||||
|
||||
epub_parser = EpubParser()
|
||||
sections = epub_parser(name, binary=blob)
|
||||
|
||||
if conf.get("output_format") == "json":
|
||||
json_results = [{"text": s} for s in sections if s]
|
||||
self.set_output("json", json_results)
|
||||
else:
|
||||
self.set_output("text", "\n".join(s for s in sections if s))
|
||||
|
||||
async def _invoke(self, **kwargs):
|
||||
function_map = {
|
||||
"pdf": self._pdf,
|
||||
@@ -1024,6 +1053,7 @@ class Parser(ProcessBase):
|
||||
"audio": self._audio,
|
||||
"video": self._video,
|
||||
"email": self._email,
|
||||
"epub": self._epub,
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user