zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-23 17:06:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(html_parser): keep text before the first block element (#17073)

Browse Source
This commit is contained in:
Andrew Chen
2026-07-23 12:43:16 +08:00
committed by GitHub
parent 5478f955ab
commit 8af2467112
2 changed files with 26 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
deepdoc/parser/html_parser.py
View File

@@ -159,7 +159,10 @@ class RAGFlowHtmlParser:
if title_flag:
content = f"{TITLE_TAGS[tag_name]} {content}"
if last_block_id != block_id:
if last_block_id is not None:
# Flush whatever is buffered, including loose text that
# precedes the first block (last_block_id is None there),
# which was otherwise overwritten and lost.
if current_content:
block_content.append(current_content)
current_content = content
last_block_id = block_id

22
test/unit_test/deepdoc/parser/test_html_parser.py
View File

@@ -148,3 +148,25 @@ def test_parser_txt_extracts_bodyless_html_fragment():
joined = "\n".join(chunks)
assert "# Title" in joined
assert "Fragment text" in joined
def test_parser_txt_keeps_text_before_first_block():
# Loose text (or an inline element) that precedes the first block element
# must not be dropped. The block-change flush was guarded on there being a
# previous block, so the buffered preamble was overwritten and lost.
joined = "\n".join(RAGFlowHtmlParser.parser_txt("<body>Preamble text.<h1>Title</h1><p>Body.</p></body>", chunk_token_num=512))
assert "Preamble text." in joined
assert "# Title" in joined
assert "Body." in joined
inline = "\n".join(RAGFlowHtmlParser.parser_txt("<body><span>Notice.</span><h2>Section</h2><p>Body.</p></body>", chunk_token_num=512))
assert "Notice." in inline
def test_parser_txt_keeps_loose_text_between_and_after_blocks():
# Control: loose text surrounded by / following blocks was already kept and
# must stay that way.
between = "\n".join(RAGFlowHtmlParser.parser_txt("<p>One.</p>loose<p>Two.</p>", chunk_token_num=512))
assert "loose" in between
trailing = "\n".join(RAGFlowHtmlParser.parser_txt("<p>Only.</p>tail", chunk_token_num=512))
assert "tail" in trailing