diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py index 334d1e897e..b593ad849a 100644 --- a/deepdoc/parser/html_parser.py +++ b/deepdoc/parser/html_parser.py @@ -159,7 +159,10 @@ class RAGFlowHtmlParser: if title_flag: content = f"{TITLE_TAGS[tag_name]} {content}" if last_block_id != block_id: - if last_block_id is not None: + # Flush whatever is buffered, including loose text that + # precedes the first block (last_block_id is None there), + # which was otherwise overwritten and lost. + if current_content: block_content.append(current_content) current_content = content last_block_id = block_id diff --git a/test/unit_test/deepdoc/parser/test_html_parser.py b/test/unit_test/deepdoc/parser/test_html_parser.py index e3a135a845..02f36bed3f 100644 --- a/test/unit_test/deepdoc/parser/test_html_parser.py +++ b/test/unit_test/deepdoc/parser/test_html_parser.py @@ -148,3 +148,25 @@ def test_parser_txt_extracts_bodyless_html_fragment(): joined = "\n".join(chunks) assert "# Title" in joined assert "Fragment text" in joined + + +def test_parser_txt_keeps_text_before_first_block(): + # Loose text (or an inline element) that precedes the first block element + # must not be dropped. The block-change flush was guarded on there being a + # previous block, so the buffered preamble was overwritten and lost. + joined = "\n".join(RAGFlowHtmlParser.parser_txt("Preamble text.

Title

Body.

", chunk_token_num=512)) + assert "Preamble text." in joined + assert "# Title" in joined + assert "Body." in joined + + inline = "\n".join(RAGFlowHtmlParser.parser_txt("Notice.

Section

Body.

", chunk_token_num=512)) + assert "Notice." in inline + + +def test_parser_txt_keeps_loose_text_between_and_after_blocks(): + # Control: loose text surrounded by / following blocks was already kept and + # must stay that way. + between = "\n".join(RAGFlowHtmlParser.parser_txt("

One.

loose

Two.

", chunk_token_num=512)) + assert "loose" in between + trailing = "\n".join(RAGFlowHtmlParser.parser_txt("

Only.

tail", chunk_token_num=512)) + assert "tail" in trailing