mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-14 20:54:30 +08:00
Fix: preserve markdown tables during delimiter extraction (#15632)
### What problem does this PR solve? Markdown extraction can split tables row by row when delimiter-based extraction uses a newline delimiter. That loses table structure during chunking even though delimiters should still split normally outside tables. This PR keeps the follow-up to #15482 intentionally narrow: - preserve Markdown pipe tables during delimiter-based extraction - preserve borderless pipe tables during delimiter-based extraction - preserve multiline HTML tables during delimiter-based extraction - keep delimiter splitting unchanged outside protected table ranges Refs #15482 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ### Testing - `ruff check deepdoc/parser/markdown_parser.py test/unit_test/deepdoc/parser/test_markdown_parser.py` - `python3 run_tests.py -t test/unit_test/deepdoc/parser/test_markdown_parser.py` - `git diff --check`
This commit is contained in:
@@ -106,3 +106,41 @@ class TestMarkdownElementExtractorFences:
|
||||
"````markdown\n```python\nprint('inner')\n```\n````",
|
||||
"After",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.p2
|
||||
class TestMarkdownElementExtractorTables:
|
||||
def test_custom_delimiter_preserves_pipe_table(self, markdown_element_extractor):
|
||||
text = "# Title\n\n| Name | Value |\n| --- | --- |\n| A | 1 |\n| B | 2 |\n\nAfter"
|
||||
|
||||
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`", include_meta=True)
|
||||
|
||||
assert [section["content"] for section in sections] == [
|
||||
"# Title",
|
||||
"| Name | Value |\n| --- | --- |\n| A | 1 |\n| B | 2 |",
|
||||
"After",
|
||||
]
|
||||
assert sections[1]["start_line"] == 2
|
||||
assert sections[1]["end_line"] == 5
|
||||
|
||||
def test_custom_delimiter_preserves_borderless_pipe_table(self, markdown_element_extractor):
|
||||
text = "Before\nName | Value\n--- | ---\nA | 1\nB | 2\nAfter"
|
||||
|
||||
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`")
|
||||
|
||||
assert sections == [
|
||||
"Before",
|
||||
"Name | Value\n--- | ---\nA | 1\nB | 2",
|
||||
"After",
|
||||
]
|
||||
|
||||
def test_custom_delimiter_preserves_html_table(self, markdown_element_extractor):
|
||||
text = "Before\n<table>\n<tr><td>A</td></tr>\n<tr><td>B</td></tr>\n</table>\nAfter"
|
||||
|
||||
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`")
|
||||
|
||||
assert sections == [
|
||||
"Before",
|
||||
"<table>\n<tr><td>A</td></tr>\n<tr><td>B</td></tr>\n</table>",
|
||||
"After",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user