Fix: preserve markdown tables during delimiter extraction (#15632)

### What problem does this PR solve?

Markdown extraction can split tables row by row when delimiter-based
extraction uses a newline delimiter. That loses table structure during
chunking even though delimiters should still split normally outside
tables.

This PR keeps the follow-up to #15482 intentionally narrow:

- preserve Markdown pipe tables during delimiter-based extraction
- preserve borderless pipe tables during delimiter-based extraction
- preserve multiline HTML tables during delimiter-based extraction
- keep delimiter splitting unchanged outside protected table ranges

Refs #15482

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

### Testing

- `ruff check deepdoc/parser/markdown_parser.py
test/unit_test/deepdoc/parser/test_markdown_parser.py`
- `python3 run_tests.py -t
test/unit_test/deepdoc/parser/test_markdown_parser.py`
- `git diff --check`
This commit is contained in:
bitloi
2026-06-04 23:35:33 -03:00
committed by GitHub
parent 431f52a5d4
commit 9f3e289b78
2 changed files with 114 additions and 2 deletions

View File

@@ -106,3 +106,41 @@ class TestMarkdownElementExtractorFences:
"````markdown\n```python\nprint('inner')\n```\n````",
"After",
]
@pytest.mark.p2
class TestMarkdownElementExtractorTables:
def test_custom_delimiter_preserves_pipe_table(self, markdown_element_extractor):
text = "# Title\n\n| Name | Value |\n| --- | --- |\n| A | 1 |\n| B | 2 |\n\nAfter"
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`", include_meta=True)
assert [section["content"] for section in sections] == [
"# Title",
"| Name | Value |\n| --- | --- |\n| A | 1 |\n| B | 2 |",
"After",
]
assert sections[1]["start_line"] == 2
assert sections[1]["end_line"] == 5
def test_custom_delimiter_preserves_borderless_pipe_table(self, markdown_element_extractor):
text = "Before\nName | Value\n--- | ---\nA | 1\nB | 2\nAfter"
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`")
assert sections == [
"Before",
"Name | Value\n--- | ---\nA | 1\nB | 2",
"After",
]
def test_custom_delimiter_preserves_html_table(self, markdown_element_extractor):
text = "Before\n<table>\n<tr><td>A</td></tr>\n<tr><td>B</td></tr>\n</table>\nAfter"
sections = markdown_element_extractor(text).extract_elements(delimiter="`\n`")
assert sections == [
"Before",
"<table>\n<tr><td>A</td></tr>\n<tr><td>B</td></tr>\n</table>",
"After",
]