From 64be8585aa64449d37a72fe5818ec8a531cb0706 Mon Sep 17 00:00:00 2001 From: Yurii214 Date: Thu, 23 Jul 2026 07:31:35 +0200 Subject: [PATCH] fix(deepdoc): keep pipe tables inside markdown code fences (#17201) --- deepdoc/parser/markdown_parser.py | 28 +++++++++++ .../deepdoc/parser/test_markdown_parser.py | 49 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/deepdoc/parser/markdown_parser.py b/deepdoc/parser/markdown_parser.py index 583e4ffd2b..7e8bf32e6d 100644 --- a/deepdoc/parser/markdown_parser.py +++ b/deepdoc/parser/markdown_parser.py @@ -20,6 +20,8 @@ import re from markdown import markdown +logger = logging.getLogger(__name__) + class RAGFlowMarkdownParser: def __init__(self, chunk_token_num=128): @@ -29,10 +31,26 @@ class RAGFlowMarkdownParser: tables = [] working_text = markdown_text + # a fenced code block can legitimately contain a pipe table, e.g. docs that show + # markdown syntax. extracting it would strip the example out of the fence and + # re-emit it as a bogus table, leaving a hollow fence behind. + # MarkdownElementExtractor already shields fences (see _fenced_code_ranges), so + # keep table extraction consistent with it. + fence_pattern = re.compile( + r"^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^[ \t]{0,3}\1[ \t]*$|\Z)", + re.MULTILINE | re.DOTALL, + ) + def replace_tables_with_rendered_html(pattern, table_list, render=True): new_text = "" last_end = 0 + fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)] for match in pattern.finditer(working_text): + if any(start <= match.start() < end for start, end in fenced_spans): + # inside a code fence: leave it in place. last_end is untouched, so + # the block is copied through verbatim. + logger.debug("markdown table pass: skipping match inside a code fence at %d", match.start()) + continue raw_table = match.group() table_list.append(raw_table) if separate_tables: @@ -75,7 +93,13 @@ class RAGFlowMarkdownParser: TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"] table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE) + tag_fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)] + def replace_tag(m): + if any(start <= m.start() < end for start, end in tag_fenced_spans): + # an html example inside a fence keeps its attributes verbatim. + logger.debug("html tag pass: preserving tag inside a code fence at %d", m.start()) + return m.group() tag_name = re.match(r"<(\w+)", m.group()).group(1) return "<{}>".format(tag_name) @@ -107,7 +131,11 @@ class RAGFlowMarkdownParser: nonlocal working_text new_text = "" last_end = 0 + fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)] for match in html_table_pattern.finditer(working_text): + if any(start <= match.start() < end for start, end in fenced_spans): + logger.debug("html table pass: skipping match inside a code fence at %d", match.start()) + continue raw_table = match.group() tables.append(raw_table) if separate_tables: diff --git a/test/unit_test/deepdoc/parser/test_markdown_parser.py b/test/unit_test/deepdoc/parser/test_markdown_parser.py index a649307e7c..1395a80978 100644 --- a/test/unit_test/deepdoc/parser/test_markdown_parser.py +++ b/test/unit_test/deepdoc/parser/test_markdown_parser.py @@ -194,6 +194,55 @@ class TestMarkdownTableDedup: assert "After" in sections[0] assert "| Name | Value |" not in sections[0] + def test_pipe_table_inside_code_fence_is_not_extracted(self, markdown_parser_module): + """A pipe table shown as an example inside a fenced code block must stay put; + extracting it would hollow out the fence and emit a bogus table chunk.""" + text = "# Guide\n\nWrite a table like this:\n\n```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\nDone.\n" + + parser = markdown_parser_module.RAGFlowMarkdownParser() + remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True) + + assert tables == [] + assert "| Name | Value |" in remainder + assert "```markdown" in remainder + + def test_html_table_inside_code_fence_is_left_intact(self, markdown_parser_module): + """An HTML example inside a fence must keep its attributes and its body: + the tag-stripping and html-table passes have to respect fences too.""" + text = '# Guide\n\nEmbed a table like this:\n\n```html\n\n\n
A
\n```\n\nDone.\n' + + parser = markdown_parser_module.RAGFlowMarkdownParser() + remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True) + + assert tables == [] + assert 'border="1"' in remainder + assert "A" in remainder + + def test_bare_html_table_inside_code_fence_is_not_extracted(self, markdown_parser_module): + """A bare (no attributes) inside a fence hits the html-table + extraction pass (the `
` fast-path guard fires); that pass must + skip fenced matches too, not just the tag-stripping pass.""" + text = "# Guide\n\n```html\n
\n\n
A
\n```\n\nDone.\n" + + parser = markdown_parser_module.RAGFlowMarkdownParser() + remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True) + + assert tables == [] + assert "" in remainder + assert "" in remainder + + def test_real_table_still_extracted_alongside_a_fenced_example(self, markdown_parser_module): + """Shielding fences must not stop genuine tables outside them from being split.""" + text = "```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\n| X | Y |\n| --- | --- |\n| 9 | 8 |\n\ntail\n" + + parser = markdown_parser_module.RAGFlowMarkdownParser() + remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True) + + assert len(tables) == 1 + assert "| X | Y |" in tables[0] + assert "| X | Y |" not in remainder + assert "| Name | Value |" in remainder + class TestMarkdownElementExtractorDelimiterHeaders: def test_custom_delimiter_merges_consecutive_lone_headers_with_body(self, markdown_element_extractor):
A