fix(deepdoc): keep pipe tables inside markdown code fences (#17201)

deepdoc/parser/markdown_parser.py

@@ -20,6 +20,8 @@ import re
 
 from markdown import markdown
 
+logger = logging.getLogger(__name__)
+
 
 class RAGFlowMarkdownParser:
     def __init__(self, chunk_token_num=128):
@@ -29,10 +31,26 @@ class RAGFlowMarkdownParser:
         tables = []
         working_text = markdown_text
 
+        # a fenced code block can legitimately contain a pipe table, e.g. docs that show
+        # markdown syntax. extracting it would strip the example out of the fence and
+        # re-emit it as a bogus table, leaving a hollow fence behind.
+        # MarkdownElementExtractor already shields fences (see _fenced_code_ranges), so
+        # keep table extraction consistent with it.
+        fence_pattern = re.compile(
+            r"^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^[ \t]{0,3}\1[ \t]*$|\Z)",
+            re.MULTILINE | re.DOTALL,
+        )
+
         def replace_tables_with_rendered_html(pattern, table_list, render=True):
             new_text = ""
             last_end = 0
+            fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
             for match in pattern.finditer(working_text):
+                if any(start <= match.start() < end for start, end in fenced_spans):
+                    # inside a code fence: leave it in place. last_end is untouched, so
+                    # the block is copied through verbatim.
+                    logger.debug("markdown table pass: skipping match inside a code fence at %d", match.start())
+                    continue
                 raw_table = match.group()
                 table_list.append(raw_table)
                 if separate_tables:
@@ -75,7 +93,13 @@ class RAGFlowMarkdownParser:
         TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"]
         table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE)
 
+        tag_fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
+
         def replace_tag(m):
+            if any(start <= m.start() < end for start, end in tag_fenced_spans):
+                # an html example inside a fence keeps its attributes verbatim.
+                logger.debug("html tag pass: preserving tag inside a code fence at %d", m.start())
+                return m.group()
             tag_name = re.match(r"<(\w+)", m.group()).group(1)
             return "<{}>".format(tag_name)
 
@@ -107,7 +131,11 @@ class RAGFlowMarkdownParser:
                 nonlocal working_text
                 new_text = ""
                 last_end = 0
+                fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
                 for match in html_table_pattern.finditer(working_text):
+                    if any(start <= match.start() < end for start, end in fenced_spans):
+                        logger.debug("html table pass: skipping match inside a code fence at %d", match.start())
+                        continue
                     raw_table = match.group()
                     tables.append(raw_table)
                     if separate_tables:

test/unit_test/deepdoc/parser/test_markdown_parser.py

@@ -194,6 +194,55 @@ class TestMarkdownTableDedup:
         assert "After" in sections[0]
         assert "| Name | Value |" not in sections[0]
 
+    def test_pipe_table_inside_code_fence_is_not_extracted(self, markdown_parser_module):
+        """A pipe table shown as an example inside a fenced code block must stay put;
+        extracting it would hollow out the fence and emit a bogus table chunk."""
+        text = "# Guide\n\nWrite a table like this:\n\n```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\nDone.\n"
+
+        parser = markdown_parser_module.RAGFlowMarkdownParser()
+        remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+        assert tables == []
+        assert "| Name | Value |" in remainder
+        assert "```markdown" in remainder
+
+    def test_html_table_inside_code_fence_is_left_intact(self, markdown_parser_module):
+        """An HTML example inside a fence must keep its attributes and its body:
+        the tag-stripping and html-table passes have to respect fences too."""
+        text = '# Guide\n\nEmbed a table like this:\n\n```html\n<table border="1">\n<tr><td>A</td></tr>\n</table>\n```\n\nDone.\n'
+
+        parser = markdown_parser_module.RAGFlowMarkdownParser()
+        remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+        assert tables == []
+        assert 'border="1"' in remainder
+        assert "<td>A</td>" in remainder
+
+    def test_bare_html_table_inside_code_fence_is_not_extracted(self, markdown_parser_module):
+        """A bare <table> (no attributes) inside a fence hits the html-table
+        extraction pass (the `<table>` fast-path guard fires); that pass must
+        skip fenced matches too, not just the tag-stripping pass."""
+        text = "# Guide\n\n```html\n<table>\n<tr><td>A</td></tr>\n</table>\n```\n\nDone.\n"
+
+        parser = markdown_parser_module.RAGFlowMarkdownParser()
+        remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+        assert tables == []
+        assert "<table>" in remainder
+        assert "<td>A</td>" in remainder
+
+    def test_real_table_still_extracted_alongside_a_fenced_example(self, markdown_parser_module):
+        """Shielding fences must not stop genuine tables outside them from being split."""
+        text = "```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\n| X | Y |\n| --- | --- |\n| 9 | 8 |\n\ntail\n"
+
+        parser = markdown_parser_module.RAGFlowMarkdownParser()
+        remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+        assert len(tables) == 1
+        assert "| X | Y |" in tables[0]
+        assert "| X | Y |" not in remainder
+        assert "| Name | Value |" in remainder
+
 
 class TestMarkdownElementExtractorDelimiterHeaders:
     def test_custom_delimiter_merges_consecutive_lone_headers_with_body(self, markdown_element_extractor):