diff --git a/deepdoc/parser/markdown_parser.py b/deepdoc/parser/markdown_parser.py
index 583e4ffd2b..7e8bf32e6d 100644
--- a/deepdoc/parser/markdown_parser.py
+++ b/deepdoc/parser/markdown_parser.py
@@ -20,6 +20,8 @@ import re
from markdown import markdown
+logger = logging.getLogger(__name__)
+
class RAGFlowMarkdownParser:
def __init__(self, chunk_token_num=128):
@@ -29,10 +31,26 @@ class RAGFlowMarkdownParser:
tables = []
working_text = markdown_text
+ # a fenced code block can legitimately contain a pipe table, e.g. docs that show
+ # markdown syntax. extracting it would strip the example out of the fence and
+ # re-emit it as a bogus table, leaving a hollow fence behind.
+ # MarkdownElementExtractor already shields fences (see _fenced_code_ranges), so
+ # keep table extraction consistent with it.
+ fence_pattern = re.compile(
+ r"^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^[ \t]{0,3}\1[ \t]*$|\Z)",
+ re.MULTILINE | re.DOTALL,
+ )
+
def replace_tables_with_rendered_html(pattern, table_list, render=True):
new_text = ""
last_end = 0
+ fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
for match in pattern.finditer(working_text):
+ if any(start <= match.start() < end for start, end in fenced_spans):
+ # inside a code fence: leave it in place. last_end is untouched, so
+ # the block is copied through verbatim.
+ logger.debug("markdown table pass: skipping match inside a code fence at %d", match.start())
+ continue
raw_table = match.group()
table_list.append(raw_table)
if separate_tables:
@@ -75,7 +93,13 @@ class RAGFlowMarkdownParser:
TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"]
table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE)
+ tag_fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
+
def replace_tag(m):
+ if any(start <= m.start() < end for start, end in tag_fenced_spans):
+ # an html example inside a fence keeps its attributes verbatim.
+ logger.debug("html tag pass: preserving tag inside a code fence at %d", m.start())
+ return m.group()
tag_name = re.match(r"<(\w+)", m.group()).group(1)
return "<{}>".format(tag_name)
@@ -107,7 +131,11 @@ class RAGFlowMarkdownParser:
nonlocal working_text
new_text = ""
last_end = 0
+ fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
for match in html_table_pattern.finditer(working_text):
+ if any(start <= match.start() < end for start, end in fenced_spans):
+ logger.debug("html table pass: skipping match inside a code fence at %d", match.start())
+ continue
raw_table = match.group()
tables.append(raw_table)
if separate_tables:
diff --git a/test/unit_test/deepdoc/parser/test_markdown_parser.py b/test/unit_test/deepdoc/parser/test_markdown_parser.py
index a649307e7c..1395a80978 100644
--- a/test/unit_test/deepdoc/parser/test_markdown_parser.py
+++ b/test/unit_test/deepdoc/parser/test_markdown_parser.py
@@ -194,6 +194,55 @@ class TestMarkdownTableDedup:
assert "After" in sections[0]
assert "| Name | Value |" not in sections[0]
+ def test_pipe_table_inside_code_fence_is_not_extracted(self, markdown_parser_module):
+ """A pipe table shown as an example inside a fenced code block must stay put;
+ extracting it would hollow out the fence and emit a bogus table chunk."""
+ text = "# Guide\n\nWrite a table like this:\n\n```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\nDone.\n"
+
+ parser = markdown_parser_module.RAGFlowMarkdownParser()
+ remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+ assert tables == []
+ assert "| Name | Value |" in remainder
+ assert "```markdown" in remainder
+
+ def test_html_table_inside_code_fence_is_left_intact(self, markdown_parser_module):
+ """An HTML example inside a fence must keep its attributes and its body:
+ the tag-stripping and html-table passes have to respect fences too."""
+ text = '# Guide\n\nEmbed a table like this:\n\n```html\n\n```\n\nDone.\n'
+
+ parser = markdown_parser_module.RAGFlowMarkdownParser()
+ remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+ assert tables == []
+ assert 'border="1"' in remainder
+ assert "| A | " in remainder
+
+ def test_bare_html_table_inside_code_fence_is_not_extracted(self, markdown_parser_module):
+ """A bare (no attributes) inside a fence hits the html-table
+ extraction pass (the `` fast-path guard fires); that pass must
+ skip fenced matches too, not just the tag-stripping pass."""
+ text = "# Guide\n\n```html\n\n```\n\nDone.\n"
+
+ parser = markdown_parser_module.RAGFlowMarkdownParser()
+ remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+ assert tables == []
+ assert "" in remainder
+ assert "| A | " in remainder
+
+ def test_real_table_still_extracted_alongside_a_fenced_example(self, markdown_parser_module):
+ """Shielding fences must not stop genuine tables outside them from being split."""
+ text = "```markdown\n| Name | Value |\n| --- | --- |\n| A | 1 |\n```\n\n| X | Y |\n| --- | --- |\n| 9 | 8 |\n\ntail\n"
+
+ parser = markdown_parser_module.RAGFlowMarkdownParser()
+ remainder, tables = parser.extract_tables_and_remainder(text, separate_tables=True)
+
+ assert len(tables) == 1
+ assert "| X | Y |" in tables[0]
+ assert "| X | Y |" not in remainder
+ assert "| Name | Value |" in remainder
+
class TestMarkdownElementExtractorDelimiterHeaders:
def test_custom_delimiter_merges_consecutive_lone_headers_with_body(self, markdown_element_extractor):