zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-24 09:26:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(deepdoc): keep pipe tables inside markdown code fences (#17201)

Browse Source
This commit is contained in:
Yurii214
2026-07-23 07:31:35 +02:00
committed by GitHub
parent 8af2467112
commit 64be8585aa
2 changed files with 77 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
deepdoc/parser/markdown_parser.py
View File

@@ -20,6 +20,8 @@ import re
from markdown import markdown
logger = logging.getLogger(__name__)
class RAGFlowMarkdownParser:
def __init__(self, chunk_token_num=128):
@@ -29,10 +31,26 @@ class RAGFlowMarkdownParser:
tables = []
working_text = markdown_text
# a fenced code block can legitimately contain a pipe table, e.g. docs that show
# markdown syntax. extracting it would strip the example out of the fence and
# re-emit it as a bogus table, leaving a hollow fence behind.
# MarkdownElementExtractor already shields fences (see _fenced_code_ranges), so
# keep table extraction consistent with it.
fence_pattern = re.compile(
r"^[ \t]{0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^[ \t]{0,3}\1[ \t]*$|\Z)",
re.MULTILINE | re.DOTALL,
)
def replace_tables_with_rendered_html(pattern, table_list, render=True):
new_text = ""
last_end = 0
fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
for match in pattern.finditer(working_text):
if any(start <= match.start() < end for start, end in fenced_spans):
# inside a code fence: leave it in place. last_end is untouched, so
# the block is copied through verbatim.
logger.debug("markdown table pass: skipping match inside a code fence at %d", match.start())
continue
raw_table = match.group()
table_list.append(raw_table)
if separate_tables:
@@ -75,7 +93,13 @@ class RAGFlowMarkdownParser:
TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"]
table_with_attributes_pattern = re.compile(rf"<(?:{'|'.join(TAGS)})[^>]*>", re.IGNORECASE)
tag_fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
def replace_tag(m):
if any(start <= m.start() < end for start, end in tag_fenced_spans):
# an html example inside a fence keeps its attributes verbatim.
logger.debug("html tag pass: preserving tag inside a code fence at %d", m.start())
return m.group()
tag_name = re.match(r"<(\w+)", m.group()).group(1)
return "<{}>".format(tag_name)
@@ -107,7 +131,11 @@ class RAGFlowMarkdownParser:
nonlocal working_text
new_text = ""
last_end = 0
fenced_spans = [(m.start(), m.end()) for m in fence_pattern.finditer(working_text)]
for match in html_table_pattern.finditer(working_text):
if any(start <= match.start() < end for start, end in fenced_spans):
logger.debug("html table pass: skipping match inside a code fence at %d", match.start())
continue
raw_table = match.group()
tables.append(raw_table)
if separate_tables: