diff --git a/deepdoc/parser/markdown_parser.py b/deepdoc/parser/markdown_parser.py index 7e8bf32e6d..b09e5adb13 100644 --- a/deepdoc/parser/markdown_parser.py +++ b/deepdoc/parser/markdown_parser.py @@ -55,11 +55,11 @@ class RAGFlowMarkdownParser: table_list.append(raw_table) if separate_tables: # Skip this match (i.e., remove it) - new_text += working_text[last_end : match.start()] + "\n\n" + new_text += working_text[last_end : match.start()] + "\n" else: # Replace with rendered HTML html_table = markdown(raw_table, extensions=["markdown.extensions.tables"]) if render else raw_table - new_text += working_text[last_end : match.start()] + html_table + "\n\n" + new_text += working_text[last_end : match.start()] + html_table + "\n" last_end = match.end() new_text += working_text[last_end:] return new_text @@ -75,7 +75,7 @@ class RAGFlowMarkdownParser: """, re.VERBOSE, ) - working_text = replace_tables_with_rendered_html(border_table_pattern, tables, render=separate_tables) + working_text = replace_tables_with_rendered_html(border_table_pattern, tables) # Borderless Markdown table no_border_table_pattern = re.compile( @@ -87,7 +87,7 @@ class RAGFlowMarkdownParser: """, re.VERBOSE, ) - working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables, render=separate_tables) + working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables) # Replace any TAGS e.g. to
TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"] @@ -139,9 +139,9 @@ class RAGFlowMarkdownParser: raw_table = match.group() tables.append(raw_table) if separate_tables: - new_text += working_text[last_end : match.start()] + "\n\n" + new_text += working_text[last_end : match.start()] + "\n" else: - new_text += working_text[last_end : match.start()] + raw_table + "\n\n" + new_text += working_text[last_end : match.start()] + raw_table + "\n" last_end = match.end() new_text += working_text[last_end:] working_text = new_text diff --git a/rag/app/naive.py b/rag/app/naive.py index 0ae8cc801f..c94cd8aa05 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -899,7 +899,7 @@ class Markdown(MarkdownParser): txt = f.read() remainder, tables = self.extract_tables_and_remainder(f"{txt}\n", separate_tables=separate_tables) - parsing_text = remainder if separate_tables else txt + parsing_text = remainder extractor = MarkdownElementExtractor(parsing_text) image_refs = self.extract_image_urls_with_lines(parsing_text) element_sections = extractor.extract_elements(delimiter, include_meta=True) @@ -922,8 +922,9 @@ class Markdown(MarkdownParser): section_images.append(combined_image) tbls = [] - for table in tables: - tbls.append(((None, markdown(table, extensions=["markdown.extensions.tables"])), "")) + if separate_tables: + for table in tables: + tbls.append(((None, markdown(table, extensions=["markdown.extensions.tables"])), "")) if return_section_images: return sections, tbls, section_images return sections, tbls @@ -1128,7 +1129,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= sections, tables, section_images = markdown_parser( filename, binary, - separate_tables=True, + separate_tables=False, delimiter=parser_config.get("delimiter", "\n!?;。;!?"), return_section_images=True, ) diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py index 0aabc89f4f..7306232575 100644 --- a/rag/flow/parser/parser.py +++ b/rag/flow/parser/parser.py @@ -1074,7 +1074,7 @@ class Parser(ProcessBase): sections, tables, section_images = markdown_parser( name, blob, - separate_tables=True, + separate_tables=False, delimiter=conf.get("delimiter"), return_section_images=True, )