mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-06-29 15:31:05 +08:00
Fix: markdown table double extraction in parser (#13892)
### What problem does this PR solve? Fixes markdown tables being parsed twice (once as markdown and again as generated HTML), which caused duplicate table chunks in the chunk list UI. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
@@ -56,7 +56,7 @@ class RAGFlowMarkdownParser:
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
working_text = replace_tables_with_rendered_html(border_table_pattern, tables)
|
||||
working_text = replace_tables_with_rendered_html(border_table_pattern, tables, render=separate_tables)
|
||||
|
||||
# Borderless Markdown table
|
||||
no_border_table_pattern = re.compile(
|
||||
@@ -68,7 +68,7 @@ class RAGFlowMarkdownParser:
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables)
|
||||
working_text = replace_tables_with_rendered_html(no_border_table_pattern, tables, render=separate_tables)
|
||||
|
||||
# Replace any TAGS e.g. <table ...> to <table>
|
||||
TAGS = ["table", "td", "tr", "th", "tbody", "thead", "div"]
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#
|
||||
|
||||
|
||||
from time import sleep
|
||||
|
||||
import pytest
|
||||
from common import batch_add_chunks, delete_all_chunks
|
||||
@@ -32,6 +31,10 @@ def condition(_dataset: DataSet):
|
||||
return False
|
||||
return True
|
||||
|
||||
@wait_for(30, 1, "Chunk indexing timeout")
|
||||
def chunks_visible(_document: Document, _chunk_ids: list[str]):
|
||||
visible_ids = {chunk.id for chunk in _document.list_chunks(page_size=1000)}
|
||||
return set(_chunk_ids).issubset(visible_ids)
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def add_chunks_func(request: FixtureRequest, add_document: tuple[DataSet, Document]) -> tuple[DataSet, Document, list[Chunk]]:
|
||||
@@ -47,6 +50,5 @@ def add_chunks_func(request: FixtureRequest, add_document: tuple[DataSet, Docume
|
||||
dataset.async_parse_documents([document.id])
|
||||
condition(dataset)
|
||||
chunks = batch_add_chunks(document, 4)
|
||||
# issues/6487
|
||||
sleep(1)
|
||||
chunks_visible(document, [chunk.id for chunk in chunks])
|
||||
return dataset, document, chunks
|
||||
|
||||
Reference in New Issue
Block a user