mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-23 00:46:42 +08:00
Feature/table parser column roles (#13710)
### What problem does this PR solve? The table file parser (CSV/Excel) currently treats all columns identically — every column is both vectorized (embedded in chunk text) and stored as filterable metadata. There's no way for users to control which columns should be searchable by semantic meaning versus which should only be filterable attributes. For example, when ingesting a news articles CSV with columns like title, content, country, category, source, etc., the embedding includes metadata fields like country: Brazil and source: Reuters in the chunk text, which dilutes the semantic quality of the embedding without adding retrieval value. The RDBMS connector (MySQL/PostgreSQL) already supports content_columns / metadata_columns, but this capability was missing for file-based table ingestion. This PR adds column-level control (vectorize / metadata / both) for the table file parser, following RAGFlow's existing patterns. Backward compatible: Datasets without table_column_roles or with table_column_mode: auto behave exactly as before (all columns = both). ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -18,14 +18,15 @@
|
||||
|
||||
from unittest.mock import Mock
|
||||
from api.utils.validation_utils import (
|
||||
validate_immutable_fields,
|
||||
ParserConfig,
|
||||
UpdateDocumentReq,
|
||||
validate_chunk_method,
|
||||
validate_document_name,
|
||||
validate_chunk_method
|
||||
validate_immutable_fields,
|
||||
)
|
||||
from api.constants import FILE_NAME_LEN_LIMIT
|
||||
from api.db import FileType
|
||||
from common.constants import RetCode
|
||||
from api.utils.validation_utils import UpdateDocumentReq
|
||||
|
||||
|
||||
def test_validate_immutable_fields_no_changes():
|
||||
@@ -299,4 +300,15 @@ def test_validate_chunk_method_other_extensions_still_valid():
|
||||
|
||||
error_msg, error_code = validate_chunk_method(doc)
|
||||
assert error_msg is None
|
||||
assert error_code is None
|
||||
assert error_code is None
|
||||
|
||||
|
||||
def test_parser_config_normalizes_legacy_vectorize_table_column_role():
|
||||
p = ParserConfig(
|
||||
table_column_roles={"title": "vectorize", "country": "metadata", "x": "both"},
|
||||
)
|
||||
assert p.table_column_roles == {
|
||||
"title": "indexing",
|
||||
"country": "metadata",
|
||||
"x": "both",
|
||||
}
|
||||
Reference in New Issue
Block a user