Feature/table parser column roles (#13710)

### What problem does this PR solve?

The table file parser (CSV/Excel) currently treats all columns
identically — every column is both vectorized (embedded in chunk text)
and stored as filterable metadata. There's no way for users to control
which columns should be searchable by semantic meaning versus which
should only be filterable attributes.

For example, when ingesting a news articles CSV with columns like title,
content, country, category, source, etc., the embedding includes
metadata fields like country: Brazil and source: Reuters in the chunk
text, which dilutes the semantic quality of the embedding without adding
retrieval value.

The RDBMS connector (MySQL/PostgreSQL) already supports content_columns
/ metadata_columns, but this capability was missing for file-based table
ingestion.

This PR adds column-level control (vectorize / metadata / both) for the
table file parser, following RAGFlow's existing patterns.

Backward compatible: Datasets without table_column_roles or with
table_column_mode: auto behave exactly as before (all columns = both).

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Ahmad Intisar
2026-05-11 07:06:04 +05:00
committed by GitHub
parent 889aba6a32
commit 3c4d1da98f
13 changed files with 1270 additions and 36 deletions

View File

@@ -713,6 +713,21 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
portugueseBr: 'Portuguese (Brazil)',
embeddingModelPlaceholder: 'Please select a embedding model.',
chunkMethodPlaceholder: 'Please select a chunking method.',
tableColumnMode: 'Column mode',
tableColumnModeAuto: 'Auto',
tableColumnModeManual: 'Manual',
tableColumnModeAutoDescription:
'All columns are included in chunk text and stored as metadata (RAGFlow default).',
tableColumnRoles: 'Column roles',
tableColumnRolesTip:
'Choose which columns to include in chunk text (indexed for vector and full-text search), in metadata only (filterable), or both. Changes apply to new parses; re-parse existing documents for roles to take effect.',
tableColumnRoleIndexing: 'Indexing',
tableColumnRoleMetadata: 'Metadata',
tableColumnRoleBoth: 'Both',
tableColumnRolesEmpty:
'Upload and parse a CSV or Excel file to begin configuring column roles.',
tableColumnRolesReparseTip:
'Re-parse existing documents for the new column roles to take effect.',
parserLabel: {
naive: 'General',
qa: 'Q&A',

View File

@@ -1,12 +1,155 @@
import { FormControl, FormItem, FormLabel } from '@/components/ui/form';
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { useTranslate } from '@/hooks/common-hooks';
import { useFormContext, useWatch } from 'react-hook-form';
import { ConfigurationFormContainer } from '../configuration-form-container';
const ROLE_OPTIONS = [
{ value: 'both', labelKey: 'tableColumnRoleBoth' },
{ value: 'indexing', labelKey: 'tableColumnRoleIndexing' },
{ value: 'metadata', labelKey: 'tableColumnRoleMetadata' },
] as const;
function selectTableColumnRoleValue(raw: string | undefined): string {
if (!raw) return 'both';
return raw === 'vectorize' ? 'indexing' : raw;
}
export function TableConfiguration() {
const form = useFormContext();
const { t } = useTranslate('knowledgeConfiguration');
const tableColumnMode = useWatch({
control: form.control,
name: 'parser_config.table_column_mode',
defaultValue: 'auto',
});
const tableColumnNames = useWatch({
control: form.control,
name: 'parser_config.table_column_names',
defaultValue: [],
});
const tableColumnRoles = useWatch({
control: form.control,
name: 'parser_config.table_column_roles',
defaultValue: {},
});
const mode = tableColumnMode === 'manual' ? 'manual' : 'auto';
const columns: string[] = Array.isArray(tableColumnNames)
? tableColumnNames
: [];
const handleModeChange = (value: string) => {
form.setValue(
'parser_config.table_column_mode',
value as 'auto' | 'manual',
);
};
const handleRoleChange = (columnName: string, role: string) => {
const current =
(form.getValues('parser_config.table_column_roles') as Record<
string,
string
>) || {};
form.setValue('parser_config.table_column_roles', {
...current,
[columnName]: role,
});
};
return (
<ConfigurationFormContainer>
{/* <ChunkMethodItem></ChunkMethodItem>
<EmbeddingModelItem></EmbeddingModelItem>
<FormItem className="space-y-2">
<FormLabel className="text-sm font-medium">
{t('tableColumnMode')}
</FormLabel>
<FormControl>
<RadioGroup
value={mode}
onValueChange={handleModeChange}
className="flex gap-4"
>
<div className="flex items-center space-x-2">
<RadioGroupItem value="auto" id="table-mode-auto" />
<label
htmlFor="table-mode-auto"
className="text-sm font-normal cursor-pointer"
>
{t('tableColumnModeAuto')}
</label>
</div>
<div className="flex items-center space-x-2">
<RadioGroupItem value="manual" id="table-mode-manual" />
<label
htmlFor="table-mode-manual"
className="text-sm font-normal cursor-pointer"
>
{t('tableColumnModeManual')}
</label>
</div>
</RadioGroup>
</FormControl>
</FormItem>
<PageRankFormField></PageRankFormField> */}
{mode === 'auto' && (
<p className="text-sm text-muted-foreground">
{t('tableColumnModeAutoDescription')}
</p>
)}
{mode === 'manual' && columns.length === 0 && (
<p className="text-sm text-muted-foreground">
{t('tableColumnRolesEmpty')}
</p>
)}
{mode === 'manual' && columns.length > 0 && (
<>
<p className="text-sm text-muted-foreground mb-3">
{t('tableColumnRolesTip')}
</p>
<div className="space-y-3">
{columns.map((col) => (
<FormItem key={col} className="flex flex-row items-center gap-4">
<FormLabel className="min-w-[120px] shrink-0 text-sm font-normal">
{col}
</FormLabel>
<FormControl>
<Select
value={selectTableColumnRoleValue(
tableColumnRoles && tableColumnRoles[col],
)}
onValueChange={(value) => handleRoleChange(col, value)}
>
<SelectTrigger className="w-[160px]">
<SelectValue />
</SelectTrigger>
<SelectContent>
{ROLE_OPTIONS.map((opt) => (
<SelectItem key={opt.value} value={opt.value}>
{t(opt.labelKey)}
</SelectItem>
))}
</SelectContent>
</Select>
</FormControl>
</FormItem>
))}
</div>
<p className="text-xs text-muted-foreground mt-3">
{t('tableColumnRolesReparseTip')}
</p>
</>
)}
</ConfigurationFormContainer>
);
}

View File

@@ -94,6 +94,18 @@ export const formSchema = z
.optional(),
enable_metadata: z.boolean().optional(),
llm_id: z.string().optional(),
// Table parser: "auto" = all columns both, "manual" = use column role selector
table_column_mode: z.enum(['auto', 'manual']).optional(),
// Table parser: column name -> role (indexing | metadata | both); legacy "vectorize" -> indexing
table_column_roles: z
.record(
z
.enum(['indexing', 'metadata', 'both', 'vectorize'])
.transform((role) => (role === 'vectorize' ? 'indexing' : role)),
)
.optional(),
// Table parser: column names list (set by backend after first parse)
table_column_names: z.array(z.string()).optional(),
})
.optional(),
pagerank: z.number(),