mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-31 13:03:49 +08:00
Feature/table parser column roles (#13710)
### What problem does this PR solve? The table file parser (CSV/Excel) currently treats all columns identically — every column is both vectorized (embedded in chunk text) and stored as filterable metadata. There's no way for users to control which columns should be searchable by semantic meaning versus which should only be filterable attributes. For example, when ingesting a news articles CSV with columns like title, content, country, category, source, etc., the embedding includes metadata fields like country: Brazil and source: Reuters in the chunk text, which dilutes the semantic quality of the embedding without adding retrieval value. The RDBMS connector (MySQL/PostgreSQL) already supports content_columns / metadata_columns, but this capability was missing for file-based table ingestion. This PR adds column-level control (vectorize / metadata / both) for the table file parser, following RAGFlow's existing patterns. Backward compatible: Datasets without table_column_roles or with table_column_mode: auto behave exactly as before (all columns = both). ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -713,6 +713,21 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
|
||||
portugueseBr: 'Portuguese (Brazil)',
|
||||
embeddingModelPlaceholder: 'Please select a embedding model.',
|
||||
chunkMethodPlaceholder: 'Please select a chunking method.',
|
||||
tableColumnMode: 'Column mode',
|
||||
tableColumnModeAuto: 'Auto',
|
||||
tableColumnModeManual: 'Manual',
|
||||
tableColumnModeAutoDescription:
|
||||
'All columns are included in chunk text and stored as metadata (RAGFlow default).',
|
||||
tableColumnRoles: 'Column roles',
|
||||
tableColumnRolesTip:
|
||||
'Choose which columns to include in chunk text (indexed for vector and full-text search), in metadata only (filterable), or both. Changes apply to new parses; re-parse existing documents for roles to take effect.',
|
||||
tableColumnRoleIndexing: 'Indexing',
|
||||
tableColumnRoleMetadata: 'Metadata',
|
||||
tableColumnRoleBoth: 'Both',
|
||||
tableColumnRolesEmpty:
|
||||
'Upload and parse a CSV or Excel file to begin configuring column roles.',
|
||||
tableColumnRolesReparseTip:
|
||||
'Re-parse existing documents for the new column roles to take effect.',
|
||||
parserLabel: {
|
||||
naive: 'General',
|
||||
qa: 'Q&A',
|
||||
|
||||
@@ -1,12 +1,155 @@
|
||||
import { FormControl, FormItem, FormLabel } from '@/components/ui/form';
|
||||
import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { useTranslate } from '@/hooks/common-hooks';
|
||||
import { useFormContext, useWatch } from 'react-hook-form';
|
||||
import { ConfigurationFormContainer } from '../configuration-form-container';
|
||||
|
||||
const ROLE_OPTIONS = [
|
||||
{ value: 'both', labelKey: 'tableColumnRoleBoth' },
|
||||
{ value: 'indexing', labelKey: 'tableColumnRoleIndexing' },
|
||||
{ value: 'metadata', labelKey: 'tableColumnRoleMetadata' },
|
||||
] as const;
|
||||
|
||||
function selectTableColumnRoleValue(raw: string | undefined): string {
|
||||
if (!raw) return 'both';
|
||||
return raw === 'vectorize' ? 'indexing' : raw;
|
||||
}
|
||||
|
||||
export function TableConfiguration() {
|
||||
const form = useFormContext();
|
||||
const { t } = useTranslate('knowledgeConfiguration');
|
||||
|
||||
const tableColumnMode = useWatch({
|
||||
control: form.control,
|
||||
name: 'parser_config.table_column_mode',
|
||||
defaultValue: 'auto',
|
||||
});
|
||||
const tableColumnNames = useWatch({
|
||||
control: form.control,
|
||||
name: 'parser_config.table_column_names',
|
||||
defaultValue: [],
|
||||
});
|
||||
const tableColumnRoles = useWatch({
|
||||
control: form.control,
|
||||
name: 'parser_config.table_column_roles',
|
||||
defaultValue: {},
|
||||
});
|
||||
|
||||
const mode = tableColumnMode === 'manual' ? 'manual' : 'auto';
|
||||
const columns: string[] = Array.isArray(tableColumnNames)
|
||||
? tableColumnNames
|
||||
: [];
|
||||
|
||||
const handleModeChange = (value: string) => {
|
||||
form.setValue(
|
||||
'parser_config.table_column_mode',
|
||||
value as 'auto' | 'manual',
|
||||
);
|
||||
};
|
||||
|
||||
const handleRoleChange = (columnName: string, role: string) => {
|
||||
const current =
|
||||
(form.getValues('parser_config.table_column_roles') as Record<
|
||||
string,
|
||||
string
|
||||
>) || {};
|
||||
form.setValue('parser_config.table_column_roles', {
|
||||
...current,
|
||||
[columnName]: role,
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<ConfigurationFormContainer>
|
||||
{/* <ChunkMethodItem></ChunkMethodItem>
|
||||
<EmbeddingModelItem></EmbeddingModelItem>
|
||||
<FormItem className="space-y-2">
|
||||
<FormLabel className="text-sm font-medium">
|
||||
{t('tableColumnMode')}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<RadioGroup
|
||||
value={mode}
|
||||
onValueChange={handleModeChange}
|
||||
className="flex gap-4"
|
||||
>
|
||||
<div className="flex items-center space-x-2">
|
||||
<RadioGroupItem value="auto" id="table-mode-auto" />
|
||||
<label
|
||||
htmlFor="table-mode-auto"
|
||||
className="text-sm font-normal cursor-pointer"
|
||||
>
|
||||
{t('tableColumnModeAuto')}
|
||||
</label>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<RadioGroupItem value="manual" id="table-mode-manual" />
|
||||
<label
|
||||
htmlFor="table-mode-manual"
|
||||
className="text-sm font-normal cursor-pointer"
|
||||
>
|
||||
{t('tableColumnModeManual')}
|
||||
</label>
|
||||
</div>
|
||||
</RadioGroup>
|
||||
</FormControl>
|
||||
</FormItem>
|
||||
|
||||
<PageRankFormField></PageRankFormField> */}
|
||||
{mode === 'auto' && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{t('tableColumnModeAutoDescription')}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{mode === 'manual' && columns.length === 0 && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{t('tableColumnRolesEmpty')}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{mode === 'manual' && columns.length > 0 && (
|
||||
<>
|
||||
<p className="text-sm text-muted-foreground mb-3">
|
||||
{t('tableColumnRolesTip')}
|
||||
</p>
|
||||
<div className="space-y-3">
|
||||
{columns.map((col) => (
|
||||
<FormItem key={col} className="flex flex-row items-center gap-4">
|
||||
<FormLabel className="min-w-[120px] shrink-0 text-sm font-normal">
|
||||
{col}
|
||||
</FormLabel>
|
||||
<FormControl>
|
||||
<Select
|
||||
value={selectTableColumnRoleValue(
|
||||
tableColumnRoles && tableColumnRoles[col],
|
||||
)}
|
||||
onValueChange={(value) => handleRoleChange(col, value)}
|
||||
>
|
||||
<SelectTrigger className="w-[160px]">
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{ROLE_OPTIONS.map((opt) => (
|
||||
<SelectItem key={opt.value} value={opt.value}>
|
||||
{t(opt.labelKey)}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</FormControl>
|
||||
</FormItem>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground mt-3">
|
||||
{t('tableColumnRolesReparseTip')}
|
||||
</p>
|
||||
</>
|
||||
)}
|
||||
</ConfigurationFormContainer>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -94,6 +94,18 @@ export const formSchema = z
|
||||
.optional(),
|
||||
enable_metadata: z.boolean().optional(),
|
||||
llm_id: z.string().optional(),
|
||||
// Table parser: "auto" = all columns both, "manual" = use column role selector
|
||||
table_column_mode: z.enum(['auto', 'manual']).optional(),
|
||||
// Table parser: column name -> role (indexing | metadata | both); legacy "vectorize" -> indexing
|
||||
table_column_roles: z
|
||||
.record(
|
||||
z
|
||||
.enum(['indexing', 'metadata', 'both', 'vectorize'])
|
||||
.transform((role) => (role === 'vectorize' ? 'indexing' : role)),
|
||||
)
|
||||
.optional(),
|
||||
// Table parser: column names list (set by backend after first parse)
|
||||
table_column_names: z.array(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
pagerank: z.number(),
|
||||
|
||||
Reference in New Issue
Block a user