feat(ingestion,web): modularize extractor configuration with sub-tabs, independent prompts, and metadata integration (#18383)

This PR modularizes the **Extractor** component configuration with dedicated feature subtabs, adds independent system prompt configuration, fixes multi-node execution determinism and parameter persistence across save and page refresh, and ensures backward compatibility with legacy flat fields.
This commit is contained in:
jay77721
2026-08-18 11:38:52 +08:00
committed by GitHub
parent 602f55deed
commit c71991bb7a
18 changed files with 1445 additions and 525 deletions

View File

@@ -3332,8 +3332,11 @@ The Indexer will store the content in the corresponding data structures for the
summary: 'Summary',
keywords: 'Keywords',
questions: 'Questions',
tags: 'Tags',
metadata: 'Metadata',
fieldName: 'Result destination',
enableSummary: 'Enable Summary',
useBuiltInTemplate: 'Use built-in template',
prompts: {
system: {
keywords: `Role

View File

@@ -2914,8 +2914,11 @@ Tokenizer 会根据所选方式将内容存储为对应的数据结构。`,
summary: '增强上下文',
keywords: '关键词',
questions: '问题',
tags: '标签',
metadata: '元数据',
fieldName: '结果目的地',
enableSummary: '启用增强上下文',
useBuiltInTemplate: '使用内置模板',
prompts: {
system: {
keywords: `角色

View File

@@ -92,13 +92,6 @@ export const InitialOutputFormatMap = {
[FileType.Audio]: AudioOutputFormat.Text,
};
export enum ContextGeneratorFieldName {
Summary = 'summary',
Keywords = 'keywords',
Questions = 'questions',
Metadata = 'metadata',
}
export const FileId = 'File'; // BeginId
export enum TokenizerSearchMethod {
@@ -350,11 +343,39 @@ export const initialTitleChunkerValues = {
export const initialExtractorValues = {
...initialLlmBaseValues,
field_name: ContextGeneratorFieldName.Summary,
keywords: {
top_n: 0,
system_prompt: '',
},
questions: {
top_n: 0,
system_prompt: '',
},
tags: {
top_n: 0,
tag_file_id: '',
},
summary: {
enabled: false,
system_prompt: '',
},
metadata_config: {
enabled: false,
metadata: [],
built_in_metadata: [],
},
metadata: [],
built_in_metadata: [],
field_name: '',
auto_keywords: 0,
auto_questions: 0,
auto_tags: 1,
auto_tags: 0,
tag_file_id: '',
enable_summary: 0,
enable_metadata: 0,
keywords_sys_prompt: '',
questions_sys_prompt: '',
sys_prompt: '',
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},

View File

@@ -62,6 +62,7 @@ const Nodes: Array<Klass<LexicalNode>> = [
type PromptContentProps = {
enablePathQueryAutoMerge: boolean;
showToolbar?: boolean;
showMergePath?: boolean;
multiLine?: boolean;
onBlur?: () => void;
onEnablePathQueryAutoMergeChange: (checked: boolean) => void;
@@ -70,6 +71,7 @@ type PromptContentProps = {
type IProps = {
enablePathQueryAutoMerge?: boolean;
showToolbar?: boolean;
showMergePath?: boolean;
multiLine?: boolean;
value?: string;
onChange?: (value?: string) => void;
@@ -81,6 +83,7 @@ type IProps = {
function PromptContent({
enablePathQueryAutoMerge,
showToolbar = true,
showMergePath = true,
multiLine = true,
onBlur,
onEnablePathQueryAutoMergeChange,
@@ -128,28 +131,30 @@ function PromptContent({
<p>{t('flow.insertVariableTip')}</p>
</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">
<span className="sr-only">{t('flow.mergePath')}</span>
<div className="origin-right scale-75">
<Switch
checked={enablePathQueryAutoMerge}
onCheckedChange={onEnablePathQueryAutoMergeChange}
aria-label={t('flow.mergePath')}
/>
</div>
</label>
</TooltipTrigger>
<TooltipContent>
<p>{t('flow.mergePath')}</p>
<p>{t('flow.mergePathTip')}</p>
</TooltipContent>
</Tooltip>
{showMergePath && (
<Tooltip>
<TooltipTrigger asChild>
<label className="flex cursor-pointer items-center rounded-sm border border-border bg-bg-base/95 px-1 py-0.5 shadow-sm backdrop-blur-sm">
<span className="sr-only">{t('flow.mergePath')}</span>
<div className="origin-right scale-75">
<Switch
checked={enablePathQueryAutoMerge}
onCheckedChange={onEnablePathQueryAutoMergeChange}
aria-label={t('flow.mergePath')}
/>
</div>
</label>
</TooltipTrigger>
<TooltipContent>
<p>{t('flow.mergePath')}</p>
<p>{t('flow.mergePathTip')}</p>
</TooltipContent>
</Tooltip>
)}
</div>
)}
<div className="relative">
{!showToolbar && (
{!showToolbar && showMergePath && (
<div className="absolute inset-y-0 right-2 z-10 flex items-center">
<Tooltip>
<TooltipTrigger asChild>
@@ -173,8 +178,9 @@ function PromptContent({
)}
<ContentEditable
className={cn(
'relative px-2 py-1 pr-14 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',
'relative px-2 py-1 focus-visible:outline-none max-h-[50vh] overflow-auto text-sm',
{
'pr-14': !showToolbar && showMergePath,
'min-h-40': multiLine,
},
)}
@@ -193,6 +199,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
onBlur,
placeholder,
showToolbar = true,
showMergePath = true,
multiLine = true,
enablePathQueryAutoMerge = true,
extraOptions,
@@ -222,11 +229,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
const onValueChange = useCallback(
(editorState: EditorState) => {
editorState?.read(() => {
// const listNodes = $nodesOfType(VariableNode); // to be removed
// const allNodes = $dfs();
const text = $getRoot().getTextContent();
onChange?.(text);
});
},
@@ -241,6 +244,7 @@ export const PromptEditor = forwardRef(function PromptEditor(
<PromptContent
enablePathQueryAutoMerge={isPathQueryAutoMergeEnabled}
showToolbar={showToolbar}
showMergePath={showMergePath}
multiLine={multiLine}
onBlur={onBlur}
onEnablePathQueryAutoMergeChange={setIsPathQueryAutoMergeEnabled}

View File

@@ -1,17 +1,34 @@
/*
* Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
AutoKeywordsFormField,
AutoQuestionsFormField,
} from '@/components/auto-keywords-form-field';
import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog';
import { LargeModelFormField } from '@/components/large-model-form-field';
import { LlmSettingSchema } from '@/components/llm-setting-items/next';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { SliderInputFormField } from '@/components/slider-input-form-field';
import { AsyncTreeSelect } from '@/components/ui/async-tree-select';
import { Form } from '@/components/ui/form';
import { Button } from '@/components/ui/button';
import { Form } from '@/components/ui/form';
import { Switch } from '@/components/ui/switch';
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
import { FormLayout } from '@/constants/form';
import { RAGFlowNodeType } from '@/interfaces/database/agent';
import { PromptEditor } from '@/pages/agent/form/components/prompt-editor';
import { MetadataType } from '@/pages/dataset/components/metedata/constant';
import {
@@ -23,42 +40,67 @@ import {
IMetaDataReturnJSONSettings,
} from '@/pages/dataset/components/metedata/interface';
import { ManageMetadataModal } from '@/pages/dataset/components/metedata/manage-modal';
import { isGoBackend } from '@/utils/backend-runtime';
import { buildOptions } from '@/utils/form';
import { zodResolver } from '@hookform/resolvers/zod';
import { Settings } from 'lucide-react';
import { memo, useCallback } from 'react';
import { memo, useCallback, useEffect, useMemo, useState } from 'react';
import { useForm, useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import {
ContextGeneratorFieldName,
initialExtractorValues,
} from '../../constant/pipeline';
import { initialExtractorValues } from '../../constant/pipeline';
import { useOwnerTenantId } from '../../context';
import { useBuildNodeOutputOptions } from '../../hooks/use-build-options';
import { useFormChangeCallback } from '../../hooks/use-form-change-callback';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
import { INextOperatorForm } from '../../interface';
import { buildOutputList } from '../../utils/build-output-list';
import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';
import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
import { useSwitchPrompt } from './use-switch-prompt';
import { canSelectTagFile, useTagFileTree } from './use-tag-file-tree';
import { FormLayout } from '@/constants/form';
export const FormSchema = z.object({
field_name: z.string(),
sys_prompt: z.string(),
keywords: z
.object({
top_n: z.number().optional(),
system_prompt: z.string().optional(),
})
.optional(),
questions: z
.object({
top_n: z.number().optional(),
system_prompt: z.string().optional(),
})
.optional(),
tags: z
.object({
top_n: z.number().optional(),
tag_file_id: z.string().optional(),
})
.optional(),
summary: z
.object({
enabled: z.union([z.number(), z.boolean()]).optional(),
system_prompt: z.string().optional(),
})
.optional(),
metadata_config: z
.object({
enabled: z.union([z.number(), z.boolean()]).optional(),
metadata: z.any().optional(),
built_in_metadata: z.any().optional(),
})
.optional(),
// Legacy flat fields for backward compatibility
field_name: z.string().optional(),
sys_prompt: z.string().optional(),
prompts: z.string().optional(),
keywords_sys_prompt: z.string().optional(),
questions_sys_prompt: z.string().optional(),
auto_keywords: z.number().optional(),
auto_questions: z.number().optional(),
auto_tags: z.number().optional(),
tag_file_id: z.string().optional(),
// Builtin auto-metadata (mirrors Python's Auto metadata): enable_metadata
// toggle + metadata / built_in_metadata field schema, consumed by the Go
// extractor's runEnableMetadata at parse time.
enable_summary: z.union([z.number(), z.boolean()]).optional(),
enable_metadata: z.number().optional(),
metadata: z.any().optional(),
built_in_metadata: z.any().optional(),
@@ -67,11 +109,13 @@ export const FormSchema = z.object({
export type ExtractorFormSchemaType = z.infer<typeof FormSchema>;
// Builtin auto-extract node id hardcoded in every builtin ingestion DSL
// template (internal/ingestion/pipeline/template/*.json). Only this node
// shows the Auto metadata option; canvas / user-pipeline extractor nodes do
// not.
const BuiltinAutoExtractNodeId = 'Extractor:AutoExtractDefault';
enum ExtractorSubTab {
Keywords = 'keywords',
Questions = 'questions',
Tags = 'tags',
Summary = 'summary',
Metadata = 'metadata',
}
// ExtractorAutoMetadata mirrors Python's dataset "Auto metadata" control: an
// enable_metadata switch plus a field-schema editor (custom + built-in).
@@ -91,11 +135,16 @@ function ExtractorAutoMetadata() {
const handleOpen = useCallback(() => {
showManageMetadataModal({
metadata: util.metaDataSettingJSONToMetaDataTableData(
form.getValues('metadata') || [],
form.getValues('metadata_config.metadata') ||
form.getValues('metadata') ||
[],
),
isCanAdd: true,
type: MetadataType.Setting,
builtInMetadata: form.getValues('built_in_metadata') || [],
builtInMetadata:
form.getValues('metadata_config.built_in_metadata') ||
form.getValues('built_in_metadata') ||
[],
});
}, [form, showManageMetadataModal]);
@@ -104,9 +153,19 @@ function ExtractorAutoMetadata() {
metadata?: IMetaDataReturnJSONSettings;
builtInMetadata?: IBuiltInMetadataItem[];
}) => {
form.setValue('metadata', data?.metadata || []);
form.setValue('built_in_metadata', data?.builtInMetadata || []);
form.setValue('enable_metadata', 1);
const metaList = data?.metadata || [];
const builtInList = data?.builtInMetadata || [];
form.setValue('metadata_config.metadata', metaList, {
shouldDirty: true,
});
form.setValue('metadata_config.built_in_metadata', builtInList, {
shouldDirty: true,
});
form.setValue('metadata_config.enabled', true, { shouldDirty: true });
// Also keep flat fields for backward compatibility
form.setValue('metadata', metaList, { shouldDirty: true });
form.setValue('built_in_metadata', builtInList, { shouldDirty: true });
form.setValue('enable_metadata', 1, { shouldDirty: true });
},
[form],
);
@@ -115,7 +174,7 @@ function ExtractorAutoMetadata() {
<>
<RAGFlowFormItem
label={t('knowledgeConfiguration.autoMetadata')}
name="enable_metadata"
name="metadata_config.enabled"
>
{(field) => (
<div className="flex items-center justify-between">
@@ -132,7 +191,12 @@ function ExtractorAutoMetadata() {
</Button>
<Switch
checked={field.value === 1 || field.value === true}
onCheckedChange={(checked) => field.onChange(checked ? 1 : 0)}
onCheckedChange={(checked) => {
field.onChange(checked);
form.setValue('enable_metadata', checked ? 1 : 0, {
shouldDirty: true,
});
}}
data-testid="extractor-metadata-switch"
/>
</div>
@@ -162,38 +226,67 @@ function ExtractorAutoMetadata() {
const outputList = buildOutputList(initialExtractorValues.outputs);
const useNormalizedExtractorFormValues = (node?: RAGFlowNodeType) => {
return useMemo(() => {
const raw = (node?.data?.form as Record<string, any>) || {};
return {
...initialExtractorValues,
...transformExtractorConfigToForm(raw),
};
}, [node?.data?.form]);
};
const ExtractorForm = ({
node,
onValuesChange,
hideOutputs,
}: INextOperatorForm) => {
const defaultValues = useFormValues(initialExtractorValues, node);
const defaultValues = useNormalizedExtractorFormValues(node);
const { t } = useTranslation();
const form = useForm<ExtractorFormSchemaType>({
defaultValues,
resolver: zodResolver(FormSchema),
// mode: 'onChange',
});
const promptOptions = useBuildNodeOutputOptions(node?.id);
useEffect(() => {
form.reset(defaultValues);
}, [defaultValues, form]);
const options = buildOptions(ContextGeneratorFieldName, t, 'flow');
const {
handleFieldNameChange,
confirmSwitch,
hideModal,
visible,
cancelSwitch,
} = useSwitchPrompt(form);
const [activeTab, setActiveTab] = useState<ExtractorSubTab>(
ExtractorSubTab.Keywords,
);
useWatchFormChange(node?.id, form);
useFormChangeCallback(form, onValuesChange);
const ownerTenantId = useOwnerTenantId();
const { treeData, loadData } = useTagFileTree(form.watch('tag_file_id'));
const tagFileIdWatch =
form.watch('tags.tag_file_id') || form.watch('tag_file_id');
const { treeData, loadData } = useTagFileTree(tagFileIdWatch);
useEffect(() => {
if (!form.getValues('keywords.system_prompt')) {
form.setValue(
'keywords.system_prompt',
t('flow.prompts.system.keywords'),
);
}
if (!form.getValues('questions.system_prompt')) {
form.setValue(
'questions.system_prompt',
t('flow.prompts.system.questions'),
);
}
if (!form.getValues('summary.system_prompt')) {
form.setValue('summary.system_prompt', t('flow.prompts.system.summary'));
}
}, [form, t]);
const handleTabChange = useCallback((tab: string) => {
setActiveTab(tab as ExtractorSubTab);
}, []);
return (
<Form {...form}>
@@ -201,20 +294,74 @@ const ExtractorForm = ({
<LargeModelFormField
ownerTenantId={ownerTenantId}
></LargeModelFormField>
<AutoKeywordsFormField name="auto_keywords"></AutoKeywordsFormField>
<AutoQuestionsFormField name="auto_questions"></AutoQuestionsFormField>
{isGoBackend() && (
<>
<SliderInputFormField
name="auto_tags"
label={t('knowledgeDetails.autoTags')}
min={1}
max={10}
defaultValue={1}
layout={FormLayout.Vertical}
></SliderInputFormField>
<RAGFlowFormItem label={t('flow.tagFile')} name="tag_file_id">
<Tabs
value={activeTab}
onValueChange={handleTabChange}
className="w-full"
>
<TabsList className="w-full justify-start">
<TabsTrigger value={ExtractorSubTab.Keywords}>
{t('flow.keywords')}
</TabsTrigger>
<TabsTrigger value={ExtractorSubTab.Questions}>
{t('flow.questions')}
</TabsTrigger>
<TabsTrigger value={ExtractorSubTab.Tags}>
{t('flow.tags') || t('knowledgeDetails.autoTags')}
</TabsTrigger>
<TabsTrigger value={ExtractorSubTab.Summary}>
{t('flow.summary')}
</TabsTrigger>
<TabsTrigger value={ExtractorSubTab.Metadata}>
{t('flow.metadata')}
</TabsTrigger>
</TabsList>
<TabsContent
value={ExtractorSubTab.Keywords}
className="space-y-4 pt-2"
>
<AutoKeywordsFormField name="keywords.top_n" />
<RAGFlowFormItem
label={t('flow.systemPrompt')}
name="keywords.system_prompt"
>
<PromptEditor
placeholder={t('flow.messagePlaceholder')}
showToolbar={false}
showMergePath={false}
/>
</RAGFlowFormItem>
</TabsContent>
<TabsContent
value={ExtractorSubTab.Questions}
className="space-y-4 pt-2"
>
<AutoQuestionsFormField name="questions.top_n" />
<RAGFlowFormItem
label={t('flow.systemPrompt')}
name="questions.system_prompt"
>
<PromptEditor
placeholder={t('flow.messagePlaceholder')}
showToolbar={false}
showMergePath={false}
/>
</RAGFlowFormItem>
</TabsContent>
<TabsContent value={ExtractorSubTab.Tags} className="space-y-4 pt-2">
<SliderInputFormField
name="tags.top_n"
label={t('knowledgeDetails.autoTags')}
min={0}
max={10}
defaultValue={0}
layout={FormLayout.Vertical}
/>
<RAGFlowFormItem label={t('flow.tagFile')} name="tags.tag_file_id">
{(field) => (
<AsyncTreeSelect
treeData={treeData}
@@ -222,55 +369,59 @@ const ExtractorForm = ({
onChange={field.onChange}
loadData={loadData}
canSelect={canSelectTagFile}
></AsyncTreeSelect>
/>
)}
</RAGFlowFormItem>
</>
)}
</TabsContent>
<RAGFlowFormItem label={t('flow.fieldName')} name="field_name">
{(field) => (
<SelectWithSearch
onChange={(value) => {
field.onChange(value);
handleFieldNameChange(value);
}}
value={field.value}
placeholder={t('dataFlowPlaceholder')}
options={options}
></SelectWithSearch>
)}
</RAGFlowFormItem>
<TabsContent
value={ExtractorSubTab.Summary}
className="space-y-4 pt-2"
>
<RAGFlowFormItem
label={t('flow.enableSummary')}
name="summary.enabled"
horizontal
valueClassName="w-auto flex justify-end"
>
{(field) => (
<Switch
checked={field.value === 1 || field.value === true}
onCheckedChange={(checked) => {
field.onChange(checked);
form.setValue('field_name', checked ? 'summary' : '', {
shouldDirty: true,
});
form.setValue('enable_summary', checked ? 1 : 0, {
shouldDirty: true,
});
}}
data-testid="extractor-summary-switch"
/>
)}
</RAGFlowFormItem>
<RAGFlowFormItem
label={t('flow.systemPrompt')}
name="summary.system_prompt"
>
<PromptEditor
placeholder={t('flow.messagePlaceholder')}
showToolbar={false}
showMergePath={false}
/>
</RAGFlowFormItem>
</TabsContent>
{(node?.data as Record<string, any>)?.operatorId ===
BuiltinAutoExtractNodeId && <ExtractorAutoMetadata />}
<RAGFlowFormItem label={t('flow.systemPrompt')} name="sys_prompt">
<PromptEditor
placeholder={t('flow.messagePlaceholder')}
showToolbar={true}
baseOptions={promptOptions}
></PromptEditor>
</RAGFlowFormItem>
<RAGFlowFormItem label={t('flow.userPrompt')} name="prompts">
<PromptEditor
showToolbar={true}
baseOptions={promptOptions}
></PromptEditor>
</RAGFlowFormItem>
<TabsContent
value={ExtractorSubTab.Metadata}
className="space-y-4 pt-2"
>
<ExtractorAutoMetadata />
</TabsContent>
</Tabs>
{!hideOutputs && <Output list={outputList}></Output>}
</FormWrapper>
{visible && (
<ConfirmDeleteDialog
title={t('flow.switchPromptMessage')}
open
onOpenChange={hideModal}
onOk={confirmSwitch}
onCancel={cancelSwitch}
></ConfirmDeleteDialog>
)}
</Form>
);
};

View File

@@ -1,68 +0,0 @@
import { useSetModalState } from '@/hooks/common-hooks';
import { useCallback, useRef } from 'react';
import { useTranslation } from 'react-i18next';
type SwitchPromptField = 'field_name' | 'sys_prompt' | 'prompts';
type SwitchPromptForm = {
getValues(name: 'field_name'): string;
setValue(
name: SwitchPromptField,
value: string,
options?: { shouldDirty?: boolean; shouldValidate?: boolean },
): void;
};
export function useSwitchPrompt(form: SwitchPromptForm) {
const { visible, showModal, hideModal } = useSetModalState();
const { t } = useTranslation();
const previousFieldNames = useRef<string[]>([form.getValues('field_name')]);
const setPromptValue = useCallback(
(field: SwitchPromptField, key: string, value: string) => {
form.setValue(field, t(`flow.prompts.${key}.${value}`), {
shouldDirty: true,
shouldValidate: true,
});
},
[form, t],
);
const handleFieldNameChange = useCallback(
(value: string) => {
if (value) {
const names = previousFieldNames.current;
if (names.length > 1) {
names.shift();
}
names.push(value);
showModal();
}
},
[showModal],
);
const confirmSwitch = useCallback(() => {
const value = form.getValues('field_name');
setPromptValue('sys_prompt', 'system', value);
setPromptValue('prompts', 'user', value);
}, [form, setPromptValue]);
const cancelSwitch = useCallback(() => {
const previousValue = previousFieldNames.current.at(-2);
if (previousValue) {
form.setValue('field_name', previousValue, {
shouldDirty: true,
shouldValidate: true,
});
}
}, [form]);
return {
handleFieldNameChange,
confirmSwitch,
hideModal,
visible,
cancelSwitch,
};
}

View File

@@ -9,7 +9,7 @@ export function useFormChangeCallback(
useEffect(() => {
if (onValuesChange) {
onValuesChange(values);
onValuesChange(form.getValues());
}
}, [onValuesChange, values]);
}, [form, onValuesChange, values]);
}

View File

@@ -400,7 +400,78 @@ export function transformTitleChunkerParams(
}
export function transformExtractorParams(params: ExtractorFormSchemaType) {
return { ...params, prompts: [{ content: params.prompts, role: 'user' }] };
const isMetadataEnabled =
params.metadata_config?.enabled !== undefined
? Boolean(params.metadata_config?.enabled)
: params.enable_metadata === 1 || params.enable_metadata === true;
const isSummaryEnabled =
params.summary?.enabled !== undefined
? Boolean(params.summary?.enabled)
: params.enable_summary === 1 ||
params.enable_summary === true ||
params.field_name === 'summary';
const metadataList =
params.metadata_config?.metadata ?? params.metadata ?? [];
const builtInMetadataList =
params.metadata_config?.built_in_metadata ??
params.built_in_metadata ??
[];
const summarySysPrompt =
params.summary?.system_prompt ?? params.sys_prompt ?? '';
const keywordsTopN = params.keywords?.top_n ?? params.auto_keywords ?? 0;
const keywordsSysPrompt =
params.keywords?.system_prompt ?? params.keywords_sys_prompt ?? '';
const questionsTopN = params.questions?.top_n ?? params.auto_questions ?? 0;
const questionsSysPrompt =
params.questions?.system_prompt ?? params.questions_sys_prompt ?? '';
const tagsTopN = params.tags?.top_n ?? params.auto_tags ?? 0;
const tagFileId = params.tags?.tag_file_id ?? params.tag_file_id ?? '';
return {
...params,
prompts: [{ content: params.prompts, role: 'user' }],
auto_keywords: keywordsTopN,
keywords_sys_prompt: keywordsSysPrompt,
keywords: {
top_n: keywordsTopN,
system_prompt: keywordsSysPrompt,
},
auto_questions: questionsTopN,
questions_sys_prompt: questionsSysPrompt,
questions: {
top_n: questionsTopN,
system_prompt: questionsSysPrompt,
},
auto_tags: tagsTopN,
tag_file_id: tagFileId,
tags: {
top_n: tagsTopN,
tag_file_id: tagFileId,
},
enable_summary: isSummaryEnabled ? 1 : 0,
sys_prompt: summarySysPrompt,
field_name: isSummaryEnabled
? (params.field_name || 'summary')
: (params.field_name === 'summary' ? '' : (params.field_name || '')),
summary: {
enabled: isSummaryEnabled,
system_prompt: summarySysPrompt,
},
enable_metadata: isMetadataEnabled ? 1 : 0,
metadata: metadataList,
built_in_metadata: builtInMetadataList,
metadata_config: {
enabled: isMetadataEnabled,
metadata: metadataList,
built_in_metadata: builtInMetadataList,
},
};
}
function transformDataOperationsParams(params: DataOperationsFormSchemaType) {

View File

@@ -0,0 +1,115 @@
import { transformExtractorConfigToForm } from '@/utils/pipeline-operator';
import { transformExtractorParams } from '../../utils';
describe('Extractor parameter transformations & precedence', () => {
describe('transformExtractorParams', () => {
it('synchronizes nested modular configs to flat fields and preserves nested objects', () => {
const input: any = {
summary: {
enabled: true,
system_prompt: 'Custom summary prompt',
},
metadata_config: {
enabled: true,
metadata: [{ key: 'category', type: 'string' }],
built_in_metadata: [{ key: 'update_time', type: 'time' }],
},
keywords: {
top_n: 5,
system_prompt: 'KW prompt',
},
questions: {
top_n: 3,
system_prompt: 'Q prompt',
},
tags: {
top_n: 2,
tag_file_id: 'tag-123',
},
llm_id: 'gpt-4',
};
const result = transformExtractorParams(input);
expect(result.enable_summary).toBe(1);
expect(result.summary).toEqual({
enabled: true,
system_prompt: 'Custom summary prompt',
});
expect(result.enable_metadata).toBe(1);
expect(result.metadata_config.enabled).toBe(true);
expect(result.metadata_config.metadata).toHaveLength(1);
expect(result.auto_keywords).toBe(5);
expect(result.auto_questions).toBe(3);
expect(result.auto_tags).toBe(2);
expect(result.tag_file_id).toBe('tag-123');
});
it('gives nested modular enabled: false precedence over legacy flat enable_*: 1', () => {
const input: any = {
summary: {
enabled: false,
system_prompt: '',
},
enable_summary: 1,
metadata_config: {
enabled: false,
metadata: [],
built_in_metadata: [],
},
enable_metadata: 1,
};
const result = transformExtractorParams(input);
expect(result.summary.enabled).toBe(false);
expect(result.enable_summary).toBe(0);
expect(result.metadata_config.enabled).toBe(false);
expect(result.enable_metadata).toBe(0);
});
it('preserves custom field_name when summary is disabled', () => {
const input: any = {
summary: {
enabled: false,
system_prompt: '',
},
field_name: 'custom_chunk_field',
};
const result = transformExtractorParams(input);
expect(result.field_name).toBe('custom_chunk_field');
});
});
describe('transformExtractorConfigToForm', () => {
it('normalizes legacy flat API format into nested form schema', () => {
const config = {
enable_summary: 1,
sys_prompt: 'Old summary prompt',
enable_metadata: 1,
metadata: [{ key: 'author', type: 'string' }],
built_in_metadata: [{ key: 'file_name', type: 'string' }],
auto_keywords: 4,
auto_questions: 2,
auto_tags: 1,
tag_file_id: 'tag-file-1',
};
const result = transformExtractorConfigToForm(config);
expect(result.summary).toEqual({
enabled: true,
system_prompt: 'Old summary prompt',
});
expect(result.metadata_config).toEqual({
enabled: true,
metadata: [{ key: 'author', type: 'string' }],
built_in_metadata: [{ key: 'file_name', type: 'string' }],
});
expect(result.keywords.top_n).toBe(4);
expect(result.questions.top_n).toBe(2);
expect(result.tags.top_n).toBe(1);
});
});
});

View File

@@ -86,7 +86,7 @@ function transformLevelsToRules(
* DSL: { prompts: [{ content: "text", role: "user" }] }
* Form: { prompts: "text" }
*/
function transformExtractorConfigToForm(
export function transformExtractorConfigToForm(
config: Record<string, any> | undefined,
): Record<string, any> {
if (!config) return {};
@@ -95,6 +95,61 @@ function transformExtractorConfigToForm(
if (Array.isArray(config.prompts) && config.prompts.length > 0) {
result.prompts = config.prompts[0]?.content ?? '';
}
const isSummaryEnabled =
config.summary?.enabled !== undefined
? Boolean(config.summary?.enabled)
: config.enable_summary === 1 || config.enable_summary === true;
const isMetadataEnabled =
config.metadata_config?.enabled !== undefined
? Boolean(config.metadata_config?.enabled)
: config.enable_metadata === 1 || config.enable_metadata === true;
result.keywords = {
top_n:
config.keywords?.top_n ??
config.auto_keywords ??
initialExtractorValues.keywords.top_n,
system_prompt:
config.keywords?.system_prompt ?? config.keywords_sys_prompt ?? '',
};
result.questions = {
top_n:
config.questions?.top_n ??
config.auto_questions ??
initialExtractorValues.questions.top_n,
system_prompt:
config.questions?.system_prompt ?? config.questions_sys_prompt ?? '',
};
result.tags = {
top_n:
config.tags?.top_n ??
config.auto_tags ??
initialExtractorValues.tags.top_n,
tag_file_id: config.tags?.tag_file_id ?? config.tag_file_id ?? '',
};
result.summary = {
enabled: isSummaryEnabled,
system_prompt: config.summary?.system_prompt ?? config.sys_prompt ?? '',
};
result.enable_summary = isSummaryEnabled ? 1 : 0;
result.field_name = isSummaryEnabled
? (config.field_name || 'summary')
: (config.field_name === 'summary' ? '' : (config.field_name || ''));
result.metadata_config = {
enabled: isMetadataEnabled,
metadata: config.metadata_config?.metadata ?? config.metadata ?? [],
built_in_metadata:
config.metadata_config?.built_in_metadata ??
config.built_in_metadata ??
[],
};
result.enable_metadata = isMetadataEnabled ? 1 : 0;
result.metadata = result.metadata_config.metadata;
result.built_in_metadata = result.metadata_config.built_in_metadata;
return result;
}