Feat: Support pipeline-related configuration at the document level. (#17212)

### Summary

Feat: Support pipeline-related configuration at the document level.
This commit is contained in:
balibabu
2026-07-22 22:24:50 +08:00
committed by GitHub
parent dccc0a4bee
commit b3d394954d
19 changed files with 702 additions and 263 deletions

View File

@@ -0,0 +1,67 @@
import { SelectWithSearch } from '@/components/originui/select-with-search';
import {
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from '@/components/ui/form';
import { useFetchBuiltinPipelines } from '@/hooks/use-agent-request';
import { cn } from '@/lib/utils';
import { useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
export function BuiltinPipelineItem({
line = 2,
name = 'parser_id',
}: {
line?: 1 | 2;
name?: string;
}) {
const { t } = useTranslation();
const form = useFormContext();
const { options: builtinPipelineOptions } = useFetchBuiltinPipelines();
return (
<FormField
control={form.control}
name={name}
render={({ field }) => (
<FormItem className="items-center space-y-0">
<div
className={cn('flex', {
'items-center': line === 1,
'flex-col gap-1': line === 2,
})}
>
<FormLabel
required
className={cn('text-sm whitespace-wrap', {
'w-1/4': line === 1,
})}
>
{t('knowledgeConfiguration.builtIn')}
</FormLabel>
<div
className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
>
<FormControl>
<SelectWithSearch
{...field}
placeholder={t(
'knowledgeConfiguration.chunkMethodPlaceholder',
)}
options={builtinPipelineOptions}
/>
</FormControl>
</div>
</div>
<div className="flex pt-1">
<div className={line === 1 ? 'w-1/4' : ''}></div>
<FormMessage />
</div>
</FormItem>
)}
/>
);
}

View File

@@ -24,7 +24,6 @@ import {
ChunkMethodItem,
EnableTocToggle,
ImageContextWindow,
ParseTypeItem,
} from '@/pages/dataset/dataset-setting/configuration/common-item';
import { zodResolver } from '@hookform/resolvers/zod';
import omit from 'lodash/omit';
@@ -44,6 +43,7 @@ import { ExcelToHtmlFormField } from '../excel-to-html-form-field';
import { LayoutRecognizeFormField } from '../layout-recognize-form-field';
import { MaxTokenNumberFormField } from '../max-token-number-from-field';
import { MinerUOptionsFormField } from '../mineru-options-form-field';
import { ParseTypeItem } from '../parse-type-form-field';
import { ButtonLoading } from '../ui/button';
import { Input } from '../ui/input';
import { DynamicPageRange } from './dynamic-page-range';

View File

@@ -0,0 +1,109 @@
import { BuiltinPipelineItem } from '@/components/builtin-pipeline-form-field';
import { DataFlowSelect } from '@/components/data-pipeline-select';
import { ParseTypeItem } from '@/components/parse-type-form-field';
import PipelineOperatorTabs from '@/components/pipeline-operator-tabs';
import { ButtonLoading } from '@/components/ui/button';
import {
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
} from '@/components/ui/dialog';
import { Form } from '@/components/ui/form';
import { ParseType } from '@/constants/knowledge';
import { IModalProps } from '@/interfaces/common';
import { IChangeParserRequestBody } from '@/interfaces/request/document';
import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import {
IDocumentPipelineDialogProps,
useDocumentPipelineForm,
} from './use-document-pipeline-form';
const FormId = 'DocumentPipelineDialogForm';
interface IProps
extends IModalProps<IChangeParserRequestBody>, IDocumentPipelineDialogProps {
loading: boolean;
}
export function DocumentPipelineDialog({
hideModal,
onOk,
parserId,
pipelineId,
parserConfig,
loading,
}: IProps) {
const { t } = useTranslation();
const {
form,
parseType,
operatorNodes,
operatorNodesLoading,
activeTab,
setActiveTab,
handleOperatorValuesChange,
showOperatorTabs,
buildSubmitData,
} = useDocumentPipelineForm({ parserId, pipelineId, parserConfig });
const onSubmit = useCallback(
async (data: Parameters<typeof buildSubmitData>[0]) => {
const ret = await onOk?.(buildSubmitData(data));
if (ret) {
hideModal?.();
}
},
[buildSubmitData, hideModal, onOk],
);
return (
<Dialog open onOpenChange={hideModal}>
<DialogContent className="max-w-[50vw] text-text-primary">
<DialogHeader>
<DialogTitle>{t('knowledgeDetails.chunkMethod')}</DialogTitle>
</DialogHeader>
<Form {...form}>
<form
onSubmit={form.handleSubmit(onSubmit)}
className="space-y-6 max-h-[70vh] overflow-auto -mx-6 px-10 py-5"
id={FormId}
>
<ParseTypeItem />
{parseType === ParseType.BuiltIn && <BuiltinPipelineItem />}
{parseType === ParseType.Pipeline && (
<DataFlowSelect
isMult={false}
showToDataPipeline={true}
formFieldName="pipeline_id"
/>
)}
{showOperatorTabs && (
<PipelineOperatorTabs
nodes={operatorNodes}
value={activeTab}
onValueChange={setActiveTab}
onOperatorValuesChange={handleOperatorValuesChange}
/>
)}
</form>
</Form>
<DialogFooter>
<ButtonLoading
type="submit"
form={FormId}
loading={
loading || (operatorNodesLoading && operatorNodes.length === 0)
}
>
{t('common.save')}
</ButtonLoading>
</DialogFooter>
</DialogContent>
</Dialog>
);
}

View File

@@ -0,0 +1,204 @@
import { ParseType } from '@/constants/knowledge';
import {
useActiveTab,
usePipelineOperatorNodes,
useResetParserConfigOnPipelineChange,
} from '@/hooks/use-pipeline-operator';
import { IChangeParserRequestBody } from '@/interfaces/request/document';
import {
getOperatorType,
transformApiConfigToForm,
transformFormConfigToApi,
} from '@/utils/pipeline-operator';
import { zodResolver } from '@hookform/resolvers/zod';
import { useCallback, useEffect, useMemo } from 'react';
import { useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
export interface IDocumentPipelineDialogProps {
parserId: string;
pipelineId?: string;
parserConfig?: Record<string, any>;
}
/**
* Converts the saved parser_config (API format, keyed by operator id) to the
* form format expected by the operator tabs, mirroring
* useFetchDatasetSettingOnMount on the dataset setting page.
*/
function transformSavedParserConfigToForm(
parserConfig?: Record<string, any>,
): Record<string, any> {
if (
!parserConfig ||
typeof parserConfig !== 'object' ||
Array.isArray(parserConfig)
) {
return {};
}
const hasPipelineKeys = Object.keys(parserConfig).some((key) =>
key.includes(':'),
);
if (!hasPipelineKeys) {
return parserConfig;
}
const formParserConfig: Record<string, any> = {};
for (const [operatorId, config] of Object.entries(parserConfig)) {
formParserConfig[operatorId] = transformApiConfigToForm(
getOperatorType(operatorId),
config as Record<string, any>,
);
}
return formParserConfig;
}
export function useDocumentPipelineForm({
parserId,
pipelineId,
parserConfig,
}: IDocumentPipelineDialogProps) {
const { t } = useTranslation();
const FormSchema = useMemo(
() =>
z
.object({
parseType: z.nativeEnum(ParseType),
parser_id: z.string(),
pipeline_id: z.string().optional(),
parser_config: z.record(z.string(), z.any()).optional(),
})
.superRefine((data, ctx) => {
if (data.parseType === ParseType.BuiltIn && !data.parser_id.trim()) {
ctx.addIssue({
path: ['parser_id'],
message: t('common.pleaseSelect'),
code: 'custom',
});
}
if (data.parseType === ParseType.Pipeline && !data.pipeline_id) {
ctx.addIssue({
path: ['pipeline_id'],
message: t('common.pleaseSelect'),
code: 'custom',
});
}
}),
[t],
);
const form = useForm<z.infer<typeof FormSchema>>({
resolver: zodResolver(FormSchema),
defaultValues: {
parseType: pipelineId ? ParseType.Pipeline : ParseType.BuiltIn,
parser_id: parserId || '',
pipeline_id: pipelineId || '',
parser_config: transformSavedParserConfigToForm(parserConfig),
},
});
const parseType = useWatch({
control: form.control,
name: 'parseType',
});
const selectedDataFlowId = useWatch({
control: form.control,
name: 'pipeline_id',
});
const selectedBuiltinId = useWatch({
control: form.control,
name: 'parser_id',
});
const isPipelineMode = parseType === ParseType.Pipeline;
const selectedPipelineId = isPipelineMode
? selectedDataFlowId
: selectedBuiltinId;
// The saved parser_config only belongs to the pipeline (and parse type) it
// was saved with — expose its id only while that exact pipeline is selected,
// so that after switching, defaults come purely from the new pipeline's DSL.
const savedParseType = pipelineId ? ParseType.Pipeline : ParseType.BuiltIn;
const savedPipelineId =
parseType === savedParseType
? isPipelineMode
? pipelineId
: parserId
: undefined;
const { operatorNodes, loading: operatorNodesLoading } =
usePipelineOperatorNodes(
selectedPipelineId,
selectedPipelineId && selectedPipelineId === savedPipelineId
? parserConfig
: undefined,
!isPipelineMode,
);
useResetParserConfigOnPipelineChange(
form,
selectedPipelineId,
savedPipelineId,
operatorNodes,
);
const { activeTab, setActiveTab } = useActiveTab(operatorNodes);
useEffect(() => {
if (parseType === ParseType.BuiltIn) {
form.setValue('pipeline_id', '');
}
}, [parseType, form]);
const handleOperatorValuesChange = useCallback(
(operatorId: string, values: any) => {
const currentParserConfig = form.getValues('parser_config') || {};
form.setValue('parser_config', {
...currentParserConfig,
[operatorId]: values,
});
},
[form],
);
const showOperatorTabs =
operatorNodes.length > 0 &&
((parseType === ParseType.Pipeline && !!selectedDataFlowId) ||
(parseType === ParseType.BuiltIn && !!selectedBuiltinId));
const buildSubmitData = useCallback(
(data: z.infer<typeof FormSchema>): IChangeParserRequestBody => {
const transformedConfig: Record<string, any> = {};
for (const [operatorId, config] of Object.entries(
data.parser_config ?? {},
)) {
transformedConfig[operatorId] = transformFormConfigToApi(
getOperatorType(operatorId),
config as Record<string, any>,
);
}
const isPipeline = data.parseType === ParseType.Pipeline;
return {
parser_id: isPipeline ? '' : data.parser_id,
pipeline_id: isPipeline ? data.pipeline_id : '',
parser_config: transformedConfig,
};
},
[],
);
return {
form,
parseType,
operatorNodes,
operatorNodesLoading,
activeTab,
setActiveTab,
handleOperatorValuesChange,
showOperatorTabs,
buildSubmitData,
};
}

View File

@@ -0,0 +1,73 @@
import {
FormControl,
FormField,
FormItem,
FormLabel,
FormMessage,
} from '@/components/ui/form';
import { Radio } from '@/components/ui/radio';
import { ParseType } from '@/constants/knowledge';
import { cn } from '@/lib/utils';
import { useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
export function ParseTypeItem({
line = 2,
name = 'parseType',
}: {
line?: number;
name?: string;
}) {
const { t } = useTranslation();
const form = useFormContext();
return (
<FormField
control={form.control}
name={name}
render={({ field }) => (
<FormItem className="items-center space-y-0">
<div
className={cn('flex', {
'items-center': line === 1,
'flex-col gap-1': line === 2,
})}
>
<FormLabel
className={cn('text-sm whitespace-wrap ', {
'w-1/4': line === 1,
})}
>
{t('knowledgeConfiguration.parseType')}
</FormLabel>
<div
className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
>
<FormControl>
<Radio.Group {...field}>
<div
className={cn(
'flex gap-2 justify-between text-muted-foreground',
line === 1 ? 'w-1/2' : 'w-3/4',
)}
>
<Radio value={ParseType.BuiltIn}>
{t('knowledgeConfiguration.builtIn')}
</Radio>
<Radio value={ParseType.Pipeline}>
{t('knowledgeConfiguration.manualSetup')}
</Radio>
</div>
</Radio.Group>
</FormControl>
</div>
</div>
<div className="flex pt-1">
<div className={line === 1 ? 'w-1/4' : ''}></div>
<FormMessage />
</div>
</FormItem>
)}
/>
);
}

View File

@@ -1,12 +1,12 @@
import { Operator } from '@/constants/agent';
import { RAGFlowNodeType } from '@/interfaces/database/agent';
import ExtractorForm from '@/pages/agent/form/extractor-form';
import ParserForm from '@/pages/agent/form/parser-form';
import TitleChunkerForm from '@/pages/agent/form/title-chunker-form';
import TokenChunkerForm from '@/pages/agent/form/token-chunker-form';
import TokenizerForm from '@/pages/agent/form/tokenizer-form';
import { getOperatorType } from '@/utils/pipeline-operator';
import { memo, useCallback } from 'react';
import ExtractorForm from '../../agent/form/extractor-form';
import ParserForm from '../../agent/form/parser-form';
import TitleChunkerForm from '../../agent/form/title-chunker-form';
import TokenChunkerForm from '../../agent/form/token-chunker-form';
import TokenizerForm from '../../agent/form/tokenizer-form';
import { getOperatorType } from './utils';
type PipelineOperatorFormProps = {
node: RAGFlowNodeType;

View File

@@ -4,6 +4,20 @@
* and merge unknown fields into the `ext` field for flexible configuration.
*/
/**
* Pipeline parser configs are keyed by operator id (e.g. "Parser:xxx"), so a
* top-level key containing ":" marks the pipeline structure, which must be
* sent as-is instead of being reshaped by extractParserConfigExt.
*/
export const isPipelineParserConfig = (
parserConfig: Record<string, any> | undefined,
): boolean => {
if (!parserConfig || typeof parserConfig !== 'object') {
return false;
}
return Object.keys(parserConfig).some((key) => key.includes(':'));
};
/**
* Extracts Raptor configuration with extra fields merged into ext.
* @param raptorConfig - The raptor configuration object

View File

@@ -39,7 +39,10 @@ import {
useGetPaginationWithRouter,
useHandleSearchChange,
} from './logic-hooks';
import { extractParserConfigExt } from './parser-config-utils';
import {
extractParserConfigExt,
isPipelineParserConfig,
} from './parser-config-utils';
import {
useGetKnowledgeSearchParams,
useSetPaginationParams,
@@ -502,6 +505,66 @@ export const useSetDocumentParser = () => {
return { setDocumentParser: mutateAsync, data, loading };
};
/**
* Go-backend variant of useSetDocumentParser. The Go document endpoint takes
* `parser_id` (instead of the legacy `chunk_method`) and expects the
* pipeline-shaped parser_config (keyed by operator id) to be sent as-is.
* Keep it parallel to the Python version — the original hook stays untouched
* and can be dropped once the Python backend is retired.
*/
export const useSetDocumentPipelineParser = () => {
const queryClient = useQueryClient();
const {
data,
isPending: loading,
mutateAsync,
} = useMutation({
mutationKey: [DocumentApiAction.SetDocumentParser, 'pipeline'],
mutationFn: async ({
parserId,
pipelineId,
documentId,
datasetId,
parserConfig,
}: {
parserId: string;
pipelineId: string;
documentId: string;
datasetId: string;
parserConfig?: IChangeParserConfigRequestBody;
}) => {
const updateData: Record<string, unknown> = {
parser_id: parserId,
pipeline_id: pipelineId,
};
if (parserConfig) {
updateData.parser_config = isPipelineParserConfig(parserConfig)
? parserConfig
: extractParserConfigExt(parserConfig);
}
const { data } = await changeDocumentParser(
datasetId,
documentId,
updateData,
);
if (data.code === 0) {
queryClient.invalidateQueries({
queryKey: [DocumentApiAction.FetchDocumentList],
});
message.success(i18n.t('message.modified'));
}
return data.code;
},
});
return { setDocumentPipelineParser: mutateAsync, data, loading };
};
export const useSetDocumentMeta = () => {
const queryClient = useQueryClient();

View File

@@ -57,7 +57,10 @@ import {
useGetPaginationWithRouter,
useHandleSearchChange,
} from './logic-hooks';
import { extractParserConfigExt } from './parser-config-utils';
import {
extractParserConfigExt,
isPipelineParserConfig,
} from './parser-config-utils';
import { useSetPaginationParams } from './route-hook';
export const enum KnowledgeApiAction {
@@ -263,15 +266,6 @@ export const useDeleteKnowledge = () => {
return { data, loading, deleteKnowledge: mutateAsync };
};
function isPipelineParserConfig(
parserConfig: Record<string, any> | undefined,
): boolean {
if (!parserConfig || typeof parserConfig !== 'object') {
return false;
}
return Object.keys(parserConfig).some((key) => key.includes(':'));
}
export const useUpdateKnowledge = (shouldFetchList = false) => {
const knowledgeBaseId = useKnowledgeBaseId();
const queryClient = useQueryClient();

View File

@@ -0,0 +1,98 @@
import { useFetchPipelineDslByPipelineId } from '@/hooks/use-agent-request';
import { RAGFlowNodeType } from '@/interfaces/database/agent';
import {
buildParserConfigFromNodes,
buildPipelineOperatorNodes,
} from '@/utils/pipeline-operator';
import { useEffect, useMemo, useRef, useState } from 'react';
import { UseFormReturn } from 'react-hook-form';
export const usePipelineOperatorNodes = (
pipelineId?: string,
pipelineParserConfig?: Record<string, any>,
isBuiltin = false,
) => {
const { dsl, loading } = useFetchPipelineDslByPipelineId(
pipelineId,
isBuiltin,
);
const operatorNodes = useMemo(() => {
return buildPipelineOperatorNodes(dsl, pipelineParserConfig);
}, [dsl, pipelineParserConfig]);
return { operatorNodes, loading };
};
/**
* Resets parser_config when the selected pipeline changes, so stale configs
* from the previous pipeline are never submitted. Once the new pipeline's
* DSL has loaded, parser_config is seeded with the DSL defaults (i.e. what
* the operator tabs display).
*
* The very first pipeline seen is treated as the initial load: the form was
* already reset with the saved parser_config on mount, so it is left
* untouched.
*/
export const useResetParserConfigOnPipelineChange = (
form: UseFormReturn<any>,
pipelineId: string | undefined,
savedPipelineId: string | undefined,
operatorNodes: RAGFlowNodeType[],
) => {
const previousPipelineIdRef = useRef<string>();
useEffect(() => {
if (!pipelineId) {
// Selection cleared (e.g. parse type switched): drop configs that
// belonged to the previously selected pipeline.
if (previousPipelineIdRef.current) {
previousPipelineIdRef.current = '';
form.setValue('parser_config', {});
}
return;
}
const isInitialLoad =
previousPipelineIdRef.current === undefined &&
pipelineId === savedPipelineId;
if (isInitialLoad || previousPipelineIdRef.current === pipelineId) {
previousPipelineIdRef.current = pipelineId;
return;
}
if (operatorNodes.length === 0) {
// The new pipeline's DSL is still loading — clear the previous
// pipeline's configs right away; defaults are seeded once it arrives.
form.setValue('parser_config', {});
return;
}
previousPipelineIdRef.current = pipelineId;
form.setValue('parser_config', buildParserConfigFromNodes(operatorNodes));
}, [form, pipelineId, savedPipelineId, operatorNodes]);
};
export const useActiveTab = (operatorNodes: RAGFlowNodeType[]) => {
const [activeTab, setActiveTab] = useState('');
useEffect(() => {
if (operatorNodes.length > 0) {
const firstTab =
(operatorNodes[0].data as Record<string, any>)?.operatorId ||
operatorNodes[0].data?.label ||
'';
const validTabs = operatorNodes.map(
(node) =>
(node.data as Record<string, any>)?.operatorId ||
node.data?.label ||
'',
);
setActiveTab((prev) => (validTabs.includes(prev) ? prev : firstTab));
} else {
setActiveTab('');
}
}, [operatorNodes]);
return { activeTab, setActiveTab };
};

View File

@@ -14,12 +14,9 @@ import {
FormLabel,
FormMessage,
} from '@/components/ui/form';
import { Radio } from '@/components/ui/radio';
import { Spin } from '@/components/ui/spin';
import { Switch } from '@/components/ui/switch';
import { ParseType } from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
import { useFetchBuiltinPipelines } from '@/hooks/use-agent-request';
import { cn } from '@/lib/utils';
import { history } from '@/utils/simple-history-util';
import { t } from 'i18next';
@@ -30,7 +27,6 @@ import {
FieldValues,
useFormContext,
} from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { useLocation } from 'react-router';
import { DataSetContext } from '..';
import { MetadataType } from '../../components/metedata/constant';
@@ -102,61 +98,6 @@ export function ChunkMethodItem(props: IProps) {
);
}
export function BuiltinPipelineItem({
line = 2,
name = 'parser_id',
}: {
line?: 1 | 2;
name?: string;
}) {
const { t } = useTranslation();
const form = useFormContext();
const { options: builtinPipelineOptions } = useFetchBuiltinPipelines();
return (
<FormField
control={form.control}
name={name}
render={({ field }) => (
<FormItem className="items-center space-y-0">
<div
className={cn('flex', {
'items-center': line === 1,
'flex-col gap-1': line === 2,
})}
>
<FormLabel
required
className={cn('text-sm whitespace-wrap', {
'w-1/4': line === 1,
})}
>
{t('knowledgeConfiguration.builtIn')}
</FormLabel>
<div
className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
>
<FormControl>
<SelectWithSearch
{...field}
placeholder={t(
'knowledgeConfiguration.chunkMethodPlaceholder',
)}
options={builtinPipelineOptions}
/>
</FormControl>
</div>
</div>
<div className="flex pt-1">
<div className={line === 1 ? 'w-1/4' : ''}></div>
<FormMessage />
</div>
</FormItem>
)}
/>
);
}
export const EmbeddingSelect = ({
isEdit,
field,
@@ -268,64 +209,6 @@ export function EmbeddingModelItem({
);
}
export function ParseTypeItem({
line = 2,
name = 'parseType',
}: {
line?: number;
name?: string;
}) {
const { t } = useTranslate('knowledgeConfiguration');
const form = useFormContext();
return (
<FormField
control={form.control}
name={name}
render={({ field }) => (
<FormItem className="items-center space-y-0">
<div
className={cn('flex', {
'items-center': line === 1,
'flex-col gap-1': line === 2,
})}
>
<FormLabel
// tooltip={t('parseTypeTip')}
className={cn('text-sm whitespace-wrap ', {
'w-1/4': line === 1,
})}
>
{t('parseType')}
</FormLabel>
<div
className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
>
<FormControl>
<Radio.Group {...field}>
<div
className={cn(
'flex gap-2 justify-between text-muted-foreground',
line === 1 ? 'w-1/2' : 'w-3/4',
)}
>
<Radio value={ParseType.BuiltIn}>{t('builtIn')}</Radio>
<Radio value={ParseType.Pipeline}>{t('manualSetup')}</Radio>
</div>
</Radio.Group>
</FormControl>
</div>
</div>
<div className="flex pt-1">
<div className={line === 1 ? 'w-1/4' : ''}></div>
<FormMessage />
</div>
</FormItem>
)}
/>
);
}
export function EnableAutoGenerateItem() {
const { t } = useTranslate('knowledgeConfiguration');
const form = useFormContext();

View File

@@ -25,8 +25,9 @@ import ChunkMethodLearnMore from './chunk-method-learn-more';
import LinkDataSource, {
IDataSourceNodeProps,
} from './components/link-data-source';
import { ParseTypeItem } from '@/components/parse-type-form-field';
import { MainContainer } from './configuration-form-container';
import { ChunkMethodItem, ParseTypeItem } from './configuration/common-item';
import { ChunkMethodItem } from './configuration/common-item';
import { formSchema } from './form-schema';
import { GeneralForm } from './general-form';
import { useFetchKnowledgeConfigurationOnMount } from './hooks';

View File

@@ -14,6 +14,7 @@ import {
import * as React from 'react';
import { ChunkMethodDialog } from '@/components/chunk-method-dialog';
import { DocumentPipelineDialog } from '@/components/document-pipeline-dialog';
import { EmptyType } from '@/components/empty/constant';
import Empty from '@/components/empty/empty';
import { RenameDialog } from '@/components/rename-dialog';
@@ -28,6 +29,7 @@ import {
} from '@/components/ui/table';
import { UseRowSelectionType } from '@/hooks/logic-hooks/use-row-selection';
import { useFetchDocumentList } from '@/hooks/use-document-request';
import { isGoBackend } from '@/utils/backend-runtime';
import { getExtension } from '@/utils/document-util';
import { t } from 'i18next';
import { pick } from 'lodash';
@@ -187,19 +189,29 @@ export function DatasetTable({
></RAGFlowPagination>
</div>
</div>
{changeParserVisible && (
<ChunkMethodDialog
documentId={changeParserRecord.id}
parserId={changeParserRecord.chunk_method}
pipelineId={changeParserRecord.pipeline_id}
parserConfig={changeParserRecord.parser_config}
documentExtension={getExtension(changeParserRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
></ChunkMethodDialog>
)}
{changeParserVisible &&
(isGoBackend() ? (
<DocumentPipelineDialog
parserId={changeParserRecord.chunk_method}
pipelineId={changeParserRecord.pipeline_id}
parserConfig={changeParserRecord.parser_config}
onOk={onChangeParserOk}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
></DocumentPipelineDialog>
) : (
<ChunkMethodDialog
documentId={changeParserRecord.id}
parserId={changeParserRecord.chunk_method}
pipelineId={changeParserRecord.pipeline_id}
parserConfig={changeParserRecord.parser_config}
documentExtension={getExtension(changeParserRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
></ChunkMethodDialog>
))}
{renameVisible && (
<RenameDialog

View File

@@ -1,11 +1,17 @@
import { useSetModalState } from '@/hooks/common-hooks';
import { useSetDocumentParser } from '@/hooks/use-document-request';
import {
useSetDocumentParser,
useSetDocumentPipelineParser,
} from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
import { IChangeParserRequestBody } from '@/interfaces/request/document';
import { isGoBackend } from '@/utils/backend-runtime';
import { useCallback, useState } from 'react';
export const useChangeDocumentParser = () => {
const { setDocumentParser, loading } = useSetDocumentParser();
const { setDocumentPipelineParser, loading: pipelineParserLoading } =
useSetDocumentPipelineParser();
const [record, setRecord] = useState<IDocumentInfo>({} as IDocumentInfo);
const {
@@ -17,7 +23,12 @@ export const useChangeDocumentParser = () => {
const onChangeParserOk = useCallback(
async (parserConfigInfo: IChangeParserRequestBody) => {
if (record?.id && record?.dataset_id) {
const ret = await setDocumentParser({
// The Go document endpoint takes `parser_id` and a pipeline-shaped
// parser_config; the Python one keeps the legacy payload shape.
const setParser = isGoBackend()
? setDocumentPipelineParser
: setDocumentParser;
const ret = await setParser({
parserId: parserConfigInfo.parser_id,
pipelineId: parserConfigInfo.pipeline_id || '',
documentId: record?.id,
@@ -29,7 +40,13 @@ export const useChangeDocumentParser = () => {
}
}
},
[record?.id, record?.dataset_id, setDocumentParser, hideChangeParserModal],
[
record?.id,
record?.dataset_id,
setDocumentParser,
setDocumentPipelineParser,
hideChangeParserModal,
],
);
const handleShowChangeParserModal = useCallback(
@@ -41,7 +58,7 @@ export const useChangeDocumentParser = () => {
);
return {
changeParserLoading: loading,
changeParserLoading: loading || pipelineParserLoading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,

View File

@@ -1,13 +1,16 @@
import { ParseType } from '@/constants/knowledge';
import { useFetchPipelineDslByPipelineId } from '@/hooks/use-agent-request';
import {
useFetchDatasetPipelineConfiguration,
useUpdateKnowledge,
} from '@/hooks/use-knowledge-request';
import { RAGFlowNodeType } from '@/interfaces/database/agent';
import { IConnector } from '@/interfaces/database/dataset';
import { useDataSourceInfo } from '@/pages/user-setting/data-source/constant';
import { checkEmbedding } from '@/services/knowledge-service';
import {
getOperatorType,
transformApiConfigToForm,
transformFormConfigToApi,
} from '@/utils/pipeline-operator';
import { pick } from 'lodash';
import {
Dispatch,
@@ -15,20 +18,11 @@ import {
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { UseFormReturn } from 'react-hook-form';
import { useParams, useSearchParams } from 'react-router';
import { z } from 'zod';
import { formSchema } from './form-schema';
import {
buildParserConfigFromNodes,
buildPipelineOperatorNodes,
getOperatorType,
transformApiConfigToForm,
transformFormConfigToApi,
} from './utils';
export function useHasParsedDocument(isEdit?: boolean) {
const { data: knowledgeDetails } = useFetchDatasetPipelineConfiguration({
@@ -125,72 +119,6 @@ export const useFetchDatasetSettingOnMount = (
return { knowledgeDetails, loading, sourceData };
};
export const usePipelineOperatorNodes = (
pipelineId?: string,
pipelineParserConfig?: Record<string, any>,
isBuiltin = false,
) => {
const { dsl, loading } = useFetchPipelineDslByPipelineId(
pipelineId,
isBuiltin,
);
const operatorNodes = useMemo(() => {
return buildPipelineOperatorNodes(dsl, pipelineParserConfig);
}, [dsl, pipelineParserConfig]);
return { operatorNodes, loading };
};
/**
* Resets parser_config when the selected pipeline changes, so stale configs
* from the previous pipeline are never submitted. Once the new pipeline's
* DSL has loaded, parser_config is seeded with the DSL defaults (i.e. what
* the operator tabs display).
*
* The very first pipeline seen is treated as the initial load: the form was
* already reset with the saved parser_config in useFetchDatasetSettingOnMount,
* so it is left untouched.
*/
export const useResetParserConfigOnPipelineChange = (
form: UseFormReturn<z.infer<typeof formSchema>>,
pipelineId: string | undefined,
savedPipelineId: string | undefined,
operatorNodes: RAGFlowNodeType[],
) => {
const previousPipelineIdRef = useRef<string>();
useEffect(() => {
if (!pipelineId) {
// Selection cleared (e.g. parse type switched): drop configs that
// belonged to the previously selected pipeline.
if (previousPipelineIdRef.current) {
previousPipelineIdRef.current = '';
form.setValue('parser_config', {});
}
return;
}
const isInitialLoad =
previousPipelineIdRef.current === undefined &&
pipelineId === savedPipelineId;
if (isInitialLoad || previousPipelineIdRef.current === pipelineId) {
previousPipelineIdRef.current = pipelineId;
return;
}
if (operatorNodes.length === 0) {
// The new pipeline's DSL is still loading — clear the previous
// pipeline's configs right away; defaults are seeded once it arrives.
form.setValue('parser_config', {});
return;
}
previousPipelineIdRef.current = pipelineId;
form.setValue('parser_config', buildParserConfigFromNodes(operatorNodes));
}, [form, pipelineId, savedPipelineId, operatorNodes]);
};
export const useSaveDatasetSetting = () => {
const { saveKnowledgeConfiguration, loading } = useUpdateKnowledge();
@@ -226,30 +154,6 @@ export const useSaveDatasetSetting = () => {
return { handleSave, loading };
};
export const useActiveTab = (operatorNodes: RAGFlowNodeType[]) => {
const [activeTab, setActiveTab] = useState('');
useEffect(() => {
if (operatorNodes.length > 0) {
const firstTab =
(operatorNodes[0].data as Record<string, any>)?.operatorId ||
operatorNodes[0].data?.label ||
'';
const validTabs = operatorNodes.map(
(node) =>
(node.data as Record<string, any>)?.operatorId ||
node.data?.label ||
'',
);
setActiveTab((prev) => (validTabs.includes(prev) ? prev : firstTab));
} else {
setActiveTab('');
}
}, [operatorNodes]);
return { activeTab, setActiveTab };
};
export const usePipelineDataList = (sourceData: any[] | undefined) => {
const { dataSourceInfo } = useDataSourceInfo();

View File

@@ -1,4 +1,7 @@
import { BuiltinPipelineItem } from '@/components/builtin-pipeline-form-field';
import { DataFlowSelect } from '@/components/data-pipeline-select';
import { ParseTypeItem } from '@/components/parse-type-form-field';
import PipelineOperatorTabs from '@/components/pipeline-operator-tabs';
import { Button, ButtonLoading } from '@/components/ui/button';
import {
Card,
@@ -12,9 +15,10 @@ import { Form } from '@/components/ui/form';
import { FormLayout } from '@/constants/form';
import { ParseType } from '@/constants/knowledge';
import {
BuiltinPipelineItem,
ParseTypeItem,
} from '@/pages/dataset/dataset-setting/configuration/common-item';
useActiveTab,
usePipelineOperatorNodes,
useResetParserConfigOnPipelineChange,
} from '@/hooks/use-pipeline-operator';
import { zodResolver } from '@hookform/resolvers/zod';
import { useCallback, useEffect, useState } from 'react';
import { useForm, useWatch } from 'react-hook-form';
@@ -26,15 +30,11 @@ import LinkDataSource, {
import { formSchema } from './form-schema';
import { GeneralForm } from './general-form';
import {
useActiveTab,
useConnectorHandlers,
useFetchDatasetSettingOnMount,
usePipelineDataList,
usePipelineOperatorNodes,
useResetParserConfigOnPipelineChange,
useSaveDatasetSetting,
} from './hooks';
import PipelineOperatorTabs from './pipeline-operator-tabs';
export default function DatasetSetting() {
const { t } = useTranslation();

View File

@@ -1,4 +1,6 @@
import { BuiltinPipelineItem } from '@/components/builtin-pipeline-form-field';
import { DataFlowSelect } from '@/components/data-pipeline-select';
import { ParseTypeItem } from '@/components/parse-type-form-field';
import { ButtonLoading } from '@/components/ui/button';
import {
Dialog,
@@ -28,10 +30,8 @@ import { useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
import {
BuiltinPipelineItem,
ChunkMethodItem,
EmbeddingModelItem,
ParseTypeItem,
} from '../dataset/dataset-setting/configuration/common-item';
import { isGoBackend } from '@/utils/backend-runtime';