diff --git a/web/src/components/auto-keywords-form-field.tsx b/web/src/components/auto-keywords-form-field.tsx index 33373b1e1d..7b55dcac5f 100644 --- a/web/src/components/auto-keywords-form-field.tsx +++ b/web/src/components/auto-keywords-form-field.tsx @@ -2,34 +2,45 @@ import { FormLayout } from '@/constants/form'; import { useTranslate } from '@/hooks/common-hooks'; import { SliderInputFormField } from './slider-input-form-field'; -export function AutoKeywordsFormField() { +interface AutoFieldProps { + name?: string; + layout?: FormLayout; +} + +export function AutoKeywordsFormField({ + name = 'parser_config.auto_keywords', + layout = FormLayout.Vertical, +}: AutoFieldProps) { const { t } = useTranslate('knowledgeDetails'); return ( ); } -export function AutoQuestionsFormField() { +export function AutoQuestionsFormField({ + name = 'parser_config.auto_questions', + layout = FormLayout.Vertical, +}: AutoFieldProps) { const { t } = useTranslate('knowledgeDetails'); return ( diff --git a/web/src/hooks/use-agent-request.ts b/web/src/hooks/use-agent-request.ts index 27107f1504..b9ede5ec69 100644 --- a/web/src/hooks/use-agent-request.ts +++ b/web/src/hooks/use-agent-request.ts @@ -7,6 +7,7 @@ import { IAgentLogResponse, IAgentLogsRequest, IAgentLogsResponse, + IBuiltinPipelineListResponse, IFlow, IFlowTemplate, IPipeLineListRequest, @@ -77,6 +78,9 @@ export const enum AgentApiAction { FetchSharedAgent = 'fetchSharedAgent', FetchAgentTags = 'fetchAgentTags', UpdateAgentTags = 'updateAgentTags', + FetchPipelineNodes = 'fetchPipelineNodes', + FetchBuiltinPipelineList = 'fetchBuiltinPipelineList', + FetchBuiltinPipelineDetail = 'fetchBuiltinPipelineDetail', } export const useFetchAgentTemplates = () => { @@ -867,6 +871,37 @@ export const useFetchAgentList = ({ return { data, loading }; }; +export const BuiltinPipelineKeys = { + list: (type: string) => + [AgentApiAction.FetchBuiltinPipelineList, type] as const, + detail: (id: string) => + [AgentApiAction.FetchBuiltinPipelineDetail, id] as const, +}; + +export const useFetchBuiltinPipelines = (type = 'builtin', enabled = true) => { + const { data, isFetching: loading } = useQuery({ + queryKey: BuiltinPipelineKeys.list(type), + initialData: { canvas: [], total: 0 }, + gcTime: 0, + enabled, + queryFn: async () => { + const { data } = await agentService.listBuiltinPipelines( + { params: { type } }, + true, + ); + return data?.data ?? { canvas: [], total: 0 }; + }, + }); + + const options = + data?.canvas?.map((item) => ({ + label: item.title, + value: item.id, + })) ?? []; + + return { data, loading, options }; +}; + export const useCancelDataflow = () => { const { data, @@ -1116,3 +1151,26 @@ export const useExportAgentLog = () => { return { exportLogs: mutateAsync, loading }; }; + +export const useFetchPipelineDslByPipelineId = ( + pipelineId?: string, + isBuiltin = false, +) => { + const { data: dsl, isFetching: loading } = useQuery({ + queryKey: isBuiltin + ? BuiltinPipelineKeys.detail(pipelineId!) + : [AgentApiAction.FetchPipelineNodes, pipelineId], + initialData: {}, + gcTime: 0, + enabled: !!pipelineId, + queryFn: async () => { + const { data } = isBuiltin + ? await agentService.getBuiltinPipeline(pipelineId!) + : await agentService.getAgent(pipelineId!); + const flow = data?.data; + return flow?.dsl ?? {}; + }, + }); + + return { dsl, loading }; +}; diff --git a/web/src/hooks/use-dataflow-request.ts b/web/src/hooks/use-dataflow-request.ts deleted file mode 100644 index b628047100..0000000000 --- a/web/src/hooks/use-dataflow-request.ts +++ /dev/null @@ -1,91 +0,0 @@ -import message from '@/components/ui/message'; -import { IFlow } from '@/interfaces/database/agent'; -import dataflowService from '@/services/dataflow-service'; -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; -import { useTranslation } from 'react-i18next'; -import { useParams } from 'react-router'; - -export const enum DataflowApiAction { - ListDataflow = 'listDataflow', - RemoveDataflow = 'removeDataflow', - FetchDataflow = 'fetchDataflow', - RunDataflow = 'runDataflow', - SetDataflow = 'setDataflow', -} - -export const useRemoveDataflow = () => { - const queryClient = useQueryClient(); - const { t } = useTranslation(); - - const { - data, - isPending: loading, - mutateAsync, - } = useMutation({ - mutationKey: [DataflowApiAction.RemoveDataflow], - mutationFn: async (ids: string[]) => { - const { data } = await dataflowService.removeDataflow({ - canvas_ids: ids, - }); - if (data.code === 0) { - queryClient.invalidateQueries({ - queryKey: [DataflowApiAction.ListDataflow], - }); - - message.success(t('message.deleted')); - } - return data.code; - }, - }); - - return { data, loading, removeDataflow: mutateAsync }; -}; - -export const useSetDataflow = () => { - const queryClient = useQueryClient(); - const { t } = useTranslation(); - - const { - data, - isPending: loading, - mutateAsync, - } = useMutation({ - mutationKey: [DataflowApiAction.SetDataflow], - mutationFn: async (params: Partial) => { - const { data } = await dataflowService.setDataflow(params); - if (data.code === 0) { - queryClient.invalidateQueries({ - queryKey: [DataflowApiAction.FetchDataflow], - }); - - message.success(t(`message.${params.id ? 'modified' : 'created'}`)); - } - return data?.code; - }, - }); - - return { data, loading, setDataflow: mutateAsync }; -}; - -export const useFetchDataflow = () => { - const { id } = useParams(); - - const { - data, - isFetching: loading, - refetch, - } = useQuery({ - queryKey: [DataflowApiAction.FetchDataflow, id], - gcTime: 0, - initialData: {} as IFlow, - enabled: !!id, - refetchOnWindowFocus: false, - queryFn: async () => { - const { data } = await dataflowService.fetchDataflow(id); - - return data?.data ?? ({} as IFlow); - }, - }); - - return { data, loading, refetch }; -}; diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts index 823dcac177..5b58a258fb 100644 --- a/web/src/hooks/use-knowledge-request.ts +++ b/web/src/hooks/use-knowledge-request.ts @@ -66,6 +66,7 @@ export const enum KnowledgeApiAction { DeleteKnowledge = 'deleteKnowledge', SaveKnowledge = 'saveKnowledge', FetchKnowledgeDetail = 'fetchKnowledgeDetail', + FetchDatasetPipelineConfiguration = 'fetchDatasetPipelineConfiguration', FetchKnowledgeGraph = 'fetchKnowledgeGraph', FetchArtifactList = 'fetchArtifactList', FetchArtifactTopicList = 'fetchArtifactTopicList', @@ -264,6 +265,15 @@ export const useDeleteKnowledge = () => { return { data, loading, deleteKnowledge: mutateAsync }; }; +function isPipelineParserConfig( + parserConfig: Record | undefined, +): boolean { + if (!parserConfig || typeof parserConfig !== 'object') { + return false; + } + return Object.keys(parserConfig).some((key) => key.includes(':')); +} + export const useUpdateKnowledge = (shouldFetchList = false) => { const knowledgeBaseId = useKnowledgeBaseId(); const queryClient = useQueryClient(); @@ -308,7 +318,9 @@ export const useUpdateKnowledge = (shouldFetchList = false) => { description, permission, pagerank, - parser_config: extractParserConfigExt(parser_config), + parser_config: isPipelineParserConfig(parser_config) + ? parser_config + : extractParserConfigExt(parser_config), ...omit(ext, ['kb_id']), }; @@ -352,6 +364,36 @@ export const useFetchKnowledgeBaseConfiguration = (props?: { return { data, loading }; }; +export const DatasetPipelineConfigurationKeys = { + detail: (knowledgeBaseId: string | null | undefined) => + [ + KnowledgeApiAction.FetchDatasetPipelineConfiguration, + knowledgeBaseId, + ] as const, +}; + +export const useFetchDatasetPipelineConfiguration = (props?: { + isEdit?: boolean; +}) => { + const { isEdit = true } = props || { isEdit: true }; + const { id } = useParams(); + const [searchParams] = useSearchParams(); + const knowledgeBaseId = searchParams.get('id') || id; + + const { data, isFetching: loading } = useQuery({ + queryKey: DatasetPipelineConfigurationKeys.detail(knowledgeBaseId), + initialData: {} as IDataset, + gcTime: 0, + enabled: !!knowledgeBaseId && isEdit, + queryFn: async () => { + const { data } = await getKbDetail(knowledgeBaseId || ''); + return data?.data ?? {}; + }, + }); + + return { data, loading }; +}; + export const ArtifactKeys = { list: ( datasetId: string, diff --git a/web/src/interfaces/database/agent.ts b/web/src/interfaces/database/agent.ts index 691946bcf8..abf84eebd5 100644 --- a/web/src/interfaces/database/agent.ts +++ b/web/src/interfaces/database/agent.ts @@ -199,6 +199,7 @@ export type BaseNodeData = { name: string; // operator name color?: string; form?: TForm; + operatorId?: string; }; export type BaseNode = Node>; @@ -301,6 +302,18 @@ export interface IPipeLineListRequest { ext?: string; } +export interface IBuiltinPipeline { + id: string; + title: string; + description?: string; + filename?: string; +} + +export interface IBuiltinPipelineListResponse { + canvas: IBuiltinPipeline[]; + total: number; +} + export interface GlobalVariableType { name: string; value: any; diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 2eb41ca8b9..985ec52e94 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -499,6 +499,7 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim testing: 'Retrieval testing', files: 'files', configuration: 'Configuration', + nextConfiguration: 'Next configuration', knowledgeGraph: 'Knowledge graph', compilation: 'Compilation', export: 'Export', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 8d6cc784f9..0cf87e4c6d 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -445,6 +445,7 @@ export default { dataset: '知识库', testing: '检索测试', configuration: '配置', + nextConfiguration: '下个配置', knowledgeGraph: '知识图谱', compilation: '编译', export: '导出', diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx index f13baf214c..006ccd657c 100644 --- a/web/src/pages/agent/constant/pipeline.tsx +++ b/web/src/pages/agent/constant/pipeline.tsx @@ -351,6 +351,8 @@ export const initialTitleChunkerValues = { export const initialExtractorValues = { ...initialLlmBaseValues, field_name: ContextGeneratorFieldName.Summary, + auto_keywords: 0, + auto_questions: 0, outputs: { chunks: { type: 'Array', value: [] }, }, diff --git a/web/src/pages/agent/form/extractor-form/index.tsx b/web/src/pages/agent/form/extractor-form/index.tsx index 4a7a60011d..22748fb384 100644 --- a/web/src/pages/agent/form/extractor-form/index.tsx +++ b/web/src/pages/agent/form/extractor-form/index.tsx @@ -1,3 +1,7 @@ +import { + AutoKeywordsFormField, + AutoQuestionsFormField, +} from '@/components/auto-keywords-form-field'; import { ConfirmDeleteDialog } from '@/components/confirm-delete-dialog'; import { LargeModelFormField } from '@/components/large-model-form-field'; import { LlmSettingSchema } from '@/components/llm-setting-items/next'; @@ -15,10 +19,11 @@ import { ContextGeneratorFieldName, initialExtractorValues, } from '../../constant/pipeline'; +import { useOwnerTenantId } from '../../context'; import { useBuildNodeOutputOptions } from '../../hooks/use-build-options'; +import { useFormChangeCallback } from '../../hooks/use-form-change-callback'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; -import { useOwnerTenantId } from '../../context'; import { INextOperatorForm } from '../../interface'; import { buildOutputList } from '../../utils/build-output-list'; import { FormWrapper } from '../components/form-wrapper'; @@ -29,6 +34,8 @@ export const FormSchema = z.object({ field_name: z.string(), sys_prompt: z.string(), prompts: z.string().optional(), + auto_keywords: z.number().optional(), + auto_questions: z.number().optional(), ...LlmSettingSchema, }); @@ -36,7 +43,11 @@ export type ExtractorFormSchemaType = z.infer; const outputList = buildOutputList(initialExtractorValues.outputs); -const ExtractorForm = ({ node }: INextOperatorForm) => { +const ExtractorForm = ({ + node, + onValuesChange, + hideOutputs, +}: INextOperatorForm) => { const defaultValues = useFormValues(initialExtractorValues, node); const { t } = useTranslation(); @@ -59,6 +70,7 @@ const ExtractorForm = ({ node }: INextOperatorForm) => { } = useSwitchPrompt(form); useWatchFormChange(node?.id, form); + useFormChangeCallback(form, onValuesChange); const ownerTenantId = useOwnerTenantId(); const isToc = form.getValues('field_name') === 'toc'; @@ -66,7 +78,11 @@ const ExtractorForm = ({ node }: INextOperatorForm) => { return (
- + + + {(field) => ( { > - + {!hideOutputs && } {visible && ( { +const ParserForm = ({ + node, + onValuesChange, + hideOutputs, +}: INextOperatorForm) => { const { t } = useTranslation(); const defaultValues = useFormValues(initialParserValues, node); @@ -360,6 +365,7 @@ const ParserForm = ({ node }: INextOperatorForm) => { }, [append]); useWatchFormChange(node?.id, form); + useFormChangeCallback(form, onValuesChange); return ( @@ -382,9 +388,11 @@ const ParserForm = ({ node }: INextOperatorForm) => { )} -
- -
+ {!hideOutputs && ( +
+ +
+ )} ); }; diff --git a/web/src/pages/agent/form/title-chunker-form/index.tsx b/web/src/pages/agent/form/title-chunker-form/index.tsx index 1e56df20c3..689ffd7194 100644 --- a/web/src/pages/agent/form/title-chunker-form/index.tsx +++ b/web/src/pages/agent/form/title-chunker-form/index.tsx @@ -9,7 +9,7 @@ import { Switch } from '@/components/ui/switch'; import { cn } from '@/lib/utils'; import { zodResolver } from '@hookform/resolvers/zod'; import { ChevronDown, ChevronUp, Trash2 } from 'lucide-react'; -import { memo, useState } from 'react'; +import { memo, useCallback, useState } from 'react'; import { useFieldArray, useForm, @@ -22,6 +22,7 @@ import { initialTitleChunkerValues, TitleChunkerMethod, } from '../../constant/pipeline'; +import { useFormChangeCallback } from '../../hooks/use-form-change-callback'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; @@ -91,6 +92,10 @@ function LevelItem({ const name = `${parentName}.${index}.expression`; + const handleRemove = useCallback(() => { + removeParent(index); + }, [removeParent, index]); + return (
@@ -108,7 +113,7 @@ function LevelItem({ type="button" variant={'ghost'} size="sm" - onClick={() => removeParent(index)} + onClick={handleRemove} > @@ -137,6 +142,10 @@ function CardBody({ cardName }: CardBodyProps) { control: form.control, }); + const handleAppendLevel = useCallback(() => { + appendLevel({ expression: '' }); + }, [appendLevel]); + return (
@@ -151,10 +160,7 @@ function CardBody({ cardName }: CardBodyProps) { ))}
- appendLevel({ expression: '' })} - className="mt-4" - > + {t('flow.addRegularExpressions')} @@ -173,46 +179,53 @@ function RulesFieldArray({ name }: RulesFieldArrayProps) { control: form.control, }); + const handleAppendRule = useCallback(() => { + append({ + levels: [{ expression: '' }], + }); + }, [append]); + return (
- {fields.map((cardField, cardIndex) => ( - - -
- - {t('flow.rule', 'Rule')} {cardIndex + 1} - -
- {fields.length > 1 && ( - - )} - - - - ))} - - append({ - levels: [{ expression: '' }], - }) - } - className="mt-4" - > + {fields.map((cardField, cardIndex) => { + const handleRemoveCard = () => remove(cardIndex); + + return ( + + +
+ + {t('flow.rule', 'Rule')} {cardIndex + 1} + +
+ {fields.length > 1 && ( + + )} + + + + ); + })} + {t('flow.addRule', 'Add Rule')}
); } -const TitleChunkerForm = ({ node }: INextOperatorForm) => { +const TitleChunkerForm = ({ + node, + onValuesChange, + hideOutputs, +}: INextOperatorForm) => { const { t } = useTranslation(); const initialValues = useFormValues(initialTitleChunkerValues, node); @@ -233,6 +246,11 @@ const TitleChunkerForm = ({ node }: INextOperatorForm) => { const hierarchyOptions = useDynamicHierarchyOptions(form, activeRulesName); useWatchFormChange(node?.id, form); + useFormChangeCallback(form, onValuesChange); + + const handleToggleShowAllTip = useCallback(() => { + setShowAllTip((prev) => !prev); + }, []); return (
@@ -257,7 +275,7 @@ const TitleChunkerForm = ({ node }: INextOperatorForm) => { />
setShowAllTip(!showAllTip)} + onClick={handleToggleShowAllTip} >
{ {(field) => ( { - field.onChange?.(checked); - }} + onCheckedChange={field.onChange} /> )} @@ -327,9 +343,7 @@ const TitleChunkerForm = ({ node }: INextOperatorForm) => { {(field) => ( { - field.onChange?.(checked); - }} + onCheckedChange={field.onChange} /> )} @@ -349,9 +363,11 @@ const TitleChunkerForm = ({ node }: INextOperatorForm) => {
{/* )} */} -
- -
+ {!hideOutputs && ( +
+ +
+ )} ); }; diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx index 1e0ceaf010..2790d5d202 100644 --- a/web/src/pages/agent/form/token-chunker-form/index.tsx +++ b/web/src/pages/agent/form/token-chunker-form/index.tsx @@ -12,6 +12,7 @@ import { useFieldArray, useForm } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; import { initialTokenChunkerValues } from '../../constant/pipeline'; +import { useFormChangeCallback } from '../../hooks/use-form-change-callback'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; @@ -41,7 +42,11 @@ export const FormSchema = z.object({ export type TokenChunkerFormSchemaType = z.infer; -const TokenChunkerForm = ({ node }: INextOperatorForm) => { +const TokenChunkerForm = ({ + node, + onValuesChange, + hideOutputs, +}: INextOperatorForm) => { const defaultValues = useFormValues(initialTokenChunkerValues, node); const { t } = useTranslation(); @@ -69,6 +74,7 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => { }); useWatchFormChange(node?.id, form); + useFormChangeCallback(form, onValuesChange); return (
@@ -136,7 +142,7 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => { ))}
- append({ value: '\n' })}> + append({ value: '\n' })}> {t('common.add')} @@ -211,9 +217,11 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => { )} -
- -
+ {!hideOutputs && ( +
+ +
+ )} ); }; diff --git a/web/src/pages/agent/form/tokenizer-form/index.tsx b/web/src/pages/agent/form/tokenizer-form/index.tsx index 8b3eb414d4..a4cdcfdddb 100644 --- a/web/src/pages/agent/form/tokenizer-form/index.tsx +++ b/web/src/pages/agent/form/tokenizer-form/index.tsx @@ -14,6 +14,7 @@ import { TokenizerFields, TokenizerSearchMethod, } from '../../constant'; +import { useFormChangeCallback } from '../../hooks/use-form-change-callback'; import { useFormValues } from '../../hooks/use-form-values'; import { useWatchFormChange } from '../../hooks/use-watch-form-change'; import { INextOperatorForm } from '../../interface'; @@ -31,7 +32,11 @@ export const FormSchema = z.object({ export type TokenizerFormSchemaType = z.infer; -const TokenizerForm = ({ node }: INextOperatorForm) => { +const TokenizerForm = ({ + node, + onValuesChange, + hideOutputs, +}: INextOperatorForm) => { const { t } = useTranslation(); const defaultValues = useFormValues(initialTokenizerValues, node); @@ -53,6 +58,7 @@ const TokenizerForm = ({ node }: INextOperatorForm) => { }); useWatchFormChange(node?.id, form); + useFormChangeCallback(form, onValuesChange); return (
@@ -81,9 +87,11 @@ const TokenizerForm = ({ node }: INextOperatorForm) => { {(field) => } -
- -
+ {!hideOutputs && ( +
+ +
+ )} ); }; diff --git a/web/src/pages/agent/hooks/use-form-change-callback.ts b/web/src/pages/agent/hooks/use-form-change-callback.ts new file mode 100644 index 0000000000..8b4b95d6b0 --- /dev/null +++ b/web/src/pages/agent/hooks/use-form-change-callback.ts @@ -0,0 +1,15 @@ +import { useEffect } from 'react'; +import { UseFormReturn, useWatch } from 'react-hook-form'; + +export function useFormChangeCallback( + form: UseFormReturn, + onValuesChange?: (values: any) => void, +) { + const values = useWatch({ control: form.control }); + + useEffect(() => { + if (onValuesChange) { + onValuesChange(values); + } + }, [onValuesChange, values]); +} diff --git a/web/src/pages/agent/interface.ts b/web/src/pages/agent/interface.ts index ac405f87db..4e7a7de67c 100644 --- a/web/src/pages/agent/interface.ts +++ b/web/src/pages/agent/interface.ts @@ -11,6 +11,8 @@ export interface IOperatorForm { export interface INextOperatorForm { node?: RAGFlowNodeType; nodeId?: string; + onValuesChange?(values: any): void; + hideOutputs?: boolean; } export interface IGenerateParameter { diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts index be9f1acbe2..fd7f2c136c 100644 --- a/web/src/pages/agent/utils.ts +++ b/web/src/pages/agent/utils.ts @@ -8,6 +8,7 @@ import { ICategorizeItemResult, RAGFlowNodeType, } from '@/interfaces/database/agent'; +import { getBackendLanguage } from '@/utils/backend-runtime'; import { buildSelectOptions } from '@/utils/component-util'; import { buildOptions, removeUselessFieldsFromValues } from '@/utils/form'; import { Edge, Node, XYPosition } from '@xyflow/react'; @@ -202,10 +203,10 @@ function transformObjectArrayToPureArray( : []; } -function transformParserParams(params: ParserFormSchemaType) { +export function transformParserParams(params: ParserFormSchemaType) { const setups = params.setups.reduce< Record - >((pre, cur) => { + >((pre, cur, index) => { if (cur.fileFormat) { let filteredSetup: Partial< ParserFormSchemaType['setups'][0] & { suffix: string[] } & { @@ -318,15 +319,26 @@ function transformParserParams(params: ParserFormSchemaType) { break; } - pre[cur.fileFormat] = filteredSetup; + pre[cur.fileFormat] = { + ...filteredSetup, + order_index: index, + } as any; } return pre; }, {}); + // The Go backend expects the setups map flattened into top-level params, + // while the Python backend reads them from the nested `setups` object. + // Default to the Python shape while the language probe is unresolved. + if (getBackendLanguage() === 'go') { + return { ...omit(params, ['setups']), ...setups }; + } return { ...params, setups }; } -function transformTokenChunkerParams(params: TokenChunkerFormSchemaType) { +export function transformTokenChunkerParams( + params: TokenChunkerFormSchemaType, +) { const { image_table_context_window, ...rest } = params; const imageTableContextWindow = Number(image_table_context_window || 0); return { @@ -349,7 +361,9 @@ function transformTokenChunkerParams(params: TokenChunkerFormSchemaType) { }; } -function transformTitleChunkerParams(params: TitleChunkerFormSchemaType) { +export function transformTitleChunkerParams( + params: TitleChunkerFormSchemaType, +) { const activeRules = (params.method === TitleChunkerMethod.Group ? params.groupRules @@ -379,7 +393,7 @@ function transformTitleChunkerParams(params: TitleChunkerFormSchemaType) { }; } -function transformExtractorParams(params: ExtractorFormSchemaType) { +export function transformExtractorParams(params: ExtractorFormSchemaType) { return { ...params, prompts: [{ content: params.prompts, role: 'user' }] }; } diff --git a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx index afd8459b37..9430a0f662 100644 --- a/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx +++ b/web/src/pages/dataset/dataset-setting/configuration/common-item.tsx @@ -19,6 +19,7 @@ import { Spin } from '@/components/ui/spin'; import { Switch } from '@/components/ui/switch'; import { ParseType } from '@/constants/knowledge'; import { useTranslate } from '@/hooks/common-hooks'; +import { useFetchBuiltinPipelines } from '@/hooks/use-agent-request'; import { cn } from '@/lib/utils'; import { history } from '@/utils/simple-history-util'; import { t } from 'i18next'; @@ -29,6 +30,7 @@ import { FieldValues, useFormContext, } from 'react-hook-form'; +import { useTranslation } from 'react-i18next'; import { useLocation } from 'react-router'; import { DataSetContext } from '..'; import { MetadataType } from '../../components/metedata/constant'; @@ -100,6 +102,61 @@ export function ChunkMethodItem(props: IProps) { ); } +export function BuiltinPipelineItem({ + line = 2, + name = 'parser_id', +}: { + line?: 1 | 2; + name?: string; +}) { + const { t } = useTranslation(); + const form = useFormContext(); + const { options: builtinPipelineOptions } = useFetchBuiltinPipelines(); + + return ( + ( + +
+ + {t('knowledgeConfiguration.builtIn')} + +
+ + + +
+
+
+
+ +
+ + )} + /> + ); +} + export const EmbeddingSelect = ({ isEdit, field, diff --git a/web/src/pages/dataset/setting/components/added-source-card.tsx b/web/src/pages/dataset/setting/components/added-source-card.tsx new file mode 100644 index 0000000000..f2cb11902c --- /dev/null +++ b/web/src/pages/dataset/setting/components/added-source-card.tsx @@ -0,0 +1,83 @@ +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { cn } from '@/lib/utils'; +import { + IDataSorceInfo, + IDataSourceBase, +} from '@/pages/user-setting/data-source/interface'; +import { Check } from 'lucide-react'; +import { useMemo } from 'react'; + +export type IAddedSourceCardProps = IDataSorceInfo & { + filterString: string; + list: IDataSourceBase[]; + selectedList: IDataSourceBase[]; + setSelectedList: (list: IDataSourceBase[]) => void; +}; +export const AddedSourceCard = (props: IAddedSourceCardProps) => { + const { + list: originList, + name, + icon, + filterString, + selectedList, + setSelectedList, + } = props; + + const list = useMemo(() => { + return originList.map((item) => { + const checked = selectedList?.some((i) => i.id === item.id) || false; + return { + ...item, + checked: checked, + }; + }); + }, [originList, selectedList]); + + const filterList = useMemo( + () => list.filter((item) => item.name.indexOf(filterString) > -1), + [filterString, list], + ); + + const onCheck = (item: IDataSourceBase & { checked: boolean }) => { + if (item.checked) { + setSelectedList(selectedList.filter((i) => i.id !== item.id)); + } else { + setSelectedList([...(selectedList || []), item]); + } + }; + return ( + <> + {filterList.length > 0 && ( + + + + {icon} + {name} + + + + {filterList.map((item) => ( +
{ + console.log('item--->', item); + onCheck(item); + }} + > +
{item.name}
+
+ {item.checked && ( + + )} +
+
+ ))} + + + )} + + ); +}; diff --git a/web/src/pages/dataset/setting/components/link-data-source-modal.tsx b/web/src/pages/dataset/setting/components/link-data-source-modal.tsx new file mode 100644 index 0000000000..9cde80eab0 --- /dev/null +++ b/web/src/pages/dataset/setting/components/link-data-source-modal.tsx @@ -0,0 +1,85 @@ +import { Button } from '@/components/ui/button'; +import { SearchInput } from '@/components/ui/input'; +import { Modal } from '@/components/ui/modal/modal'; +import { IConnector } from '@/interfaces/database/dataset'; +import { useListDataSource } from '@/pages/user-setting/data-source/hooks'; +import { IDataSourceBase } from '@/pages/user-setting/data-source/interface'; +import { t } from 'i18next'; +import { useEffect, useState } from 'react'; +import { AddedSourceCard } from './added-source-card'; + +const LinkDataSourceModal = ({ + selectedList, + open, + setOpen, + onSubmit, +}: { + selectedList: IConnector[]; + open: boolean; + setOpen: (open: boolean) => void; + onSubmit?: (list: IDataSourceBase[] | undefined) => void; +}) => { + const [list, setList] = useState(); + const [fileterString, setFileterString] = useState(''); + + useEffect(() => { + setList(selectedList); + }, [selectedList]); + + const { categorizedList } = useListDataSource(); + const handleFormSubmit = (values: any) => { + console.log(values, selectedList); + onSubmit?.(list); + }; + return ( + { + setList(selectedList); + }} + onOpenChange={setOpen} + showfooter={false} + > +
+ setFileterString(e.target.value)} + /> +
+ {categorizedList.map((item, index) => ( + setList(list)} + filterString={fileterString} + {...item} + /> + ))} +
+
+ + +
+
+ + ); +}; +export default LinkDataSourceModal; diff --git a/web/src/pages/dataset/setting/components/link-data-source.tsx b/web/src/pages/dataset/setting/components/link-data-source.tsx new file mode 100644 index 0000000000..fe12c61836 --- /dev/null +++ b/web/src/pages/dataset/setting/components/link-data-source.tsx @@ -0,0 +1,213 @@ +import { IconFontFill } from '@/components/icon-font'; +import { Button } from '@/components/ui/button'; +import { Switch } from '@/components/ui/switch'; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks'; +import { IConnector } from '@/interfaces/database/dataset'; +import { delSourceModal } from '@/pages/user-setting/data-source/component/delete-source-modal'; +import { useDataSourceInfo } from '@/pages/user-setting/data-source/constant'; +import { useDataSourceRebuild } from '@/pages/user-setting/data-source/hooks'; +import { IDataSourceBase } from '@/pages/user-setting/data-source/interface'; +import { Link, Settings, Unlink } from 'lucide-react'; +import { useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import LinkDataSourceModal from './link-data-source-modal'; + +export type IDataSourceNodeProps = IConnector & { + icon: React.ReactNode; +}; + +export interface ILinkDataSourceProps { + data?: IConnector[]; + handleLinkOrEditSubmit?: (data: IDataSourceBase[] | undefined) => void; + unbindFunc?: (item: DataSourceItemProps) => void; + handleAutoParse?: (option: { + source_id: string; + isAutoParse: boolean; + }) => void; +} + +interface DataSourceItemProps extends IDataSourceNodeProps { + openLinkModalFunc?: (open: boolean, data?: IDataSourceNodeProps) => void; + unbindFunc?: (item: DataSourceItemProps) => void; + handleAutoParse?: (option: { + source_id: string; + isAutoParse: boolean; + }) => void; +} + +const DataSourceItem = (props: DataSourceItemProps) => { + const { dataSourceInfo } = useDataSourceInfo(); + const { t } = useTranslation(); + const { id, name, icon, source, auto_parse, unbindFunc, handleAutoParse } = + props; + + const { navigateToDataSourceDetail } = useNavigatePage(); + const { handleRebuild } = useDataSourceRebuild(); + const toDetail = (id: string) => { + navigateToDataSourceDetail(id); + }; + + return ( +
+
+
{icon}
+
+ {dataSourceInfo[source].name} +
+
{name}
+
+
+
+
+ {t('knowledgeConfiguration.autoParse')} +
+ { + handleAutoParse?.({ source_id: id, isAutoParse }); + }} + className="w-8 h-4" + /> +
+ + + + + + {t('knowledgeConfiguration.rebuildTip')} + + + + <> + + +
+
+ ); +}; + +const LinkDataSource = (props: ILinkDataSourceProps) => { + const { + data, + handleLinkOrEditSubmit: submit, + unbindFunc, + handleAutoParse, + } = props; + const { t } = useTranslation(); + const { dataSourceInfo } = useDataSourceInfo(); + const [openLinkModal, setOpenLinkModal] = useState(false); + + const pipelineNode: IDataSourceNodeProps[] = useMemo(() => { + if (data && data.length > 0) { + return data.map((item) => { + return { + ...item, + id: item?.id, + name: item?.name, + icon: + dataSourceInfo[item?.source as keyof typeof dataSourceInfo]?.icon || + '', + } as IDataSourceNodeProps; + }); + } + return []; + }, [data, dataSourceInfo]); + + const openLinkModalFunc = (open: boolean, data?: IDataSourceNodeProps) => { + console.log('open', open, data); + setOpenLinkModal(open); + }; + + const handleLinkOrEditSubmit = (data: IDataSourceBase[] | undefined) => { + console.log('handleLinkOrEditSubmit', data); + submit?.(data); + setOpenLinkModal(false); + }; + + return ( +
+
+
+ {t('knowledgeConfiguration.dataSource')} +
+
+
+ {t('knowledgeConfiguration.linkSourceSetTip')} +
+ +
+
+
+ {pipelineNode.map( + (item) => + item.id && ( + + ), + )} +
+ { + openLinkModalFunc(open); + }} + onSubmit={handleLinkOrEditSubmit} + /> +
+ ); +}; +export default LinkDataSource; diff --git a/web/src/pages/dataset/setting/embedding-model-form-field.tsx b/web/src/pages/dataset/setting/embedding-model-form-field.tsx new file mode 100644 index 0000000000..e472c594e7 --- /dev/null +++ b/web/src/pages/dataset/setting/embedding-model-form-field.tsx @@ -0,0 +1,130 @@ +import { ModelTreeSelect, ModelTypeMap } from '@/components/model-tree-select'; +import { + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Spin } from '@/components/ui/spin'; +import { useTranslate } from '@/hooks/common-hooks'; +import { cn } from '@/lib/utils'; +import { useMemo, useState } from 'react'; +import { FieldValues, useFormContext } from 'react-hook-form'; +import { useHandleKbEmbedding, useHasParsedDocument } from './hooks'; + +interface IProps { + line?: 1 | 2; + isEdit?: boolean; +} + +export const EmbeddingSelect = ({ + isEdit, + field, + name, + disabled = false, + testId, + ownerTenantId, +}: { + isEdit: boolean; + field: FieldValues; + name?: string; + disabled?: boolean; + testId?: string; + ownerTenantId?: string; +}) => { + const { t } = useTranslate('knowledgeConfiguration'); + const form = useFormContext(); + const { handleChange } = useHandleKbEmbedding(); + + const oldValue = useMemo(() => { + const embdStr = form.getValues(name || 'embedding_model'); + return embdStr || ''; + }, [form, name]); + const [loading, setLoading] = useState(false); + return ( + + { + field.onChange(value); + if (isEdit && disabled) { + setLoading(true); + const res = await handleChange({ + embed_id: value, + }); + if (res.code !== 0) { + field.onChange(oldValue); + } + setLoading(false); + } + }} + ownerTenantId={ownerTenantId} + disabled={disabled && !isEdit} + value={field.value} + placeholder={t('embeddingModelPlaceholder')} + testId={testId} + /> + + ); +}; + +export function EmbeddingModelItem({ + line = 1, + isEdit, + ownerTenantId, +}: IProps & { ownerTenantId?: string }) { + const { t } = useTranslate('knowledgeConfiguration'); + const form = useFormContext(); + const disabled = useHasParsedDocument(isEdit); + return ( + <> + ( + +
+ + {t('embeddingModel')} + +
+ + + +
+
+
+
+ +
+ + )} + /> + + ); +} diff --git a/web/src/pages/dataset/setting/form-schema.ts b/web/src/pages/dataset/setting/form-schema.ts new file mode 100644 index 0000000000..3f39737dc8 --- /dev/null +++ b/web/src/pages/dataset/setting/form-schema.ts @@ -0,0 +1,41 @@ +import { ParseType } from '@/constants/knowledge'; +import { t } from 'i18next'; +import { z } from 'zod'; + +export const formSchema = z + .object({ + parse_type: z.nativeEnum(ParseType).optional(), + pipeline_id: z.string().optional(), + pipeline_name: z.string().optional(), + pipeline_avatar: z.string().optional(), + name: z.string().min(1, { + message: 'Username must be at least 2 characters.', + }), + description: z.string().optional(), + parser_id: z.string().optional(), + avatar: z.any().nullish(), + permission: z.string().optional(), + embedding_model: z.string(), + pagerank: z.number(), + parser_config: z.record(z.string(), z.any()).optional(), + connectors: z + .array( + z.object({ + id: z.string().optional(), + name: z.string().optional(), + source: z.string().optional(), + ststus: z.string().optional(), + auto_parse: z.string().optional(), + }), + ) + .optional(), + }) + .superRefine((data, ctx) => { + if (data.parse_type === ParseType.Pipeline && !data.pipeline_id) { + ctx.addIssue({ + path: ['pipeline_id'], + message: t('common.pleaseSelect'), + code: 'custom', + }); + } + }); diff --git a/web/src/pages/dataset/setting/general-form.tsx b/web/src/pages/dataset/setting/general-form.tsx new file mode 100644 index 0000000000..8972bd27f7 --- /dev/null +++ b/web/src/pages/dataset/setting/general-form.tsx @@ -0,0 +1,110 @@ +import { AvatarUpload } from '@/components/avatar-upload'; +import PageRankFormField from '@/components/page-rank-form-field'; +import { + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import { useFormContext } from 'react-hook-form'; +import { useTranslation } from 'react-i18next'; +import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context'; +import { EmbeddingModelItem } from './embedding-model-form-field'; +import { PermissionFormField } from './permission-form-field'; + +export function GeneralForm() { + const form = useFormContext(); + const { t } = useTranslation(); + + return ( + <> + ( + +
+ + * + {t('common.name')} + + + + +
+
+
+ +
+ + )} + /> + ( + +
+ + {t('setting.avatar')} + + + + +
+
+
+ +
+ + )} + /> + { + // null initialize empty string + if (typeof field.value === 'object' && !field.value) { + form.setValue('description', ''); + } + return ( + +
+ + {t('flow.description')} + + + + +
+
+
+ +
+ + ); + }} + /> + + + + + ); +} diff --git a/web/src/pages/dataset/setting/hooks.ts b/web/src/pages/dataset/setting/hooks.ts new file mode 100644 index 0000000000..bc00d6bc89 --- /dev/null +++ b/web/src/pages/dataset/setting/hooks.ts @@ -0,0 +1,280 @@ +import { ParseType } from '@/constants/knowledge'; +import { useFetchPipelineDslByPipelineId } from '@/hooks/use-agent-request'; +import { + useFetchDatasetPipelineConfiguration, + useUpdateKnowledge, +} from '@/hooks/use-knowledge-request'; +import { RAGFlowNodeType } from '@/interfaces/database/agent'; +import { IConnector } from '@/interfaces/database/dataset'; +import { useDataSourceInfo } from '@/pages/user-setting/data-source/constant'; +import { checkEmbedding } from '@/services/knowledge-service'; +import { pick } from 'lodash'; +import { + Dispatch, + SetStateAction, + useCallback, + useEffect, + useMemo, + useState, +} from 'react'; +import { UseFormReturn } from 'react-hook-form'; +import { useParams, useSearchParams } from 'react-router'; +import { z } from 'zod'; +import { formSchema } from './form-schema'; +import { + buildPipelineOperatorNodes, + getOperatorType, + transformApiConfigToForm, + transformFormConfigToApi, +} from './utils'; + +export function useHasParsedDocument(isEdit?: boolean) { + const { data: knowledgeDetails } = useFetchDatasetPipelineConfiguration({ + isEdit, + }); + return knowledgeDetails.chunk_count > 0; +} + +export const useHandleKbEmbedding = () => { + const { id } = useParams(); + const [searchParams] = useSearchParams(); + const knowledgeBaseId = searchParams.get('id') || id; + const handleChange = useCallback( + async ({ embed_id }: { embed_id: string }) => { + const res = await checkEmbedding(knowledgeBaseId || '', { + embd_id: embed_id, + }); + return res.data; + }, + [knowledgeBaseId], + ); + return { + handleChange, + }; +}; + +export const useFetchDatasetSettingOnMount = ( + form: UseFormReturn>, +) => { + const { data: knowledgeDetails, loading } = + useFetchDatasetPipelineConfiguration(); + const { dataSourceInfo } = useDataSourceInfo(); + + const sourceData = useMemo(() => { + return (knowledgeDetails?.connectors ?? []).map( + (connector: IConnector) => ({ + ...connector, + icon: + dataSourceInfo[connector.source as keyof typeof dataSourceInfo] + ?.icon || '', + }), + ); + }, [knowledgeDetails?.connectors, dataSourceInfo]); + + useEffect(() => { + const parserConfig = knowledgeDetails.parser_config as + | Record + | undefined; + let formParserConfig: Record | undefined = parserConfig; + + // Convert parser_config to form format if in pipeline mode + if ( + parserConfig && + typeof parserConfig === 'object' && + !Array.isArray(parserConfig) + ) { + const keys = Object.keys(parserConfig); + const hasPipelineKeys = keys.some((key) => key.includes(':')); + if (hasPipelineKeys) { + formParserConfig = {}; + for (const [operatorId, config] of Object.entries(parserConfig)) { + const operatorType = getOperatorType(operatorId); + formParserConfig[operatorId] = transformApiConfigToForm( + operatorType, + config as Record, + ); + } + } + } + + const formValues = { + ...pick(knowledgeDetails, [ + 'description', + 'name', + 'permission', + 'connectors', + 'pagerank', + 'avatar', + 'pipeline_id', + 'pipeline_name', + 'pipeline_avatar', + 'parser_id', + ]), + embedding_model: knowledgeDetails.embedding_model, + connectors: sourceData, + parse_type: knowledgeDetails.pipeline_id + ? ParseType.Pipeline + : ParseType.BuiltIn, + parser_config: formParserConfig, + } as z.infer; + form.reset(formValues); + }, [form, knowledgeDetails, sourceData]); + + return { knowledgeDetails, loading, sourceData }; +}; + +export const usePipelineOperatorNodes = ( + pipelineId?: string, + pipelineParserConfig?: Record, + isBuiltin = false, +) => { + const { dsl, loading } = useFetchPipelineDslByPipelineId( + pipelineId, + isBuiltin, + ); + + const operatorNodes = useMemo(() => { + return buildPipelineOperatorNodes(dsl, pipelineParserConfig); + }, [dsl, pipelineParserConfig]); + + return { operatorNodes, loading }; +}; + +export const useSaveDatasetSetting = () => { + const { saveKnowledgeConfiguration, loading } = useUpdateKnowledge(); + + const handleSave = useCallback( + async (values: z.infer) => { + const payload = { ...values }; + + // Apply forward transforms to parser_config if in pipeline mode + if (payload.parser_config) { + const transformedConfig: Record = {}; + for (const [operatorId, config] of Object.entries( + payload.parser_config, + )) { + const operatorType = getOperatorType(operatorId); + transformedConfig[operatorId] = transformFormConfigToApi( + operatorType, + config as Record, + ); + } + payload.parser_config = transformedConfig; + } + + if (payload.parse_type === ParseType.BuiltIn) { + payload.pipeline_id = undefined; + } else { + payload.parser_id = undefined; + } + return saveKnowledgeConfiguration(payload); + }, + [saveKnowledgeConfiguration], + ); + + return { handleSave, loading }; +}; + +export const useActiveTab = (operatorNodes: RAGFlowNodeType[]) => { + const [activeTab, setActiveTab] = useState(''); + + useEffect(() => { + if (operatorNodes.length > 0) { + const firstTab = + (operatorNodes[0].data as Record)?.operatorId || + operatorNodes[0].data?.label || + ''; + const validTabs = operatorNodes.map( + (node) => + (node.data as Record)?.operatorId || + node.data?.label || + '', + ); + setActiveTab((prev) => (validTabs.includes(prev) ? prev : firstTab)); + } else { + setActiveTab(''); + } + }, [operatorNodes]); + + return { activeTab, setActiveTab }; +}; + +export const usePipelineDataList = (sourceData: any[] | undefined) => { + const { dataSourceInfo } = useDataSourceInfo(); + + return useMemo( + () => + sourceData?.map((item) => ({ + ...item, + icon: + dataSourceInfo[item.source as keyof typeof dataSourceInfo]?.icon || + '', + })), + [sourceData, dataSourceInfo], + ); +}; + +export const useConnectorHandlers = ( + form: UseFormReturn, + sourceData?: any[], + setSourceData?: Dispatch>, +) => { + const { dataSourceInfo } = useDataSourceInfo(); + + const handleLinkOrEditSubmit = useCallback( + (data: any[] | undefined) => { + if (data) { + const connectors = data.map((connector) => ({ + ...connector, + auto_parse: (connector as IConnector).auto_parse === '0' ? '0' : '1', + icon: + dataSourceInfo[connector.source as keyof typeof dataSourceInfo] + ?.icon || '', + })); + setSourceData?.(connectors); + form.setValue('connectors', connectors || []); + } + }, + [dataSourceInfo, form, setSourceData], + ); + + const unbindFunc = useCallback( + (data: any) => { + if (data) { + const connectors = sourceData?.filter( + (connector) => connector.id !== data.id, + ); + setSourceData?.(connectors); + form.setValue('connectors', connectors || []); + } + }, + [sourceData, form, setSourceData], + ); + + const handleAutoParse = useCallback( + ({ + source_id, + isAutoParse, + }: { + source_id: string; + isAutoParse: boolean; + }) => { + if (source_id) { + const connectors = sourceData?.map((connector) => { + if (connector.id === source_id) { + return { + ...connector, + auto_parse: isAutoParse ? '1' : '0', + }; + } + return connector; + }); + setSourceData?.(connectors); + form.setValue('connectors', connectors || []); + } + }, + [sourceData, form, setSourceData], + ); + + return { handleLinkOrEditSubmit, unbindFunc, handleAutoParse }; +}; diff --git a/web/src/pages/dataset/setting/index.tsx b/web/src/pages/dataset/setting/index.tsx new file mode 100644 index 0000000000..dd5833bd2a --- /dev/null +++ b/web/src/pages/dataset/setting/index.tsx @@ -0,0 +1,225 @@ +import { DataFlowSelect } from '@/components/data-pipeline-select'; +import { Button, ButtonLoading } from '@/components/ui/button'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import Divider from '@/components/ui/divider'; +import { Form } from '@/components/ui/form'; +import { FormLayout } from '@/constants/form'; +import { ParseType } from '@/constants/knowledge'; +import { + BuiltinPipelineItem, + ParseTypeItem, +} from '@/pages/dataset/dataset-setting/configuration/common-item'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { useCallback, useEffect, useState } from 'react'; +import { useForm, useWatch } from 'react-hook-form'; +import { useTranslation } from 'react-i18next'; +import { z } from 'zod'; +import LinkDataSource, { + IDataSourceNodeProps, +} from './components/link-data-source'; +import { formSchema } from './form-schema'; +import { GeneralForm } from './general-form'; +import { + useActiveTab, + useConnectorHandlers, + useFetchDatasetSettingOnMount, + usePipelineDataList, + usePipelineOperatorNodes, + useSaveDatasetSetting, +} from './hooks'; +import PipelineOperatorTabs from './pipeline-operator-tabs'; + +export default function DatasetSetting() { + const { t } = useTranslation(); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + parse_type: ParseType.BuiltIn, + pipeline_id: '', + pipeline_name: '', + pipeline_avatar: '', + parser_id: '', + parser_config: {}, + name: '', + description: '', + avatar: null, + permission: '', + embedding_model: '', + pagerank: 0, + connectors: [], + }, + }); + + const { + knowledgeDetails, + loading: datasetSettingLoading, + sourceData, + } = useFetchDatasetSettingOnMount(form); + const { handleSave, loading: saveLoading } = useSaveDatasetSetting(); + + const [sourceDataState, setSourceDataState] = + useState(); + + useEffect(() => { + setSourceDataState(sourceData); + }, [sourceData]); + + const parseType = useWatch({ + control: form.control, + name: 'parse_type', + defaultValue: ParseType.BuiltIn, + }); + const pipelineId = useWatch({ + control: form.control, + name: 'pipeline_id', + defaultValue: '', + }); + + const builtinPipelineId = useWatch({ + control: form.control, + name: 'parser_id', + defaultValue: '', + }); + + const pipelineParserConfig = knowledgeDetails?.parser_config as + | Record + | undefined; + + const { operatorNodes } = usePipelineOperatorNodes( + parseType === ParseType.Pipeline ? pipelineId : builtinPipelineId, + pipelineParserConfig, + parseType === ParseType.BuiltIn, + ); + + const { activeTab, setActiveTab } = useActiveTab(operatorNodes); + + useEffect(() => { + if (parseType === ParseType.BuiltIn) { + form.setValue('pipeline_id', ''); + form.setValue('pipeline_name', ''); + form.setValue('pipeline_avatar', ''); + } + }, [parseType, form]); + + const handleSubmit = useCallback( + async (data: z.infer) => { + await handleSave(data); + }, + [handleSave], + ); + + const { handleLinkOrEditSubmit, unbindFunc, handleAutoParse } = + useConnectorHandlers(form, sourceDataState, setSourceDataState); + + const handleOperatorValuesChange = useCallback( + (operatorId: string, values: any) => { + const currentParserConfig = form.getValues('parser_config') || {}; + form.setValue('parser_config', { + ...currentParserConfig, + [operatorId]: values, + }); + }, + [form], + ); + + const pipelineDataList = usePipelineDataList(sourceDataState); + + const showOperatorTabs = + operatorNodes.length > 0 && + ((parseType === ParseType.Pipeline && !!pipelineId) || + (parseType === ParseType.BuiltIn && !!builtinPipelineId)); + + return ( +
+ + +
+ + {t('knowledgeDetails.nextConfiguration')} + + + {t('knowledgeConfiguration.titleDescription')} + +
+ + + +
+ +
+
+
+ {t('knowledgeConfiguration.baseInfo')} +
+ + + +
+ {t('knowledgeConfiguration.dataPipeline')} +
+ + {parseType === ParseType.BuiltIn && ( + + )} + {parseType === ParseType.Pipeline && ( + + )} + {showOperatorTabs && ( + + )} + + + +
+
+ +
+ + + + {t('knowledgeConfiguration.save')} + +
+
+ + + +
+ ); +} diff --git a/web/src/pages/dataset/setting/permission-form-field.tsx b/web/src/pages/dataset/setting/permission-form-field.tsx new file mode 100644 index 0000000000..33336a8241 --- /dev/null +++ b/web/src/pages/dataset/setting/permission-form-field.tsx @@ -0,0 +1,30 @@ +import { SelectWithSearch } from '@/components/originui/select-with-search'; +import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { PermissionRole } from '@/constants/permission'; +import { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +export function PermissionFormField() { + const { t } = useTranslation(); + const teamOptions = useMemo(() => { + return Object.values(PermissionRole).map((x) => ({ + label: t('knowledgeConfiguration.' + x), + value: x, + })); + }, [t]); + + return ( + + + + ); +} diff --git a/web/src/pages/dataset/setting/pipeline-operator-form.tsx b/web/src/pages/dataset/setting/pipeline-operator-form.tsx new file mode 100644 index 0000000000..4f2d700a46 --- /dev/null +++ b/web/src/pages/dataset/setting/pipeline-operator-form.tsx @@ -0,0 +1,77 @@ +import { Operator } from '@/constants/agent'; +import { RAGFlowNodeType } from '@/interfaces/database/agent'; +import { memo, useCallback } from 'react'; +import ExtractorForm from '../../agent/form/extractor-form'; +import ParserForm from '../../agent/form/parser-form'; +import TitleChunkerForm from '../../agent/form/title-chunker-form'; +import TokenChunkerForm from '../../agent/form/token-chunker-form'; +import TokenizerForm from '../../agent/form/tokenizer-form'; +import { getOperatorType } from './utils'; + +type PipelineOperatorFormProps = { + node: RAGFlowNodeType; + onValuesChange?: (values: any) => void; +}; + +const PipelineOperatorForm = ({ + node, + onValuesChange, +}: PipelineOperatorFormProps) => { + const operatorType = getOperatorType( + (node.data as Record)?.operatorId || node.data?.label || '', + ); + + const handleValuesChange = useCallback( + (values: any) => { + onValuesChange?.(values); + }, + [onValuesChange], + ); + + switch (operatorType) { + case Operator.Parser: + return ( + + ); + case Operator.TokenChunker: + return ( + + ); + case Operator.TitleChunker: + return ( + + ); + case Operator.Extractor: + return ( + + ); + case Operator.Tokenizer: + return ( + + ); + default: + return null; + } +}; + +export default memo(PipelineOperatorForm); diff --git a/web/src/pages/dataset/setting/pipeline-operator-tabs.tsx b/web/src/pages/dataset/setting/pipeline-operator-tabs.tsx new file mode 100644 index 0000000000..0bd8a44478 --- /dev/null +++ b/web/src/pages/dataset/setting/pipeline-operator-tabs.tsx @@ -0,0 +1,66 @@ +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { RAGFlowNodeType } from '@/interfaces/database/agent'; +import { memo, useCallback } from 'react'; +import PipelineOperatorForm from './pipeline-operator-form'; + +type PipelineOperatorTabsProps = { + nodes: RAGFlowNodeType[]; + value: string; + onValueChange: (value: string) => void; + onOperatorValuesChange: (operatorId: string, values: any) => void; +}; + +const PipelineOperatorTabs = ({ + nodes, + value, + onValueChange, + onOperatorValuesChange, +}: PipelineOperatorTabsProps) => { + const getOperatorId = useCallback((node: RAGFlowNodeType) => { + return ( + (node.data as Record)?.operatorId || node.data?.label || '' + ); + }, []); + + const getTabValue = useCallback( + (node: RAGFlowNodeType, index: number) => { + return getOperatorId(node) || String(index); + }, + [getOperatorId], + ); + + const handleValuesChange = useCallback( + (node: RAGFlowNodeType) => (values: any) => { + onOperatorValuesChange(getOperatorId(node), values); + }, + [getOperatorId, onOperatorValuesChange], + ); + + return ( + + + {nodes.map((node, index) => { + const tabValue = getTabValue(node, index); + return ( + + {node.data?.name || node.data?.label || tabValue} + + ); + })} + + {nodes.map((node, index) => { + const tabValue = getTabValue(node, index); + return ( + + + + ); + })} + + ); +}; + +export default memo(PipelineOperatorTabs); diff --git a/web/src/pages/dataset/setting/utils.ts b/web/src/pages/dataset/setting/utils.ts new file mode 100644 index 0000000000..67e8b7bd57 --- /dev/null +++ b/web/src/pages/dataset/setting/utils.ts @@ -0,0 +1,345 @@ +import { Operator } from '@/constants/agent'; +import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent'; +import { + initialExtractorValues, + initialParserValues, + initialTitleChunkerValues, + initialTokenChunkerValues, + initialTokenizerValues, +} from '@/pages/agent/constant/pipeline'; +import { + transformExtractorParams, + transformParserParams, + transformTitleChunkerParams, + transformTokenChunkerParams, +} from '@/pages/agent/utils'; +import { cloneDeep } from 'lodash'; + +export const FileNodeId = 'File'; + +export function getOperatorType(operatorId: string): Operator { + return (operatorId.split(':')[0] || operatorId) as Operator; +} + +export function transformParserConfigSetups( + setups: Record | undefined, +): any[] { + if (!setups || typeof setups !== 'object') { + return []; + } + + return Object.entries(setups) + .map(([fileFormat, config]) => ({ + fileFormat, + ...config, + })) + .sort((a, b) => (a.order_index ?? Infinity) - (b.order_index ?? Infinity)); +} + +function transformLevelsToRules( + levels: any[], +): Array<{ levels: Array<{ expression: string }> }> { + if (!Array.isArray(levels)) return []; + return levels + .map((levelGroup) => { + if (Array.isArray(levelGroup)) { + const filteredExpressions = levelGroup.filter( + (expr: string) => expr && expr.trim() !== '', + ); + if (filteredExpressions.length === 0) return null; + return { + levels: filteredExpressions.map((expression: string) => ({ + expression, + })), + }; + } + return { levels: [{ expression: '' }] }; + }) + .filter((rule) => rule !== null); +} + +/** + * Converts Extractor config from API/DSL format to form format. + * DSL: { prompts: [{ content: "text", role: "user" }] } + * Form: { prompts: "text" } + */ +function transformExtractorConfigToForm( + config: Record | undefined, +): Record { + if (!config) return {}; + + const result = { ...config }; + if (Array.isArray(config.prompts) && config.prompts.length > 0) { + result.prompts = config.prompts[0]?.content ?? ''; + } + return result; +} + +/** + * Converts TokenChunker config from API/DSL format to form format. + * DSL: { delimiters: ["\n"], overlapped_percent: 0.1, table_context_size: 10, image_context_size: 20 } + * Form: { delimiters: [{ value: "\n" }], overlapped_percent: 10, image_table_context_window: 15, enable_children: false } + */ +function transformTokenChunkerConfigToForm( + config: Record | undefined, +): Record { + if (!config) return {}; + + const result = { ...config }; + + // Convert string array delimiters to object array + if (Array.isArray(config.delimiters)) { + result.delimiters = config.delimiters.map((d: string) => ({ value: d })); + } + if (Array.isArray(config.children_delimiters)) { + result.children_delimiters = config.children_delimiters.map( + (d: string) => ({ value: d }), + ); + } + + // Convert overlapped_percent from 0-1 scale to 0-100 scale + if (typeof config.overlapped_percent === 'number') { + result.overlapped_percent = Math.round(config.overlapped_percent * 100); + } + + // Merge table_context_size and image_context_size into image_table_context_window + const tableSize = Number(config.table_context_size ?? 0); + const imageSize = Number(config.image_context_size ?? 0); + result.image_table_context_window = Math.max(tableSize, imageSize); + + // Derive delimiter_mode from data + if (config.delimiter_mode === undefined) { + const hasDelimiters = + Array.isArray(config.delimiters) && config.delimiters.length > 0; + result.delimiter_mode = hasDelimiters ? 'delimiter' : 'token_size'; + } + + // Derive enable_children from presence of children_delimiters + if (config.enable_children === undefined) { + result.enable_children = + Array.isArray(config.children_delimiters) && + config.children_delimiters.length > 0; + } + + // Clean up DSL-only fields not in form schema + delete result.table_context_size; + delete result.image_context_size; + + return result; +} + +/** + * Converts TitleChunker config from API/DSL format to form format. + * DSL: { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading } + * Form: { method, hierarchyHierarchy, hierarchyGroup, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules } + */ +function transformTitleChunkerConfigToForm( + config: Record | undefined, +): Record { + if (!config) return {}; + + const result = { ...config }; + const method = config.method ?? 'hierarchy'; + + // Convert legacy `hierarchy` (single field) to split fields + let hierarchy = config.hierarchy; + if (typeof hierarchy === 'number') { + hierarchy = String(hierarchy); + } + + if (method === 'hierarchy') { + result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3'; + result.hierarchyGroup = config.hierarchyGroup || '0'; + } else { + result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3'; + result.hierarchyGroup = config.hierarchyGroup || hierarchy || '0'; + } + + // Convert `levels` to `hierarchyRules`/`groupRules` + if (config.levels && Array.isArray(config.levels) && !config.hierarchyRules) { + result.hierarchyRules = transformLevelsToRules(config.levels); + } + if (config.levels && Array.isArray(config.levels) && !config.groupRules) { + result.groupRules = transformLevelsToRules(config.levels); + } + + // Clean up DSL-only fields + delete result.hierarchy; + delete result.levels; + delete result.rules; + + return result; +} + +/** + * Tokenizer is a passthrough — both API and form formats are identical. + */ +function transformTokenizerConfigToForm( + config: Record | undefined, +): Record { + return config ?? {}; +} + +/** + * Dispatches reverse transform by operator type. + * Converts DSL-format config to form-format config. + */ +export function transformApiConfigToForm( + operatorType: string, + config: Record | undefined, +): Record { + switch (operatorType) { + case Operator.Parser: + return { setups: transformParserConfigSetups(config) }; + case Operator.Extractor: + return transformExtractorConfigToForm(config); + case Operator.Tokenizer: + return transformTokenizerConfigToForm(config); + case Operator.TokenChunker: + return transformTokenChunkerConfigToForm(config); + case Operator.TitleChunker: + return transformTitleChunkerConfigToForm(config); + default: + return config ?? {}; + } +} + +/** + * Dispatches forward transform by operator type. + * Converts form-format config to DSL format for saving. + */ +export function transformFormConfigToApi( + operatorType: string, + config: Record | undefined, +): Record { + if (!config) return {}; + + switch (operatorType) { + case Operator.Parser: + return transformParserParams(config as any); + case Operator.Extractor: + return transformExtractorParams(config as any); + case Operator.Tokenizer: + return config; // passthrough for Tokenizer + case Operator.TokenChunker: + return transformTokenChunkerParams(config as any); + case Operator.TitleChunker: + return transformTitleChunkerParams(config as any); + default: + return config; + } +} + +function normalizeOperatorForm( + operatorId: string, + rawForm: Record | undefined, +): Record { + const operatorType = getOperatorType(operatorId); + + switch (operatorType) { + case Operator.Parser: { + return { + ...cloneDeep(initialParserValues), + ...rawForm, + setups: rawForm?.setups?.length + ? rawForm.setups + : cloneDeep(initialParserValues.setups), + }; + } + case Operator.TitleChunker: + return { + ...cloneDeep(initialTitleChunkerValues), + ...rawForm, + }; + case Operator.TokenChunker: + return { + ...cloneDeep(initialTokenChunkerValues), + ...rawForm, + }; + case Operator.Extractor: + return { + ...cloneDeep(initialExtractorValues), + ...rawForm, + }; + case Operator.Tokenizer: + return { + ...cloneDeep(initialTokenizerValues), + ...rawForm, + }; + default: + return rawForm ?? {}; + } +} + +export function buildOperatorNode( + dslNode: RAGFlowNodeType, + pipelineParserConfig: Record = {}, +): RAGFlowNodeType { + const operatorId = dslNode.id; + const operatorType = getOperatorType(operatorId); + + // DSL-format config from API's parser_config + const configFromApi = operatorId + ? pipelineParserConfig[operatorId] + : undefined; + + // Form-format config from DSL graph + const configFromDsl = dslNode.data?.form; + + // Convert API config to form format, then merge (DSL template is baseline, API overrides) + const convertedApiConfig = transformApiConfigToForm( + operatorType, + configFromApi, + ); + const rawForm = { + ...configFromDsl, // template defaults from DSL (form format) + ...convertedApiConfig, // user overrides from API (now also form format) + }; + + return { + ...dslNode, + id: '', + data: { + ...dslNode.data, + label: operatorType, + operatorId, + form: normalizeOperatorForm(operatorId, rawForm), + }, + }; +} + +export function buildPipelineOperatorNodes( + dsl?: DSL, + pipelineParserConfig: Record = {}, +): RAGFlowNodeType[] { + if (!dsl?.graph?.nodes || !dsl?.graph?.edges) { + return []; + } + + // Build source → target map from edges (pipeline is linear) + const sourceToTarget = new Map(); + for (const edge of dsl.graph.edges) { + sourceToTarget.set(edge.source, edge.target); + } + + // Follow the chain starting from File, collecting node IDs in order + const orderedIds: string[] = []; + let currentId: string | undefined = FileNodeId; + while (currentId) { + orderedIds.push(currentId); + currentId = sourceToTarget.get(currentId); + } + + // Build a lookup from node ID → node + const nodeById = new Map(); + for (const node of dsl.graph.nodes) { + nodeById.set(node.id, node); + } + + // Map ordered IDs to nodes, excluding File + return orderedIds + .filter((id) => id !== FileNodeId) + .map((id) => nodeById.get(id)) + .filter((node): node is RAGFlowNodeType => node !== undefined) + .map((node) => buildOperatorNode(node, pipelineParserConfig)); +} diff --git a/web/src/pages/dataset/sidebar/index.tsx b/web/src/pages/dataset/sidebar/index.tsx index 6df9c2f9f3..3ba4ab890e 100644 --- a/web/src/pages/dataset/sidebar/index.tsx +++ b/web/src/pages/dataset/sidebar/index.tsx @@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next'; import { LucideBookText, + LucideCog, LucideFolderOpen, LucideLogs, LucideSettings, @@ -18,6 +19,7 @@ import { formatPureDate } from '@/utils/date'; import { IDataset } from '@/interfaces/database/dataset'; import { useParams } from 'react-router'; +import { getBackendLanguage } from '@/utils/backend-runtime'; type PropType = { refreshCount?: number; @@ -46,11 +48,24 @@ export function SideBar({ dataset: data }: PropType) { label: t(`knowledgeDetails.overview`), key: Routes.DataSetOverview, }, - { - icon: , - label: t(`knowledgeDetails.configuration`), - key: Routes.DataSetSetting, - }, + ...(getBackendLanguage() === 'python' + ? [ + { + icon: , + label: t(`knowledgeDetails.configuration`), + key: Routes.DataSetSetting, + }, + ] + : []), + ...(getBackendLanguage() === 'go' + ? [ + { + icon: , + label: t(`knowledgeDetails.configuration`), + key: Routes.DataSetSettingNext, + }, + ] + : []), { icon: , label: t(`knowledgeDetails.compilation`), diff --git a/web/src/pages/datasets/dataset-creating-dialog.tsx b/web/src/pages/datasets/dataset-creating-dialog.tsx index f43ce5e924..964fb61fd4 100644 --- a/web/src/pages/datasets/dataset-creating-dialog.tsx +++ b/web/src/pages/datasets/dataset-creating-dialog.tsx @@ -28,14 +28,14 @@ import { useForm, useWatch } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; import { - ChunkMethodItem, + BuiltinPipelineItem, EmbeddingModelItem, ParseTypeItem, } from '../dataset/dataset-setting/configuration/common-item'; const FormId = 'dataset-creating-form'; -const ChunkMethodName = 'chunk_method'; +const ChunkMethodName = 'parser_id'; export function InputForm({ onOk }: IModalProps) { const { t } = useTranslation(); @@ -98,7 +98,9 @@ export function InputForm({ onOk }: IModalProps) { function onSubmit(data: z.infer) { const nextData = - parseType === ParseType.BuiltIn ? data : omit(data, ChunkMethodName); + parseType === ParseType.BuiltIn + ? omit(data, ['pipeline_id']) + : omit(data, [ChunkMethodName]); onOk?.(nextData); } @@ -140,7 +142,7 @@ export function InputForm({ onOk }: IModalProps) { {parseType === ParseType.BuiltIn && ( - + )} {parseType === ParseType.Pipeline && ( import('@/pages/dataset/dataset-setting'), }, + { + path: `${Routes.DatasetBase}${Routes.DataSetSettingNext}/:id`, + Component: () => import('@/pages/dataset/setting'), + }, ], }, { diff --git a/web/src/services/agent-service.ts b/web/src/services/agent-service.ts index 78fbfd82b3..4c9d56111c 100644 --- a/web/src/services/agent-service.ts +++ b/web/src/services/agent-service.ts @@ -27,6 +27,8 @@ const { prompt, cancelDataflow, cancelCanvas, + listBuiltinPipelines, + getBuiltinPipeline, } = api; const methods = { @@ -122,6 +124,14 @@ const methods = { url: api.createAgentSession, method: 'post', }, + listBuiltinPipelines: { + url: listBuiltinPipelines, + method: 'get', + }, + getBuiltinPipeline: { + url: getBuiltinPipeline, + method: 'get', + }, } as const; const agentService = registerNextServer(methods); diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 892c119df6..a63034197d 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -408,6 +408,8 @@ export default { removeDataflow: `${webAPI}/dataflow/rm`, listDataflow: `${webAPI}/dataflow/list`, runDataflow: `${webAPI}/dataflow/run`, + listBuiltinPipelines: `${restAPIv1}/pipelines`, + getBuiltinPipeline: (id: string) => `${restAPIv1}/pipelines/${id}`, // admin adminLogin: `${restAPIv1}/admin/login`,