Feat: Support pipeline-related configuration at the document level. (#17212)

### Summary

Feat: Support pipeline-related configuration at the document level.

web/src/pages/dataset/dataset-setting/configuration/common-item.tsx

@@ -14,12 +14,9 @@ import {
   FormLabel,
   FormMessage,
 } from '@/components/ui/form';
-import { Radio } from '@/components/ui/radio';
 import { Spin } from '@/components/ui/spin';
 import { Switch } from '@/components/ui/switch';
-import { ParseType } from '@/constants/knowledge';
 import { useTranslate } from '@/hooks/common-hooks';
-import { useFetchBuiltinPipelines } from '@/hooks/use-agent-request';
 import { cn } from '@/lib/utils';
 import { history } from '@/utils/simple-history-util';
 import { t } from 'i18next';
@@ -30,7 +27,6 @@ import {
   FieldValues,
   useFormContext,
 } from 'react-hook-form';
-import { useTranslation } from 'react-i18next';
 import { useLocation } from 'react-router';
 import { DataSetContext } from '..';
 import { MetadataType } from '../../components/metedata/constant';
@@ -102,61 +98,6 @@ export function ChunkMethodItem(props: IProps) {
   );
 }
 
-export function BuiltinPipelineItem({
-  line = 2,
-  name = 'parser_id',
-}: {
-  line?: 1 | 2;
-  name?: string;
-}) {
-  const { t } = useTranslation();
-  const form = useFormContext();
-  const { options: builtinPipelineOptions } = useFetchBuiltinPipelines();
-
-  return (
-    <FormField
-      control={form.control}
-      name={name}
-      render={({ field }) => (
-        <FormItem className="items-center space-y-0">
-          <div
-            className={cn('flex', {
-              'items-center': line === 1,
-              'flex-col gap-1': line === 2,
-            })}
-          >
-            <FormLabel
-              required
-              className={cn('text-sm whitespace-wrap', {
-                'w-1/4': line === 1,
-              })}
-            >
-              {t('knowledgeConfiguration.builtIn')}
-            </FormLabel>
-            <div
-              className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
-            >
-              <FormControl>
-                <SelectWithSearch
-                  {...field}
-                  placeholder={t(
-                    'knowledgeConfiguration.chunkMethodPlaceholder',
-                  )}
-                  options={builtinPipelineOptions}
-                />
-              </FormControl>
-            </div>
-          </div>
-          <div className="flex pt-1">
-            <div className={line === 1 ? 'w-1/4' : ''}></div>
-            <FormMessage />
-          </div>
-        </FormItem>
-      )}
-    />
-  );
-}
-
 export const EmbeddingSelect = ({
   isEdit,
   field,
@@ -268,64 +209,6 @@ export function EmbeddingModelItem({
   );
 }
 
-export function ParseTypeItem({
-  line = 2,
-  name = 'parseType',
-}: {
-  line?: number;
-  name?: string;
-}) {
-  const { t } = useTranslate('knowledgeConfiguration');
-  const form = useFormContext();
-
-  return (
-    <FormField
-      control={form.control}
-      name={name}
-      render={({ field }) => (
-        <FormItem className="items-center space-y-0">
-          <div
-            className={cn('flex', {
-              'items-center': line === 1,
-              'flex-col gap-1': line === 2,
-            })}
-          >
-            <FormLabel
-              // tooltip={t('parseTypeTip')}
-              className={cn('text-sm  whitespace-wrap ', {
-                'w-1/4': line === 1,
-              })}
-            >
-              {t('parseType')}
-            </FormLabel>
-            <div
-              className={cn('text-muted-foreground', { 'w-3/4': line === 1 })}
-            >
-              <FormControl>
-                <Radio.Group {...field}>
-                  <div
-                    className={cn(
-                      'flex gap-2 justify-between text-muted-foreground',
-                      line === 1 ? 'w-1/2' : 'w-3/4',
-                    )}
-                  >
-                    <Radio value={ParseType.BuiltIn}>{t('builtIn')}</Radio>
-                    <Radio value={ParseType.Pipeline}>{t('manualSetup')}</Radio>
-                  </div>
-                </Radio.Group>
-              </FormControl>
-            </div>
-          </div>
-          <div className="flex pt-1">
-            <div className={line === 1 ? 'w-1/4' : ''}></div>
-            <FormMessage />
-          </div>
-        </FormItem>
-      )}
-    />
-  );
-}
-
 export function EnableAutoGenerateItem() {
   const { t } = useTranslate('knowledgeConfiguration');
   const form = useFormContext();

web/src/pages/dataset/dataset-setting/index.tsx

@@ -25,8 +25,9 @@ import ChunkMethodLearnMore from './chunk-method-learn-more';
 import LinkDataSource, {
   IDataSourceNodeProps,
 } from './components/link-data-source';
+import { ParseTypeItem } from '@/components/parse-type-form-field';
 import { MainContainer } from './configuration-form-container';
-import { ChunkMethodItem, ParseTypeItem } from './configuration/common-item';
+import { ChunkMethodItem } from './configuration/common-item';
 import { formSchema } from './form-schema';
 import { GeneralForm } from './general-form';
 import { useFetchKnowledgeConfigurationOnMount } from './hooks';

web/src/pages/dataset/dataset/dataset-table.tsx

@@ -14,6 +14,7 @@ import {
 import * as React from 'react';
 
 import { ChunkMethodDialog } from '@/components/chunk-method-dialog';
+import { DocumentPipelineDialog } from '@/components/document-pipeline-dialog';
 import { EmptyType } from '@/components/empty/constant';
 import Empty from '@/components/empty/empty';
 import { RenameDialog } from '@/components/rename-dialog';
@@ -28,6 +29,7 @@ import {
 } from '@/components/ui/table';
 import { UseRowSelectionType } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchDocumentList } from '@/hooks/use-document-request';
+import { isGoBackend } from '@/utils/backend-runtime';
 import { getExtension } from '@/utils/document-util';
 import { t } from 'i18next';
 import { pick } from 'lodash';
@@ -187,19 +189,29 @@ export function DatasetTable({
           ></RAGFlowPagination>
         </div>
       </div>
-      {changeParserVisible && (
-        <ChunkMethodDialog
-          documentId={changeParserRecord.id}
-          parserId={changeParserRecord.chunk_method}
-          pipelineId={changeParserRecord.pipeline_id}
-          parserConfig={changeParserRecord.parser_config}
-          documentExtension={getExtension(changeParserRecord.name)}
-          onOk={onChangeParserOk}
-          visible={changeParserVisible}
-          hideModal={hideChangeParserModal}
-          loading={changeParserLoading}
-        ></ChunkMethodDialog>
-      )}
+      {changeParserVisible &&
+        (isGoBackend() ? (
+          <DocumentPipelineDialog
+            parserId={changeParserRecord.chunk_method}
+            pipelineId={changeParserRecord.pipeline_id}
+            parserConfig={changeParserRecord.parser_config}
+            onOk={onChangeParserOk}
+            hideModal={hideChangeParserModal}
+            loading={changeParserLoading}
+          ></DocumentPipelineDialog>
+        ) : (
+          <ChunkMethodDialog
+            documentId={changeParserRecord.id}
+            parserId={changeParserRecord.chunk_method}
+            pipelineId={changeParserRecord.pipeline_id}
+            parserConfig={changeParserRecord.parser_config}
+            documentExtension={getExtension(changeParserRecord.name)}
+            onOk={onChangeParserOk}
+            visible={changeParserVisible}
+            hideModal={hideChangeParserModal}
+            loading={changeParserLoading}
+          ></ChunkMethodDialog>
+        ))}
 
       {renameVisible && (
         <RenameDialog

web/src/pages/dataset/dataset/use-change-document-parser.ts

@@ -1,11 +1,17 @@
 import { useSetModalState } from '@/hooks/common-hooks';
-import { useSetDocumentParser } from '@/hooks/use-document-request';
+import {
+  useSetDocumentParser,
+  useSetDocumentPipelineParser,
+} from '@/hooks/use-document-request';
 import { IDocumentInfo } from '@/interfaces/database/document';
 import { IChangeParserRequestBody } from '@/interfaces/request/document';
+import { isGoBackend } from '@/utils/backend-runtime';
 import { useCallback, useState } from 'react';
 
 export const useChangeDocumentParser = () => {
   const { setDocumentParser, loading } = useSetDocumentParser();
+  const { setDocumentPipelineParser, loading: pipelineParserLoading } =
+    useSetDocumentPipelineParser();
   const [record, setRecord] = useState<IDocumentInfo>({} as IDocumentInfo);
 
   const {
@@ -17,7 +23,12 @@ export const useChangeDocumentParser = () => {
   const onChangeParserOk = useCallback(
     async (parserConfigInfo: IChangeParserRequestBody) => {
       if (record?.id && record?.dataset_id) {
-        const ret = await setDocumentParser({
+        // The Go document endpoint takes `parser_id` and a pipeline-shaped
+        // parser_config; the Python one keeps the legacy payload shape.
+        const setParser = isGoBackend()
+          ? setDocumentPipelineParser
+          : setDocumentParser;
+        const ret = await setParser({
           parserId: parserConfigInfo.parser_id,
           pipelineId: parserConfigInfo.pipeline_id || '',
           documentId: record?.id,
@@ -29,7 +40,13 @@ export const useChangeDocumentParser = () => {
         }
       }
     },
-    [record?.id, record?.dataset_id, setDocumentParser, hideChangeParserModal],
+    [
+      record?.id,
+      record?.dataset_id,
+      setDocumentParser,
+      setDocumentPipelineParser,
+      hideChangeParserModal,
+    ],
   );
 
   const handleShowChangeParserModal = useCallback(
@@ -41,7 +58,7 @@ export const useChangeDocumentParser = () => {
   );
 
   return {
-    changeParserLoading: loading,
+    changeParserLoading: loading || pipelineParserLoading,
     onChangeParserOk,
     changeParserVisible,
     hideChangeParserModal,

web/src/pages/dataset/setting/hooks.ts

@@ -1,13 +1,16 @@
 import { ParseType } from '@/constants/knowledge';
-import { useFetchPipelineDslByPipelineId } from '@/hooks/use-agent-request';
 import {
   useFetchDatasetPipelineConfiguration,
   useUpdateKnowledge,
 } from '@/hooks/use-knowledge-request';
-import { RAGFlowNodeType } from '@/interfaces/database/agent';
 import { IConnector } from '@/interfaces/database/dataset';
 import { useDataSourceInfo } from '@/pages/user-setting/data-source/constant';
 import { checkEmbedding } from '@/services/knowledge-service';
+import {
+  getOperatorType,
+  transformApiConfigToForm,
+  transformFormConfigToApi,
+} from '@/utils/pipeline-operator';
 import { pick } from 'lodash';
 import {
   Dispatch,
@@ -15,20 +18,11 @@ import {
   useCallback,
   useEffect,
   useMemo,
-  useRef,
-  useState,
 } from 'react';
 import { UseFormReturn } from 'react-hook-form';
 import { useParams, useSearchParams } from 'react-router';
 import { z } from 'zod';
 import { formSchema } from './form-schema';
-import {
-  buildParserConfigFromNodes,
-  buildPipelineOperatorNodes,
-  getOperatorType,
-  transformApiConfigToForm,
-  transformFormConfigToApi,
-} from './utils';
 
 export function useHasParsedDocument(isEdit?: boolean) {
   const { data: knowledgeDetails } = useFetchDatasetPipelineConfiguration({
@@ -125,72 +119,6 @@ export const useFetchDatasetSettingOnMount = (
   return { knowledgeDetails, loading, sourceData };
 };
 
-export const usePipelineOperatorNodes = (
-  pipelineId?: string,
-  pipelineParserConfig?: Record<string, any>,
-  isBuiltin = false,
-) => {
-  const { dsl, loading } = useFetchPipelineDslByPipelineId(
-    pipelineId,
-    isBuiltin,
-  );
-
-  const operatorNodes = useMemo(() => {
-    return buildPipelineOperatorNodes(dsl, pipelineParserConfig);
-  }, [dsl, pipelineParserConfig]);
-
-  return { operatorNodes, loading };
-};
-
-/**
- * Resets parser_config when the selected pipeline changes, so stale configs
- * from the previous pipeline are never submitted. Once the new pipeline's
- * DSL has loaded, parser_config is seeded with the DSL defaults (i.e. what
- * the operator tabs display).
- *
- * The very first pipeline seen is treated as the initial load: the form was
- * already reset with the saved parser_config in useFetchDatasetSettingOnMount,
- * so it is left untouched.
- */
-export const useResetParserConfigOnPipelineChange = (
-  form: UseFormReturn<z.infer<typeof formSchema>>,
-  pipelineId: string | undefined,
-  savedPipelineId: string | undefined,
-  operatorNodes: RAGFlowNodeType[],
-) => {
-  const previousPipelineIdRef = useRef<string>();
-
-  useEffect(() => {
-    if (!pipelineId) {
-      // Selection cleared (e.g. parse type switched): drop configs that
-      // belonged to the previously selected pipeline.
-      if (previousPipelineIdRef.current) {
-        previousPipelineIdRef.current = '';
-        form.setValue('parser_config', {});
-      }
-      return;
-    }
-
-    const isInitialLoad =
-      previousPipelineIdRef.current === undefined &&
-      pipelineId === savedPipelineId;
-    if (isInitialLoad || previousPipelineIdRef.current === pipelineId) {
-      previousPipelineIdRef.current = pipelineId;
-      return;
-    }
-
-    if (operatorNodes.length === 0) {
-      // The new pipeline's DSL is still loading — clear the previous
-      // pipeline's configs right away; defaults are seeded once it arrives.
-      form.setValue('parser_config', {});
-      return;
-    }
-
-    previousPipelineIdRef.current = pipelineId;
-    form.setValue('parser_config', buildParserConfigFromNodes(operatorNodes));
-  }, [form, pipelineId, savedPipelineId, operatorNodes]);
-};
-
 export const useSaveDatasetSetting = () => {
   const { saveKnowledgeConfiguration, loading } = useUpdateKnowledge();
 
@@ -226,30 +154,6 @@ export const useSaveDatasetSetting = () => {
   return { handleSave, loading };
 };
 
-export const useActiveTab = (operatorNodes: RAGFlowNodeType[]) => {
-  const [activeTab, setActiveTab] = useState('');
-
-  useEffect(() => {
-    if (operatorNodes.length > 0) {
-      const firstTab =
-        (operatorNodes[0].data as Record<string, any>)?.operatorId ||
-        operatorNodes[0].data?.label ||
-        '';
-      const validTabs = operatorNodes.map(
-        (node) =>
-          (node.data as Record<string, any>)?.operatorId ||
-          node.data?.label ||
-          '',
-      );
-      setActiveTab((prev) => (validTabs.includes(prev) ? prev : firstTab));
-    } else {
-      setActiveTab('');
-    }
-  }, [operatorNodes]);
-
-  return { activeTab, setActiveTab };
-};
-
 export const usePipelineDataList = (sourceData: any[] | undefined) => {
   const { dataSourceInfo } = useDataSourceInfo();
 

web/src/pages/dataset/setting/index.tsx

@@ -1,4 +1,7 @@
+import { BuiltinPipelineItem } from '@/components/builtin-pipeline-form-field';
 import { DataFlowSelect } from '@/components/data-pipeline-select';
+import { ParseTypeItem } from '@/components/parse-type-form-field';
+import PipelineOperatorTabs from '@/components/pipeline-operator-tabs';
 import { Button, ButtonLoading } from '@/components/ui/button';
 import {
   Card,
@@ -12,9 +15,10 @@ import { Form } from '@/components/ui/form';
 import { FormLayout } from '@/constants/form';
 import { ParseType } from '@/constants/knowledge';
 import {
-  BuiltinPipelineItem,
-  ParseTypeItem,
-} from '@/pages/dataset/dataset-setting/configuration/common-item';
+  useActiveTab,
+  usePipelineOperatorNodes,
+  useResetParserConfigOnPipelineChange,
+} from '@/hooks/use-pipeline-operator';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { useCallback, useEffect, useState } from 'react';
 import { useForm, useWatch } from 'react-hook-form';
@@ -26,15 +30,11 @@ import LinkDataSource, {
 import { formSchema } from './form-schema';
 import { GeneralForm } from './general-form';
 import {
-  useActiveTab,
   useConnectorHandlers,
   useFetchDatasetSettingOnMount,
   usePipelineDataList,
-  usePipelineOperatorNodes,
-  useResetParserConfigOnPipelineChange,
   useSaveDatasetSetting,
 } from './hooks';
-import PipelineOperatorTabs from './pipeline-operator-tabs';
 
 export default function DatasetSetting() {
   const { t } = useTranslation();

web/src/pages/dataset/setting/pipeline-operator-form.tsx

@@ -1,77 +0,0 @@
-import { Operator } from '@/constants/agent';
-import { RAGFlowNodeType } from '@/interfaces/database/agent';
-import { memo, useCallback } from 'react';
-import ExtractorForm from '../../agent/form/extractor-form';
-import ParserForm from '../../agent/form/parser-form';
-import TitleChunkerForm from '../../agent/form/title-chunker-form';
-import TokenChunkerForm from '../../agent/form/token-chunker-form';
-import TokenizerForm from '../../agent/form/tokenizer-form';
-import { getOperatorType } from './utils';
-
-type PipelineOperatorFormProps = {
-  node: RAGFlowNodeType;
-  onValuesChange?: (values: any) => void;
-};
-
-const PipelineOperatorForm = ({
-  node,
-  onValuesChange,
-}: PipelineOperatorFormProps) => {
-  const operatorType = getOperatorType(
-    (node.data as Record<string, any>)?.operatorId || node.data?.label || '',
-  );
-
-  const handleValuesChange = useCallback(
-    (values: any) => {
-      onValuesChange?.(values);
-    },
-    [onValuesChange],
-  );
-
-  switch (operatorType) {
-    case Operator.Parser:
-      return (
-        <ParserForm
-          node={node}
-          onValuesChange={handleValuesChange}
-          hideOutputs
-        />
-      );
-    case Operator.TokenChunker:
-      return (
-        <TokenChunkerForm
-          node={node}
-          onValuesChange={handleValuesChange}
-          hideOutputs
-        />
-      );
-    case Operator.TitleChunker:
-      return (
-        <TitleChunkerForm
-          node={node}
-          onValuesChange={handleValuesChange}
-          hideOutputs
-        />
-      );
-    case Operator.Extractor:
-      return (
-        <ExtractorForm
-          node={node}
-          onValuesChange={handleValuesChange}
-          hideOutputs
-        />
-      );
-    case Operator.Tokenizer:
-      return (
-        <TokenizerForm
-          node={node}
-          onValuesChange={handleValuesChange}
-          hideOutputs
-        />
-      );
-    default:
-      return null;
-  }
-};
-
-export default memo(PipelineOperatorForm);

web/src/pages/dataset/setting/pipeline-operator-tabs.tsx

@@ -1,66 +0,0 @@
-import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
-import { RAGFlowNodeType } from '@/interfaces/database/agent';
-import { memo, useCallback } from 'react';
-import PipelineOperatorForm from './pipeline-operator-form';
-
-type PipelineOperatorTabsProps = {
-  nodes: RAGFlowNodeType[];
-  value: string;
-  onValueChange: (value: string) => void;
-  onOperatorValuesChange: (operatorId: string, values: any) => void;
-};
-
-const PipelineOperatorTabs = ({
-  nodes,
-  value,
-  onValueChange,
-  onOperatorValuesChange,
-}: PipelineOperatorTabsProps) => {
-  const getOperatorId = useCallback((node: RAGFlowNodeType) => {
-    return (
-      (node.data as Record<string, any>)?.operatorId || node.data?.label || ''
-    );
-  }, []);
-
-  const getTabValue = useCallback(
-    (node: RAGFlowNodeType, index: number) => {
-      return getOperatorId(node) || String(index);
-    },
-    [getOperatorId],
-  );
-
-  const handleValuesChange = useCallback(
-    (node: RAGFlowNodeType) => (values: any) => {
-      onOperatorValuesChange(getOperatorId(node), values);
-    },
-    [getOperatorId, onOperatorValuesChange],
-  );
-
-  return (
-    <Tabs value={value} onValueChange={onValueChange} className="w-full">
-      <TabsList className="w-full justify-start">
-        {nodes.map((node, index) => {
-          const tabValue = getTabValue(node, index);
-          return (
-            <TabsTrigger key={tabValue} value={tabValue}>
-              {node.data?.name || node.data?.label || tabValue}
-            </TabsTrigger>
-          );
-        })}
-      </TabsList>
-      {nodes.map((node, index) => {
-        const tabValue = getTabValue(node, index);
-        return (
-          <TabsContent key={tabValue} value={tabValue}>
-            <PipelineOperatorForm
-              node={node}
-              onValuesChange={handleValuesChange(node)}
-            />
-          </TabsContent>
-        );
-      })}
-    </Tabs>
-  );
-};
-
-export default memo(PipelineOperatorTabs);

web/src/pages/dataset/setting/utils.ts

@@ -1,358 +0,0 @@
-import { Operator } from '@/constants/agent';
-import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';
-import {
-  initialExtractorValues,
-  initialParserValues,
-  initialTitleChunkerValues,
-  initialTokenChunkerValues,
-  initialTokenizerValues,
-} from '@/pages/agent/constant/pipeline';
-import {
-  transformExtractorParams,
-  transformParserParams,
-  transformTitleChunkerParams,
-  transformTokenChunkerParams,
-} from '@/pages/agent/utils';
-import { cloneDeep, isEmpty } from 'lodash';
-
-export const FileNodeId = 'File';
-
-export function getOperatorType(operatorId: string): Operator {
-  return (operatorId.split(':')[0] || operatorId) as Operator;
-}
-
-export function transformParserConfigSetups(
-  setups: Record<string, any> | undefined,
-): any[] {
-  if (!setups || typeof setups !== 'object') {
-    return [];
-  }
-
-  return Object.entries(setups)
-    .map(([fileFormat, config]) => ({
-      fileFormat,
-      ...config,
-    }))
-    .sort((a, b) => (a.order_index ?? Infinity) - (b.order_index ?? Infinity));
-}
-
-function transformLevelsToRules(
-  levels: any[],
-): Array<{ levels: Array<{ expression: string }> }> {
-  if (!Array.isArray(levels)) return [];
-  return levels
-    .map((levelGroup) => {
-      if (Array.isArray(levelGroup)) {
-        const filteredExpressions = levelGroup.filter(
-          (expr: string) => expr && expr.trim() !== '',
-        );
-        if (filteredExpressions.length === 0) return null;
-        return {
-          levels: filteredExpressions.map((expression: string) => ({
-            expression,
-          })),
-        };
-      }
-      return { levels: [{ expression: '' }] };
-    })
-    .filter((rule) => rule !== null);
-}
-
-/**
- * Converts Extractor config from API/DSL format to form format.
- * DSL:  { prompts: [{ content: "text", role: "user" }] }
- * Form: { prompts: "text" }
- */
-function transformExtractorConfigToForm(
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  if (!config) return {};
-
-  const result = { ...config };
-  if (Array.isArray(config.prompts) && config.prompts.length > 0) {
-    result.prompts = config.prompts[0]?.content ?? '';
-  }
-  return result;
-}
-
-/**
- * Converts TokenChunker config from API/DSL format to form format.
- * DSL:  { delimiters: ["\n"], overlapped_percent: 0.1, table_context_size: 10, image_context_size: 20 }
- * Form: { delimiters: [{ value: "\n" }], overlapped_percent: 10, image_table_context_window: 15, enable_children: false }
- */
-function transformTokenChunkerConfigToForm(
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  if (!config) return {};
-
-  const result = { ...config };
-
-  // Convert string array delimiters to object array
-  if (Array.isArray(config.delimiters)) {
-    result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
-  }
-  if (Array.isArray(config.children_delimiters)) {
-    result.children_delimiters = config.children_delimiters.map(
-      (d: string) => ({ value: d }),
-    );
-  }
-
-  // Convert overlapped_percent from 0-1 scale to 0-100 scale
-  if (typeof config.overlapped_percent === 'number') {
-    result.overlapped_percent = Math.round(config.overlapped_percent * 100);
-  }
-
-  // Merge table_context_size and image_context_size into image_table_context_window
-  const tableSize = Number(config.table_context_size ?? 0);
-  const imageSize = Number(config.image_context_size ?? 0);
-  result.image_table_context_window = Math.max(tableSize, imageSize);
-
-  // Derive delimiter_mode from data
-  if (config.delimiter_mode === undefined) {
-    const hasDelimiters =
-      Array.isArray(config.delimiters) && config.delimiters.length > 0;
-    result.delimiter_mode = hasDelimiters ? 'delimiter' : 'token_size';
-  }
-
-  // Derive enable_children from presence of children_delimiters
-  if (config.enable_children === undefined) {
-    result.enable_children =
-      Array.isArray(config.children_delimiters) &&
-      config.children_delimiters.length > 0;
-  }
-
-  // Clean up DSL-only fields not in form schema
-  delete result.table_context_size;
-  delete result.image_context_size;
-
-  return result;
-}
-
-/**
- * Converts TitleChunker config from API/DSL format to form format.
- * DSL:  { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading }
- * Form: { method, hierarchyHierarchy, hierarchyGroup, include_heading_content, root_chunk_as_heading, hierarchyRules, groupRules }
- */
-function transformTitleChunkerConfigToForm(
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  if (!config) return {};
-
-  const result = { ...config };
-  const method = config.method ?? 'hierarchy';
-
-  // Convert legacy `hierarchy` (single field) to split fields
-  let hierarchy = config.hierarchy;
-  if (typeof hierarchy === 'number') {
-    hierarchy = String(hierarchy);
-  }
-
-  if (method === 'hierarchy') {
-    result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
-    result.hierarchyGroup = config.hierarchyGroup || '0';
-  } else {
-    result.hierarchyHierarchy = config.hierarchyHierarchy || hierarchy || '3';
-    result.hierarchyGroup = config.hierarchyGroup || hierarchy || '0';
-  }
-
-  // Convert `levels` to `hierarchyRules`/`groupRules`
-  if (config.levels && Array.isArray(config.levels) && !config.hierarchyRules) {
-    result.hierarchyRules = transformLevelsToRules(config.levels);
-  }
-  if (config.levels && Array.isArray(config.levels) && !config.groupRules) {
-    result.groupRules = transformLevelsToRules(config.levels);
-  }
-
-  // Clean up DSL-only fields
-  delete result.hierarchy;
-  delete result.levels;
-  delete result.rules;
-
-  return result;
-}
-
-/**
- * Tokenizer is a passthrough — both API and form formats are identical.
- */
-function transformTokenizerConfigToForm(
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  return config ?? {};
-}
-
-/**
- * Dispatches reverse transform by operator type.
- * Converts DSL-format config to form-format config.
- */
-export function transformApiConfigToForm(
-  operatorType: string,
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  switch (operatorType) {
-    case Operator.Parser:
-      return { setups: transformParserConfigSetups(config) };
-    case Operator.Extractor:
-      return transformExtractorConfigToForm(config);
-    case Operator.Tokenizer:
-      return transformTokenizerConfigToForm(config);
-    case Operator.TokenChunker:
-      return transformTokenChunkerConfigToForm(config);
-    case Operator.TitleChunker:
-      return transformTitleChunkerConfigToForm(config);
-    default:
-      return config ?? {};
-  }
-}
-
-/**
- * Dispatches forward transform by operator type.
- * Converts form-format config to DSL format for saving.
- */
-export function transformFormConfigToApi(
-  operatorType: string,
-  config: Record<string, any> | undefined,
-): Record<string, any> {
-  if (!config) return {};
-
-  switch (operatorType) {
-    case Operator.Parser:
-      return transformParserParams(config as any);
-    case Operator.Extractor:
-      return transformExtractorParams(config as any);
-    case Operator.Tokenizer:
-      return config; // passthrough for Tokenizer
-    case Operator.TokenChunker:
-      return transformTokenChunkerParams(config as any);
-    case Operator.TitleChunker:
-      return transformTitleChunkerParams(config as any);
-    default:
-      return config;
-  }
-}
-
-function normalizeOperatorForm(
-  operatorId: string,
-  rawForm: Record<string, any> | undefined,
-): Record<string, any> {
-  const operatorType = getOperatorType(operatorId);
-
-  switch (operatorType) {
-    case Operator.Parser: {
-      return {
-        ...cloneDeep(initialParserValues),
-        ...rawForm,
-        setups: rawForm?.setups?.length
-          ? rawForm.setups
-          : cloneDeep(initialParserValues.setups),
-      };
-    }
-    case Operator.TitleChunker:
-      return {
-        ...cloneDeep(initialTitleChunkerValues),
-        ...rawForm,
-      };
-    case Operator.TokenChunker:
-      return {
-        ...cloneDeep(initialTokenChunkerValues),
-        ...rawForm,
-      };
-    case Operator.Extractor:
-      return {
-        ...cloneDeep(initialExtractorValues),
-        ...rawForm,
-      };
-    case Operator.Tokenizer:
-      return {
-        ...cloneDeep(initialTokenizerValues),
-        ...rawForm,
-      };
-    default:
-      return rawForm ?? {};
-  }
-}
-
-export function buildOperatorNode(
-  dslNode: RAGFlowNodeType,
-  pipelineParserConfig: Record<string, any> = {},
-): RAGFlowNodeType {
-  const operatorId = dslNode.id;
-  const operatorType = getOperatorType(operatorId);
-
-  // Form-format config from DSL graph
-  const configFromDsl = dslNode.data?.form;
-
-  const rawForm = {
-    ...configFromDsl, // template defaults from DSL (form format)
-  };
-
-  if (!isEmpty(pipelineParserConfig)) {
-    // user overrides from API (now also form format)
-    Object.assign(
-      rawForm,
-      transformApiConfigToForm(operatorType, pipelineParserConfig[operatorId]), // Convert API config to form format, then merge (DSL template is baseline, API overrides)
-    );
-  }
-
-  return {
-    ...dslNode,
-    id: '',
-    data: {
-      ...dslNode.data,
-      label: operatorType,
-      operatorId,
-      form: normalizeOperatorForm(operatorId, rawForm),
-    },
-  };
-}
-
-export function buildPipelineOperatorNodes(
-  dsl?: DSL,
-  pipelineParserConfig: Record<string, any> = {},
-): RAGFlowNodeType[] {
-  if (!dsl?.graph?.nodes || !dsl?.graph?.edges) {
-    return [];
-  }
-
-  // Build source → target map from edges (pipeline is linear)
-  const sourceToTarget = new Map<string, string>();
-  for (const edge of dsl.graph.edges) {
-    sourceToTarget.set(edge.source, edge.target);
-  }
-
-  // Follow the chain starting from File, collecting node IDs in order
-  const orderedIds: string[] = [];
-  let currentId: string | undefined = FileNodeId;
-  while (currentId) {
-    orderedIds.push(currentId);
-    currentId = sourceToTarget.get(currentId);
-  }
-
-  // Build a lookup from node ID → node
-  const nodeById = new Map<string, RAGFlowNodeType>();
-  for (const node of dsl.graph.nodes) {
-    nodeById.set(node.id, node);
-  }
-
-  // Map ordered IDs to nodes, excluding File
-  return orderedIds
-    .filter((id) => id !== FileNodeId)
-    .map((id) => nodeById.get(id))
-    .filter((node): node is RAGFlowNodeType => node !== undefined)
-    .map((node) => buildOperatorNode(node, pipelineParserConfig));
-}
-
-/**
- * Builds a parser_config record from operator nodes, keyed by operatorId.
- * Each entry is the node's merged form (DSL template defaults + overrides),
- * i.e. exactly what the operator tabs display.
- */
-export function buildParserConfigFromNodes(
-  operatorNodes: RAGFlowNodeType[],
-): Record<string, any> {
-  return Object.fromEntries(
-    operatorNodes.map((node) => [
-      (node.data as Record<string, any>)?.operatorId || node.data?.label || '',
-      node.data?.form ?? {},
-    ]),
-  );
-}

web/src/pages/datasets/dataset-creating-dialog.tsx

@@ -1,4 +1,6 @@
+import { BuiltinPipelineItem } from '@/components/builtin-pipeline-form-field';
 import { DataFlowSelect } from '@/components/data-pipeline-select';
+import { ParseTypeItem } from '@/components/parse-type-form-field';
 import { ButtonLoading } from '@/components/ui/button';
 import {
   Dialog,
@@ -28,10 +30,8 @@ import { useForm, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
 import {
-  BuiltinPipelineItem,
   ChunkMethodItem,
   EmbeddingModelItem,
-  ParseTypeItem,
 } from '../dataset/dataset-setting/configuration/common-item';
 import { isGoBackend } from '@/utils/backend-runtime';