Feat: pipeline support ONE chunking method (#14024)

### What problem does this PR solve?

Feat: pipeline support ONE chunking method

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

---------

Co-authored-by: Yingfeng <yingfeng.zhang@gmail.com>

web/src/locales/en.ts

@@ -1554,6 +1554,10 @@ Example: Virtual Hosted Style`,
       tab: 'Tab',
       space: 'Space',
       delimiters: 'Delimiters',
+      one: 'One',
+      oneChunkTitle: 'Note',
+      oneChunkDescription:
+        'All parsed sections will be merged in order into a single chunk.',
       enableChildrenDelimiters: 'Child chunk are used for retrieval',
       merge: 'Merge',
       split: 'Split',

web/src/locales/zh.ts

@@ -1303,6 +1303,10 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       tab: '制表符',
       space: '空格',
       delimiters: '分隔符',
+      one: 'One',
+      oneChunkTitle: 'Note',
+      oneChunkDescription:
+        '所有解析后的 sections 会按原始顺序合并为 1 个 chunk。',
       merge: '合并',
       split: '拆分',
       script: '脚本',

web/src/pages/agent/constant/pipeline.tsx

@@ -255,6 +255,7 @@ export const initialTokenChunkerValues = {
   outputs: {
     chunks: { type: 'Array<Object>', value: [] },
   },
+  delimiter_mode: 'token_size',
   chunk_token_size: 512,
   overlapped_percent: 0,
   delimiters: [{ value: '\n' }],

web/src/pages/agent/form/token-chunker-form/index.tsx

@@ -6,7 +6,7 @@ import { BlockButton, Button } from '@/components/ui/button';
 import { Form, FormControl, FormField, FormItem } from '@/components/ui/form';
 import { Switch } from '@/components/ui/switch';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { Trash2 } from 'lucide-react';
+import { Info, Trash2 } from 'lucide-react';
 import { memo } from 'react';
 import { useFieldArray, useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
@@ -36,7 +36,7 @@ export const FormSchema = z.object({
     }),
   ),
   overlapped_percent: z.number(),
-  delimiter_mode: z.enum(['token_size', 'delimiter']).optional(),
+  delimiter_mode: z.enum(['token_size', 'delimiter', 'one']).optional(),
 });
 
 export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;
@@ -50,7 +50,7 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => {
     delimiter_mode: defaultValues.delimiter_mode || 'token_size',
   };
 
-  const form = useForm<SplitterFormSchemaType>({
+  const form = useForm<TokenChunkerFormSchemaType>({
     defaultValues: formDefaultValues,
     resolver: zodResolver(FormSchema),
   });
@@ -81,6 +81,7 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => {
             options: [
               { label: 'Token Size', value: 'token_size' },
               { label: t('flow.delimiters'), value: 'delimiter' },
+              { label: t('flow.one'), value: 'one' },
             ],
           }}
         />
@@ -141,58 +142,74 @@ const TokenChunkerForm = ({ node }: INextOperatorForm) => {
           </>
         )}
 
-        <fieldset>
-          <div className="mb-2 flex justify-between items-center gap-1">
-            <span>{t('flow.enableChildrenDelimiters')}</span>
-
-            <FormField
-              control={form.control}
-              name="enable_children"
-              render={({ field: { value, onChange, ...restProps } }) => (
-                <FormItem>
-                  <FormControl>
-                    <Switch
-                      checked={value}
-                      onCheckedChange={onChange}
-                      {...restProps}
-                    />
-                  </FormControl>
-                </FormItem>
-              )}
-            />
-          </div>
-
-          {form.getValues('enable_children') && (
-            <div className="space-y-4">
-              {childrenDelimiters.fields.map((field, index) => (
-                <div key={field.id} className="flex items-center gap-2">
-                  <RAGFlowFormItem
-                    name={`children_delimiters.${index}.value`}
-                    label="children_delimiter"
-                    labelClassName="!hidden"
-                    className="flex-auto space-y-0"
-                  >
-                    <DelimiterInput className="!m-0"></DelimiterInput>
-                  </RAGFlowFormItem>
-
-                  <Button
-                    type="button"
-                    variant="ghost"
-                    onClick={() => childrenDelimiters.remove(index)}
-                  >
-                    <Trash2 />
-                  </Button>
-                </div>
-              ))}
-
-              <BlockButton
-                onClick={() => childrenDelimiters.append({ value: '\n' })}
-              >
-                {t('common.add')}
-              </BlockButton>
+        {delimiterMode === 'one' && (
+          <div className="flex items-start gap-3 rounded-lg border bg-muted/50 px-4 py-3">
+            <Info className="mt-0.5 h-4 w-4 shrink-0 text-foreground/70" />
+            <div className="space-y-1">
+              <div className="text-sm font-medium">
+                {t('flow.oneChunkTitle')}
+              </div>
+              <p className="text-sm text-muted-foreground">
+                {t('flow.oneChunkDescription')}
+              </p>
             </div>
-          )}
-        </fieldset>
+          </div>
+        )}
+
+        {delimiterMode !== 'one' && (
+          <fieldset>
+            <div className="mb-2 flex justify-between items-center gap-1">
+              <span>{t('flow.enableChildrenDelimiters')}</span>
+
+              <FormField
+                control={form.control}
+                name="enable_children"
+                render={({ field: { value, onChange, ...restProps } }) => (
+                  <FormItem>
+                    <FormControl>
+                      <Switch
+                        checked={value}
+                        onCheckedChange={onChange}
+                        {...restProps}
+                      />
+                    </FormControl>
+                  </FormItem>
+                )}
+              />
+            </div>
+
+            {form.getValues('enable_children') && (
+              <div className="space-y-4">
+                {childrenDelimiters.fields.map((field, index) => (
+                  <div key={field.id} className="flex items-center gap-2">
+                    <RAGFlowFormItem
+                      name={`children_delimiters.${index}.value`}
+                      label="children_delimiter"
+                      labelClassName="!hidden"
+                      className="flex-auto space-y-0"
+                    >
+                      <DelimiterInput className="!m-0"></DelimiterInput>
+                    </RAGFlowFormItem>
+
+                    <Button
+                      type="button"
+                      variant="ghost"
+                      onClick={() => childrenDelimiters.remove(index)}
+                    >
+                      <Trash2 />
+                    </Button>
+                  </div>
+                ))}
+
+                <BlockButton
+                  onClick={() => childrenDelimiters.append({ value: '\n' })}
+                >
+                  {t('common.add')}
+                </BlockButton>
+              </div>
+            )}
+          </fieldset>
+        )}
       </FormWrapper>
       <div className="p-5">
         <Output list={outputList}></Output>

web/src/pages/agent/utils.ts

@@ -303,7 +303,10 @@ function transformTokenChunkerParams(params: TokenChunkerFormSchemaType) {
   const imageTableContextWindow = Number(image_table_context_window || 0);
   return {
     ...rest,
-    overlapped_percent: Number(params.overlapped_percent) / 100,
+    overlapped_percent:
+      params.delimiter_mode === 'one'
+        ? 0
+        : Number(params.overlapped_percent) / 100,
     delimiters:
       params.delimiter_mode === 'delimiter'
         ? transformObjectArrayToPureArray(params.delimiters, 'value')