From 398d81f587fa7c5f48b9c87cf6c67993b5e1e4d5 Mon Sep 17 00:00:00 2001
From: balibabu <cike8899@users.noreply.github.com>
Date: Tue, 11 Aug 2026 14:07:15 +0800
Subject: [PATCH] Fix: Move the content of the "Token Size" tab from the token
 chunker operator to the "Delimiter" tab.  #17888 (#17975)

---
 web/src/pages/agent/constant/pipeline.tsx     |  2 +-
 .../agent/form/token-chunker-form/index.tsx   | 20 ++++++++++---------
 web/src/utils/pipeline-operator.ts            | 15 +++++++-------
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 03dbb7e6f5..3b26171529 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -262,7 +262,7 @@ export const initialTokenChunkerValues = {
   outputs: {
     chunks: { type: 'Array<Object>', value: [] },
   },
-  delimiter_mode: 'token_size',
+  delimiter_mode: 'delimiter',
   chunk_token_size: 512,
   overlapped_percent: 0,
   delimiters: [{ value: '\n' }],
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx
index b61af84747..1b17fb480c 100644
--- a/web/src/pages/agent/form/token-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx
@@ -6,6 +6,7 @@ import { BlockButton, Button } from '@/components/ui/button';
 import { Form, FormControl, FormField, FormItem } from '@/components/ui/form';
 import { Switch } from '@/components/ui/switch';
 import { zodResolver } from '@hookform/resolvers/zod';
+import { isEmpty } from 'lodash';
 import { Info, Trash2 } from 'lucide-react';
 import { memo } from 'react';
 import { useFieldArray, useForm } from 'react-hook-form';
@@ -37,7 +38,7 @@ export const FormSchema = z.object({
     }),
   ),
   overlapped_percent: z.number(),
-  delimiter_mode: z.enum(['token_size', 'delimiter', 'one']).optional(),
+  delimiter_mode: z.enum(['delimiter', 'one']).optional(),
 });
 
 export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;
@@ -50,9 +51,16 @@ const TokenChunkerForm = ({
   const defaultValues = useFormValues(initialTokenChunkerValues, node);
   const { t } = useTranslation();
 
+  // Normalize legacy values: 'token_size' (removed tab) and empty fall back
+  // to 'delimiter'; nodes saved in the removed tab may carry empty delimiters,
+  // so seed the default '\n' row.
   const formDefaultValues = {
     ...defaultValues,
-    delimiter_mode: defaultValues.delimiter_mode || 'token_size',
+    delimiter_mode:
+      defaultValues.delimiter_mode === 'one' ? 'one' : 'delimiter',
+    delimiters: isEmpty(defaultValues.delimiters)
+      ? [{ value: '\n' }]
+      : defaultValues.delimiters,
   };
 
   const form = useForm<TokenChunkerFormSchemaType>({
@@ -85,14 +93,13 @@ const TokenChunkerForm = ({
             type: FormFieldType.Segmented,
             label: '',
             options: [
-              { label: 'Token Size', value: 'token_size' },
               { label: t('flow.delimiters'), value: 'delimiter' },
               { label: t('flow.one'), value: 'one' },
             ],
           }}
         />
 
-        {delimiterMode === 'token_size' && (
+        {delimiterMode === 'delimiter' && (
           <>
             <SliderInputFormField
               name="chunk_token_size"
@@ -113,11 +120,6 @@ const TokenChunkerForm = ({
               label={t('knowledgeConfiguration.imageTableContextWindow')}
               tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}
             />
-          </>
-        )}
-
-        {delimiterMode === 'delimiter' && (
-          <>
             <section>
               <span className="mb-2 inline-block">{t('flow.delimiters')}</span>
               <div className="space-y-4">
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index a4305114e2..e436ca61d0 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -110,9 +110,13 @@ function transformTokenChunkerConfigToForm(
 
   const result = { ...config };
 
-  // Convert string array delimiters to object array
+  // Convert string array delimiters to object array; seed the default '\n'
+  // row when the saved list is empty (legacy token_size nodes saved []).
   if (Array.isArray(config.delimiters)) {
     result.delimiters = config.delimiters.map((d: string) => ({ value: d }));
+    if (result.delimiters.length === 0) {
+      result.delimiters = [{ value: '\n' }];
+    }
   }
   if (Array.isArray(config.children_delimiters)) {
     result.children_delimiters = config.children_delimiters.map(
@@ -130,12 +134,9 @@ function transformTokenChunkerConfigToForm(
   const imageSize = Number(config.image_context_size ?? 0);
   result.image_table_context_window = Math.max(tableSize, imageSize);
 
-  // Derive delimiter_mode from data
-  if (config.delimiter_mode === undefined) {
-    const hasDelimiters =
-      Array.isArray(config.delimiters) && config.delimiters.length > 0;
-    result.delimiter_mode = hasDelimiters ? 'delimiter' : 'token_size';
-  }
+  // Normalize delimiter_mode: the 'token_size' tab was removed from the form,
+  // so legacy configs (explicit 'token_size' or absent) load as 'delimiter'.
+  result.delimiter_mode = config.delimiter_mode === 'one' ? 'one' : 'delimiter';
 
   // Derive enable_children from presence of children_delimiters
   if (config.enable_children === undefined) {