feat: add LLM-guided semantic rechunking for knowledge compilation (#17546)

2026-08-01 21:37:33 +08:00 · 2026-07-31 13:33:30 +08:00
parent 0969c04eca
commit 44e13d1cb6
20 changed files with 827 additions and 117 deletions
--- a/web/src/interfaces/database/compilation-template.ts
+++ b/web/src/interfaces/database/compilation-template.ts
@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfig {
  relation?: ICompilationTemplateSection;
  raptor?: ICompilationTemplateRaptorConfig;
  global_rules?: string;
+  rechunk?: boolean;
+  rechunk_rules?: string;
  [section: string]:
    | ICompilationTemplateSection
    | ICompilationTemplateRaptorConfig
--- a/web/src/interfaces/request/compilation-template.ts
+++ b/web/src/interfaces/request/compilation-template.ts
@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfigRequest {
  relation?: ICompilationTemplateSectionRequest;
  raptor?: ICompilationTemplateRaptorConfigRequest;
  global_rules?: string;
+  rechunk?: boolean;
+  rechunk_rules?: string;
  [section: string]:
    | ICompilationTemplateSectionRequest
    | ICompilationTemplateRaptorConfigRequest
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1900,6 +1900,12 @@ Example: Virtual Hosted Style`,
      rechunkByTreeLeaves: 'Re-chunk by tree leaves',
      rechunkByTreeLeavesTip:
        "Merge each leaf cluster's source chunks into a single replacement chunk. Originals are kept but marked unavailable for retrieval. Only one tree template per group may enable this.",
+      rechunkInput: 'Re-chunk parser output',
+      rechunkInputTip:
+        'Let the LLM determine chunk boundaries based on the knowledge compilation task.',
+      rechunkRules: 'Rechunking rules',
+      rechunkRulesPlaceholder:
+        'Describe how the LLM should group source chunks for this compilation task.',
      jsonPreview: 'JSON preview',
      processFlow: 'Process flow',
      processFlowComingSoon: 'Process flow preview coming soon',
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1590,6 +1590,11 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
      rechunkByTreeLeaves: '按树叶重新分块',
      rechunkByTreeLeavesTip:
        '将每个叶簇的源数据块合并为单个替换数据块。原始数据块保留但标记为不可检索。每个分组最多只能有一个树模板启用此功能。',
+      rechunkInput: '重新切分 Parser 输出',
+      rechunkInputTip: '根据知识编译任务，由 LLM 决定 chunk 边界。',
+      rechunkRules: 'Rechunk 规则',
+      rechunkRulesPlaceholder:
+        '描述此知识编译任务下，LLM 应如何合并和划分源 chunk。',
      jsonPreview: 'JSON 预览',
      processFlow: '流程视图',
      processFlowComingSoon: '流程视图预览即将到来',
--- a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx
+++ b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx
@@ -1,6 +1,7 @@
 import { ModelTreeSelectFormField } from '@/components/model-tree-select';
 import { SelectWithSearch } from '@/components/originui/select-with-search';
 import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SwitchFormField } from '@/components/switch-fom-field';
 import { Button } from '@/components/ui/button';
 import { Input } from '@/components/ui/input';
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
@@ -65,6 +66,10 @@ export function TemplateConfiguration({
    control: form.control,
    name: `templates.${selectedTemplateIndex}.kind`,
  });
+  const rechunk = useWatch({
+    control: form.control,
+    name: `templates.${selectedTemplateIndex}.config.rechunk`,
+  });

  const availableKindOptions = useAvailableKindOptions(
    form,
@@ -198,39 +203,62 @@ export function TemplateConfiguration({
          {kind === CompilationTemplateKind.Tree ? (
            <TreeTemplateFields index={selectedTemplateIndex} />
          ) : (
-            sectionNames.length > 0 &&
-            activeSectionTab && (
-              <Tabs
-                value={activeSectionTab}
-                onValueChange={setActiveSectionTab}
-                className="w-full"
-              >
-                <TabsList className="w-full justify-start">
+            <>
+              {kind !== CompilationTemplateKind.Artifacts && (
+                <>
+                  <SwitchFormField
+                    name={`templates.${selectedTemplateIndex}.config.rechunk`}
+                    label={t('setting.rechunkInput')}
+                    tooltip={t('setting.rechunkInputTip')}
+                    vertical={false}
+                  />
+                  {rechunk && (
+                    <RAGFlowFormItem
+                      name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}
+                      label={t('setting.rechunkRules')}
+                    >
+                      <Textarea
+                        placeholder={t('setting.rechunkRulesPlaceholder')}
+                        rows={6}
+                        resize="vertical"
+                      />
+                    </RAGFlowFormItem>
+                  )}
+                </>
+              )}
+              {sectionNames.length > 0 && activeSectionTab && (
+                <Tabs
+                  value={activeSectionTab}
+                  onValueChange={setActiveSectionTab}
+                  className="w-full"
+                >
+                  <TabsList className="w-full justify-start">
+                    {sectionNames.map((sectionName) => (
+                      <TabsTrigger
+                        key={sectionName}
+                        value={sectionName}
+                        className="flex-1"
+                      >
+                        {t(
+                          SectionTitleKeyMap[sectionName] ??
+                            startCase(sectionName),
+                        )}
+                      </TabsTrigger>
+                    ))}
+                  </TabsList>
+
                  {sectionNames.map((sectionName) => (
-                    <TabsTrigger
+                    <TabsContent
                      key={sectionName}
                      value={sectionName}
-                      className="flex-1"
+                      className="mt-4"
                    >
-                      {t(
-                        SectionTitleKeyMap[sectionName] ??
-                          startCase(sectionName),
-                      )}
-                    </TabsTrigger>
+                      {renderSectionTabs(sectionName)}
+                    </TabsContent>
                  ))}
-                </TabsList>
-
-                {sectionNames.map((sectionName) => (
-                  <TabsContent
-                    key={sectionName}
-                    value={sectionName}
-                    className="mt-4"
-                  >
-                    {renderSectionTabs(sectionName)}
-                  </TabsContent>
-                ))}
-              </Tabs>
-            )
+                </Tabs>
+              )}
+            </>
          )}
        </div>
      </div>
--- a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
@@ -61,6 +61,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {
    instruction: '',
    page_example: '',
    use_blueprint: false,
+    rechunk: false,
+    rechunk_rules: '',
  },
 };

@@ -81,6 +83,8 @@ export const isConfigMetaKey = (key: string) =>
    'page_example',
    'synthesis',
    'use_blueprint',
+    'rechunk',
+    'rechunk_rules',
  ].includes(key);

 export const createEmptyField = (keys: string[]) =>
@@ -132,6 +136,15 @@ export const buildConfigFromBuiltin = (
      : {}),
    use_blueprint:
      kind === CompilationTemplateKind.Artifacts && example.length > 0,
+    ...(kind !== CompilationTemplateKind.Tree
+      ? {
+          rechunk: builtinTemplate.config?.rechunk === true,
+          rechunk_rules:
+            typeof builtinTemplate.config?.rechunk_rules === 'string'
+              ? builtinTemplate.config.rechunk_rules
+              : '',
+        }
+      : {}),
  };

  if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -183,6 +196,15 @@ export const transformDetailToForm = (
      : {}),
    use_blueprint:
      detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,
+    ...(detail.kind !== CompilationTemplateKind.Tree
+      ? {
+          rechunk: config.rechunk === true,
+          rechunk_rules:
+            typeof config.rechunk_rules === 'string'
+              ? config.rechunk_rules
+              : '',
+        }
+      : {}),
  };

  if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -258,7 +280,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {
      return;
    }
    if (isConfigMetaKey(key)) {
-      if (typeof value === 'string') config[key] = value;
+      if (typeof value === 'string' || typeof value === 'boolean')
+        config[key] = value;
    } else {
      config[key] = value as ICompilationTemplateConfigRequest[string];
    }
--- a/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx
@@ -1,6 +1,7 @@
 import { ModelTreeSelectFormField } from '@/components/model-tree-select';
 import { SelectWithSearch } from '@/components/originui/select-with-search';
 import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SwitchFormField } from '@/components/switch-fom-field';
 import { Input } from '@/components/ui/input';
 import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
 import { Textarea } from '@/components/ui/textarea';
@@ -58,6 +59,10 @@ export function TemplateConfiguration({
    control: form.control,
    name: `templates.${selectedTemplateIndex}.kind`,
  });
+  const rechunk = useWatch({
+    control: form.control,
+    name: `templates.${selectedTemplateIndex}.config.rechunk`,
+  });

  const availableKindOptions = useAvailableKindOptions(
    form,
@@ -191,39 +196,62 @@ export function TemplateConfiguration({
          {kind === CompilationTemplateKind.Tree ? (
            <TreeTemplateFields index={selectedTemplateIndex} />
          ) : (
-            sectionNames.length > 0 &&
-            activeSectionTab && (
-              <Tabs
-                value={activeSectionTab}
-                onValueChange={setActiveSectionTab}
-                className="w-full"
-              >
-                <TabsList className="w-full justify-start">
+            <>
+              {kind !== CompilationTemplateKind.Artifacts && (
+                <>
+                  <SwitchFormField
+                    name={`templates.${selectedTemplateIndex}.config.rechunk`}
+                    label={t('setting.rechunkInput')}
+                    tooltip={t('setting.rechunkInputTip')}
+                    vertical={false}
+                  />
+                  {rechunk && (
+                    <RAGFlowFormItem
+                      name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}
+                      label={t('setting.rechunkRules')}
+                    >
+                      <Textarea
+                        placeholder={t('setting.rechunkRulesPlaceholder')}
+                        rows={6}
+                        resize="vertical"
+                      />
+                    </RAGFlowFormItem>
+                  )}
+                </>
+              )}
+              {sectionNames.length > 0 && activeSectionTab && (
+                <Tabs
+                  value={activeSectionTab}
+                  onValueChange={setActiveSectionTab}
+                  className="w-full"
+                >
+                  <TabsList className="w-full justify-start">
+                    {sectionNames.map((sectionName) => (
+                      <TabsTrigger
+                        key={sectionName}
+                        value={sectionName}
+                        className="flex-1"
+                      >
+                        {t(
+                          SectionTitleKeyMap[sectionName] ??
+                            startCase(sectionName),
+                        )}
+                      </TabsTrigger>
+                    ))}
+                  </TabsList>
+
                  {sectionNames.map((sectionName) => (
-                    <TabsTrigger
+                    <TabsContent
                      key={sectionName}
                      value={sectionName}
-                      className="flex-1"
+                      className="mt-4"
                    >
-                      {t(
-                        SectionTitleKeyMap[sectionName] ??
-                          startCase(sectionName),
-                      )}
-                    </TabsTrigger>
+                      {renderSectionTabs(sectionName)}
+                    </TabsContent>
                  ))}
-                </TabsList>
-
-                {sectionNames.map((sectionName) => (
-                  <TabsContent
-                    key={sectionName}
-                    value={sectionName}
-                    className="mt-4"
-                  >
-                    {renderSectionTabs(sectionName)}
-                  </TabsContent>
-                ))}
-              </Tabs>
-            )
+                </Tabs>
+              )}
+            </>
          )}

          {children}
--- a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
@@ -57,6 +57,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {
    instruction: '',
    page_example: '',
    use_blueprint: false,
+    rechunk: false,
+    rechunk_rules: '',
  },
 };

@@ -77,6 +79,8 @@ export const isConfigMetaKey = (key: string) =>
    'page_example',
    'synthesis',
    'use_blueprint',
+    'rechunk',
+    'rechunk_rules',
  ].includes(key);

 export const createEmptyField = (keys: string[]) =>
@@ -128,6 +132,15 @@ export const buildConfigFromBuiltin = (
      : {}),
    use_blueprint:
      kind === CompilationTemplateKind.Artifacts && example.length > 0,
+    ...(kind !== CompilationTemplateKind.Tree
+      ? {
+          rechunk: builtinTemplate.config?.rechunk === true,
+          rechunk_rules:
+            typeof builtinTemplate.config?.rechunk_rules === 'string'
+              ? builtinTemplate.config.rechunk_rules
+              : '',
+        }
+      : {}),
  };

  if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -179,6 +192,15 @@ export const transformDetailToForm = (
      : {}),
    use_blueprint:
      detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,
+    ...(detail.kind !== CompilationTemplateKind.Tree
+      ? {
+          rechunk: config.rechunk === true,
+          rechunk_rules:
+            typeof config.rechunk_rules === 'string'
+              ? config.rechunk_rules
+              : '',
+        }
+      : {}),
  };

  if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -254,7 +276,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {
      return;
    }
    if (isConfigMetaKey(key)) {
-      if (typeof value === 'string') config[key] = value;
+      if (typeof value === 'string' || typeof value === 'boolean')
+        config[key] = value;
    } else {
      config[key] = value as ICompilationTemplateConfigRequest[string];
    }