feat(api): add unified index API and dataset management endpoints (#14222)

### What problem does this PR solve? ## Summary Refactor the dataset API layer into a clean service/REST separation pattern, add a unified `/index` API for graph/raptor/mindmap operations, and introduce several new dataset management endpoints with full test coverage. ## Changes ### Service Layer (`dataset_api_service.py`) - Added `trace_index(dataset_id, tenant_id, index_type)` — unified trace function for all index types - Added `run_index`, `delete_index` service functions - Added `get_dataset`, `get_ingestion_summary`, `list_ingestion_logs`, `get_ingestion_log` - Added `run_embedding`, `list_tags`, `aggregate_tags`, `delete_tags`, `rename_tag` - Added `get_flattened_metadata`, `get_auto_metadata`, `update_auto_metadata` ### REST API Layer (`dataset_api.py`) **New unified routes:** | Method | Route | Description | |--------|-------|-------------| | POST | `/datasets/<id>/index?type=graph\|raptor\|mindmap` | Run index task | | GET | `/datasets/<id>/index?type=graph\|raptor\|mindmap` | Trace index task | | DELETE | `/datasets/<id>/<index_type>` | Delete index | | GET | `/datasets/<id>` | Get dataset details | | GET | `/datasets/<id>/ingestions/summary` | Ingestion summary | | GET | `/datasets/<id>/ingestions` | List ingestion logs | | GET | `/datasets/<id>/ingestions/<log_id>` | Get single ingestion log | | POST | `/datasets/<id>/embedding` | Run embedding | | GET | `/datasets/<id>/tags` | List tags | | GET | `/datasets/tags/aggregation` | Aggregate tags across datasets | | DELETE | `/datasets/<id>/tags` | Delete tags | | PUT | `/datasets/<id>/tags` | Rename tag | | GET | `/datasets/metadata/flattened` | Get flattened metadata | | GET/PUT | `/datasets/<id>/metadata/config` | New metadata config path | **Removed routes (replaced by unified `/index`):** - `POST /datasets/<id>/mindmap` - `GET /datasets/<id>/mindmap` **Preserved legacy routes (backward compatibility):** - `/run_graphrag`, `/trace_graphrag`, `/run_raptor`, `/trace_raptor` - `/auto_metadata` GET/PUT ### Test Suite - Updated `common.py` helpers: added `trace_index`, removed `run_mindmap`/`trace_mindmap` - Added 7 new test files with 39 test cases total: | Test File | Cases | |-----------|-------| | `test_get_dataset.py` | 4 | | `test_ingestion_summary.py` | 2 | | `test_ingestion_logs.py` | 5 | | `test_index_api.py` | 14 | | `test_embedding.py` | 2 | | `test_tags.py` | 8 | | `test_flattened_metadata.py` | 4 | - Deleted `test_mindmap_tasks.py` (covered by unified index tests) ## Design Decisions 1. **Unified `/index?type=...`** — single endpoint replaces 3 separate route pairs for graph/raptor/mindmap 2. **Backward compatibility** — old routes (`/run_graphrag`, `/run_raptor`, `/auto_metadata`) preserved alongside new paths 3. **`_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap"}`** — input validation via constant set 4. **`_INDEX_TYPE_TO_TASK_ID_FIELD`** — maps index type to KB model task ID field for clean dispatch ## Files Changed - `api/apps/restful_apis/dataset_api.py` - `api/apps/services/dataset_api_service.py` - `sdk/python/ragflow_sdk/modules/dataset.py` - `test/testcases/test_http_api/common.py` - `test/testcases/test_http_api/test_dataset_management/` (7 new files) ### Type of change - [x] New Feature (non-breaking change which adds functionality) - [x] Refactoring --------- Signed-off-by: noob <yixiao121314@outlook.com>
2026-08-08 00:18:12 +08:00 · 2026-04-27 01:38:01 +00:00
parent fb95136f39
commit 4dcc42e0e1
51 changed files with 1765 additions and 4381 deletions
--- a/web/src/hooks/use-knowledge-request.ts
+++ b/web/src/hooks/use-knowledge-request.ts
@@ -14,6 +14,7 @@ import { ITestRetrievalRequestBody } from '@/interfaces/request/knowledge';
 import i18n from '@/locales/config';
 import kbService, {
  deleteKnowledgeGraph,
+  getKbDetail,
  getKnowledgeGraph,
  listDataset,
  listTag,
@@ -407,9 +408,7 @@ export const useFetchKnowledgeBaseConfiguration = (props?: {
    gcTime: 0,
    enabled: !!knowledgeBaseId && isEdit,
    queryFn: async () => {
-      const { data } = await kbService.getKbDetail({
-        kb_id: knowledgeBaseId,
-      });
+      const { data } = await getKbDetail(knowledgeBaseId || '');
      return data?.data ?? {};
    },
  });
@@ -443,7 +442,9 @@ export function useFetchKnowledgeMetadata(kbIds: string[] = []) {
    enabled: kbIds.length > 0,
    gcTime: 0,
    queryFn: async () => {
-      const { data } = await kbService.getMeta({ kb_ids: kbIds.join(',') });
+      const { data } = await kbService.getMeta({
+        dataset_ids: kbIds.join(','),
+      });
      return data?.data ?? {};
    },
  });
@@ -549,7 +550,7 @@ export const useFetchTagListByKnowledgeIds = () => {
    gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
    queryFn: async () => {
      const { data } = await kbService.listTagByKnowledgeIds({
-        kb_ids: knowledgeIds.join(','),
+        dataset_ids: knowledgeIds.join(','),
      });
      const list = data?.data || [];
      return list;
--- a/web/src/interfaces/database/dataset.ts
+++ b/web/src/interfaces/database/dataset.ts
@@ -1,5 +1,5 @@
 // for the dataset list
-// The data structures returned by the `datasets` interface and `kb/detail` are inconsistent.
+// The data structures returned by the `datasets` interface and `/api/v1/datasets/{id}` are inconsistent.

 export interface IDataset {
  avatar?: string;
--- a/web/src/pages/dataset/components/metedata/hooks/use-manage-modal.ts
+++ b/web/src/pages/dataset/components/metedata/hooks/use-manage-modal.ts
@@ -2,8 +2,9 @@ import message from '@/components/ui/message';
 import { useSetModalState } from '@/hooks/common-hooks';
 import { useSelectedIds } from '@/hooks/logic-hooks/use-row-selection';
 import { DocumentApiAction } from '@/hooks/use-document-request';
-import kbService, {
+import {
  getMetaDataService,
+  kbUpdateMetaData,
  updateDocumentMetaDataConfig,
  updateDocumentsMetadata,
 } from '@/services/knowledge-service';
@@ -413,8 +414,7 @@ export const useManageMetaDataModal = (
  const handleSaveSettings = useCallback(
    async (callback: () => void, builtInMetadata?: IBuiltInMetadataItem[]) => {
      const data = util.tableDataToMetaDataSettingJSON(tableData);
-      const { data: res } = await kbService.kbUpdateMetaData({
-        kb_id: id,
+      const { data: res } = await kbUpdateMetaData(id || '', {
        metadata: data,
        builtInMetadata: builtInMetadata || [],
      });
@@ -434,14 +434,11 @@ export const useManageMetaDataModal = (
  const handleSaveSingleFileSettings = useCallback(
    async (callback: () => void) => {
      const data = util.tableDataToMetaDataSettingJSON(tableData);
-      // otherData contains: documentId
-      if (otherData?.documentId && id) {
+      if (otherData?.documentId) {
        const { data: res } = await updateDocumentMetaDataConfig({
-          kb_id: id,
+          kb_id: id || '',
          doc_id: otherData.documentId,
-          data: {
-            metadata: data,
-          },
+          data: { metadata: data },
        });
        if (res.code === 0) {
          message.success(t('message.operated'));
--- a/web/src/pages/dataset/dataset-overview/hook.ts
+++ b/web/src/pages/dataset/dataset-overview/hook.ts
@@ -3,7 +3,8 @@ import {
  useGetPaginationWithRouter,
  useHandleSearchChange,
 } from '@/hooks/logic-hooks';
-import kbService, {
+import {
+  getKnowledgeBasicInfo,
  listDataPipelineLogDocument,
  listPipelineDatasetLogs,
 } from '@/services/knowledge-service';
@@ -20,9 +21,9 @@ const useFetchOverviewTotal = () => {
  const { data } = useQuery<IOverviewTotal>({
    queryKey: ['overviewTotal'],
    queryFn: async () => {
-      const { data: res = {} } = await kbService.getKnowledgeBasicInfo({
-        kb_id: knowledgeBaseId,
-      });
+      const { data: res = {} } = await getKnowledgeBasicInfo(
+        knowledgeBaseId || '',
+      );
      return res.data || [];
    },
  });
@@ -61,16 +62,12 @@ const useFetchFileLogList = () => {
    },
    enabled: true,
    queryFn: async () => {
-      const { data: res = {} } = await fetchFunc(
-        {
-          kb_id: knowledgeBaseId,
-          page: pagination.current,
-          page_size: pagination.pageSize,
-          keywords: searchString,
-          // order_by: '',
-        },
-        { ...filterValue },
-      );
+      const { data: res = {} } = await fetchFunc(knowledgeBaseId || '', {
+        page: pagination.current,
+        page_size: pagination.pageSize,
+        keywords: searchString,
+        ...filterValue,
+      });
      return res.data || [];
    },
  });
--- a/web/src/pages/dataset/dataset-setting/hooks.ts
+++ b/web/src/pages/dataset/dataset-setting/hooks.ts
@@ -4,7 +4,7 @@ import { useSetModalState } from '@/hooks/common-hooks';
 import { useFetchKnowledgeBaseConfiguration } from '@/hooks/use-knowledge-request';
 import { useSelectLlmOptionsByModelType } from '@/hooks/use-llm-request';
 import { useSelectParserList } from '@/hooks/use-user-setting-request';
-import kbService from '@/services/knowledge-service';
+import { checkEmbedding } from '@/services/knowledge-service';
 import { useIsFetching } from '@tanstack/react-query';
 import { pick } from 'lodash';
 import { useCallback, useEffect, useState } from 'react';
@@ -108,8 +108,7 @@ export const useHandleKbEmbedding = () => {
  const knowledgeBaseId = searchParams.get('id') || id;
  const handleChange = useCallback(
    async ({ embed_id }: { embed_id: string }) => {
-      const res = await kbService.checkEmbedding({
-        kb_id: knowledgeBaseId,
+      const res = await checkEmbedding(knowledgeBaseId || '', {
        embd_id: embed_id,
      });
      return res.data;
--- a/web/src/pages/dataset/dataset/generate-button/hook.ts
+++ b/web/src/pages/dataset/dataset/generate-button/hook.ts
@@ -2,10 +2,8 @@ import message from '@/components/ui/message';
 import agentService from '@/services/agent-service';
 import {
  deletePipelineTask,
-  runGraphRag,
-  runRaptor,
-  traceGraphRag,
-  traceRaptor,
+  runIndex,
+  traceIndex,
 } from '@/services/knowledge-service';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { t } from 'i18next';
@@ -59,7 +57,7 @@ export const useTraceGenerate = ({ open }: { open: boolean }) => {
      retryDelay: 1000,
      enabled: open,
      queryFn: async () => {
-        const { data } = await traceGraphRag(id);
+        const { data } = await traceIndex(id, 'graph');
        return data?.data || {};
      },
    });
@@ -74,7 +72,7 @@ export const useTraceGenerate = ({ open }: { open: boolean }) => {
      retryDelay: 1000,
      enabled: open,
      queryFn: async () => {
-        const { data } = await traceRaptor(id);
+        const { data } = await traceIndex(id, 'raptor');
        return data?.data || {};
      },
    });
@@ -134,9 +132,9 @@ export const useDatasetGenerate = () => {
  } = useMutation({
    mutationKey: [DatasetKey.generate],
    mutationFn: async ({ type }: { type: GenerateType }) => {
-      const func =
-        type === GenerateType.KnowledgeGraph ? runGraphRag : runRaptor;
-      const { data } = await func(id);
+      const indexType =
+        type === GenerateType.KnowledgeGraph ? 'graph' : 'raptor';
+      const { data } = await runIndex(id, indexType);
      if (data.code === 0) {
        message.success(t('message.operated'));
        queryClient.invalidateQueries({
--- a/web/src/services/knowledge-service.ts
+++ b/web/src/services/knowledge-service.ts
@@ -8,33 +8,25 @@ import { ProcessingType } from '@/pages/dataset/dataset-overview/dataset-common'
 import api from '@/utils/api';
 import { getAuthorization } from '@/utils/authorization-util';
 import registerServer from '@/utils/register-server';
-import request, { post } from '@/utils/request';
+import request from '@/utils/request';
 import axios from 'axios';

 const {
  createKb,
  rmKb,
-  getKbDetail,
  kbList,
-  getDocumentList,
  documentChangeStatus,
  documentCreate,
  documentChangeParser,
  documentThumbnails,
  retrievalTest,
  documentRun,
-  documentUpload,
  webCrawl,
  knowledgeGraph,
  listTagByKnowledgeIds,
  setMeta,
  getMeta,
  retrievalTestShare,
-  getKnowledgeBasicInfo,
-  fetchDataPipelineLog,
-  fetchPipelineDatasetLogs,
-  checkEmbedding,
-  kbUpdateMetaData,
 } = api;

 const methods = {
@@ -46,19 +38,11 @@ const methods = {
    url: rmKb,
    method: 'delete',
  },
-  getKbDetail: {
-    url: getKbDetail,
-    method: 'get',
-  },
  getList: {
    url: kbList,
    method: 'get',
  },
  // document manager
-  getDocumentList: {
-    url: getDocumentList,
-    method: 'get',
-  },
  documentChangeStatus: {
    url: documentChangeStatus,
    method: 'post',
@@ -79,10 +63,6 @@ const methods = {
    url: documentThumbnails,
    method: 'get',
  },
-  documentUpload: {
-    url: documentUpload,
-    method: 'post',
-  },
  webCrawl: {
    url: webCrawl,
    method: 'post',
@@ -115,36 +95,10 @@ const methods = {
    url: retrievalTestShare,
    method: 'post',
  },
-  getKnowledgeBasicInfo: {
-    url: getKnowledgeBasicInfo,
-    method: 'get',
-  },
-  fetchDataPipelineLog: {
-    url: fetchDataPipelineLog,
-    method: 'post',
-  },
-  fetchPipelineDatasetLogs: {
-    url: fetchPipelineDatasetLogs,
-    method: 'post',
-  },
-  getPipelineDetail: {
-    url: api.getPipelineDetail,
-    method: 'get',
-  },
-
  pipelineRerun: {
    url: api.pipelineRerun,
    method: 'post',
  },
-
-  checkEmbedding: {
-    url: checkEmbedding,
-    method: 'post',
-  },
-  kbUpdateMetaData: {
-    url: kbUpdateMetaData,
-    method: 'post',
-  },
 };

 const baseKbService = registerServer<keyof typeof methods>(methods, request);
@@ -281,16 +235,19 @@ const kbService = {
  ...chunkService,
 };

+export const getKbDetail = (datasetId: string) =>
+  request.get(api.getKbDetail(datasetId));
+
 export const listTag = (knowledgeId: string) =>
  request.get(api.listTag(knowledgeId));

 export const removeTag = (knowledgeId: string, tags: string[]) =>
-  post(api.removeTag(knowledgeId), { tags });
+  request.delete(api.removeTag(knowledgeId), { data: { tags } });

 export const renameTag = (
  knowledgeId: string,
  { fromTag, toTag }: IRenameTag,
-) => post(api.renameTag(knowledgeId), { fromTag, toTag });
+) => request.put(api.renameTag(knowledgeId), { data: { fromTag, toTag } });

 export function getKnowledgeGraph(knowledgeId: string) {
  return request.get(api.getKnowledgeGraph(knowledgeId));
@@ -306,17 +263,11 @@ export const listDataset = (params?: IFetchKnowledgeListRequestParams) =>
 export const updateKb = (datasetId: string, data: Record<string, any>) =>
  request.put(api.updateKb(datasetId), { data });

-export const runGraphRag = (datasetId: string) =>
-  request.post(api.runGraphRag(datasetId));
+export const runIndex = (datasetId: string, indexType: string) =>
+  request.post(api.runIndex(datasetId, indexType));

-export const traceGraphRag = (datasetId: string) =>
-  request.get(api.traceGraphRag(datasetId));
-
-export const runRaptor = (datasetId: string) =>
-  request.post(api.runRaptor(datasetId));
-
-export const traceRaptor = (datasetId: string) =>
-  request.get(api.traceRaptor(datasetId));
+export const traceIndex = (datasetId: string, indexType: string) =>
+  request.get(api.traceIndex(datasetId, indexType));

 // Using RESTful API: GET /api/v1/datasets/{dataset_id}/documents
 export const listDocument = (
@@ -403,16 +354,28 @@ export const updateDocumentMetaDataConfig = ({
  });

 export const listDataPipelineLogDocument = (
-  params?: IFetchKnowledgeListRequestParams,
-  body?: IFetchDocumentListRequestBody,
-) => request.post(api.fetchDataPipelineLog, { data: body || {}, params });
+  datasetId: string,
+  params?: Record<string, any>,
+) => request.get(api.fetchDataPipelineLog(datasetId), { params });
+
 export const listPipelineDatasetLogs = (
-  params?: IFetchKnowledgeListRequestParams & {
-    kb_id?: string;
-    keywords?: string;
-  },
-  body?: IFetchDocumentListRequestBody,
-) => request.post(api.fetchPipelineDatasetLogs, { data: body || {}, params });
+  datasetId: string,
+  params?: Record<string, any>,
+) => request.get(api.fetchPipelineDatasetLogs(datasetId), { params });
+
+export const getPipelineDetail = (datasetId: string, logId: string) =>
+  request.get(api.getPipelineDetail(datasetId, logId));
+
+export const getKnowledgeBasicInfo = (datasetId: string) =>
+  request.get(api.getKnowledgeBasicInfo(datasetId));
+
+export const checkEmbedding = (datasetId: string, data: Record<string, any>) =>
+  request.post(api.checkEmbedding(datasetId), { data });
+
+export const kbUpdateMetaData = (
+  datasetId: string,
+  data: Record<string, any>,
+) => request.put(api.kbUpdateMetaData(datasetId), { data });

 export function deletePipelineTask({
  kb_id,
@@ -421,7 +384,7 @@ export function deletePipelineTask({
  kb_id: string;
  type: ProcessingType;
 }) {
-  return request.delete(api.unbindPipelineTask({ kb_id, type }));
+  return request.delete(api.unbindPipelineTask(kb_id, type));
 }

 export default kbService;
--- a/web/src/utils/api.ts
+++ b/web/src/utils/api.ts
@@ -57,46 +57,50 @@ export default {

  // knowledge base

-  checkEmbedding: `${webAPI}/kb/check_embedding`,
+  checkEmbedding: (datasetId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/embedding`,
  kbList: `${restAPIv1}/datasets`,
  createKb: `${restAPIv1}/datasets`,
  updateKb: (datasetId: string) => `${restAPIv1}/datasets/${datasetId}`,
  rmKb: `${restAPIv1}/datasets`,
-  getKbDetail: `${webAPI}/kb/detail`,
+  getKbDetail: (datasetId: string) => `${restAPIv1}/datasets/${datasetId}`,
  getKnowledgeGraph: (knowledgeId: string) =>
-    `${restAPIv1}/datasets/${knowledgeId}/knowledge_graph`,
+    `${restAPIv1}/datasets/${knowledgeId}/graph/search`,
  deleteKnowledgeGraph: (knowledgeId: string) =>
-    `${restAPIv1}/datasets/${knowledgeId}/knowledge_graph`,
-  getMeta: `${webAPI}/kb/get_meta`,
-  getKnowledgeBasicInfo: `${webAPI}/kb/basic_info`,
+    `${restAPIv1}/datasets/${knowledgeId}/graph`,
+  getMeta: `${restAPIv1}/datasets/metadata/flattened`,
+  getKnowledgeBasicInfo: (datasetId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/ingestions/summary`,
  // data pipeline log
-  fetchDataPipelineLog: `${webAPI}/kb/list_pipeline_logs`,
-  getPipelineDetail: `${webAPI}/kb/pipeline_log_detail`,
-  fetchPipelineDatasetLogs: `${webAPI}/kb/list_pipeline_dataset_logs`,
-  runGraphRag: (datasetId: string) =>
-    `${restAPIv1}/datasets/${datasetId}/run_graphrag`,
-  traceGraphRag: (datasetId: string) =>
-    `${restAPIv1}/datasets/${datasetId}/trace_graphrag`,
-  runRaptor: (datasetId: string) =>
-    `${restAPIv1}/datasets/${datasetId}/run_raptor`,
-  traceRaptor: (datasetId: string) =>
-    `${restAPIv1}/datasets/${datasetId}/trace_raptor`,
-  unbindPipelineTask: ({ kb_id, type }: { kb_id: string; type: string }) =>
-    `${webAPI}/kb/unbind_task?kb_id=${kb_id}&pipeline_task_type=${type}`,
-  pipelineRerun: `${restAPIv1}/agents/rerun`,
+  fetchDataPipelineLog: (datasetId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/ingestions`,
+  getPipelineDetail: (datasetId: string, logId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/ingestions/${logId}`,
+  fetchPipelineDatasetLogs: (datasetId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/ingestions`,
+  runIndex: (datasetId: string, indexType: string) =>
+    `${restAPIv1}/datasets/${datasetId}/index?type=${indexType}`,
+  traceIndex: (datasetId: string, indexType: string) =>
+    `${restAPIv1}/datasets/${datasetId}/index?type=${indexType}`,
+  unbindPipelineTask: (datasetId: string, indexType: string) =>
+    `${restAPIv1}/datasets/${datasetId}/${indexType}`,
+  pipelineRerun: `${webAPI}/canvas/rerun`,
  getMetaData: (datasetId: string) =>
    `${restAPIv1}/datasets/${datasetId}/metadata/summary`,
  updateDocumentsMetadata: (datasetId: string) =>
    `${restAPIv1}/datasets/${datasetId}/documents/metadatas`,
-  kbUpdateMetaData: `${webAPI}/kb/update_metadata_setting`,
+  kbUpdateMetaData: (datasetId: string) =>
+    `${restAPIv1}/datasets/${datasetId}/metadata/config`,
  documentUpdateMetaDataConfig: (datasetId: string, documentId: string) =>
    `${restAPIv1}/datasets/${datasetId}/documents/${documentId}/metadata/config`,

  // tags
-  listTag: (knowledgeId: string) => `${webAPI}/kb/${knowledgeId}/tags`,
-  listTagByKnowledgeIds: `${webAPI}/kb/tags`,
-  removeTag: (knowledgeId: string) => `${webAPI}/kb/${knowledgeId}/rm_tags`,
-  renameTag: (knowledgeId: string) => `${webAPI}/kb/${knowledgeId}/rename_tag`,
+  listTag: (knowledgeId: string) => `${restAPIv1}/datasets/${knowledgeId}/tags`,
+  listTagByKnowledgeIds: `${restAPIv1}/datasets/tags/aggregation`,
+  removeTag: (knowledgeId: string) =>
+    `${restAPIv1}/datasets/${knowledgeId}/tags`,
+  renameTag: (knowledgeId: string) =>
+    `${restAPIv1}/datasets/${knowledgeId}/tags`,

  // chunk
  chunkList: (datasetId: string, documentId: string) =>
--- a/web/src/utils/llm-util.ts
+++ b/web/src/utils/llm-util.ts
@@ -84,8 +84,7 @@ const API_WHITELIST = [
  '/v1/canvas/setting',
  '/api/v1/searches/',
  '/api/v1/memories',
-  '/v1/kb/create',
-  '/v1/kb/update',
+  '/api/v1/datasets',
  '/v1/dataflow/set',
 ];