fix: fetch all knowledge bases via pagination in link-to-dataset dialog (#17170)

web/src/components/knowledge-base-item.tsx

@@ -28,10 +28,12 @@ export function useDisableDifferenceEmbeddingDataset(name: string) {
   const datasetId = useWatch({ name, control: form.control });
   const [searchString, setSearchString] = useState('');
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
-  const { list: datasetListOrigin, loading } = useFetchKnowledgeList(
-    true,
-    debouncedSearchString,
-  );
+  const {
+    list: datasetListOrigin,
+    loading,
+    handleScroll,
+    hasNextPage,
+  } = useFetchKnowledgeList(true, debouncedSearchString);
   const datasetCacheRef = useRef(new Map<string, IDataset>());
 
   const datasetList = useMemo(() => {
@@ -96,6 +98,8 @@ export function useDisableDifferenceEmbeddingDataset(name: string) {
     handleSearchChange,
     loading,
     searchString,
+    handleScroll,
+    hasNextPage,
   };
 }
 
@@ -110,8 +114,14 @@ export function KnowledgeBaseFormField({
 }) {
   const { t } = useTranslation();
 
-  const { datasetOptions, handleSearchChange, loading, searchString } =
-    useDisableDifferenceEmbeddingDataset(name);
+  const {
+    datasetOptions,
+    handleSearchChange,
+    loading,
+    searchString,
+    handleScroll,
+    hasNextPage,
+  } = useDisableDifferenceEmbeddingDataset(name);
 
   const nextOptions = buildQueryVariableOptionsByShowVariable(showVariable)();
 
@@ -178,6 +188,7 @@ export function KnowledgeBaseFormField({
           onSearchChange={handleSearchChange}
           isSearching={loading}
           shouldFilter={false}
+          onListScroll={hasNextPage ? handleScroll : undefined}
           {...field}
         />
       )}

web/src/components/ui/multi-select.tsx

@@ -192,6 +192,7 @@ interface MultiSelectProps
   onSearchChange?: (value: string) => void;
   isSearching?: boolean;
   shouldFilter?: boolean;
+  onListScroll?: (e: React.UIEvent<HTMLDivElement>) => void;
 }
 
 export const MultiSelect = React.forwardRef<
@@ -217,6 +218,7 @@ export const MultiSelect = React.forwardRef<
       onSearchChange,
       isSearching = false,
       shouldFilter,
+      onListScroll,
       ...props
     },
     ref,
@@ -451,7 +453,7 @@ export const MultiSelect = React.forwardRef<
                 onValueChange={onSearchChange}
               />
             )}
-            <CommandList className="mt-2">
+            <CommandList className="mt-2" onScroll={onListScroll}>
               <CommandEmpty>
                 {isSearching ? t('common.searching') : t('common.noDataFound')}
               </CommandEmpty>

web/src/hooks/use-knowledge-request.ts

@@ -51,7 +51,7 @@ import {
 } from '@tanstack/react-query';
 import { useDebounce } from 'ahooks';
 import { omit } from 'lodash';
-import { useCallback, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
 import { useParams, useSearchParams } from 'react-router';
 import {
   useGetPaginationWithRouter,
@@ -234,7 +234,9 @@ export const useCreateKnowledge = () => {
         message.success(
           i18n.t(`message.${params?.id ? 'modified' : 'created'}`),
         );
-        queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] });
+        queryClient.invalidateQueries({
+          queryKey: [KnowledgeApiAction.FetchKnowledgeList],
+        });
       }
       return data;
     },
@@ -818,43 +820,114 @@ export const useClearWiki = () => {
   return { data, loading, clearWiki: mutateAsync };
 };
 
-export const useFetchKnowledgeList = (
-  shouldFilterListWithoutDocument: boolean = false,
-  keywords = '',
-): {
-  list: IDataset[];
-  loading: boolean;
-} => {
-  const { data, isFetching: loading } = useQuery({
-    queryKey: [
+const KNOWLEDGE_LIST_PAGE_SIZE = 10;
+
+export const KnowledgeListKeys = {
+  list: (
+    shouldFilterListWithoutDocument: boolean,
+    keywords: string,
+    pageSize: number,
+  ) =>
+    [
       KnowledgeApiAction.FetchKnowledgeList,
       shouldFilterListWithoutDocument,
       keywords,
-    ],
-    initialData: [],
-    gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3
-    queryFn: async () => {
-      const { data } = await listDataset(
-        keywords
-          ? {
-              ext: {
-                keywords,
-              },
-            }
-          : undefined,
-      );
-      const list = data?.data ?? [];
-      return shouldFilterListWithoutDocument
-        ? list.filter((x: IDataset) => x.chunk_count > 0)
-        : list;
-    },
-  });
+      pageSize,
+    ] as const,
+};
 
-  return { list: data, loading };
+export const useFetchKnowledgeList = (
+  shouldFilterListWithoutDocument: boolean = false,
+  keywords = '',
+  pageSize: number = KNOWLEDGE_LIST_PAGE_SIZE,
+): {
+  list: IDataset[];
+  loading: boolean;
+  fetchNextPage: () => void;
+  hasNextPage: boolean;
+  isFetchingNextPage: boolean;
+  handleScroll: (e: React.UIEvent<HTMLDivElement>) => void;
+} => {
+  const { data, fetchNextPage, hasNextPage, isFetching, isFetchingNextPage } =
+    useInfiniteQuery<{ items: IDataset[] }>({
+      queryKey: KnowledgeListKeys.list(
+        shouldFilterListWithoutDocument,
+        keywords,
+        pageSize,
+      ),
+      gcTime: 0,
+      initialPageParam: 1,
+      queryFn: async ({ pageParam }) => {
+        const page = pageParam as number;
+        const { data } = await listDataset({
+          page,
+          page_size: pageSize,
+          ...(keywords ? { ext: { keywords } } : {}),
+        });
+        return { items: (data?.data ?? []) as IDataset[] };
+      },
+      getNextPageParam: (lastPage, allPages) =>
+        lastPage.items.length >= pageSize ? allPages.length + 1 : undefined,
+    });
+
+  const list = useMemo(() => {
+    const all = data?.pages.flatMap((page) => page.items) ?? [];
+    return shouldFilterListWithoutDocument
+      ? all.filter((x) => x.chunk_count > 0)
+      : all;
+  }, [data, shouldFilterListWithoutDocument]);
+
+  const handleScroll = useCallback(
+    (e: React.UIEvent<HTMLDivElement>) => {
+      const { scrollTop, scrollHeight, clientHeight } = e.currentTarget;
+      const threshold = 50;
+      if (
+        scrollHeight - scrollTop - clientHeight <= threshold &&
+        hasNextPage &&
+        !isFetchingNextPage
+      ) {
+        fetchNextPage();
+      }
+    },
+    [fetchNextPage, hasNextPage, isFetchingNextPage],
+  );
+
+  return {
+    list,
+    loading: isFetching,
+    fetchNextPage,
+    hasNextPage,
+    isFetchingNextPage,
+    handleScroll,
+  };
+};
+
+/**
+ * For consumers that need the COMPLETE list (no scroll UI).
+ * Uses a large page size to minimize round-trips, and auto-loads
+ * until exhausted.
+ */
+export const useFetchAllKnowledgeList = (
+  shouldFilterListWithoutDocument: boolean = false,
+  keywords = '',
+): { list: IDataset[]; loading: boolean } => {
+  const { list, loading, hasNextPage, fetchNextPage } = useFetchKnowledgeList(
+    shouldFilterListWithoutDocument,
+    keywords,
+    200,
+  );
+
+  useEffect(() => {
+    if (hasNextPage && !loading) {
+      fetchNextPage();
+    }
+  }, [hasNextPage, loading, fetchNextPage]);
+
+  return { list, loading };
 };
 
 export const useSelectKnowledgeOptions = () => {
-  const { list } = useFetchKnowledgeList();
+  const { list } = useFetchAllKnowledgeList();
 
   const options = list?.map((item) => ({
     label: item.name,

web/src/pages/agent/canvas/node/retrieval-node.tsx

@@ -1,6 +1,6 @@
 import { NodeCollapsible } from '@/components/collapse';
 import { RAGFlowAvatar } from '@/components/ragflow-avatar';
-import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
+import { useFetchAllKnowledgeList } from '@/hooks/use-knowledge-request';
 import { useFetchAllMemoryList } from '@/hooks/use-memory-request';
 import { BaseNode } from '@/interfaces/database/agent';
 import { NodeProps, Position } from '@xyflow/react';
@@ -25,7 +25,7 @@ function InnerRetrievalNode({
 }: NodeProps<BaseNode<RetrievalFormSchemaType>>) {
   const knowledgeBaseIds: string[] = get(data, 'form.dataset_ids', []);
   const memoryIds: string[] = get(data, 'form.memory_ids', []);
-  const { list: knowledgeList } = useFetchKnowledgeList(true);
+  const { list: knowledgeList } = useFetchAllKnowledgeList(true);
 
   const { getLabel } = useGetVariableLabelOrTypeByValue({ nodeId: id });
 

web/src/pages/dataset/dataset-setting/components/tag-item.tsx

@@ -10,7 +10,9 @@ import {
 import { MultiSelect } from '@/components/ui/multi-select';
 import { FormLayout } from '@/constants/form';
 import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
+import { useDebounce } from 'ahooks';
 import DOMPurify from 'dompurify';
+import { useState } from 'react';
 import { useFormContext, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 
@@ -18,7 +20,15 @@ export const TagSetItem = () => {
   const { t } = useTranslation();
   const form = useFormContext();
 
-  const { list: knowledgeList } = useFetchKnowledgeList(true);
+  const [searchString, setSearchString] = useState('');
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+
+  const {
+    list: knowledgeList,
+    handleScroll,
+    hasNextPage,
+    loading,
+  } = useFetchKnowledgeList(true, debouncedSearchString);
 
   const knowledgeOptions = knowledgeList
     .filter((x) => x.chunk_method === 'tag')
@@ -60,6 +70,11 @@ export const TagSetItem = () => {
                 <MultiSelect
                   options={knowledgeOptions}
                   onValueChange={field.onChange}
+                  searchValue={searchString}
+                  onSearchChange={setSearchString}
+                  isSearching={loading}
+                  shouldFilter={false}
+                  onListScroll={hasNextPage ? handleScroll : undefined}
                   // placeholder={t('chat.knowledgeBasesMessage')}
                   variant="inverted"
                   maxCount={10}

web/src/pages/datasets/use-select-owners.ts

@@ -1,10 +1,10 @@
 import { FilterCollection } from '@/components/list-filter-bar/interface';
-import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
+import { useFetchAllKnowledgeList } from '@/hooks/use-knowledge-request';
 import { buildOwnersFilter } from '@/utils/list-filter-util';
 import { useTranslation } from 'react-i18next';
 
 export function useSelectOwners() {
-  const { list } = useFetchKnowledgeList();
+  const { list } = useFetchAllKnowledgeList();
   const { t } = useTranslation();
 
   const filters: FilterCollection[] = [

web/src/pages/files/hooks.ts

@@ -86,7 +86,7 @@ export const useHandleConnectToKnowledge = () => {
   const [record, setRecord] = useState<IFile>({} as IFile);
   const [documentIds, setDocumentIds] = useState<string[]>([]);
   const [mode, setMode] = useState<ConnectFileToKnowledgeMode>('replace');
-  const knowledgeOptions = useSelectKnowledgeOptions();
+  const { options: knowledgeOptions } = useSelectKnowledgeOptions();
 
   const initialValue = useMemo(() => {
     return Array.isArray(record?.kbs_info)

web/src/pages/files/link-to-dataset-dialog.tsx

@@ -15,10 +15,12 @@ import {
   FormMessage,
 } from '@/components/ui/form';
 import { MultiSelect } from '@/components/ui/multi-select';
-import { useSelectKnowledgeOptions } from '@/hooks/use-knowledge-request';
+import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request';
 import { IModalProps } from '@/interfaces/common';
+import { useDebounce } from 'ahooks';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { Link2 } from 'lucide-react';
+import { useMemo, useState } from 'react';
 import { useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
@@ -47,7 +49,21 @@ function LinkToDatasetForm({
     },
   });
 
-  const options = useSelectKnowledgeOptions();
+  const [searchString, setSearchString] = useState('');
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const { list, loading, handleScroll, hasNextPage } = useFetchKnowledgeList(
+    false,
+    debouncedSearchString,
+  );
+
+  const options = useMemo(
+    () =>
+      list.map((item) => ({
+        label: item.name,
+        value: item.id,
+      })),
+    [list],
+  );
 
   function onSubmit(data: z.infer<typeof FormSchema>) {
     onConnectToKnowledgeOk(data.knowledgeIds);
@@ -77,6 +93,11 @@ function LinkToDatasetForm({
                   defaultValue={field.value}
                   placeholder={t('fileManager.pleaseSelect')}
                   maxCount={100}
+                  searchValue={searchString}
+                  onSearchChange={setSearchString}
+                  isSearching={loading}
+                  shouldFilter={false}
+                  onListScroll={hasNextPage ? handleScroll : undefined}
                   //   {...field}
                   modalPopover
                 />