From 418d3c8cefc98ad2571f122abb1760cc41ae791f Mon Sep 17 00:00:00 2001 From: euvre <93761161+euvre@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:48:05 +0800 Subject: [PATCH] fix: fetch all knowledge bases via pagination in link-to-dataset dialog (#17170) --- web/src/components/knowledge-base-item.tsx | 23 ++- web/src/components/ui/multi-select.tsx | 4 +- web/src/hooks/use-knowledge-request.ts | 137 ++++++++++++++---- .../agent/canvas/node/retrieval-node.tsx | 4 +- .../dataset-setting/components/tag-item.tsx | 17 ++- web/src/pages/datasets/use-select-owners.ts | 4 +- web/src/pages/files/hooks.ts | 2 +- .../pages/files/link-to-dataset-dialog.tsx | 25 +++- 8 files changed, 169 insertions(+), 47 deletions(-) diff --git a/web/src/components/knowledge-base-item.tsx b/web/src/components/knowledge-base-item.tsx index c657059375..c80ba23b21 100644 --- a/web/src/components/knowledge-base-item.tsx +++ b/web/src/components/knowledge-base-item.tsx @@ -28,10 +28,12 @@ export function useDisableDifferenceEmbeddingDataset(name: string) { const datasetId = useWatch({ name, control: form.control }); const [searchString, setSearchString] = useState(''); const debouncedSearchString = useDebounce(searchString, { wait: 500 }); - const { list: datasetListOrigin, loading } = useFetchKnowledgeList( - true, - debouncedSearchString, - ); + const { + list: datasetListOrigin, + loading, + handleScroll, + hasNextPage, + } = useFetchKnowledgeList(true, debouncedSearchString); const datasetCacheRef = useRef(new Map()); const datasetList = useMemo(() => { @@ -96,6 +98,8 @@ export function useDisableDifferenceEmbeddingDataset(name: string) { handleSearchChange, loading, searchString, + handleScroll, + hasNextPage, }; } @@ -110,8 +114,14 @@ export function KnowledgeBaseFormField({ }) { const { t } = useTranslation(); - const { datasetOptions, handleSearchChange, loading, searchString } = - useDisableDifferenceEmbeddingDataset(name); + const { + datasetOptions, + handleSearchChange, + loading, + searchString, + handleScroll, + hasNextPage, + } = useDisableDifferenceEmbeddingDataset(name); const nextOptions = buildQueryVariableOptionsByShowVariable(showVariable)(); @@ -178,6 +188,7 @@ export function KnowledgeBaseFormField({ onSearchChange={handleSearchChange} isSearching={loading} shouldFilter={false} + onListScroll={hasNextPage ? handleScroll : undefined} {...field} /> )} diff --git a/web/src/components/ui/multi-select.tsx b/web/src/components/ui/multi-select.tsx index ff1af70db0..d2a191b3e8 100644 --- a/web/src/components/ui/multi-select.tsx +++ b/web/src/components/ui/multi-select.tsx @@ -192,6 +192,7 @@ interface MultiSelectProps onSearchChange?: (value: string) => void; isSearching?: boolean; shouldFilter?: boolean; + onListScroll?: (e: React.UIEvent) => void; } export const MultiSelect = React.forwardRef< @@ -217,6 +218,7 @@ export const MultiSelect = React.forwardRef< onSearchChange, isSearching = false, shouldFilter, + onListScroll, ...props }, ref, @@ -451,7 +453,7 @@ export const MultiSelect = React.forwardRef< onValueChange={onSearchChange} /> )} - + {isSearching ? t('common.searching') : t('common.noDataFound')} diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts index 51d1e8ca5f..4ac38d7daa 100644 --- a/web/src/hooks/use-knowledge-request.ts +++ b/web/src/hooks/use-knowledge-request.ts @@ -51,7 +51,7 @@ import { } from '@tanstack/react-query'; import { useDebounce } from 'ahooks'; import { omit } from 'lodash'; -import { useCallback, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { useParams, useSearchParams } from 'react-router'; import { useGetPaginationWithRouter, @@ -234,7 +234,9 @@ export const useCreateKnowledge = () => { message.success( i18n.t(`message.${params?.id ? 'modified' : 'created'}`), ); - queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] }); + queryClient.invalidateQueries({ + queryKey: [KnowledgeApiAction.FetchKnowledgeList], + }); } return data; }, @@ -818,43 +820,114 @@ export const useClearWiki = () => { return { data, loading, clearWiki: mutateAsync }; }; -export const useFetchKnowledgeList = ( - shouldFilterListWithoutDocument: boolean = false, - keywords = '', -): { - list: IDataset[]; - loading: boolean; -} => { - const { data, isFetching: loading } = useQuery({ - queryKey: [ +const KNOWLEDGE_LIST_PAGE_SIZE = 10; + +export const KnowledgeListKeys = { + list: ( + shouldFilterListWithoutDocument: boolean, + keywords: string, + pageSize: number, + ) => + [ KnowledgeApiAction.FetchKnowledgeList, shouldFilterListWithoutDocument, keywords, - ], - initialData: [], - gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3 - queryFn: async () => { - const { data } = await listDataset( - keywords - ? { - ext: { - keywords, - }, - } - : undefined, - ); - const list = data?.data ?? []; - return shouldFilterListWithoutDocument - ? list.filter((x: IDataset) => x.chunk_count > 0) - : list; - }, - }); + pageSize, + ] as const, +}; - return { list: data, loading }; +export const useFetchKnowledgeList = ( + shouldFilterListWithoutDocument: boolean = false, + keywords = '', + pageSize: number = KNOWLEDGE_LIST_PAGE_SIZE, +): { + list: IDataset[]; + loading: boolean; + fetchNextPage: () => void; + hasNextPage: boolean; + isFetchingNextPage: boolean; + handleScroll: (e: React.UIEvent) => void; +} => { + const { data, fetchNextPage, hasNextPage, isFetching, isFetchingNextPage } = + useInfiniteQuery<{ items: IDataset[] }>({ + queryKey: KnowledgeListKeys.list( + shouldFilterListWithoutDocument, + keywords, + pageSize, + ), + gcTime: 0, + initialPageParam: 1, + queryFn: async ({ pageParam }) => { + const page = pageParam as number; + const { data } = await listDataset({ + page, + page_size: pageSize, + ...(keywords ? { ext: { keywords } } : {}), + }); + return { items: (data?.data ?? []) as IDataset[] }; + }, + getNextPageParam: (lastPage, allPages) => + lastPage.items.length >= pageSize ? allPages.length + 1 : undefined, + }); + + const list = useMemo(() => { + const all = data?.pages.flatMap((page) => page.items) ?? []; + return shouldFilterListWithoutDocument + ? all.filter((x) => x.chunk_count > 0) + : all; + }, [data, shouldFilterListWithoutDocument]); + + const handleScroll = useCallback( + (e: React.UIEvent) => { + const { scrollTop, scrollHeight, clientHeight } = e.currentTarget; + const threshold = 50; + if ( + scrollHeight - scrollTop - clientHeight <= threshold && + hasNextPage && + !isFetchingNextPage + ) { + fetchNextPage(); + } + }, + [fetchNextPage, hasNextPage, isFetchingNextPage], + ); + + return { + list, + loading: isFetching, + fetchNextPage, + hasNextPage, + isFetchingNextPage, + handleScroll, + }; +}; + +/** + * For consumers that need the COMPLETE list (no scroll UI). + * Uses a large page size to minimize round-trips, and auto-loads + * until exhausted. + */ +export const useFetchAllKnowledgeList = ( + shouldFilterListWithoutDocument: boolean = false, + keywords = '', +): { list: IDataset[]; loading: boolean } => { + const { list, loading, hasNextPage, fetchNextPage } = useFetchKnowledgeList( + shouldFilterListWithoutDocument, + keywords, + 200, + ); + + useEffect(() => { + if (hasNextPage && !loading) { + fetchNextPage(); + } + }, [hasNextPage, loading, fetchNextPage]); + + return { list, loading }; }; export const useSelectKnowledgeOptions = () => { - const { list } = useFetchKnowledgeList(); + const { list } = useFetchAllKnowledgeList(); const options = list?.map((item) => ({ label: item.name, diff --git a/web/src/pages/agent/canvas/node/retrieval-node.tsx b/web/src/pages/agent/canvas/node/retrieval-node.tsx index 35c07799a3..bbbd0dc473 100644 --- a/web/src/pages/agent/canvas/node/retrieval-node.tsx +++ b/web/src/pages/agent/canvas/node/retrieval-node.tsx @@ -1,6 +1,6 @@ import { NodeCollapsible } from '@/components/collapse'; import { RAGFlowAvatar } from '@/components/ragflow-avatar'; -import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request'; +import { useFetchAllKnowledgeList } from '@/hooks/use-knowledge-request'; import { useFetchAllMemoryList } from '@/hooks/use-memory-request'; import { BaseNode } from '@/interfaces/database/agent'; import { NodeProps, Position } from '@xyflow/react'; @@ -25,7 +25,7 @@ function InnerRetrievalNode({ }: NodeProps>) { const knowledgeBaseIds: string[] = get(data, 'form.dataset_ids', []); const memoryIds: string[] = get(data, 'form.memory_ids', []); - const { list: knowledgeList } = useFetchKnowledgeList(true); + const { list: knowledgeList } = useFetchAllKnowledgeList(true); const { getLabel } = useGetVariableLabelOrTypeByValue({ nodeId: id }); diff --git a/web/src/pages/dataset/dataset-setting/components/tag-item.tsx b/web/src/pages/dataset/dataset-setting/components/tag-item.tsx index 5602fa00d1..6324c1f2c4 100644 --- a/web/src/pages/dataset/dataset-setting/components/tag-item.tsx +++ b/web/src/pages/dataset/dataset-setting/components/tag-item.tsx @@ -10,7 +10,9 @@ import { import { MultiSelect } from '@/components/ui/multi-select'; import { FormLayout } from '@/constants/form'; import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request'; +import { useDebounce } from 'ahooks'; import DOMPurify from 'dompurify'; +import { useState } from 'react'; import { useFormContext, useWatch } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; @@ -18,7 +20,15 @@ export const TagSetItem = () => { const { t } = useTranslation(); const form = useFormContext(); - const { list: knowledgeList } = useFetchKnowledgeList(true); + const [searchString, setSearchString] = useState(''); + const debouncedSearchString = useDebounce(searchString, { wait: 500 }); + + const { + list: knowledgeList, + handleScroll, + hasNextPage, + loading, + } = useFetchKnowledgeList(true, debouncedSearchString); const knowledgeOptions = knowledgeList .filter((x) => x.chunk_method === 'tag') @@ -60,6 +70,11 @@ export const TagSetItem = () => { { const [record, setRecord] = useState({} as IFile); const [documentIds, setDocumentIds] = useState([]); const [mode, setMode] = useState('replace'); - const knowledgeOptions = useSelectKnowledgeOptions(); + const { options: knowledgeOptions } = useSelectKnowledgeOptions(); const initialValue = useMemo(() => { return Array.isArray(record?.kbs_info) diff --git a/web/src/pages/files/link-to-dataset-dialog.tsx b/web/src/pages/files/link-to-dataset-dialog.tsx index 9be6cf14f1..42ac659c22 100644 --- a/web/src/pages/files/link-to-dataset-dialog.tsx +++ b/web/src/pages/files/link-to-dataset-dialog.tsx @@ -15,10 +15,12 @@ import { FormMessage, } from '@/components/ui/form'; import { MultiSelect } from '@/components/ui/multi-select'; -import { useSelectKnowledgeOptions } from '@/hooks/use-knowledge-request'; +import { useFetchKnowledgeList } from '@/hooks/use-knowledge-request'; import { IModalProps } from '@/interfaces/common'; +import { useDebounce } from 'ahooks'; import { zodResolver } from '@hookform/resolvers/zod'; import { Link2 } from 'lucide-react'; +import { useMemo, useState } from 'react'; import { useForm } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; import { z } from 'zod'; @@ -47,7 +49,21 @@ function LinkToDatasetForm({ }, }); - const options = useSelectKnowledgeOptions(); + const [searchString, setSearchString] = useState(''); + const debouncedSearchString = useDebounce(searchString, { wait: 500 }); + const { list, loading, handleScroll, hasNextPage } = useFetchKnowledgeList( + false, + debouncedSearchString, + ); + + const options = useMemo( + () => + list.map((item) => ({ + label: item.name, + value: item.id, + })), + [list], + ); function onSubmit(data: z.infer) { onConnectToKnowledgeOk(data.knowledgeIds); @@ -77,6 +93,11 @@ function LinkToDatasetForm({ defaultValue={field.value} placeholder={t('fileManager.pleaseSelect')} maxCount={100} + searchValue={searchString} + onSearchChange={setSearchString} + isSearching={loading} + shouldFilter={false} + onListScroll={hasNextPage ? handleScroll : undefined} // {...field} modalPopover />