diff --git a/web/src/components/chunk-method-dialog/hooks.ts b/web/src/components/chunk-method-dialog/hooks.ts index ef24872f2a..0dd5a7fee3 100644 --- a/web/src/components/chunk-method-dialog/hooks.ts +++ b/web/src/components/chunk-method-dialog/hooks.ts @@ -1,4 +1,108 @@ -import { useCallback } from 'react'; +import { useSelectParserList } from '@/hooks/use-user-setting-request'; +import { useCallback, useMemo } from 'react'; + +const ParserListMap = new Map([ + [ + ['pdf'], + [ + 'naive', + 'resume', + 'manual', + 'paper', + 'book', + 'laws', + 'presentation', + 'one', + 'qa', + 'knowledge_graph', + ], + ], + [ + ['doc', 'docx'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'manual', + 'knowledge_graph', + ], + ], + [ + ['xlsx', 'xls'], + ['naive', 'qa', 'table', 'one', 'knowledge_graph'], + ], + [['ppt', 'pptx'], ['presentation']], + [ + ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'], + ['picture'], + ], + [ + ['txt'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'table', + 'knowledge_graph', + ], + ], + [ + ['csv'], + [ + 'naive', + 'resume', + 'book', + 'laws', + 'one', + 'qa', + 'table', + 'knowledge_graph', + ], + ], + [ + ['md', 'mdx'], + ['naive', 'qa', 'knowledge_graph'], + ], + [['json'], ['naive', 'knowledge_graph']], + [['eml'], ['email']], +]); + +const getParserList = ( + values: string[], + parserList: Array<{ + value: string; + label: string; + }>, +) => { + return parserList.filter((x) => values?.some((y) => y === x.value)); +}; + +export const useFetchParserListOnMount = (documentExtension: string) => { + const parserList = useSelectParserList(); + + const nextParserList = useMemo(() => { + const key = [...ParserListMap.keys()].find((x) => + x.some((y) => y === documentExtension), + ); + if (key) { + const values = ParserListMap.get(key); + return getParserList(values ?? [], parserList); + } + + return getParserList( + ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'], + parserList, + ); + }, [parserList, documentExtension]); + + return { parserList: nextParserList }; +}; const hideAutoKeywords = ['qa', 'table', 'resume', 'knowledge_graph', 'tag']; diff --git a/web/src/hooks/use-user-setting-request.tsx b/web/src/hooks/use-user-setting-request.tsx index 41a487e225..3b498e2cff 100644 --- a/web/src/hooks/use-user-setting-request.tsx +++ b/web/src/hooks/use-user-setting-request.tsx @@ -10,7 +10,6 @@ import { } from '@/interfaces/database/user-setting'; import { ISetLangfuseConfigRequestBody } from '@/interfaces/request/system'; import { DEFAULT_LANGUAGE_CODE, supportedLanguages } from '@/locales/config'; -import kbService from '@/services/knowledge-service'; import userService, { addTenantUser, agreeTenant, @@ -103,46 +102,51 @@ export const useSelectParserList = (): Array<{ const { data: tenantInfo } = useFetchTenantInfo(true); const { t } = useTranslation(); - // Fetch the builtin pipeline catalog from the backend (embed-registry), - // replacing the old hardcoded parser list. - const { data: pipelineListData } = useQuery({ - queryKey: ['listPipelines'], - queryFn: async () => { - const { data } = await kbService.listPipelines(); - return data; - }, - staleTime: Infinity, - }); - const pipelineList: Array<{ - parser_id: string; - title: string; - dsl: Record; - }> = pipelineListData?.data ?? []; + const defaultParsers = useMemo( + () => [ + { value: 'naive', label: t('knowledgeConfiguration.parserLabel.naive') }, + { value: 'qa', label: t('knowledgeConfiguration.parserLabel.qa') }, + { + value: 'resume', + label: t('knowledgeConfiguration.parserLabel.resume'), + }, + { + value: 'manual', + label: t('knowledgeConfiguration.parserLabel.manual'), + }, + { value: 'table', label: t('knowledgeConfiguration.parserLabel.table') }, + { value: 'paper', label: t('knowledgeConfiguration.parserLabel.paper') }, + { value: 'book', label: t('knowledgeConfiguration.parserLabel.book') }, + { value: 'laws', label: t('knowledgeConfiguration.parserLabel.laws') }, + { + value: 'presentation', + label: t('knowledgeConfiguration.parserLabel.presentation'), + }, + { + value: 'picture', + label: t('knowledgeConfiguration.parserLabel.picture'), + }, + { value: 'one', label: t('knowledgeConfiguration.parserLabel.one') }, + { value: 'audio', label: t('knowledgeConfiguration.parserLabel.audio') }, + { value: 'email', label: t('knowledgeConfiguration.parserLabel.email') }, + { value: 'tag', label: t('knowledgeConfiguration.parserLabel.tag') }, + ], + [t], + ); const parserList = useMemo(() => { const parserArray: Array = tenantInfo?.parser_ids?.split(',') ?? []; const filteredArray = parserArray.filter((x) => x.trim() !== ''); - // Tenant-level parser_ids filter (if configured), otherwise all builtins. - if (filteredArray.length > 0) { - return filteredArray.map((x) => { - const arr = x.split(':'); - return { value: arr[0], label: arr[1] }; - }); + if (filteredArray.length === 0) { + return defaultParsers; } - // Map API pipeline list to { value, label }; prefer i18n label, - // fallback to the API title when the i18n key is missing. - const labelFromAPI = (parserId: string, title: string) => { - const key = `knowledgeConfiguration.parserLabel.${parserId}`; - const translated = t(key); - return translated !== key ? translated : title; - }; - return pipelineList.map((item) => ({ - value: item.parser_id, - label: labelFromAPI(item.parser_id, item.title), - })); - }, [tenantInfo, pipelineList, t]); + return filteredArray.map((x) => { + const arr = x.split(':'); + return { value: arr[0], label: arr[1] }; + }); + }, [tenantInfo, defaultParsers]); return parserList; }; diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index b4ee9a5056..f14e8077d7 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -778,7 +778,6 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim 'Re-parse existing documents for the new column roles to take effect.', parserLabel: { naive: 'General', - general: 'General', qa: 'Q&A', resume: 'Resume', manual: 'Manual', diff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.ts index 3b21f8790b..bebfffeb31 100644 --- a/web/src/services/knowledge-service.ts +++ b/web/src/services/knowledge-service.ts @@ -19,7 +19,6 @@ const { documentThumbnails, documentIngest, listTagByKnowledgeIds, - listPipelines, setMeta, getMeta, getMetaKeys, @@ -67,10 +66,6 @@ const methods = { url: retrievalTestShare, method: 'post', }, - listPipelines: { - url: listPipelines, - method: 'get', - }, pipelineRerun: { url: api.pipelineRerun, method: 'post', diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index 892c119df6..fa212a24a1 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -181,7 +181,6 @@ export default { `${restAPIv1}/datasets/${datasetId}/ingestions/${logId}`, fetchPipelineDatasetLogs: (datasetId: string) => `${restAPIv1}/datasets/${datasetId}/ingestions`, - listPipelines: `${restAPIv1}/pipelines?type=builtin`, runIndex: (datasetId: string, indexType: string) => `${restAPIv1}/datasets/${datasetId}/index?type=${indexType.toLowerCase()}`, traceIndex: (datasetId: string, indexType: string) =>