Feat: full optimization on connector dashboard (#14979)

### What problem does this PR solve?

This PR improves the connector dashboard task management experience and
adds better visibility into connector execution logs.

### Overview:

#### Before
<img width="700" alt="image"
src="https://github.com/user-attachments/assets/e4a8ed6f-2e18-4f0f-8528-41a514550052"
/>

#### Now:
<img width="700" alt="Screenshot from 2026-05-18 16-31-30"
src="https://github.com/user-attachments/assets/d4ca193b-847a-49ae-9e4f-5fbca60ea627"
/>

### 1. Add a new logging page to the connector dashboard

A new logging page has been added so users can view connector task
execution logs directly from the connector dashboard.

### 2. Merge the Resume button into Confirm

The separate **Resume** button has been removed. The **Confirm** button
now represents different actions depending on the current task state:

- **Save**: Save form changes and reschedule tasks.
- **Stop**: Cancel currently scheduled or running tasks.
- **Resume**: Create new scheduled tasks after the previous tasks have
been stopped.
- **Start**: Start tasks when no task has been started yet.

### 3. Separate syncing and pruning tasks

Connector tasks are now separated into **syncing** and **pruning**.

Pruning is controlled by the **Sync deleted files** option:

- When **Sync deleted files** is disabled, only syncing tasks are shown.
- When **Sync deleted files** is enabled, both syncing and pruning tasks
are shown.

**Now: Sync deleted files disabled**

<img width="700" alt="Sync deleted files disabled"
src="https://github.com/user-attachments/assets/dbd9232e-614a-407f-a0b1-c109e5fa567d"
/>

**Now: Sync deleted files enabled**

<img width="700" alt="Sync deleted files enabled"
src="https://github.com/user-attachments/assets/1f527f48-ccb3-4ee8-97ca-086891489296"
/>

### 4. Update logs in backend

<img width="700" alt="image"
src="https://github.com/user-attachments/assets/10a95a3f-98c1-4e67-8afa-ddf6cda5b0b2"
/>

### 5. Remove connector resume API

- Removed: `POST /v1/connectors/<connector_id>/resume`
- Replaced by: `PATCH /v1/connectors/<connector_id>`


### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
Magicbook1108
2026-05-19 10:07:11 +08:00
committed by GitHub
parent 41a9fc0030
commit b69a6a5d80
15 changed files with 706 additions and 557 deletions

View File

@@ -9,9 +9,9 @@ import { Button } from '@/components/ui/button';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { Input } from '@/components/ui/input';
import { Separator } from '@/components/ui/separator';
import { RunningStatus } from '@/constants/knowledge';
import { RunningStatus, RunningStatusOld } from '@/constants/knowledge';
import { t } from 'i18next';
import { CirclePause, Repeat } from 'lucide-react';
import { isEqual } from 'lodash';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { FieldValues } from 'react-hook-form';
import {
@@ -25,9 +25,9 @@ import {
} from '../constant';
import {
useAddDataSource,
useDataSourceResume,
useFetchDataSourceDetail,
useTestDataSource,
useUpdateDataSourceStatus,
} from '../hooks';
import { DataSourceLogsTable } from './log-table';
@@ -35,7 +35,8 @@ const SourceDetailPage = () => {
const formRef = useRef<DynamicFormRef>(null);
const { data: detail } = useFetchDataSourceDetail();
const { handleResume } = useDataSourceResume();
const { updateStatus, loading: statusUpdateLoading } =
useUpdateDataSourceStatus();
const { dataSourceInfo } = useDataSourceInfo();
const detailInfo = useMemo(() => {
if (detail) {
@@ -44,83 +45,52 @@ const SourceDetailPage = () => {
}, [detail, dataSourceInfo]);
const [fields, setFields] = useState<FormFieldConfig[]>([]);
const [isDirty, setIsDirty] = useState(false);
const [defaultValues, setDefaultValues] = useState<FieldValues>(
DataSourceFormDefaultValues[
detail?.source as keyof typeof DataSourceFormDefaultValues
] as FieldValues,
);
const runSchedule = useCallback(() => {
handleResume({
resume:
detail?.status === RunningStatus.RUNNING ||
detail?.status === RunningStatus.SCHEDULE
? false
: true,
});
}, [detail, handleResume]);
const customFields = useMemo(() => {
return [
{
label: 'Prune Freq',
name: 'prune_freq',
type: FormFieldType.Number,
required: false,
shouldRender: (values: any) => !!values?.config?.sync_deleted_files,
render: (fieldProps: FormFieldConfig) => {
return (
<Input
{...fieldProps}
type={FormFieldType.Number}
suffix={
<span className="px-2 text-text-secondary italic">
{t('setting.minutes')}
</span>
}
/>
);
},
},
{
label: 'Refresh Freq',
name: 'refresh_freq',
type: FormFieldType.Number,
required: false,
render: (fieldProps: FormFieldConfig) => (
<div className="flex items-center gap-1 w-full relative">
<div className="flex-1">
<Input
{...fieldProps}
type={FormFieldType.Number}
suffix={
<span className="px-2 text-text-secondary italic">
{t('setting.minutes')}
</span>
}
/>
</div>
<button
type="button"
className="text-text-secondary bg-bg-input rounded-sm text-xs h-full p-2 border border-border-button hover:bg-border-button hover:text-text-primary"
onClick={() => {
runSchedule();
}}
>
{detail?.status === RunningStatus.RUNNING ||
detail?.status === RunningStatus.SCHEDULE ? (
<CirclePause size={12} />
) : (
<Repeat size={12} />
)}
</button>
</div>
<Input
{...fieldProps}
type={FormFieldType.Number}
suffix={
<span className="px-2 text-text-secondary italic">
{t('setting.minutes')}
</span>
}
/>
),
},
{
label: 'Prune Freq',
name: 'prune_freq',
type: FormFieldType.Number,
required: false,
hidden: true,
render: (fieldProps: FormFieldConfig) => {
return (
<div className="flex items-center gap-1 w-full relative">
<div className="flex-1">
<Input
{...fieldProps}
type={FormFieldType.Number}
suffix={
<span className="px-2 text-text-secondary italic">
hours
</span>
}
/>
</div>
</div>
);
},
},
{
label: 'Timeout Secs',
name: 'timeout_secs',
@@ -143,7 +113,7 @@ const SourceDetailPage = () => {
),
},
];
}, [detail, runSchedule]);
}, []);
const { addLoading, handleAddOk } = useAddDataSource({ isEdit: true });
const { loading: testLoading, handleTest } = useTestDataSource();
@@ -152,6 +122,54 @@ const SourceDetailPage = () => {
formRef?.current?.submit();
}, []);
const isUnstarted = useMemo(
() =>
detail?.status === RunningStatus.UNSTART ||
detail?.status === RunningStatusOld.UNSTART,
[detail?.status],
);
const isConnectorActive = useMemo(
() =>
detail?.status === RunningStatus.RUNNING ||
detail?.status === RunningStatus.SCHEDULE ||
detail?.status === RunningStatusOld.RUNNING ||
detail?.status === RunningStatusOld.SCHEDULE,
[detail?.status],
);
const actionMode = useMemo(() => {
if (isDirty) {
return 'save' as const;
}
if (isUnstarted) {
return 'save' as const;
}
if (isConnectorActive) {
return 'stop' as const;
}
return 'resume' as const;
}, [isConnectorActive, isDirty, isUnstarted]);
const handlePrimaryAction = useCallback(() => {
if (actionMode === 'save') {
onSubmit();
return;
}
updateStatus(
actionMode === 'resume' ? RunningStatus.SCHEDULE : RunningStatus.CANCEL,
);
}, [actionMode, onSubmit, updateStatus]);
const primaryActionLabel = useMemo(() => {
if (actionMode === 'stop') return 'Stop';
if (actionMode === 'resume') return 'Resume';
return 'Save';
}, [actionMode]);
useEffect(() => {
const baseFields = DataSourceFormBaseFields.map((field) => {
if (field.name === 'name') {
@@ -191,9 +209,20 @@ const SourceDetailPage = () => {
),
};
setDefaultValues(defaultValueTemp);
setIsDirty(false);
}
}, [detail, customFields, onSubmit]);
useEffect(() => {
const instance = formRef.current;
if (!instance) return;
setIsDirty(!isEqual(instance.getValues(), defaultValues));
return instance.watchDirty((_nextIsDirty, values) => {
setIsDirty(!isEqual(values, defaultValues));
});
}, [defaultValues, fields]);
return (
<div className="px-10 py-5">
<BackButton />
@@ -229,22 +258,21 @@ const SourceDetailPage = () => {
)}
<Button
type="button"
onClick={onSubmit}
disabled={addLoading}
loading={addLoading}
onClick={handlePrimaryAction}
disabled={addLoading || statusUpdateLoading}
loading={
(addLoading && actionMode === 'save') ||
(statusUpdateLoading && actionMode !== 'save')
}
>
{t('common.confirm')}
{/* {addLoading && <Loader2 className="mr-2 h-4 w-4 animate-spin" />}
{addLoading
? t('modal.loadingText', { defaultValue: 'Submitting...' })
: t('modal.okText', { defaultValue: 'Submit' })} */}
{primaryActionLabel}
</Button>
</div>
<section className="flex flex-col gap-2">
<div className="text-2xl text-text-primary mb-2">
{t('setting.log')}
</div>
<DataSourceLogsTable refresh_freq={detail?.refresh_freq || false} />
<DataSourceLogsTable autoRefresh={isConnectorActive} />
</section>
</CardContent>
</Card>

View File

@@ -1,6 +1,5 @@
import FileStatusBadge from '@/components/file-status-badge';
import { RAGFlowAvatar } from '@/components/ragflow-avatar';
import { Button } from '@/components/ui/button';
import { RAGFlowPagination } from '@/components/ui/ragflow-pagination';
import {
Table,
@@ -14,11 +13,6 @@ import { RunningStatusMap } from '@/constants/knowledge';
import { RunningStatus } from '@/pages/dataset/dataset/constant';
import { Routes } from '@/routes';
import { formatDate } from '@/utils/date';
import {
HoverCard,
HoverCardContent,
HoverCardTrigger,
} from '@radix-ui/react-hover-card';
import {
ColumnDef,
flexRender,
@@ -30,15 +24,86 @@ import {
} from '@tanstack/react-table';
import { t } from 'i18next';
import { pick } from 'lodash';
import { Eye } from 'lucide-react';
import { useCallback, useMemo } from 'react';
import { useCallback, useEffect, useMemo, useState } from 'react';
import { useNavigate } from 'react-router';
import { useLogListDataSource } from '../hooks';
import { IDataSourceLog } from '../interface';
const formatDuration = (seconds: number) => {
const safeSeconds = Math.max(0, seconds);
const hours = Math.floor(safeSeconds / 3600);
const minutes = Math.floor((safeSeconds % 3600) / 60);
const remainingSeconds = safeSeconds % 60;
if (hours > 0) {
return `${hours}h ${minutes}m ${remainingSeconds}s`;
}
if (minutes > 0) {
return `${minutes}m ${remainingSeconds}s`;
}
return `${remainingSeconds}s`;
};
const getTaskCountdownSeconds = (row: IDataSourceLog, now: number) => {
const freqMinutes =
row.task_type === 'prune'
? Number(row.prune_freq || 0)
: Number(row.refresh_freq || 0);
const scheduledAt = row.time_started
? new Date(row.time_started).getTime()
: 0;
if (!freqMinutes || !scheduledAt) {
return null;
}
const nextRunAt = scheduledAt + freqMinutes * 60 * 1000;
return Math.ceil((nextRunAt - now) / 1000);
};
const TaskCountdown = ({ row, now }: { row: IDataSourceLog; now: number }) => {
const remainingSeconds = getTaskCountdownSeconds(row, now);
if (remainingSeconds === null) {
return '';
}
return <span>Task starts in {formatDuration(remainingSeconds)}</span>;
};
const getSummary = (row: IDataSourceLog, now: number) => {
if (row.status === RunningStatus.SCHEDULE || row.status === '5') {
return <TaskCountdown row={row} now={now} />;
}
if (row.status === RunningStatus.RUNNING || row.status === '1') {
return row.task_type === 'prune' ? 'Prune in progress' : 'Sync in progress';
}
if (row.status === RunningStatus.FAIL || row.status === '4') {
return row.error_msg || 'Task failed';
}
if (row.status === RunningStatus.CANCEL || row.status === '2') {
return '';
}
if (row.task_type === 'prune') {
return `deleted=${row.docs_removed_from_index || 0}, error=${row.error_count || 0}`;
}
return `total=${row.total_docs_indexed || 0}, added=${row.new_docs_indexed || 0}, updated=${Math.max(
0,
(row.total_docs_indexed || 0) - (row.new_docs_indexed || 0),
)}, error=${row.error_count || 0}`;
};
const columns = ({
handleToDataSetDetail,
now,
}: {
handleToDataSetDetail: (id: string) => void;
now: number;
}) => {
return [
{
@@ -71,7 +136,6 @@ const columns = ({
<div
className="flex items-center gap-2 text-text-primary cursor-pointer"
onClick={() => {
console.log('handleToDataSetDetail', row.original.kb_id);
handleToDataSetDetail(row.original.kb_id);
}}
>
@@ -86,39 +150,16 @@ const columns = ({
},
},
{
accessorKey: 'new_docs_indexed',
header: t('setting.newDocs'),
accessorKey: 'task_type',
header: 'Task Type',
cell: ({ row }) => row.original.task_type || 'sync',
},
{
id: 'operations',
header: t('setting.errorMsg'),
id: 'summary',
header: 'Summary',
cell: ({ row }) => (
<div className="flex gap-1 items-center">
{row.original.error_msg}
{row.original.error_msg && (
<div className="flex justify-start space-x-2 opacity-0 group-hover:opacity-100 transition-opacity">
<HoverCard>
<HoverCardTrigger>
<Button
variant="ghost"
size="sm"
className="p-1"
// onClick={() => {
// showLog(row, LogTabs.FILE_LOGS);
// }}
>
<Eye />
</Button>
</HoverCardTrigger>
<HoverCardContent className="w-[40vw] max-h-[40vh] overflow-auto bg-bg-base z-[999] px-3 py-2 rounded-md border border-border-default">
<div className="space-y-2">
{row.original.full_exception_trace}
</div>
</HoverCardContent>
</HoverCard>
</div>
)}
<div className="max-w-[32rem] whitespace-normal break-words text-text-primary">
{getSummary(row.original as IDataSourceLog, now)}
</div>
),
},
@@ -131,14 +172,22 @@ const columns = ({
// total: 0,
// };
export const DataSourceLogsTable = ({
refresh_freq,
autoRefresh,
}: {
refresh_freq: number | false;
autoRefresh: boolean;
}) => {
// const [pagination, setPagination] = useState(paginationInit);
const { data, pagination, setPagination } =
useLogListDataSource(refresh_freq);
const { data, pagination, setPagination } = useLogListDataSource(autoRefresh);
const navigate = useNavigate();
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
const timer = window.setInterval(() => {
setNow(Date.now());
}, 1000);
return () => window.clearInterval(timer);
}, []);
const currentPagination = useMemo(
() => ({
pageIndex: (pagination.current || 1) - 1,
@@ -149,15 +198,14 @@ export const DataSourceLogsTable = ({
const handleToDataSetDetail = useCallback(
(id: string) => {
console.log('handleToDataSetDetail', id);
navigate(`${Routes.DatasetBase}${Routes.DatasetBase}/${id}`);
navigate(`${Routes.Dataset}/${id}`);
},
[navigate],
);
const table = useReactTable<any>({
data: data || [],
columns: columns({ handleToDataSetDetail }),
columns: columns({ handleToDataSetDetail, now }),
manualPagination: true,
getCoreRowModel: getCoreRowModel(),
getPaginationRowModel: getPaginationRowModel(),

View File

@@ -1,9 +1,9 @@
import message from '@/components/ui/message';
import { RunningStatus } from '@/constants/knowledge';
import { useSetModalState } from '@/hooks/common-hooks';
import { useGetPaginationWithRouter } from '@/hooks/logic-hooks';
import dataSourceService, {
dataSourceRebuild,
dataSourceResume,
dataSourceUpdate,
deleteDataSource,
featchDataSourceDetail,
@@ -15,7 +15,12 @@ import { t } from 'i18next';
import { useCallback, useMemo, useState } from 'react';
import { useParams, useSearchParams } from 'react-router';
import { DataSourceKey, useDataSourceInfo } from './constant';
import { IDataSorceInfo, IDataSource, IDataSourceBase } from './interface';
import {
IDataSorceInfo,
IDataSource,
IDataSourceBase,
IDataSourceLog,
} from './interface';
export const useListDataSource = () => {
const { dataSourceInfo } = useDataSourceInfo();
@@ -28,10 +33,8 @@ export const useListDataSource = () => {
});
const categorizeDataBySource = (data: IDataSourceBase[]) => {
const categorizedData: Record<DataSourceKey, any[]> = {} as Record<
DataSourceKey,
any[]
>;
const categorizedData: Partial<Record<DataSourceKey, IDataSourceBase[]>> =
{};
data.forEach((item) => {
const source = item.source;
@@ -93,17 +96,29 @@ export const useAddDataSource = ({ isEdit = false }: { isEdit?: boolean }) => {
async (data: any) => {
setAddLoading(true);
const { data: res } = isEdit
? await dataSourceUpdate(data.id, data)
? await dataSourceUpdate(data.id, {
...data,
reschedule: true,
})
: await dataSourceService.dataSourceSet(data);
console.log('🚀 ~ handleAddOk ~ code:', res.code);
if (res.code === 0) {
if (isEdit && res.data?.id) {
queryClient.setQueryData(
['data-source-detail', res.data.id],
res.data,
);
queryClient.invalidateQueries({
queryKey: ['data-source-detail', res.data.id],
});
}
queryClient.invalidateQueries({ queryKey: ['data-source'] });
message.success(t(`message.operated`));
hideAddingModal();
}
setAddLoading(false);
},
[hideAddingModal, queryClient],
[hideAddingModal, isEdit, queryClient],
);
return {
@@ -117,24 +132,25 @@ export const useAddDataSource = ({ isEdit = false }: { isEdit?: boolean }) => {
};
};
export const useLogListDataSource = (refresh_freq: number | false) => {
export const useLogListDataSource = (autoRefresh: boolean) => {
const { pagination, setPagination } = useGetPaginationWithRouter();
const [currentQueryParameters] = useSearchParams();
const id = currentQueryParameters.get('id');
const { data, isFetching } = useQuery<{ logs: IDataSource[]; total: number }>(
{
queryKey: ['data-source-logs', id, pagination, refresh_freq],
refetchInterval: refresh_freq ? refresh_freq * 60 * 1000 : false,
queryFn: async () => {
const { data } = await getDataSourceLogs(id as string, {
page_size: pagination.pageSize,
page: pagination.current,
});
return data.data;
},
const { data, isFetching } = useQuery<{
logs: IDataSourceLog[];
total: number;
}>({
queryKey: ['data-source-logs', id, pagination, autoRefresh],
refetchInterval: autoRefresh ? 15 * 1000 : false,
queryFn: async () => {
const { data } = await getDataSourceLogs(id as string, {
page_size: pagination.pageSize,
page: pagination.current,
});
return data.data;
},
);
});
return {
data: data?.logs,
isFetching,
@@ -179,21 +195,49 @@ export const useFetchDataSourceDetail = () => {
return { data };
};
export const useDataSourceResume = () => {
export const useUpdateDataSourceStatus = () => {
const [currentQueryParameters] = useSearchParams();
const id = currentQueryParameters.get('id');
const queryClient = useQueryClient();
const handleResume = useCallback(
async (param: { resume: boolean }) => {
const { data } = await dataSourceResume(id as string, param);
if (data.code === 0) {
queryClient.invalidateQueries({ queryKey: ['data-source-detail', id] });
message.success(t(`message.operated`));
const [loading, setLoading] = useState(false);
const updateStatus = useCallback(
async (status: RunningStatus.SCHEDULE | RunningStatus.CANCEL) => {
if (!id) return;
setLoading(true);
try {
const { data } = await dataSourceUpdate(id, {
status,
});
if (data.code === 0) {
queryClient.setQueryData(
['data-source-detail', id],
(previous?: IDataSource) => ({
...(previous || {}),
...(data.data || {}),
status: data.data?.status ?? status,
}),
);
await Promise.all([
queryClient.invalidateQueries({
queryKey: ['data-source-detail', id],
}),
queryClient.invalidateQueries({ queryKey: ['data-source'] }),
queryClient.invalidateQueries({
queryKey: ['data-source-logs', id],
}),
]);
message.success(t(`message.operated`));
}
} finally {
setLoading(false);
}
},
[id, queryClient],
);
return { handleResume };
return { updateStatus, loading };
};
export const useDataSourceRebuild = () => {

View File

@@ -1,5 +1,5 @@
import { RunningStatus } from '@/constants/knowledge';
import { DataSourceKey } from './contant';
import { DataSourceKey } from './constant';
export interface IDataSorceInfo {
id: DataSourceKey;
@@ -28,20 +28,20 @@ export interface IDataSourceBase {
export interface IDataSourceLog {
connector_id: string;
docs_removed_from_index?: number;
error_count: number;
error_msg: string;
id: string;
kb_id: string;
kb_name: string;
name: string;
new_docs_indexed: number;
poll_range_end: null | string;
poll_range_start: null | string;
reindex: string;
source: DataSourceKey;
prune_freq?: number;
refresh_freq?: number;
status: RunningStatus;
tenant_id: string;
timeout_secs: number;
task_type?: string;
time_started?: string | null;
total_docs_indexed?: number;
update_date: string;
}
interface IDataSourceInfoItem {