Feat: enable sync deleted files for more connectors (#14353)

### What problem does this PR solve?

Feat: enable sync delted files for connectors

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

web/src/pages/user-setting/data-source/add-datasource-modal.tsx

@@ -7,9 +7,8 @@ import { useTranslation } from 'react-i18next';
 import {
   DataSourceFormBaseFields,
   DataSourceFormDefaultValues,
-  DataSourceFormFields,
   getCommonExtraDefaultValues,
-  getCommonExtraFields,
+  getDataSourceFieldsWithExtras,
   mergeDataSourceFormValues,
 } from './constant';
 import { IDataSorceInfo } from './interface';
@@ -28,10 +27,7 @@ const AddDataSourceModal = ({
     if (sourceData) {
       setFields([
         ...DataSourceFormBaseFields,
-        ...DataSourceFormFields[
-          sourceData.id as keyof typeof DataSourceFormFields
-        ],
-        ...getCommonExtraFields(sourceData.id),
+        ...getDataSourceFieldsWithExtras(sourceData.id as any),
       ] as FormFieldConfig[]);
     }
   }, [sourceData]);

web/src/pages/user-setting/data-source/constant/index.tsx

@@ -11,36 +11,38 @@ import GoogleDriveTokenField from '../component/google-drive-token-field';
 import { IDataSourceInfoMap } from '../interface';
 import { bitbucketConstant } from './bitbucket-constant';
 import { confluenceConstant } from './confluence-constant';
+import { jiraConstant } from './jira-constant';
 import { S3Constant } from './s3-constant';
 import { seafileConstant } from './seafile-constant';
 
 export enum DataSourceKey {
-  RSS = 'rss',
   CONFLUENCE = 'confluence',
-  S3 = 's3',
   NOTION = 'notion',
-  DISCORD = 'discord',
   GOOGLE_DRIVE = 'google_drive',
-  MOODLE = 'moodle',
   GMAIL = 'gmail',
+  GOOGLE_CLOUD_STORAGE = 'google_cloud_storage',
+  OCI_STORAGE = 'oci_storage',
+  S3 = 's3',
+  R2 = 'r2',
   JIRA = 'jira',
-  WEBDAV = 'webdav',
   BOX = 'box',
   DROPBOX = 'dropbox',
-  R2 = 'r2',
-  OCI_STORAGE = 'oci_storage',
-  GOOGLE_CLOUD_STORAGE = 'google_cloud_storage',
-  AIRTABLE = 'airtable',
-  DINGTALK_AI_TABLE = 'dingtalk_ai_table',
+  BITBUCKET = 'bitbucket',
   GITLAB = 'gitlab',
+  GITHUB = 'github',
+  MOODLE = 'moodle',
+  DISCORD = 'discord',
+  ZENDESK = 'zendesk',
+  WEBDAV = 'webdav',
+  AIRTABLE = 'airtable',
   ASANA = 'asana',
   IMAP = 'imap',
-  GITHUB = 'github',
-  BITBUCKET = 'bitbucket',
-  ZENDESK = 'zendesk',
+  DINGTALK_AI_TABLE = 'dingtalk_ai_table',
   SEAFILE = 'seafile',
   MYSQL = 'mysql',
   POSTGRESQL = 'postgresql',
+  RSS = 'rss',
+
   //   SHAREPOINT = 'sharepoint',
   //   SLACK = 'slack',
   //   TEAMS = 'teams',
@@ -56,6 +58,30 @@ export const DataSourceFeatureVisibilityMap = {
   [DataSourceKey.GITHUB]: {
     syncDeletedFiles: true,
   },
+  [DataSourceKey.CONFLUENCE]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.BOX]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.S3]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.R2]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.GOOGLE_CLOUD_STORAGE]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.OCI_STORAGE]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.NOTION]: {
+    syncDeletedFiles: true,
+  },
+  [DataSourceKey.JIRA]: {
+    syncDeletedFiles: true,
+  },
 };
 
 const isDataSourceFeatureVisible = (
@@ -294,6 +320,47 @@ export const getCommonExtraDefaultValues = () => ({
   },
 });
 
+export const getDataSourceFieldsWithExtras = (
+  source?: DataSourceKey,
+): FormFieldConfig[] => {
+  if (!source) {
+    return [];
+  }
+
+  const sourceFields =
+    DataSourceFormFields[source as keyof typeof DataSourceFormFields] || [];
+  const extraFields = getCommonExtraFields(source);
+
+  if (source !== DataSourceKey.JIRA) {
+    return [...sourceFields, ...extraFields];
+  }
+
+  const modeFieldIndex = sourceFields.findIndex(
+    (field) => field.name === 'config.is_cloud',
+  );
+  if (modeFieldIndex < 0) {
+    return [...sourceFields, ...extraFields];
+  }
+
+  const sharedFields = sourceFields.slice(0, modeFieldIndex);
+  const modeFields = sourceFields.slice(modeFieldIndex);
+
+  const sharedCheckboxFieldIndex = sharedFields.findIndex(
+    (field) => field.type === FormFieldType.Checkbox,
+  );
+
+  if (sharedCheckboxFieldIndex < 0) {
+    return [...sharedFields, ...extraFields, ...modeFields];
+  }
+
+  return [
+    ...sharedFields.slice(0, sharedCheckboxFieldIndex),
+    ...sharedFields.slice(sharedCheckboxFieldIndex),
+    ...extraFields,
+    ...modeFields,
+  ];
+};
+
 export const DataSourceFormFields = {
   [DataSourceKey.RSS]: [
     {
@@ -569,106 +636,7 @@ export const DataSourceFormFields = {
       required: true,
     },
   ],
-  [DataSourceKey.JIRA]: [
-    {
-      label: 'Jira Base URL',
-      name: 'config.base_url',
-      type: FormFieldType.Text,
-      required: true,
-      placeholder: 'https://your-domain.atlassian.net',
-      tooltip: t('setting.jiraBaseUrlTip'),
-    },
-    {
-      label: 'Project Key',
-      name: 'config.project_key',
-      type: FormFieldType.Text,
-      required: false,
-      placeholder: 'RAGFlow',
-      tooltip: t('setting.jiraProjectKeyTip'),
-    },
-    {
-      label: 'Custom JQL',
-      name: 'config.jql_query',
-      type: FormFieldType.Textarea,
-      required: false,
-      placeholder: 'project = RAG AND updated >= -7d',
-      tooltip: t('setting.jiraJqlTip'),
-    },
-    {
-      label: 'Batch Size',
-      name: 'config.batch_size',
-      type: FormFieldType.Number,
-      required: false,
-      tooltip: t('setting.jiraBatchSizeTip'),
-    },
-    {
-      label: 'Include Comments',
-      name: 'config.include_comments',
-      type: FormFieldType.Checkbox,
-      required: false,
-      defaultValue: true,
-      tooltip: t('setting.jiraCommentsTip'),
-    },
-    {
-      label: 'Include Attachments',
-      name: 'config.include_attachments',
-      type: FormFieldType.Checkbox,
-      required: false,
-      defaultValue: false,
-      tooltip: t('setting.jiraAttachmentsTip'),
-    },
-    {
-      label: 'Attachment Size Limit (bytes)',
-      name: 'config.attachment_size_limit',
-      type: FormFieldType.Number,
-      required: false,
-      defaultValue: 10 * 1024 * 1024,
-      tooltip: t('setting.jiraAttachmentSizeTip'),
-    },
-    {
-      label: 'Labels to Skip',
-      name: 'config.labels_to_skip',
-      type: FormFieldType.Tag,
-      required: false,
-      tooltip: t('setting.jiraLabelsTip'),
-    },
-    {
-      label: 'Comment Email Blacklist',
-      name: 'config.comment_email_blacklist',
-      type: FormFieldType.Tag,
-      required: false,
-      tooltip: t('setting.jiraBlacklistTip'),
-    },
-    {
-      label: 'Use Scoped Token (Clould only)',
-      name: 'config.scoped_token',
-      type: FormFieldType.Checkbox,
-      required: false,
-      tooltip: t('setting.jiraScopedTokenTip'),
-    },
-    {
-      label: 'Jira User Email (Cloud) or User Name (Server)',
-      name: 'config.credentials.jira_user_email',
-      type: FormFieldType.Text,
-      required: true,
-      placeholder: 'you@example.com',
-      tooltip: t('setting.jiraEmailTip'),
-    },
-    {
-      label: 'Jira API Token (Cloud only)',
-      name: 'config.credentials.jira_api_token',
-      type: FormFieldType.Password,
-      required: false,
-      tooltip: t('setting.jiraTokenTip'),
-    },
-    {
-      label: 'Jira Password (Server only)',
-      name: 'config.credentials.jira_password',
-      type: FormFieldType.Password,
-      required: false,
-      tooltip: t('setting.jiraPasswordTip'),
-    },
-  ],
+  [DataSourceKey.JIRA]: jiraConstant(t),
   [DataSourceKey.WEBDAV]: [
     {
       label: 'WebDAV Server URL',
@@ -1247,6 +1215,7 @@ export const DataSourceFormDefaultValues = {
     name: '',
     source: DataSourceKey.JIRA,
     config: {
+      is_cloud: true,
       base_url: '',
       project_key: '',
       jql_query: '',
@@ -1259,6 +1228,7 @@ export const DataSourceFormDefaultValues = {
       scoped_token: false,
       credentials: {
         jira_user_email: '',
+        jira_username: '',
         jira_api_token: '',
         jira_password: '',
       },

web/src/pages/user-setting/data-source/constant/jira-constant.tsx

@@ -0,0 +1,149 @@
+import { FormFieldType } from '@/components/dynamic-form';
+import { TFunction } from 'i18next';
+
+export const jiraConstant = (t: TFunction) => [
+  {
+    label: 'Jira User Email',
+    name: 'config.credentials.jira_user_email',
+    type: FormFieldType.Text,
+    required: true,
+    placeholder: 'you@example.com',
+    tooltip: t('setting.jiraEmailTip'),
+    shouldRender: (formValues: any) => formValues?.config?.is_cloud !== false,
+    customValidate: (val: string, formValues: any) => {
+      if (formValues?.config?.is_cloud !== false) {
+        return Boolean(val) || 'Jira User Email is required';
+      }
+      return true;
+    },
+  },
+  {
+    label: 'Jira Username',
+    name: 'config.credentials.jira_username',
+    type: FormFieldType.Text,
+    required: true,
+    tooltip: t('setting.jiraEmailTip'),
+    shouldRender: (formValues: any) => formValues?.config?.is_cloud === false,
+    customValidate: (val: string, formValues: any) => {
+      if (formValues?.config?.is_cloud === false) {
+        return Boolean(val) || 'Jira Username is required';
+      }
+      return true;
+    },
+  },
+  {
+    label: 'Jira Base URL',
+    name: 'config.base_url',
+    type: FormFieldType.Text,
+    required: true,
+    placeholder: 'https://your-domain.atlassian.net',
+    tooltip: t('setting.jiraBaseUrlTip'),
+  },
+  {
+    label: 'Project Key',
+    name: 'config.project_key',
+    type: FormFieldType.Text,
+    required: false,
+    placeholder: 'RAGFlow',
+    tooltip: t('setting.jiraProjectKeyTip'),
+  },
+  {
+    label: 'Custom JQL',
+    name: 'config.jql_query',
+    type: FormFieldType.Textarea,
+    required: false,
+    placeholder: 'project = RAG AND updated >= -7d',
+    tooltip: t('setting.jiraJqlTip'),
+  },
+  {
+    label: 'Batch Size',
+    name: 'config.batch_size',
+    type: FormFieldType.Number,
+    required: false,
+    tooltip: t('setting.jiraBatchSizeTip'),
+  },
+  {
+    label: 'Attachment Size Limit (bytes)',
+    name: 'config.attachment_size_limit',
+    type: FormFieldType.Number,
+    required: false,
+    defaultValue: 10 * 1024 * 1024,
+    tooltip: t('setting.jiraAttachmentSizeTip'),
+  },
+  {
+    label: 'Labels to Skip',
+    name: 'config.labels_to_skip',
+    type: FormFieldType.Tag,
+    required: false,
+    tooltip: t('setting.jiraLabelsTip'),
+  },
+  {
+    label: 'Comment Email Blacklist',
+    name: 'config.comment_email_blacklist',
+    type: FormFieldType.Tag,
+    required: false,
+    tooltip: t('setting.jiraBlacklistTip'),
+  },
+  {
+    label: 'Include Comments',
+    name: 'config.include_comments',
+    type: FormFieldType.Checkbox,
+    required: false,
+    defaultValue: true,
+    tooltip: t('setting.jiraCommentsTip'),
+  },
+  {
+    label: 'Include Attachments',
+    name: 'config.include_attachments',
+    type: FormFieldType.Checkbox,
+    required: false,
+    defaultValue: false,
+    tooltip: t('setting.jiraAttachmentsTip'),
+  },
+  {
+    label: 'Mode',
+    name: 'config.is_cloud',
+    type: FormFieldType.Segmented,
+    options: [
+      { label: 'Cloud', value: true },
+      { label: 'Server', value: false },
+    ],
+    defaultValue: true,
+  },
+  {
+    label: 'Jira API Token',
+    name: 'config.credentials.jira_api_token',
+    type: FormFieldType.Password,
+    required: false,
+    tooltip: t('setting.jiraTokenTip'),
+    shouldRender: (formValues: any) => formValues?.config?.is_cloud !== false,
+    customValidate: (val: string, formValues: any) => {
+      if (formValues?.config?.is_cloud !== false) {
+        return Boolean(val) || 'Jira API Token is required';
+      }
+      return true;
+    },
+  },
+  {
+    label: 'Jira Password',
+    name: 'config.credentials.jira_password',
+    type: FormFieldType.Password,
+    required: false,
+    tooltip: t('setting.jiraPasswordTip'),
+    shouldRender: (formValues: any) => formValues?.config?.is_cloud === false,
+    customValidate: (val: string, formValues: any) => {
+      if (formValues?.config?.is_cloud === false) {
+        return Boolean(val) || 'Jira Password is required';
+      }
+      return true;
+    },
+  },
+  {
+    label: 'Use Scoped Token',
+    name: 'config.scoped_token',
+    type: FormFieldType.Checkbox,
+    required: false,
+    tooltip: t('setting.jiraScopedTokenTip'),
+    shouldRender: (formValues: any) => formValues?.config?.is_cloud !== false,
+  },
+];

web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx

@@ -17,9 +17,8 @@ import { FieldValues } from 'react-hook-form';
 import {
   DataSourceFormBaseFields,
   DataSourceFormDefaultValues,
-  DataSourceFormFields,
   getCommonExtraDefaultValues,
-  getCommonExtraFields,
+  getDataSourceFieldsWithExtras,
   mergeDataSourceFormValues,
   useDataSourceInfo,
 } from '../constant';
@@ -166,10 +165,7 @@ const SourceDetailPage = () => {
     if (detail) {
       const fields = [
         ...baseFields,
-        ...DataSourceFormFields[
-          detail.source as keyof typeof DataSourceFormFields
-        ],
-        ...getCommonExtraFields(detail.source),
+        ...getDataSourceFieldsWithExtras(detail.source as any),
         ...customFields,
       ] as FormFieldConfig[];