Feature big query connector (#15871)

### What problem does this PR solve?

This PR adds Google BigQuery as a first-class data source connector in
RAGFlow.

It enables users to ingest and sync BigQuery data using the same
row-to-document model used by relational database connectors: selected
content columns become document text, metadata columns become document
metadata, an optional ID column provides stable document IDs, and an
optional timestamp column enables cursor-based incremental sync.

The connector supports service-account JSON credentials, table mode,
custom query mode, GoogleSQL queries, cursor-based incremental sync,
deleted-row pruning support, configurable query limits such as
`maximum_bytes_billed`, dry-run validation, batch loading, stable
document IDs, and BigQuery-aware value serialization.

web/src/assets/svg/data-source/bigquery.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 128 128">
+  <path fill="#4386FA" d="M58.4 17.6C38.7 17.6 22.7 33.6 22.7 53.3c0 19.7 16 35.7 35.7 35.7 8.1 0 15.6-2.7 21.6-7.3l4.7 4.7v4.1l24.6 24.5 8.5-8.5L93.7 81.9h-4.1l-4.7-4.7c4.6-6 7.3-13.5 7.3-21.6 0-19.7-16-35.7-35.7-35.7-.1 0-.1 0-.1 0zm0 11.4c13.4 0 24.3 10.9 24.3 24.3S71.8 77.6 58.4 77.6 34.1 66.7 34.1 53.3 45 29 58.4 29z"/>
+  <path fill="#FFF" d="M44.6 50.1v8.9c1.6 2.8 3.9 5.2 6.7 6.8V50.1h-6.7zm10.4-7.4v25.8c1.1.2 2.2.3 3.4.3.9 0 1.9-.1 2.8-.2V42.7H55zm10.6 11.6v14c2.8-1.6 5.1-3.9 6.7-6.6v-7.4h-6.7z"/>
+</svg>

web/src/components/markdown-content/index.tsx

@@ -1,5 +1,6 @@
 import Image from '@/components/image';
 import SvgIcon from '@/components/svg-icon';
+import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
 import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
 import { citationMarkerReg } from '@/utils/citation-utils';
 import { getExtension } from '@/utils/document-util';
@@ -10,7 +11,6 @@ import Markdown from 'react-markdown';
 import SyntaxHighlighter from 'react-syntax-highlighter';
 import rehypeKatex from 'rehype-katex';
 import rehypeRaw from 'rehype-raw';
-import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
 import { visitParents } from 'unist-util-visit-parents';
 
 import { useTranslation } from 'react-i18next';
@@ -72,7 +72,11 @@ const MarkdownContent = ({
       text = t('chat.searching');
     }
     const nextText = replaceTextByOldReg(text);
-    return pipe(replaceThinkToSection, replaceRetrievingToSection, preprocessLaTeX)(nextText);
+    return pipe(
+      replaceThinkToSection,
+      replaceRetrievingToSection,
+      preprocessLaTeX,
+    )(nextText);
   }, [content, t]);
 
   useEffect(() => {

web/src/components/paddleocr-options-form-field.tsx

@@ -6,7 +6,14 @@ import { buildOptions } from '@/utils/form';
 import { useFormContext, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 
-const algorithmOptions = buildOptions(['PaddleOCR-VL-1.6', 'PaddleOCR-VL-1.5', 'PaddleOCR-VL', 'PP-OCRv6', 'PP-OCRv5', 'PP-StructureV3']);
+const algorithmOptions = buildOptions([
+  'PaddleOCR-VL-1.6',
+  'PaddleOCR-VL-1.5',
+  'PaddleOCR-VL',
+  'PP-OCRv6',
+  'PP-OCRv5',
+  'PP-StructureV3',
+]);
 
 export function PaddleOCROptionsFormField({
   namePrefix = 'parser_config',
@@ -57,7 +64,10 @@ export function PaddleOCROptionsFormField({
 
       <RAGFlowFormItem
         name={buildName('paddleocr_access_token')}
-        label={t('knowledgeConfiguration.paddleocrAccessToken', 'AI Studio Access Token')}
+        label={t(
+          'knowledgeConfiguration.paddleocrAccessToken',
+          'AI Studio Access Token',
+        )}
         tooltip={t(
           'knowledgeConfiguration.paddleocrAccessTokenTip',
           'Access token for PaddleOCR API (optional)',
@@ -67,14 +77,19 @@ export function PaddleOCROptionsFormField({
         {(field) => (
           <Input
             {...field}
-            placeholder={t('knowledgeConfiguration.paddleocrAccessTokenPlaceholder')}
+            placeholder={t(
+              'knowledgeConfiguration.paddleocrAccessTokenPlaceholder',
+            )}
           />
         )}
       </RAGFlowFormItem>
 
       <RAGFlowFormItem
         name={buildName('paddleocr_algorithm')}
-        label={t('knowledgeConfiguration.paddleocrAlgorithm', 'PaddleOCR Algorithm')}
+        label={t(
+          'knowledgeConfiguration.paddleocrAlgorithm',
+          'PaddleOCR Algorithm',
+        )}
         tooltip={t(
           'knowledgeConfiguration.paddleocrAlgorithmTip',
           'Algorithm to use for PaddleOCR parsing',

web/src/locales/en.ts

@@ -1405,6 +1405,32 @@ Example: Virtual Hosted Style`,
         'Column to use as unique document ID. If not specified, a hash of the content will be used.',
       postgresqlTimestampColumnTip:
         'Datetime/timestamp column for incremental sync. Only rows modified after the last sync will be fetched.',
+      bigqueryDescription:
+        'Connect to Google BigQuery to sync rows from a table or a custom GoogleSQL query.',
+      bigqueryProjectIdTip:
+        'GCP project that owns the query jobs (e.g. my-gcp-project).',
+      bigqueryLocationTip:
+        'Default location for the client and query jobs, such as US or EU. Leave empty to let BigQuery infer it.',
+      bigqueryServiceAccountJsonTip:
+        'Service account key JSON with BigQuery access. Paste the full key file contents.',
+      bigqueryDatasetIdTip:
+        'Dataset id for table mode. Required together with Table ID when no SQL query is provided.',
+      bigqueryTableIdTip:
+        'Table id for table mode. Required together with Dataset ID when no SQL query is provided.',
+      bigqueryQueryTip:
+        'Custom GoogleSQL query. Takes precedence over Dataset ID and Table ID. Standard SQL only.',
+      bigqueryContentColumnsTip:
+        'Comma-separated column names whose values will be combined as document content for vectorization.',
+      bigqueryMetadataColumnsTip:
+        'Comma-separated column names to store as document metadata (not vectorized, but searchable).',
+      bigqueryIdColumnTip:
+        'Column to use as unique document ID. If not specified, a hash of the content will be used.',
+      bigqueryTimestampColumnTip:
+        'Timestamp, datetime, date, or numeric column for incremental sync. Only rows newer than the last sync will be fetched.',
+      bigqueryMaximumBytesBilledTip:
+        'Hard cost guard applied to every query job, in bytes. Defaults to 1 GiB.',
+      bigqueryJobTimeoutMsTip:
+        'Optional per-query job timeout in milliseconds.',
       rest_apiDescription:
         'Connect any REST API endpoint as a data source using a flexible, configuration-driven connector.',
       onedriveDescription:

web/src/pages/next-search/markdown-content/index.tsx

@@ -1,5 +1,6 @@
 import Image from '@/components/image';
 import SvgIcon from '@/components/svg-icon';
+import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
 import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
 import { getExtension } from '@/utils/document-util';
 import DOMPurify from 'dompurify';
@@ -8,7 +9,6 @@ import Markdown from 'react-markdown';
 import SyntaxHighlighter from 'react-syntax-highlighter';
 import rehypeKatex from 'rehype-katex';
 import rehypeRaw from 'rehype-raw';
-import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
 import { visitParents } from 'unist-util-visit-parents';
 
 import { useTranslation } from 'react-i18next';
@@ -75,7 +75,11 @@ const MarkdownContent = ({
       text = t('chat.searching');
     }
     const nextText = replaceTextByOldReg(text);
-    return pipe(replaceThinkToSection, replaceRetrievingToSection, preprocessLaTeX)(nextText);
+    return pipe(
+      replaceThinkToSection,
+      replaceRetrievingToSection,
+      preprocessLaTeX,
+    )(nextText);
   }, [content, t]);
 
   useEffect(() => {

web/src/pages/user-setting/data-source/constant/index.tsx

@@ -41,6 +41,7 @@ export enum DataSourceKey {
   SEAFILE = 'seafile',
   MYSQL = 'mysql',
   POSTGRESQL = 'postgresql',
+  BIGQUERY = 'bigquery',
   REST_API = 'rest_api',
   RSS = 'rss',
   ONEDRIVE = 'onedrive',
@@ -160,6 +161,9 @@ export const DataSourceFeatureVisibilityMap: Partial<
   [DataSourceKey.POSTGRESQL]: {
     syncDeletedFiles: true,
   },
+  [DataSourceKey.BIGQUERY]: {
+    syncDeletedFiles: true,
+  },
 };
 
 const isDataSourceFeatureVisible = (
@@ -333,6 +337,11 @@ export const generateDataSourceInfo = (t: TFunction) => {
       description: t(`setting.${DataSourceKey.POSTGRESQL}Description`),
       icon: <SvgIcon name={'data-source/postgresql'} width={38} />,
     },
+    [DataSourceKey.BIGQUERY]: {
+      name: 'BigQuery',
+      description: t(`setting.${DataSourceKey.BIGQUERY}Description`),
+      icon: <SvgIcon name={'data-source/bigquery'} width={38} />,
+    },
     [DataSourceKey.ONEDRIVE]: {
       name: 'OneDrive',
       description: t(`setting.${DataSourceKey.ONEDRIVE}Description`),
@@ -1483,6 +1492,166 @@ export const DataSourceFormFields = {
       tooltip: t('setting.postgresqlTimestampColumnTip'),
     },
   ],
+  [DataSourceKey.BIGQUERY]: [
+    {
+      label: 'Project ID',
+      name: 'config.project_id',
+      type: FormFieldType.Text,
+      required: true,
+      placeholder: 'my-gcp-project',
+      tooltip: t('setting.bigqueryProjectIdTip'),
+    },
+    {
+      label: 'Location',
+      name: 'config.location',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'US',
+      tooltip: t('setting.bigqueryLocationTip'),
+    },
+    {
+      label: 'Service Account JSON',
+      name: 'config.credentials.service_account_json',
+      type: FormFieldType.Password,
+      required: true,
+      placeholder: '{ "type": "service_account", "project_id": "...", ... }',
+      tooltip: t('setting.bigqueryServiceAccountJsonTip'),
+    },
+    {
+      label: 'Dataset ID',
+      name: 'config.dataset_id',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'analytics',
+      tooltip: t('setting.bigqueryDatasetIdTip'),
+      customValidate: (val: string, values: any) => {
+        const hasQuery = !!(values?.config?.query ?? '').trim();
+        const hasTable = !!(values?.config?.table_id ?? '').trim();
+        if (!hasQuery && !((val ?? '').trim() && hasTable)) {
+          return 'Dataset ID is required when not using a custom SQL Query';
+        }
+        return true;
+      },
+    },
+    {
+      label: 'Table ID',
+      name: 'config.table_id',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'customers',
+      tooltip: t('setting.bigqueryTableIdTip'),
+      customValidate: (val: string, values: any) => {
+        const hasQuery = !!(values?.config?.query ?? '').trim();
+        const hasDataset = !!(values?.config?.dataset_id ?? '').trim();
+        if (!hasQuery && !(hasDataset && (val ?? '').trim())) {
+          return 'Table ID is required when not using a custom SQL Query';
+        }
+        return true;
+      },
+    },
+    {
+      label: 'SQL Query',
+      name: 'config.query',
+      type: FormFieldType.Textarea,
+      required: false,
+      placeholder: 'Leave empty to use Dataset ID + Table ID',
+      tooltip: t('setting.bigqueryQueryTip'),
+      customValidate: (val: string, values: any) => {
+        const hasQuery = !!(val ?? '').trim();
+        const hasDataset = !!(values?.config?.dataset_id ?? '').trim();
+        const hasTable = !!(values?.config?.table_id ?? '').trim();
+        if (!hasQuery && !(hasDataset && hasTable)) {
+          return 'Provide a SQL Query, or both Dataset ID and Table ID';
+        }
+        return true;
+      },
+    },
+    {
+      label: 'Content Columns',
+      name: 'config.content_columns',
+      type: FormFieldType.Text,
+      required: true,
+      placeholder: 'name,description,notes',
+      tooltip: t('setting.bigqueryContentColumnsTip'),
+    },
+    {
+      label: 'Metadata Columns',
+      name: 'config.metadata_columns',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'customer_id,status,region',
+      tooltip: t('setting.bigqueryMetadataColumnsTip'),
+    },
+    {
+      label: 'ID Column',
+      name: 'config.id_column',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'customer_id',
+      tooltip: t('setting.bigqueryIdColumnTip'),
+    },
+    {
+      label: 'Timestamp Column',
+      name: 'config.timestamp_column',
+      type: FormFieldType.Text,
+      required: false,
+      placeholder: 'updated_at',
+      tooltip: t('setting.bigqueryTimestampColumnTip'),
+    },
+    {
+      label: 'Max Bytes Billed',
+      name: 'config.maximum_bytes_billed',
+      type: FormFieldType.Number,
+      required: false,
+      placeholder: '1073741824',
+      tooltip: t('setting.bigqueryMaximumBytesBilledTip'),
+      validation: {
+        min: 1,
+        message: 'Max Bytes Billed must be at least 1',
+      },
+    },
+    {
+      label: 'Job Timeout (ms)',
+      name: 'config.job_timeout_ms',
+      type: FormFieldType.Number,
+      required: false,
+      placeholder: '300000',
+      tooltip: t('setting.bigqueryJobTimeoutMsTip'),
+      validation: {
+        min: 1,
+        message: 'Job Timeout must be at least 1',
+      },
+    },
+    {
+      label: 'Page Size',
+      name: 'config.page_size',
+      type: FormFieldType.Number,
+      required: false,
+      placeholder: '1000',
+      validation: {
+        min: 1,
+        message: 'Page Size must be at least 1',
+      },
+    },
+    {
+      label: 'Batch Size',
+      name: 'config.batch_size',
+      type: FormFieldType.Number,
+      required: false,
+      placeholder: '100',
+      validation: {
+        min: 1,
+        message: 'Batch Size must be at least 1',
+      },
+    },
+    {
+      label: 'Use Query Cache',
+      name: 'config.use_query_cache',
+      type: FormFieldType.Checkbox,
+      required: false,
+      defaultValue: true,
+    },
+  ],
   [DataSourceKey.REST_API]: [
     // ── Essential fields ──────────────────────────────────────────────
     {
@@ -2150,6 +2319,29 @@ export const DataSourceFormDefaultValues = {
       },
     },
   },
+  [DataSourceKey.BIGQUERY]: {
+    name: '',
+    source: DataSourceKey.BIGQUERY,
+    config: {
+      project_id: '',
+      dataset_id: '',
+      table_id: '',
+      location: '',
+      query: '',
+      content_columns: '',
+      metadata_columns: '',
+      id_column: '',
+      timestamp_column: '',
+      batch_size: 100,
+      page_size: 1000,
+      maximum_bytes_billed: 1073741824,
+      job_timeout_ms: 300000,
+      use_query_cache: true,
+      credentials: {
+        service_account_json: '',
+      },
+    },
+  },
   [DataSourceKey.ONEDRIVE]: {
     name: '',
     source: DataSourceKey.ONEDRIVE,

web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx

@@ -244,7 +244,8 @@ const SourceDetailPage = () => {
             />
           </div>
           <div className="max-w-[1200px] flex justify-end gap-2">
-            {detail?.source === DataSourceKey.REST_API && (
+            {(detail?.source === DataSourceKey.REST_API ||
+              detail?.source === DataSourceKey.BIGQUERY) && (
               <Button
                 type="button"
                 variant="outline"