mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-21 23:21:04 +08:00
Feature big query connector (#15871)
### What problem does this PR solve? This PR adds Google BigQuery as a first-class data source connector in RAGFlow. It enables users to ingest and sync BigQuery data using the same row-to-document model used by relational database connectors: selected content columns become document text, metadata columns become document metadata, an optional ID column provides stable document IDs, and an optional timestamp column enables cursor-based incremental sync. The connector supports service-account JSON credentials, table mode, custom query mode, GoogleSQL queries, cursor-based incremental sync, deleted-row pruning support, configurable query limits such as `maximum_bytes_billed`, dry-run validation, batch loading, stable document IDs, and BigQuery-aware value serialization.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import Image from '@/components/image';
|
||||
import SvgIcon from '@/components/svg-icon';
|
||||
import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
|
||||
import { IReference, IReferenceChunk } from '@/interfaces/database/chat';
|
||||
import { citationMarkerReg } from '@/utils/citation-utils';
|
||||
import { getExtension } from '@/utils/document-util';
|
||||
@@ -10,7 +11,6 @@ import Markdown from 'react-markdown';
|
||||
import SyntaxHighlighter from 'react-syntax-highlighter';
|
||||
import rehypeKatex from 'rehype-katex';
|
||||
import rehypeRaw from 'rehype-raw';
|
||||
import { MarkdownRemarkPlugins } from '@/constants/markdown-remark-plugins';
|
||||
import { visitParents } from 'unist-util-visit-parents';
|
||||
|
||||
import { useTranslation } from 'react-i18next';
|
||||
@@ -72,7 +72,11 @@ const MarkdownContent = ({
|
||||
text = t('chat.searching');
|
||||
}
|
||||
const nextText = replaceTextByOldReg(text);
|
||||
return pipe(replaceThinkToSection, replaceRetrievingToSection, preprocessLaTeX)(nextText);
|
||||
return pipe(
|
||||
replaceThinkToSection,
|
||||
replaceRetrievingToSection,
|
||||
preprocessLaTeX,
|
||||
)(nextText);
|
||||
}, [content, t]);
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@@ -6,7 +6,14 @@ import { buildOptions } from '@/utils/form';
|
||||
import { useFormContext, useWatch } from 'react-hook-form';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
const algorithmOptions = buildOptions(['PaddleOCR-VL-1.6', 'PaddleOCR-VL-1.5', 'PaddleOCR-VL', 'PP-OCRv6', 'PP-OCRv5', 'PP-StructureV3']);
|
||||
const algorithmOptions = buildOptions([
|
||||
'PaddleOCR-VL-1.6',
|
||||
'PaddleOCR-VL-1.5',
|
||||
'PaddleOCR-VL',
|
||||
'PP-OCRv6',
|
||||
'PP-OCRv5',
|
||||
'PP-StructureV3',
|
||||
]);
|
||||
|
||||
export function PaddleOCROptionsFormField({
|
||||
namePrefix = 'parser_config',
|
||||
@@ -57,7 +64,10 @@ export function PaddleOCROptionsFormField({
|
||||
|
||||
<RAGFlowFormItem
|
||||
name={buildName('paddleocr_access_token')}
|
||||
label={t('knowledgeConfiguration.paddleocrAccessToken', 'AI Studio Access Token')}
|
||||
label={t(
|
||||
'knowledgeConfiguration.paddleocrAccessToken',
|
||||
'AI Studio Access Token',
|
||||
)}
|
||||
tooltip={t(
|
||||
'knowledgeConfiguration.paddleocrAccessTokenTip',
|
||||
'Access token for PaddleOCR API (optional)',
|
||||
@@ -67,14 +77,19 @@ export function PaddleOCROptionsFormField({
|
||||
{(field) => (
|
||||
<Input
|
||||
{...field}
|
||||
placeholder={t('knowledgeConfiguration.paddleocrAccessTokenPlaceholder')}
|
||||
placeholder={t(
|
||||
'knowledgeConfiguration.paddleocrAccessTokenPlaceholder',
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
</RAGFlowFormItem>
|
||||
|
||||
<RAGFlowFormItem
|
||||
name={buildName('paddleocr_algorithm')}
|
||||
label={t('knowledgeConfiguration.paddleocrAlgorithm', 'PaddleOCR Algorithm')}
|
||||
label={t(
|
||||
'knowledgeConfiguration.paddleocrAlgorithm',
|
||||
'PaddleOCR Algorithm',
|
||||
)}
|
||||
tooltip={t(
|
||||
'knowledgeConfiguration.paddleocrAlgorithmTip',
|
||||
'Algorithm to use for PaddleOCR parsing',
|
||||
|
||||
Reference in New Issue
Block a user