Feat: Add a data compilation layer. (#16777)

### Summary

Feat: Add a data compilation layer.
This commit is contained in:
balibabu
2026-07-09 17:49:16 +08:00
committed by GitHub
parent 5de823eab9
commit 0083ad0deb
171 changed files with 14477 additions and 791 deletions

View File

@@ -0,0 +1,93 @@
import { type IArtifactGraphEntity } from '@/interfaces/database/dataset';
import { cn } from '@/lib/utils';
import isEmpty from 'lodash/isEmpty';
import { memo, useCallback, useEffect, useMemo, useRef } from 'react';
import ForceGraph2D, { type ForceGraphMethods } from 'react-force-graph-2d';
import { type ArtifactForceGraphProps } from './types';
import { useContainerDimensions } from './use-container-dimensions';
import { defaultMapNodeToValue, renderNodeLabel } from './utils';
function ArtifactForceGraph<TNodeValue = IArtifactGraphEntity>({
data,
show = true,
onNodeClick,
mapNodeToValue = defaultMapNodeToValue as (
node: IArtifactGraphEntity,
) => TNodeValue,
getNodeId = (node) => node.slug,
}: ArtifactForceGraphProps<TNodeValue>) {
const containerRef = useRef<HTMLDivElement>(null);
const fgRef = useRef<ForceGraphMethods<IArtifactGraphEntity> | undefined>(
undefined,
);
const hasFittedRef = useRef(false);
const dimensions = useContainerDimensions(containerRef, show);
const graphData = useMemo(() => {
if (isEmpty(data) || !data) {
return { nodes: [], links: [] };
}
const nodes = (data.entities || []).map((entity) => ({
...entity,
id: getNodeId(entity),
}));
const links = (data.relations || []).map((relation) => ({
source: relation.from,
target: relation.to,
}));
return { nodes, links };
}, [data, getNodeId]);
useEffect(() => {
hasFittedRef.current = false;
}, [graphData]);
const handleEngineStop = useCallback(() => {
if (!hasFittedRef.current && fgRef.current) {
fgRef.current.zoomToFit(400);
hasFittedRef.current = true;
}
}, []);
const handleNodeClick = useCallback(
(node: IArtifactGraphEntity) => {
onNodeClick?.(mapNodeToValue(node));
},
[onNodeClick, mapNodeToValue],
);
return (
<div
ref={containerRef}
className={cn('flex-1 min-h-0 h-full', !show && 'hidden')}
>
{dimensions.width > 0 && dimensions.height > 0 && (
<ForceGraph2D
ref={fgRef}
width={dimensions.width}
height={dimensions.height}
graphData={graphData}
nodeAutoColorBy="type"
cooldownTicks={100}
nodeLabel={''}
onEngineStop={handleEngineStop}
onNodeClick={handleNodeClick}
nodeCanvasObject={renderNodeLabel}
nodeCanvasObjectMode={() => 'after'}
/>
)}
</div>
);
}
const MemoizedArtifactForceGraph = memo(ArtifactForceGraph) as <
TNodeValue = IArtifactGraphEntity,
>(
props: ArtifactForceGraphProps<TNodeValue>,
) => React.ReactElement;
export { MemoizedArtifactForceGraph as ArtifactForceGraph };
export default MemoizedArtifactForceGraph;

View File

@@ -0,0 +1,12 @@
import {
type IArtifactGraph,
type IArtifactGraphEntity,
} from '@/interfaces/database/dataset';
export interface ArtifactForceGraphProps<TNodeValue = IArtifactGraphEntity> {
data?: IArtifactGraph;
show?: boolean;
onNodeClick?: (node: TNodeValue) => void;
mapNodeToValue?: (node: IArtifactGraphEntity) => TNodeValue;
getNodeId?: (node: IArtifactGraphEntity) => string;
}

View File

@@ -0,0 +1,25 @@
import { useEffect, useState, type RefObject } from 'react';
export function useContainerDimensions(
ref: RefObject<HTMLElement | null>,
enabled: boolean = true,
) {
const [dimensions, setDimensions] = useState({ width: 0, height: 0 });
useEffect(() => {
if (!ref.current || !enabled) return;
const observer = new ResizeObserver((entries) => {
const entry = entries[0];
if (entry) {
const { width, height } = entry.contentRect;
setDimensions({ width, height });
}
});
observer.observe(ref.current);
return () => observer.disconnect();
}, [ref, enabled]);
return dimensions;
}

View File

@@ -0,0 +1,26 @@
import { type IArtifactGraphEntity } from '@/interfaces/database/dataset';
import { type ComponentProps } from 'react';
import ForceGraph2D from 'react-force-graph-2d';
export const renderNodeLabel: NonNullable<
ComponentProps<typeof ForceGraph2D>['nodeCanvasObject']
> = (node, ctx) => {
const label = node.name;
const fontSize = 2;
ctx.font = `${fontSize}px Sans-Serif`;
ctx.textAlign = 'center';
ctx.textBaseline = 'top';
const textSecondary = getComputedStyle(document.documentElement)
.getPropertyValue('--text-secondary')
.trim();
ctx.fillStyle = `rgb(${textSecondary})`;
if (node.x && node.y) {
ctx.fillText(label, node.x, node.y + 5);
}
};
export const defaultMapNodeToValue = <TNode extends IArtifactGraphEntity>(
node: TNode,
): TNode => node;

View File

@@ -37,6 +37,7 @@ import {
AutoQuestionsFormField,
} from '../auto-keywords-form-field';
import { ChildrenDelimiterForm } from '../children-delimiter-form';
import { CompilationTemplateFormField } from '../compilation-template-form-field';
import { DataFlowSelect } from '../data-pipeline-select';
import { DelimiterFormField } from '../delimiter-form-field';
import { EntityTypesFormField } from '../entity-types-form-field';
@@ -152,6 +153,7 @@ export function ChunkMethodDialog({
)
.optional(),
enable_metadata: z.boolean().optional(),
compilation_template_group_id: z.array(z.string()).optional(),
}),
})
.superRefine((data, ctx) => {
@@ -300,6 +302,10 @@ export function ChunkMethodDialog({
<ParseTypeItem />
{parseType === ParseType.BuiltIn && <ChunkMethodItem />}
{parseType === ParseType.BuiltIn && (
<CompilationTemplateFormField></CompilationTemplateFormField>
)}
{showPages && parseType === ParseType.BuiltIn && (
<DynamicPageRange />
)}

View File

@@ -42,6 +42,7 @@ export function useDefaultParserValues() {
metadata: [],
built_in_metadata: [],
enable_metadata: false,
compilation_template_group_id: [],
};
return defaultParserValues as IParserConfig;

View File

@@ -0,0 +1,117 @@
import { RAGFlowFormItem } from '@/components/ragflow-form';
import {
MultiSelect,
MultiSelectOptionType,
} from '@/components/ui/multi-select';
import { useFetchAllCompilationTemplateGroups } from '@/hooks/use-compilation-template-group-request';
import { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
type CompilationTemplateFormFieldProps = {
horizontal?: boolean;
name?: string;
};
const ScopeTranslationKeyMap: Record<string, string> = {
file: 'scopeFile',
dataset: 'scopeDataset',
};
type CompilationTemplateMultiSelectProps = {
value?: string[];
onChange(value: string[]): void;
};
/**
* Renders a multi-select for compilation template groups.
*
* Selection rule:
* - Each group has a scope: "file" or "dataset".
* - At most one group per scope can be selected.
* - The two scopes can be combined (one file group + one dataset group).
* - Once a scope is selected, remaining options of the same scope are disabled.
* - The scope is displayed as a suffix after each option label.
*/
function CompilationTemplateMultiSelect({
value: rawValue = [],
onChange,
}: CompilationTemplateMultiSelectProps) {
const { t } = useTranslation();
const { groups } = useFetchAllCompilationTemplateGroups();
const value = useMemo(() => {
// Normalize legacy single-string values into an array during the migration.
if (Array.isArray(rawValue)) return rawValue;
if (typeof rawValue === 'string' && rawValue) return [rawValue];
return [];
}, [rawValue]);
// Collect scopes that are already selected by the current value.
const selectedScopes = useMemo(() => {
const scopes = new Set<string>();
value.forEach((id) => {
const group = groups?.find((g) => g.id === id);
if (group?.scope) {
scopes.add(group.scope);
}
});
return scopes;
}, [value, groups]);
const options = useMemo<MultiSelectOptionType[]>(() => {
return (groups ?? []).map((group) => {
const scopeTranslationKey = group.scope
? ScopeTranslationKeyMap[group.scope]
: undefined;
// Disable other options that share an already-selected scope.
const isSameScopeSelected =
!!group.scope &&
selectedScopes.has(group.scope) &&
!value.includes(group.id);
return {
label: group.name,
value: group.id,
disabled: isSameScopeSelected,
suffix: scopeTranslationKey ? (
<span className="text-text-secondary ml-1">
({t(`knowledgeConfiguration.${scopeTranslationKey}`)})
</span>
) : undefined,
};
});
}, [groups, selectedScopes, value, t]);
return (
<MultiSelect
options={options}
onValueChange={onChange}
defaultValue={value}
value={value}
showSelectAll={false}
/>
);
}
export function CompilationTemplateFormField({
horizontal,
name = 'parser_config.compilation_template_group_id',
}: CompilationTemplateFormFieldProps) {
const { t } = useTranslation();
return (
<RAGFlowFormItem
name={name}
label={t('knowledgeConfiguration.compilationTemplate')}
className="pb-4"
horizontal={horizontal}
>
{(field) => (
<CompilationTemplateMultiSelect
value={field.value ?? []}
onChange={field.onChange}
/>
)}
</RAGFlowFormItem>
);
}

View File

@@ -1,5 +1,6 @@
import { formatDate } from '@/utils/date';
import { formatBytes } from '@/utils/file-util';
import { PropsWithChildren } from 'react';
import { useTranslation } from 'react-i18next';
type Props = {
@@ -7,6 +8,7 @@ type Props = {
name: string;
create_date: string;
className?: string;
wrapperClassName?: string;
};
export default function DocumentHeader({
@@ -14,27 +16,32 @@ export default function DocumentHeader({
name,
create_date,
className,
}: Props) {
children,
wrapperClassName,
}: PropsWithChildren<Props>) {
const sizeName = formatBytes(size);
const dateStr = formatDate(create_date);
const { t } = useTranslation();
return (
<header className={className}>
<h2 className="text-2xl font-semibold truncate">{name}</h2>
<dl
className="
text-text-secondary text-sm flex
[&_dt]:after:content-[':'] [&_dt]:after:me-[.5ch]
[&_dd]:me-[2ch]"
>
<dt>{t('chunk.size')}</dt>
<dd>{sizeName}</dd>
<header className={wrapperClassName}>
<section className={className}>
<h2 className="text-2xl font-semibold truncate">{name}</h2>
<dl
className="
text-text-secondary text-sm flex truncate
[&_dt]:after:content-[':'] [&_dt]:after:me-[.5ch]
[&_dd]:me-[2ch]"
>
<dt>{t('chunk.size')}</dt>
<dd>{sizeName}</dd>
<dt>{t('chunk.uploadedTime')}</dt>
<dd>{dateStr}</dd>
</dl>
<dt>{t('chunk.uploadedTime')}</dt>
<dd>{dateStr}</dd>
</dl>
</section>
{children}
</header>
);
}

View File

@@ -0,0 +1,87 @@
import { Button } from '@/components/ui/button';
import { SearchInput } from '@/components/ui/input';
import { cn } from '@/lib/utils';
import { Search, X } from 'lucide-react';
import { useCallback, useState } from 'react';
import { useTranslation } from 'react-i18next';
interface ExpandableSearchInputProps {
value: string;
onChange: (value: string) => void;
placeholder?: string;
className?: string;
inputClassName?: string;
width?: number | string;
}
export function ExpandableSearchInput({
value,
onChange,
placeholder,
className,
inputClassName,
width = 192,
}: ExpandableSearchInputProps) {
const { t } = useTranslation();
const [isOpen, setIsOpen] = useState(false);
const handleToggle = useCallback(() => {
setIsOpen((prev) => {
const next = !prev;
if (!next) {
onChange('');
}
return next;
});
}, [onChange]);
const handleChange = useCallback(
(e: React.ChangeEvent<HTMLInputElement>) => {
onChange(e.target.value);
},
[onChange],
);
return (
<div className={cn('relative flex items-center gap-2', className)}>
<div
className={cn(
'transition-all duration-300 ease-in-out',
isOpen ? 'opacity-100' : 'w-0 overflow-hidden opacity-0',
)}
style={{ width: isOpen ? width : 0 }}
>
<SearchInput
value={value}
onChange={handleChange}
className={cn('w-full', inputClassName)}
autoFocus={isOpen}
placeholder={placeholder}
suffix={
isOpen ? (
<button
type="button"
onClick={handleToggle}
className="p-1 text-text-secondary hover:text-text-primary"
aria-label={t('common.close', 'Close')}
>
<X className="h-4 w-4" />
</button>
) : null
}
/>
</div>
{!isOpen && (
<Button
variant="ghost"
size="icon"
type="button"
onClick={handleToggle}
aria-label={t('chunk.search', 'Search')}
>
<Search className="h-5 w-5" />
</Button>
)}
</div>
);
}

View File

@@ -22,6 +22,7 @@ const JsonEditor: React.FC<JsonEditorProps> = ({
height = '400px',
className = '',
options = {},
defaultExpanded = false,
}) => {
const containerRef = useRef<HTMLDivElement>(null);
const editorRef = useRef<any>(null);
@@ -66,6 +67,10 @@ const JsonEditor: React.FC<JsonEditorProps> = ({
if (value) {
editorRef.current.set(value);
if (defaultExpanded) {
editorRef.current.expandAll?.();
}
}
setIsLoading(false);
@@ -138,6 +143,10 @@ const JsonEditor: React.FC<JsonEditorProps> = ({
editorRef.current = new JSONEditor(containerRef.current, newOptions);
editorRef.current.set(currentData);
if (defaultExpanded) {
editorRef.current.expandAll?.();
}
} catch (error) {
console.error(
'Failed to reload jsoneditor with new language:',
@@ -148,7 +157,7 @@ const JsonEditor: React.FC<JsonEditorProps> = ({
initEditorWithNewLanguage();
}
}, [i18n.language, value, onChange, options]);
}, [i18n.language, value, onChange, options, defaultExpanded]);
useEffect(() => {
if (editorRef.current && value !== undefined) {
@@ -157,14 +166,22 @@ const JsonEditor: React.FC<JsonEditorProps> = ({
const currentJson = editorRef.current.get();
if (JSON.stringify(currentJson) !== JSON.stringify(value)) {
editorRef.current.set(value);
if (defaultExpanded) {
editorRef.current.expandAll?.();
}
}
} catch (err) {
console.error(err);
// Skip update if there is a syntax error in the current editor
editorRef.current.set(value);
if (defaultExpanded) {
editorRef.current.expandAll?.();
}
}
}
}, [value]);
}, [value, defaultExpanded]);
return (
<div

View File

@@ -336,4 +336,7 @@ export interface JsonEditorProps {
// Configuration options for the JSONEditor
options?: JsonEditorOptions;
// Whether to expand all nodes by default in tree/view/form modes
defaultExpanded?: boolean;
}

View File

@@ -24,7 +24,7 @@ export const LLMLabel = ({ value }: IProps) => {
/>
<span className="font-medium truncate">{modelName}</span>
{instanceName && (
<span className="text-slate-400 truncate flex-shrink-0">
<span className="text-text-secondary truncate flex-shrink-0">
{instanceName}
</span>
)}

View File

@@ -0,0 +1,97 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import LexicalEditor from '@/lib/editor/lexical-editor';
import RawMarkdownEditor from '@/lib/editor/raw-markdown-editor';
interface MarkdownEditorProps {
content: string;
onChange?: (content: string) => void;
readOnly?: boolean;
}
export default function MarkdownEditor({
content,
onChange,
readOnly = false,
}: MarkdownEditorProps) {
const [showSource, setShowSource] = useState(false);
const [rawContent, setRawContent] = useState(content);
const contentRef = useRef(content);
useEffect(() => {
contentRef.current = content;
if (showSource) setRawContent(content);
}, [showSource, content]);
const handleWysiwygChange = useCallback(
(md: string) => {
if (readOnly) return;
contentRef.current = md;
onChange?.(md);
},
[onChange, readOnly],
);
const handleRawChange = useCallback(
(value: string) => {
if (readOnly) return;
setRawContent(value);
contentRef.current = value;
onChange?.(value);
},
[onChange, readOnly],
);
const toggleSource = useCallback(() => {
if (!showSource) setRawContent(contentRef.current);
setShowSource((prev) => !prev);
}, [showSource]);
return (
<div className="flex flex-1 flex-col min-h-0">
<div className="nim-editor-container">
<div
className="flex flex-1 flex-col min-h-0"
style={{ display: showSource ? 'none' : 'flex' }}
>
<LexicalEditor
content={content}
onChange={handleWysiwygChange}
readOnly={readOnly}
placeholder={readOnly ? '' : 'Start writing...'}
onToggleSource={toggleSource}
showSource={showSource}
/>
</div>
<div
className="flex flex-1 flex-col min-h-0"
style={{ display: showSource ? 'flex' : 'none' }}
>
{readOnly ? (
<div className="flex-1 overflow-auto whitespace-pre-wrap break-words px-5 py-4 text-[13px] leading-relaxed font-mono bg-bg-card text-text-primary">
<div className="flex items-center gap-1.5 px-3 py-1 border-b border-border bg-bg-base shrink-0 -mx-5 -mt-4 mb-3">
<button
type="button"
onClick={toggleSource}
className="px-2.5 py-1 text-xs font-medium border border-accent-primary rounded bg-accent-primary-10 text-accent-primary hover:bg-accent-primary-20 transition-colors"
>
WYSIWYG
</button>
</div>
<pre className="m-0 font-inherit text-inherit leading-inherit whitespace-pre-wrap">
{rawContent}
</pre>
</div>
) : (
<RawMarkdownEditor
content={rawContent}
onChange={handleRawChange}
readOnly={readOnly}
onToggleSource={toggleSource}
/>
)}
</div>
</div>
</div>
);
}

View File

@@ -197,12 +197,14 @@ export interface ModelTreeSelectFormFieldProps extends ModelTreeSelectProps {
name?: string;
label?: string;
tooltip?: string;
required?: boolean;
}
export function ModelTreeSelectFormField({
name = 'llm_id',
label,
tooltip,
required,
...rest
}: ModelTreeSelectFormFieldProps) {
const form = useFormContext();
@@ -214,7 +216,11 @@ export function ModelTreeSelectFormField({
name={name}
render={({ field }) => (
<FormItem>
{label && <FormLabel tooltip={tooltip}>{label}</FormLabel>}
{label && (
<FormLabel required={required} tooltip={tooltip}>
{label}
</FormLabel>
)}
<FormControl>
<ModelTreeSelect
{...rest}

View File

@@ -36,6 +36,7 @@ export type SelectWithSearchFlagOptionType = {
value?: string;
disabled?: boolean;
options?: RAGFlowSelectOptionType[];
keywords?: string[];
};
export type SelectWithSearchFlagProps = {
@@ -289,7 +290,8 @@ export const SelectWithSearch = forwardRef<
value={group.value}
disabled={group.disabled}
keywords={
typeof group.label === 'string' ? [group.label] : []
group.keywords ??
(typeof group.label === 'string' ? [group.label] : [])
}
onSelect={handleSelect}
data-testid={

View File

@@ -4,11 +4,11 @@ import { useTranslate } from '@/hooks/common-hooks';
import { cn } from '@/lib/utils';
import { useKnowledgeBaseContext } from '@/pages/dataset/contexts/knowledge-base-context';
import { LLMModelItem } from '@/pages/dataset/dataset-setting/configuration/common-item';
import { GenerateType } from '@/pages/dataset/dataset/generate-button/constants';
import {
GenerateLogButton,
GenerateType,
IGenerateLogButtonProps,
} from '@/pages/dataset/dataset/generate-button/generate';
} from '@/pages/dataset/dataset/generate-button/generate-log-button';
import { upperFirst } from 'lodash';
import { useCallback, useMemo } from 'react';
import { useFormContext, useWatch } from 'react-hook-form';
@@ -111,12 +111,12 @@ export function UseGraphRagFormField({
}
// The three types "table", "resume" and "one" do not display this configuration.
const GraphRagItems = ({
const GraphRagItems = function GraphRagItems({
marginBottom = false,
className = 'p-10',
data,
onDelete,
}: GraphRagItemsProps) => {
}: GraphRagItemsProps) {
const { t } = useTranslate('knowledgeConfiguration');
const form = useFormContext();

View File

@@ -1,11 +1,11 @@
import { FormLayout } from '@/constants/form';
import { DocumentParserType } from '@/constants/knowledge';
import { useTranslate } from '@/hooks/common-hooks';
import { GenerateType } from '@/pages/dataset/dataset/generate-button/constants';
import {
GenerateLogButton,
GenerateType,
IGenerateLogButtonProps,
} from '@/pages/dataset/dataset/generate-button/generate';
} from '@/pages/dataset/dataset/generate-button/generate-log-button';
import random from 'lodash/random';
import { Shuffle } from 'lucide-react';
import { useCallback, useEffect, useMemo } from 'react';

View File

@@ -1,3 +1,5 @@
// https://github.com/MrLightful/shadcn-tree-view
'use client';
import { cn } from '@/lib/utils';