fix: honor dataset language in Go vision dispatch (#17892)

### Summary

- Propagate the dataset language through Go DOCX, Markdown, PDF
figure-enhancement, and standalone-image vision paths.
- Explicitly render the shared figure prompt's `{{ language }}`
placeholder in Go.
- Use English when the dataset language is empty.
- Make the default standalone-image prompt request the dataset language
while preserving visible text in its original language.
- Add focused tests for caller propagation, language fallback, prompt
rendering, and prompt-cache isolation.
This commit is contained in:
taek105
2026-08-11 23:18:04 +09:00
committed by GitHub
parent 875ca966e9
commit 492d6d81a9
9 changed files with 316 additions and 54 deletions

View File

@@ -44,21 +44,21 @@ import (
)
var (
docxVisionPromptBuilder = buildDOCXVisionPrompt
visionChatInvoker = defaultVisionChatInvoker
docxVisionConcurrency uint = 10
figureVisionPromptBuilder = buildFigureVisionPrompt
visionChatInvoker = defaultVisionChatInvoker
docxVisionConcurrency uint = 10
)
const (
docxVisionPromptFile = "vision_llm_figure_describe_prompt.md"
docxVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md"
figureVisionPromptFile = "vision_llm_figure_describe_prompt.md"
figureVisionPromptWithContextFile = "vision_llm_figure_describe_prompt_with_context.md"
)
var (
docxVisionPromptsBase string
docxVisionPromptsOnce sync.Once
docxVisionPromptCache = make(map[string]string)
docxVisionPromptMu sync.RWMutex
figureVisionPromptsBase string
figureVisionPromptsOnce sync.Once
figureVisionPromptCache = make(map[string]string)
figureVisionPromptMu sync.RWMutex
)
// maybeDispatchDOCXVision enriches a DOCX parse result with vision-model
@@ -96,6 +96,7 @@ func maybeDispatchDOCXVision(
if tenantID == "" {
return dispatched, false, nil
}
language := resolveVisionLanguage(inputs, "")
// Resolve the tenant's IMAGE2TEXT model.
driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
@@ -143,7 +144,7 @@ func maybeDispatchDOCXVision(
// DOCX JSON items have no surrounding context (unlike the
// former Markdown path), so use the bare figure prompt —
// matching Python's VisionFigureParser(context_size=0).
prompt, perr := docxVisionPromptBuilder("", "")
prompt, perr := figureVisionPromptBuilder("", "", language)
if perr != nil {
return
}
@@ -175,28 +176,30 @@ func maybeDispatchDOCXVision(
return dispatched, modified, nil
}
// buildDOCXVisionPrompt loads the figure-describe prompt template
// buildFigureVisionPrompt loads the figure-describe prompt template
// and, when context text is available, renders it with the
// with-context variant. Mirrors Python:
//
// if context_above or context_below:
// prompt = vision_llm_figure_describe_prompt_with_context(context_above, context_below)
// prompt = vision_llm_figure_describe_prompt_with_context(
// context_above, context_below, language=language)
// else:
// prompt = vision_llm_figure_describe_prompt()
func buildDOCXVisionPrompt(contextAbove, contextBelow string) (string, error) {
// prompt = vision_llm_figure_describe_prompt(language=language)
func buildFigureVisionPrompt(contextAbove, contextBelow, language string) (string, error) {
hasContext := strings.TrimSpace(contextAbove) != "" || strings.TrimSpace(contextBelow) != ""
var templateName string
if hasContext {
templateName = docxVisionPromptWithContextFile
templateName = figureVisionPromptWithContextFile
} else {
templateName = docxVisionPromptFile
templateName = figureVisionPromptFile
}
template, err := loadDOCXVisionPromptFile(templateName)
template, err := loadFigureVisionPromptFile(templateName)
if err != nil {
return "", err
}
template = renderFigureVisionLanguage(template, language)
if hasContext {
template = strings.ReplaceAll(template, "{{ context_above }}", contextAbove)
@@ -205,36 +208,36 @@ func buildDOCXVisionPrompt(contextAbove, contextBelow string) (string, error) {
return template, nil
}
func loadDOCXVisionPromptFile(filename string) (string, error) {
docxVisionPromptMu.RLock()
if cached, ok := docxVisionPromptCache[filename]; ok {
docxVisionPromptMu.RUnlock()
func loadFigureVisionPromptFile(filename string) (string, error) {
figureVisionPromptMu.RLock()
if cached, ok := figureVisionPromptCache[filename]; ok {
figureVisionPromptMu.RUnlock()
return cached, nil
}
docxVisionPromptMu.RUnlock()
figureVisionPromptMu.RUnlock()
baseDir, err := docxVisionPromptsBaseDir()
baseDir, err := figureVisionPromptsBaseDir()
if err != nil {
return "", err
}
promptPath := filepath.Join(baseDir, "rag", "prompts", filename)
content, err := os.ReadFile(promptPath)
if err != nil {
return "", fmt.Errorf("docx vision prompt %q: %w", filename, err)
return "", fmt.Errorf("figure vision prompt %q: %w", filename, err)
}
cached := strings.TrimSpace(string(content))
docxVisionPromptMu.Lock()
docxVisionPromptCache[filename] = cached
docxVisionPromptMu.Unlock()
figureVisionPromptMu.Lock()
figureVisionPromptCache[filename] = cached
figureVisionPromptMu.Unlock()
return cached, nil
}
func docxVisionPromptsBaseDir() (string, error) {
func figureVisionPromptsBaseDir() (string, error) {
var initErr error
docxVisionPromptsOnce.Do(func() {
figureVisionPromptsOnce.Do(func() {
root := utility.GetProjectRoot()
if _, statErr := os.Stat(filepath.Join(root, "rag", "prompts")); statErr == nil {
docxVisionPromptsBase = root
figureVisionPromptsBase = root
return
}
initErr = fmt.Errorf("rag/prompts not found under project root %q", root)
@@ -242,7 +245,7 @@ func docxVisionPromptsBaseDir() (string, error) {
if initErr != nil {
return "", initErr
}
return docxVisionPromptsBase, nil
return figureVisionPromptsBase, nil
}
func buildVisionMessages(prompt, imageBase64 string) []modelModule.Message {

View File

@@ -75,11 +75,11 @@ func (c *docxVisionCaptureInvoker) invoke(
func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {
origResolver := resolveTenantModelByType
origInvoker := visionChatInvoker
origPrompt := docxVisionPromptBuilder
origPrompt := figureVisionPromptBuilder
defer func() {
resolveTenantModelByType = origResolver
visionChatInvoker = origInvoker
docxVisionPromptBuilder = origPrompt
figureVisionPromptBuilder = origPrompt
}()
resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
@@ -87,7 +87,11 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {
}
invoker := &docxVisionCaptureInvoker{}
visionChatInvoker = invoker.invoke
docxVisionPromptBuilder = func(string, string) (string, error) { return "describe the figure", nil }
var capturedLanguage string
figureVisionPromptBuilder = func(_, _, language string) (string, error) {
capturedLanguage = language
return "describe the figure", nil
}
dispatched := parserDispatchResult{
OutputFormat: "json",
@@ -105,7 +109,7 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {
dao.DB,
utility.FileTypeDOCX,
dispatched,
map[string]any{"tenant_id": "t1"},
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
defaultSetups(),
)
if err != nil {
@@ -130,6 +134,9 @@ func TestMaybeDispatchDOCXVision_EnhancesJSONImages(t *testing.T) {
if len(invoker.images) != 1 {
t.Fatalf("vision invoker called %d times, want 1 (only the image item)", len(invoker.images))
}
if capturedLanguage != "Japanese" {
t.Errorf("figure prompt language = %q, want Japanese", capturedLanguage)
}
if want := "data:image/png;base64,aGVsbG8taW1hZ2U="; invoker.images[0] != want {
t.Errorf("vision image data URI = %q, want %q", invoker.images[0], want)
}

View File

@@ -29,7 +29,6 @@ package component
import (
"context"
"fmt"
"strings"
"sync"
@@ -102,6 +101,7 @@ func maybeDispatchMarkdownVision(
if tenantID == "" {
return dispatched, false, nil
}
language := resolveVisionLanguage(inputs, "")
// Resolve the tenant's IMAGE2TEXT model.
driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
@@ -124,7 +124,7 @@ func maybeDispatchMarkdownVision(
// Markdown images have no context — use the
// default (no-context) prompt template.
prompt, err := buildMarkdownVisionPrompt()
prompt, err := figureVisionPromptBuilder("", "", language)
if err != nil {
return
}
@@ -157,14 +157,3 @@ func maybeDispatchMarkdownVision(
return dispatched, true, nil
}
// buildMarkdownVisionPrompt loads the default (no-context) figure
// describe prompt template, mirroring Python's
// vision_llm_figure_describe_prompt().
func buildMarkdownVisionPrompt() (string, error) {
template, err := loadDOCXVisionPromptFile(docxVisionPromptFile)
if err != nil {
return "", fmt.Errorf("markdown vision prompt: %w", err)
}
return template, nil
}

View File

@@ -151,7 +151,7 @@ func maybeDispatchImage(
// --- Phase 2: VLM description (when OCR text is short) ---
// Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32
// then use OCR text only; otherwise call cv_mdl.describe().
lang := getStringOr(setup, "lang", "")
lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
eng := strings.EqualFold(lang, "english")
if ocrText != "" {
@@ -174,7 +174,7 @@ func maybeDispatchImage(
fmt.Errorf("parser: picture image2text model: %w", err)
}
prompt := "Describe this image in detail."
prompt := defaultImageVisionPrompt(lang)
// image family's contract key is system_prompt (parser.go:295),
// mirroring Python parser.py:1119. Do NOT read setup["prompt"]
// here — that key is for the video family, not image.

View File

@@ -133,6 +133,44 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
}
}
func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
origResolver := resolveTenantModelByType
defer func() { resolveTenantModelByType = origResolver }()
drv := &imagePromptCaptureDriver{}
resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
return drv, "img-model", &modelModule.APIConfig{}, 0, nil
}
setups := defaultSetups()
setups["image"]["lang"] = "Chinese"
setups["image"]["system_prompt"] = ""
_, _, err := maybeDispatchImage(
t.Context(),
dao.DB,
utility.FileTypeVISUAL,
"test.png",
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
setups,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
}
got, ok := firstUserText(drv.captured)
if !ok {
t.Fatalf("no user text captured in VLM messages: %#v", drv.captured)
}
if !strings.Contains(got, "Respond in Japanese.") {
t.Fatalf("VLM user text = %q, want dataset language instruction", got)
}
if strings.Contains(got, "Respond in Chinese.") {
t.Fatalf("VLM user text = %q, setup fallback overrode dataset language", got)
}
}
// TestMaybeDispatchImage_ReturnsJSONWithImage pins the output-shape fix:
// the image branch must return a JSON item carrying the `image` attachment
// (data URI) and `doc_type_kwd:"image"`, mirroring Python
@@ -363,12 +401,19 @@ func TestMaybeDispatchAudio_DefaultOutputFormatJson(t *testing.T) {
// Before the fix the table item was skipped and never sent to the VLM.
func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {
origResolver := resolveTenantModelByType
defer func() { resolveTenantModelByType = origResolver }()
origPrompt := figureVisionPromptBuilder
defer func() {
resolveTenantModelByType = origResolver
figureVisionPromptBuilder = origPrompt
}()
drv := &imagePromptCaptureDriver{}
resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
return drv, "img-model", &modelModule.APIConfig{}, 0, nil
}
figureVisionPromptBuilder = func(_, _, language string) (string, error) {
return "describe in " + language, nil
}
dispatched := parserDispatchResult{
OutputFormat: "json",
@@ -383,7 +428,7 @@ func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {
dao.DB,
utility.FileTypeMarkdown,
dispatched,
map[string]any{"tenant_id": "t1"},
map[string]any{"tenant_id": "t1", "lang": "Korean"},
)
if err != nil {
t.Fatalf("maybeDispatchMarkdownVision: %v", err)
@@ -398,6 +443,10 @@ func TestMaybeDispatchMarkdownVision_EnhancesTables(t *testing.T) {
if got, _ := res.JSON[0]["text"].(string); got != "captured" {
t.Fatalf("table item text = %q, want %q (table items must be vision-enhanced)", got, "captured")
}
gotPrompt, ok := firstUserText(drv.captured)
if !ok || gotPrompt != "describe in Korean" {
t.Fatalf("VLM user text = %q, want dataset language propagated", gotPrompt)
}
}
// TestDefaultEmailOutputFormatIsJSON pins diff 2.2: the email family default

View File

@@ -543,6 +543,10 @@ func pdfVisionPromptsBaseDir() (string, error) {
return pdfVisionPromptsBase, nil
}
// renderPDFVisionPrompt only renders page metadata. The full-page PDF vision
// prompt is a transcription contract that preserves the document's original
// language; dataset-language instructions apply to figure descriptions in
// maybeDispatchPDFVisionEnhancement instead.
func renderPDFVisionPrompt(template string, page int) string {
rendered := strings.ReplaceAll(template, "{{ page }}", fmt.Sprintf("%d", page))
rendered = strings.ReplaceAll(rendered, "{{page}}", fmt.Sprintf("%d", page))
@@ -587,6 +591,7 @@ func maybeDispatchPDFVisionEnhancement(
if tenantID == "" {
return dispatched, false, nil
}
language := resolveVisionLanguage(inputs, "")
driver, modelName, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
if err != nil {
return dispatched, false, nil
@@ -620,7 +625,7 @@ func maybeDispatchPDFVisionEnhancement(
if img == "" {
return
}
prompt, perr := docxVisionPromptBuilder("", "")
prompt, perr := figureVisionPromptBuilder("", "", language)
if perr != nil {
return
}

View File

@@ -15,7 +15,67 @@
package component
import "testing"
import (
"context"
"testing"
"ragflow/internal/dao"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/utility"
"gorm.io/gorm"
)
func TestMaybeDispatchPDFVisionEnhancementForwardsDatasetLanguage(t *testing.T) {
origResolver := resolveTenantModelByType
origInvoker := visionChatInvoker
origPrompt := figureVisionPromptBuilder
t.Cleanup(func() {
resolveTenantModelByType = origResolver
visionChatInvoker = origInvoker
figureVisionPromptBuilder = origPrompt
})
resolveTenantModelByType = func(context.Context, *gorm.DB, string, entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
return &docxVisionFakeDriver{}, "pdf-vision-model", &modelModule.APIConfig{}, 0, nil
}
invoker := &docxVisionCaptureInvoker{}
visionChatInvoker = invoker.invoke
capturedLanguage := ""
figureVisionPromptBuilder = func(_, _, language string) (string, error) {
capturedLanguage = language
return "describe the figure", nil
}
dispatched := parserDispatchResult{
OutputFormat: "json",
DocType: "pdf",
JSON: []map[string]any{
{"text": "caption", "image": "data:image/png;base64,aW1hZ2U=", "doc_type_kwd": "image"},
},
}
res, modified, err := maybeDispatchPDFVisionEnhancement(
t.Context(),
dao.DB,
utility.FileTypePDF,
dispatched,
map[string]any{"tenant_id": "t1", "lang": "Dutch"},
)
if err != nil {
t.Fatalf("maybeDispatchPDFVisionEnhancement: %v", err)
}
if !modified {
t.Fatal("modified = false, want true")
}
if capturedLanguage != "Dutch" {
t.Fatalf("figure prompt language = %q, want Dutch", capturedLanguage)
}
if got := res.JSON[0]["text"]; got != "caption\na diagram of a pipeline" {
t.Fatalf("enhanced text = %q", got)
}
}
// TestIsNamedPDFParseMethodWhitelistAligned verifies that the runtime
// "named parse_method" classifier agrees with (*ParserComponent).Check()'s

View File

@@ -0,0 +1,46 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package component
import "strings"
const defaultVisionLanguage = "English"
func normalizeVisionLanguage(language string) string {
language = strings.TrimSpace(language)
if language == "" {
return defaultVisionLanguage
}
return language
}
func resolveVisionLanguage(inputs map[string]any, fallback string) string {
if language := strings.TrimSpace(getStringOr(inputs, "lang", "")); language != "" {
return language
}
return normalizeVisionLanguage(fallback)
}
func renderFigureVisionLanguage(prompt, language string) string {
language = normalizeVisionLanguage(language)
prompt = strings.ReplaceAll(prompt, "{{ language }}", language)
return strings.ReplaceAll(prompt, "{{language}}", language)
}
func defaultImageVisionPrompt(language string) string {
return "Describe this image in detail. Respond in " + normalizeVisionLanguage(language) + ". Preserve all visible text in its original language; do not translate it."
}

View File

@@ -0,0 +1,103 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package component
import (
"strings"
"testing"
)
func TestResolveVisionLanguage(t *testing.T) {
tests := []struct {
name string
inputs map[string]any
fallback string
want string
}{
{
name: "dataset language takes precedence",
inputs: map[string]any{"lang": " Japanese "},
fallback: "Chinese",
want: "Japanese",
},
{
name: "configured fallback is used",
inputs: map[string]any{"lang": ""},
fallback: " Korean ",
want: "Korean",
},
{
name: "empty values default to English",
inputs: nil,
fallback: "",
want: defaultVisionLanguage,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := resolveVisionLanguage(tt.inputs, tt.fallback); got != tt.want {
t.Fatalf("resolveVisionLanguage() = %q, want %q", got, tt.want)
}
})
}
}
func TestRenderFigureVisionLanguage(t *testing.T) {
prompt := renderFigureVisionLanguage("spaced={{ language }} compact={{language}}", "Japanese")
if prompt != "spaced=Japanese compact=Japanese" {
t.Fatalf("renderFigureVisionLanguage() = %q", prompt)
}
prompt = renderFigureVisionLanguage("language={{ language }}", "")
if prompt != "language=English" {
t.Fatalf("empty-language rendering = %q, want English fallback", prompt)
}
}
func TestBuildFigureVisionPromptRendersLanguageAndContext(t *testing.T) {
prompt, err := buildFigureVisionPrompt("Context above", "Context below", "Japanese")
if err != nil {
t.Fatalf("buildFigureVisionPrompt: %v", err)
}
for _, want := range []string{
"Write all descriptions and field values in Japanese.",
"Context above",
"Context below",
} {
if !strings.Contains(prompt, want) {
t.Fatalf("prompt does not contain %q", want)
}
}
for _, unresolved := range []string{"{{ language }}", "{{ context_above }}", "{{ context_below }}"} {
if strings.Contains(prompt, unresolved) {
t.Fatalf("prompt contains unresolved placeholder %q", unresolved)
}
}
secondPrompt, err := buildFigureVisionPrompt("", "", "Chinese")
if err != nil {
t.Fatalf("buildFigureVisionPrompt with cached template: %v", err)
}
if !strings.Contains(secondPrompt, "Write all descriptions and field values in Chinese.") {
t.Fatalf("cached prompt did not render the second request language")
}
if strings.Contains(secondPrompt, "in Japanese.") {
t.Fatalf("cached prompt leaked the previous request language")
}
}