diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go index 444730d037..e627c125d0 100644 --- a/internal/ingestion/component/media_dispatch.go +++ b/internal/ingestion/component/media_dispatch.go @@ -168,10 +168,17 @@ func maybeDispatchImage( } } - outputFormat, _ := setup["output_format"].(string) - if outputFormat == "" { - outputFormat = "text" - } + // The image family always emits a structured JSON item carrying the + // image attachment (data URI) and doc_type_kwd, mirroring Python + // rag/app/picture.py:71-72 (doc["image"]=img, doc["doc_type_kwd"]= + // "image"). picture.py has no "text" output mode — it always returns + // a structured doc — so output_format is hardcoded to "json" and any + // setup override is ignored. The former behavior returned a bare Text + // string, which dropped the image attachment, set doc_type to "text", + // and on the default json path produced JSON=nil so downstream + // Chunkers rejected the payload with errRequiredField{"json"}. + imageB64 := base64.StdEncoding.EncodeToString(binary) + dataURI := "data:" + imageMIME(filename) + ";base64," + imageB64 // --- Phase 2: VLM description (when OCR text is short) --- // Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32 @@ -184,11 +191,7 @@ func maybeDispatchImage( charCount := len(ocrText) if (eng && wordCount > 32) || charCount > 32 { // OCR returned substantial text — skip VLM. - return parserDispatchResult{ - OutputFormat: outputFormat, - DocType: "image", - Text: ocrText, - }, true, nil + return imageDispatchResult(ocrText, dataURI), true, nil } } @@ -197,20 +200,12 @@ func maybeDispatchImage( if err != nil { // If VLM is unavailable but we have OCR text, return it. if ocrText != "" { - return parserDispatchResult{ - OutputFormat: outputFormat, - DocType: "image", - Text: ocrText, - }, true, nil + return imageDispatchResult(ocrText, dataURI), true, nil } return parserDispatchResult{}, true, fmt.Errorf("Parser: picture image2text model: %w", err) } - imageB64 := base64.StdEncoding.EncodeToString(binary) - mimeType := imageMIME(filename) - dataURI := "data:" + mimeType + ";base64," + imageB64 - prompt := "Describe this image in detail." // image family's contract key is system_prompt (parser.go:295), // mirroring Python parser.py:1119. Do NOT read setup["prompt"] @@ -229,11 +224,7 @@ func maybeDispatchImage( resp, err := driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil) if err != nil { if ocrText != "" { - return parserDispatchResult{ - OutputFormat: outputFormat, - DocType: "image", - Text: ocrText, - }, true, nil + return imageDispatchResult(ocrText, dataURI), true, nil } return parserDispatchResult{}, true, fmt.Errorf("Parser: picture describe: %w", err) @@ -253,11 +244,23 @@ func maybeDispatchImage( combined = vlmText } } + return imageDispatchResult(combined, dataURI), true, nil +} + +// imageDispatchResult builds the structured JSON payload for the image +// family: a single item carrying the combined text, the image attachment +// (data URI), and doc_type_kwd "image". Mirrors Python +// rag/app/picture.py:71-72. +func imageDispatchResult(text, dataURI string) parserDispatchResult { return parserDispatchResult{ - OutputFormat: outputFormat, + OutputFormat: "json", DocType: "image", - Text: combined, - }, true, nil + JSON: []map[string]any{{ + "text": text, + "image": dataURI, + "doc_type_kwd": "image", + }}, + } } // Audio dispatch: SPEECH2TEXT transcription --- diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go index b2bde8e5dd..1d684e44de 100644 --- a/internal/ingestion/component/media_dispatch_test.go +++ b/internal/ingestion/component/media_dispatch_test.go @@ -17,6 +17,7 @@ package component import ( "context" + "strings" "sync" "testing" @@ -104,8 +105,18 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) { if !dispatched { t.Fatalf("expected dispatched=true for VISUAL file") } - if res.Text == "" { - t.Fatalf("expected non-empty combined text") + // After the output-shape fix the image branch returns JSON items + // (OutputFormat=="json"), not a bare Text field. The combined text + // now lives in JSON[0]["text"]; the legacy res.Text is no longer + // populated for the image family. + if res.OutputFormat != "json" { + t.Fatalf("OutputFormat = %q, want json (image family is always structured)", res.OutputFormat) + } + if len(res.JSON) != 1 { + t.Fatalf("JSON len = %d, want 1 (image result must be a single JSON item)", len(res.JSON)) + } + if txt, _ := res.JSON[0]["text"].(string); txt == "" { + t.Fatalf("expected non-empty combined text in JSON[0][\"text\"]") } got, ok := firstUserText(drv.captured) @@ -117,6 +128,90 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) { } } +// TestMaybeDispatchImage_ReturnsJSONWithImage pins the output-shape fix: +// the image branch must return a JSON item carrying the `image` attachment +// (data URI) and `doc_type_kwd:"image"`, mirroring Python +// rag/app/picture.py:71-72. Before the fix the branch returned a bare Text +// string with JSON=nil, dropping the image attachment and (on the default +// json path) causing OneChunker/TokenChunker to reject the payload. +func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) { + origResolver := resolveTenantModelByType + defer func() { resolveTenantModelByType = origResolver }() + + drv := &imagePromptCaptureDriver{} + resolveTenantModelByType = func(tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + return drv, "img-model", &modelModule.APIConfig{}, 0, nil + } + + setups := defaultSetups() + res, dispatched, err := maybeDispatchImage( + context.Background(), + utility.FileTypeVISUAL, + "test.png", + []byte("not-a-real-image"), + map[string]any{"tenant_id": "t1"}, + setups, + ) + if err != nil { + t.Fatalf("maybeDispatchImage: %v", err) + } + if !dispatched { + t.Fatalf("expected dispatched=true") + } + if res.OutputFormat != "json" { + t.Fatalf("OutputFormat = %q, want json", res.OutputFormat) + } + if len(res.JSON) != 1 { + t.Fatalf("JSON len = %d, want 1", len(res.JSON)) + } + item := res.JSON[0] + if got, _ := item["doc_type_kwd"].(string); got != "image" { + t.Errorf("doc_type_kwd = %q, want \"image\"", got) + } + img, _ := item["image"].(string) + if !strings.HasPrefix(img, "data:") || !strings.Contains(img, ";base64,") { + t.Errorf("image = %q, want a data URI (data:;base64,)", img) + } + if txt, _ := item["text"].(string); txt == "" { + t.Errorf("text field empty; want non-empty combined OCR+VLM text") + } +} + +// TestMaybeDispatchImage_HardcodesJSONOutput verifies the image family +// always emits json regardless of setup["output_format"]. Python +// rag/app/picture.py:chunk() has no output_format concept — it always +// returns a structured doc. Honoring a "text" override produced a bare +// Text payload that lost the image attachment and set doc_type to "text". +func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) { + origResolver := resolveTenantModelByType + defer func() { resolveTenantModelByType = origResolver }() + + drv := &imagePromptCaptureDriver{} + resolveTenantModelByType = func(tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) { + return drv, "img-model", &modelModule.APIConfig{}, 0, nil + } + + setups := defaultSetups() + setups["image"]["output_format"] = "text" // legacy/override; must be ignored + res, _, err := maybeDispatchImage( + context.Background(), + utility.FileTypeVISUAL, + "test.png", + []byte("not-a-real-image"), + map[string]any{"tenant_id": "t1"}, + setups, + ) + if err != nil { + t.Fatalf("maybeDispatchImage: %v", err) + } + if res.OutputFormat != "json" { + t.Fatalf("OutputFormat = %q, want json (image family must ignore output_format override)", res.OutputFormat) + } + if len(res.JSON) != 1 { + t.Fatalf("JSON len = %d, want 1 even when setup says text", len(res.JSON)) + } +} + // audioTranscribeDriver is a mock ModelDriver whose TranscribeAudio returns a // fixed transcription, so maybeDispatchAudio can be exercised without a real // ASR provider. diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json index cca7efc839..f04f47a273 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_picture.json @@ -50,7 +50,7 @@ } }, "image": { - "output_format": "text", + "output_format": "json", "parse_method": "ocr", "preprocess": [ "main_content" @@ -231,7 +231,7 @@ "setups": [ { "fileFormat": "image", - "output_format": "text", + "output_format": "json", "parse_method": "ocr", "preprocess": [ "main_content"