fix(ingestion): improve Extractor/LLM robustness and Python->Go parity (#17470)

## Summary

This PR hardens the Go ingestion **Extractor** and **LLM retry** paths
and closes several Python->Go parity gaps in the keyword/question/tag
extraction flow.

- **Generic retry utility** (`internal/common/retry.go`):
`RetryWithBackoff` with exponential backoff (default 3 retries, 2s
initial delay, capped at 1m), context-aware sleep, and a `maxRetries<=0`
fast path. Covered by `internal/common/retry_test.go`.
- **LLM retry reuse**: `agent/component/llm_retry.go` now delegates to
`common.RetryWithBackoff` instead of an inline loop (behavior preserved:
ctx cancellation short-circuits the backoff).
- **Extractor LLM calls** (`extractor.go`):
- `call()` now retries transient LLM failures via `RetryWithBackoff`
(retry exhaustion fails the chunk instead of silently skipping).
  - Sets `temperature = 0.2`, matching Python `generator.py:230,245`.
- Runs keyword and question extraction **concurrently** per chunk when
both are enabled (`task_executor.py:444-448`), with mutex-guarded map
writes to avoid data races.
- Substitutes `{field_name}` placeholders (including `{chunks}` -> chunk
text) in `prompt`/`system_prompt` before the call, mirroring Python
`string_format` (`extractor.py:102-103`); unmatched placeholders are
left as-is.
- Falls back to the **tenant default chat model** when `llm_id` is empty
(`task_executor.py:573-574`).
- Strips `` **greedily** (`strings.LastIndex`) in
`cleanExtractionResult`.
- **Auto-tagging** (`extractor_tag.go`): drops the `in.llmID != ""`
guards so an empty `llm_id` no longer skips tagging (uses the tenant
default model), and strips `` greedily in `parseTaggerResponse`.
- **Docs**: fixes a misleading `PresentationChunker` docstring that
claimed per-slide `image`/`position` output (the PPTX path emits none —
unlike PDF), and removes a stale `docs/migration_python_go_diff.md`
reference in `media_dispatch.go`.

## Test plan

- `bash build.sh --test ./internal/common/...` — passes (new retry
utility + tests).
- `bash build.sh --test ./internal/ingestion/component/...` — passes
(extractor/chunker/schema).
- `gofmt` and lefthook pre-commit checks pass.

Note: the personal `docs/migration_python_go_diff.md` working notebook
in the tree is intentionally **not** part of this PR.

---------

Co-authored-by: CodeBuddy <noreply@codebuddy.ai>

internal/common/retry.go

@@ -0,0 +1,76 @@
+// Package common — generic retry utility with exponential backoff.
+//
+// RetryWithBackoff calls fn up to maxRetries+1 times (one initial
+// attempt + maxRetries retries), sleeping delay*2^attempt between
+// failures (capped at 1 minute). The sleep honours ctx cancellation.
+//
+// maxRetries <= 0 yields a single attempt (no retries).
+// initialDelay <= 0 results in no delay between retries.
+//
+// Usage:
+//
+//	err := RetryWithBackoff(ctx, 3, 2*time.Second, func() error {
+//	    return someFailableOperation()
+//	})
+
+package common
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+const (
+	// DefaultRetryMax is the default number of retries (3).
+	DefaultRetryMax = 3
+	// DefaultRetryDelay is the initial backoff delay (2s).
+	DefaultRetryDelay = 2 * time.Second
+)
+
+// RetryWithBackoff retries fn on error with exponential backoff.
+// Returns nil on the first successful attempt. Returns the last
+// error wrapped with the retry count when all attempts fail.
+//
+// An optional shouldRetry predicate lets callers abort on
+// non-transient errors: when supplied and it returns false for a
+// given error, RetryWithBackoff stops immediately and returns that
+// error without further backoff. A nil predicate (or a nil function
+// in the slice) retries on every error, preserving the original
+// behavior.
+func RetryWithBackoff(ctx context.Context, maxRetries int, initialDelay time.Duration, fn func() error, shouldRetry ...func(error) bool) error {
+	if maxRetries <= 0 {
+		return fn()
+	}
+	canRetry := func(err error) bool {
+		if len(shouldRetry) == 0 || shouldRetry[0] == nil {
+			return true
+		}
+		return shouldRetry[0](err)
+	}
+	delay := initialDelay
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if !canRetry(err) {
+			return err
+		}
+		if attempt == maxRetries {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(delay):
+		}
+		delay *= 2
+		if delay > time.Minute {
+			delay = time.Minute
+		}
+	}
+	return fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr)
+}

internal/common/retry_test.go

@@ -0,0 +1,157 @@
+package common
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestRetryWithBackoff_SuccessOnFirstAttempt(t *testing.T) {
+	var calls atomic.Int32
+	err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
+		calls.Add(1)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls.Load() != 1 {
+		t.Errorf("calls = %d, want 1", calls.Load())
+	}
+}
+
+func TestRetryWithBackoff_SuccessOnThirdAttempt(t *testing.T) {
+	var calls atomic.Int32
+	err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
+		if calls.Add(1) < 3 {
+			return errors.New("transient")
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if calls.Load() != 3 {
+		t.Errorf("calls = %d, want 3", calls.Load())
+	}
+}
+
+func TestRetryWithBackoff_AllRetriesExhausted(t *testing.T) {
+	sentinel := errors.New("persistent failure")
+	err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
+		return sentinel
+	})
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if !errors.Is(err, sentinel) {
+		t.Errorf("error should wrap sentinel, got %v", err)
+	}
+	if !strings.Contains(err.Error(), "after 3 retries") {
+		t.Errorf("error should mention retry count: %v", err)
+	}
+}
+
+func TestRetryWithBackoff_ContextCancelled(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	var calls atomic.Int32
+	errChan := make(chan error, 1)
+	go func() {
+		errChan <- RetryWithBackoff(ctx, 3, 10*time.Second, func() error {
+			calls.Add(1)
+			cancel() // cancel before retry sleep
+			return errors.New("fail")
+		})
+	}()
+	err := <-errChan
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("expected context.Canceled, got %v", err)
+	}
+	if calls.Load() != 1 {
+		t.Errorf("calls = %d, want 1 (no retry after cancel)", calls.Load())
+	}
+}
+
+func TestRetryWithBackoff_ContextDeadlineExceeded(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
+	defer cancel()
+	var calls atomic.Int32
+	err := RetryWithBackoff(ctx, 3, time.Second, func() error {
+		calls.Add(1)
+		return errors.New("fail")
+	})
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("expected DeadlineExceeded, got %v", err)
+	}
+	// Should have made at least 1 call; ctx deadline stops further retries
+	if calls.Load() < 1 {
+		t.Errorf("expected at least 1 call, got %d", calls.Load())
+	}
+}
+
+func TestRetryWithBackoff_ZeroMaxRetries(t *testing.T) {
+	var calls atomic.Int32
+	err := RetryWithBackoff(t.Context(), 0, 0, func() error {
+		calls.Add(1)
+		return errors.New("fail")
+	})
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if calls.Load() != 1 {
+		t.Errorf("calls = %d, want 1 (single attempt)", calls.Load())
+	}
+}
+
+func TestRetryWithBackoff_NegativeMaxRetries(t *testing.T) {
+	var calls atomic.Int32
+	err := RetryWithBackoff(t.Context(), -1, 0, func() error {
+		calls.Add(1)
+		return errors.New("fail")
+	})
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if calls.Load() != 1 {
+		t.Errorf("calls = %d, want 1 (negative → single attempt)", calls.Load())
+	}
+}
+
+// TestRetryWithBackoff_NonRetryableStops verifies that when a
+// shouldRetry predicate returns false, RetryWithBackoff aborts
+// immediately instead of burning the remaining retries/backoff.
+func TestRetryWithBackoff_NonRetryableStops(t *testing.T) {
+	var calls atomic.Int32
+	sentinel := errors.New("auth failure")
+	err := RetryWithBackoff(t.Context(), 3, 10*time.Second, func() error {
+		calls.Add(1)
+		return sentinel
+	}, func(err error) bool {
+		return !errors.Is(err, sentinel)
+	})
+	if !errors.Is(err, sentinel) {
+		t.Errorf("expected sentinel, got %v", err)
+	}
+	if calls.Load() != 1 {
+		t.Errorf("calls = %d, want 1 (non-retryable → single attempt)", calls.Load())
+	}
+}
+
+// TestRetryWithBackoff_NilPredicateRetriesAll confirms that omitting
+// the predicate preserves the original blind-retry behavior.
+func TestRetryWithBackoff_NilPredicateRetriesAll(t *testing.T) {
+	var calls atomic.Int32
+	err := RetryWithBackoff(t.Context(), 2, time.Millisecond, func() error {
+		calls.Add(1)
+		return errors.New("transient")
+	})
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if calls.Load() != 3 {
+		t.Errorf("calls = %d, want 3 (all retries attempted)", calls.Load())
+	}
+}