fix(parser): derive eng from content for DOCX/HTML TOC removal (#17585)

## Summary
- The `eng` flag passed to `removeTOCWord`/`removeContentsTable` was
hardcoded `false`, so English documents used the CJK 3-char TOC prefix
instead of Python's 2-word English prefix. This caused English tables of
contents to be under-deleted (left in indexed text) or over-deleted
(body paragraphs sharing the 3-char prefix dropped).
- `eng` is now derived from the parsed item content via `isEnglishItems`
(a port of Python `is_english`/`_is_english`), mirroring Python's
content heuristic.

## Changes
- `internal/parser/parser/text_toc.go`: add
`isEnglishItems`/`isEnglishTexts` (ASCII-ratio >80% ⇒ English,
fullmatch-anchored regex).
- `internal/parser/parser/docx_parser.go`: pass
`isEnglishItems(sections)` / `isEnglishItems(lineItems)` to
`removeTOCWord` (json + markdown paths).
- `internal/parser/parser/html_parser.go`: pass `isEnglishItems(items)`
to `removeContentsTable`.
- `docs/migration_python_go_diff.md`: close the 1.9/1.10 `eng` residual
and resolve the contradictory Parser 2.11 "Partially fixed" note.

## Notes
- `remove_toc` is a Parser-stage, DSL-configured feature (`remove_toc:
true/false` in the parser family setup). This change only fixes the
internal English/CJK prefix decision; it does not alter the pipeline or
the DSL contract. `remove_header_footer` (precise match) is unaffected.
- `eng` is auto-derived from content rather than exposed as a new config
knob, to stay faithful to Python behavior.

## Test plan
- `CGO_ENABLED=0 go test ./internal/parser/parser/ -run
'TestRemoveContentsTable|TestIsEnglishTexts|TestRemoveTOCWordEnglishDetection'`
- Added `TestIsEnglishTexts` (ASCII-ratio cases) and
`TestRemoveTOCWordEnglishDetection` (English TOC no longer over-deletes
"Chapter 2 Method").

---------

Co-authored-by: CodeBuddy <noreply@codebuddy.ai>

internal/parser/parser/docx_parser.go

@@ -108,7 +108,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
 		// (mirrors Python parser.py:892-893 remove_toc_word).
 		if p.RemoveTOC {
 			outlines := extractDOCXOutlines(irJSON)
-			sections = removeTOCWord(sections, outlines, false)
+			sections = removeTOCWord(sections, outlines, isEnglishItems(sections))
 		}
 		if len(sections) == 0 {
 			sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
@@ -150,7 +150,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
 		for _, ln := range lines {
 			lineItems = append(lineItems, map[string]any{"text": ln})
 		}
-		filtered := removeTOCWord(lineItems, outlines, false)
+		filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems))
 		rebuilt := make([]string, 0, len(filtered))
 		for _, item := range filtered {
 			rebuilt = append(rebuilt, itemText(item))

internal/parser/parser/html_parser.go

@@ -86,7 +86,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
 	// remove_toc: post-parse text heuristic (mirrors Python
 	// parser.py:1087-1088 remove_toc → remove_contents_table).
 	if p.RemoveTOC {
-		items = removeContentsTable(items, false)
+		items = removeContentsTable(items, isEnglishItems(items))
 	}
 	if items == nil {
 		items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}

internal/parser/parser/text_toc.go

@@ -105,3 +105,44 @@ func tocPrefix(text string, eng bool) string {
 	}
 	return string(runes[:3])
 }
+
+// isEnglishTexts reports whether the given texts read as English. It returns
+// true when more than 80% of the (up to 200) sampled non-empty segments consist
+// solely of ASCII letters/digits/punctuation. The result selects the TOC-entry
+// prefix strategy: a 2-word prefix for English content, otherwise a 3-character
+// prefix (used by removeContentsTable / tocPrefix).
+func isEnglishTexts(texts []string) bool {
+	var sampled []string
+	for _, t := range texts {
+		if s := strings.TrimSpace(t); s != "" {
+			sampled = append(sampled, s)
+		}
+		if len(sampled) >= 200 {
+			break
+		}
+	}
+	if len(sampled) == 0 {
+		return false
+	}
+	pat := regexp.MustCompile("^[" + "`" + "a-zA-Z0-9\\s.,':;/\"?<>!()-]+$")
+	eng := 0
+	for _, t := range sampled {
+		if pat.MatchString(t) {
+			eng++
+		}
+	}
+	return float64(eng)/float64(len(sampled)) > 0.8
+}
+
+// isEnglishItems reports whether the parser items read as English, classifying
+// each item's text with isEnglishTexts. It is passed to removeTOCWord /
+// removeContentsTable so English documents use the 2-word TOC prefix.
+func isEnglishItems(items []map[string]any) bool {
+	texts := make([]string, 0, len(items))
+	for _, it := range items {
+		if t := itemText(it); t != "" {
+			texts = append(texts, t)
+		}
+	}
+	return isEnglishTexts(texts)
+}

internal/parser/parser/text_toc_test.go

@@ -106,3 +106,60 @@ func TestRemoveContentsTable(t *testing.T) {
 		})
 	}
 }
+
+// TestIsEnglishTexts classifies text as English when >80% of sampled segments
+// are ASCII.
+func TestIsEnglishTexts(t *testing.T) {
+	cases := []struct {
+		name  string
+		texts []string
+		want  bool
+	}{
+		{name: "empty", texts: nil, want: false},
+		{name: "all english", texts: []string{"Chapter One", "Section Two", "Body text here"}, want: true},
+		{name: "all chinese", texts: []string{"第一章 概述", "第二章 方法", "正文内容"}, want: false},
+		{name: "mostly english (>80%)", texts: []string{
+			"Introduction to the system", "Background and related work",
+			"Methodology details", "Evaluation results", "Conclusion",
+			"第一章", // single CJK line among 6 (>80% english)
+		}, want: true},
+		{name: "exactly 80% english", texts: []string{
+			"Introduction", "Background", "Method", "Conclusion", "第一章",
+		}, want: false}, // 4/5 == 80% is NOT > 80%
+		{name: "mostly chinese (<80% english)", texts: []string{
+			"第一章 概述", "第二章 方法", "第三章 结果", "English abstract only",
+		}, want: false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isEnglishTexts(tc.texts); got != tc.want {
+				t.Errorf("isEnglishTexts(%v) = %v, want %v", tc.texts, got, tc.want)
+			}
+		})
+	}
+}
+
+// TestRemoveTOCWordEnglishDetection verifies English content uses the 2-word
+// TOC prefix, so "Chapter 2 Method" is NOT over-deleted (a 3-character prefix
+// would have dropped it).
+func TestRemoveTOCWordEnglishDetection(t *testing.T) {
+	items := []map[string]any{
+		{"text": "Intro"},
+		{"text": "Contents"},
+		{"text": "Chapter 1 Overview"},
+		{"text": "Chapter 2 Method"},
+		{"text": "Body starts"},
+	}
+	if !isEnglishItems(items) {
+		t.Fatalf("expected isEnglishItems=true for English content")
+	}
+	got := removeContentsTable(items, isEnglishItems(items))
+	want := []map[string]any{
+		{"text": "Intro"},
+		{"text": "Chapter 2 Method"},
+		{"text": "Body starts"},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("English TOC detection: got %v, want %v", got, want)
+	}
+}