diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go index 6e4177ced5..4405b17096 100644 --- a/internal/parser/parser/docx_parser.go +++ b/internal/parser/parser/docx_parser.go @@ -108,7 +108,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data // (mirrors Python parser.py:892-893 remove_toc_word). if p.RemoveTOC { outlines := extractDOCXOutlines(irJSON) - sections = removeTOCWord(sections, outlines, false) + sections = removeTOCWord(sections, outlines, isEnglishItems(sections)) } if len(sections) == 0 { sections = []map[string]any{{"text": "", "doc_type_kwd": "text"}} @@ -150,7 +150,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data for _, ln := range lines { lineItems = append(lineItems, map[string]any{"text": ln}) } - filtered := removeTOCWord(lineItems, outlines, false) + filtered := removeTOCWord(lineItems, outlines, isEnglishItems(lineItems)) rebuilt := make([]string, 0, len(filtered)) for _, item := range filtered { rebuilt = append(rebuilt, itemText(item)) diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go index f613528a70..13d9ab28f3 100644 --- a/internal/parser/parser/html_parser.go +++ b/internal/parser/parser/html_parser.go @@ -86,7 +86,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data // remove_toc: post-parse text heuristic (mirrors Python // parser.py:1087-1088 remove_toc → remove_contents_table). if p.RemoveTOC { - items = removeContentsTable(items, false) + items = removeContentsTable(items, isEnglishItems(items)) } if items == nil { items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} diff --git a/internal/parser/parser/text_toc.go b/internal/parser/parser/text_toc.go index bfdedd8db3..e31e42d4d6 100644 --- a/internal/parser/parser/text_toc.go +++ b/internal/parser/parser/text_toc.go @@ -105,3 +105,44 @@ func tocPrefix(text string, eng bool) string { } return string(runes[:3]) } + +// isEnglishTexts reports whether the given texts read as English. It returns +// true when more than 80% of the (up to 200) sampled non-empty segments consist +// solely of ASCII letters/digits/punctuation. The result selects the TOC-entry +// prefix strategy: a 2-word prefix for English content, otherwise a 3-character +// prefix (used by removeContentsTable / tocPrefix). +func isEnglishTexts(texts []string) bool { + var sampled []string + for _, t := range texts { + if s := strings.TrimSpace(t); s != "" { + sampled = append(sampled, s) + } + if len(sampled) >= 200 { + break + } + } + if len(sampled) == 0 { + return false + } + pat := regexp.MustCompile("^[" + "`" + "a-zA-Z0-9\\s.,':;/\"?<>!()-]+$") + eng := 0 + for _, t := range sampled { + if pat.MatchString(t) { + eng++ + } + } + return float64(eng)/float64(len(sampled)) > 0.8 +} + +// isEnglishItems reports whether the parser items read as English, classifying +// each item's text with isEnglishTexts. It is passed to removeTOCWord / +// removeContentsTable so English documents use the 2-word TOC prefix. +func isEnglishItems(items []map[string]any) bool { + texts := make([]string, 0, len(items)) + for _, it := range items { + if t := itemText(it); t != "" { + texts = append(texts, t) + } + } + return isEnglishTexts(texts) +} diff --git a/internal/parser/parser/text_toc_test.go b/internal/parser/parser/text_toc_test.go index a4c3a66a63..01ceb8137b 100644 --- a/internal/parser/parser/text_toc_test.go +++ b/internal/parser/parser/text_toc_test.go @@ -106,3 +106,60 @@ func TestRemoveContentsTable(t *testing.T) { }) } } + +// TestIsEnglishTexts classifies text as English when >80% of sampled segments +// are ASCII. +func TestIsEnglishTexts(t *testing.T) { + cases := []struct { + name string + texts []string + want bool + }{ + {name: "empty", texts: nil, want: false}, + {name: "all english", texts: []string{"Chapter One", "Section Two", "Body text here"}, want: true}, + {name: "all chinese", texts: []string{"第一章 概述", "第二章 方法", "正文内容"}, want: false}, + {name: "mostly english (>80%)", texts: []string{ + "Introduction to the system", "Background and related work", + "Methodology details", "Evaluation results", "Conclusion", + "第一章", // single CJK line among 6 (>80% english) + }, want: true}, + {name: "exactly 80% english", texts: []string{ + "Introduction", "Background", "Method", "Conclusion", "第一章", + }, want: false}, // 4/5 == 80% is NOT > 80% + {name: "mostly chinese (<80% english)", texts: []string{ + "第一章 概述", "第二章 方法", "第三章 结果", "English abstract only", + }, want: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isEnglishTexts(tc.texts); got != tc.want { + t.Errorf("isEnglishTexts(%v) = %v, want %v", tc.texts, got, tc.want) + } + }) + } +} + +// TestRemoveTOCWordEnglishDetection verifies English content uses the 2-word +// TOC prefix, so "Chapter 2 Method" is NOT over-deleted (a 3-character prefix +// would have dropped it). +func TestRemoveTOCWordEnglishDetection(t *testing.T) { + items := []map[string]any{ + {"text": "Intro"}, + {"text": "Contents"}, + {"text": "Chapter 1 Overview"}, + {"text": "Chapter 2 Method"}, + {"text": "Body starts"}, + } + if !isEnglishItems(items) { + t.Fatalf("expected isEnglishItems=true for English content") + } + got := removeContentsTable(items, isEnglishItems(items)) + want := []map[string]any{ + {"text": "Intro"}, + {"text": "Chapter 2 Method"}, + {"text": "Body starts"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("English TOC detection: got %v, want %v", got, want) + } +}