// // Copyright 2026 The InfiniFlow Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // package parser import ( "bytes" "context" "fmt" "strings" "golang.org/x/net/html" ) type HTMLParser struct { RemoveHeaderFooter bool RemoveTOC bool } func NewHTMLParser() *HTMLParser { return &HTMLParser{} } func (p *HTMLParser) String() string { return "HTMLParser" } // ConfigureFromSetup reads the HTML family setup map. Mirrors the // Python parser.py HTML setup keys: remove_header_footer (pre-parse // tag strip) and remove_toc (post-parse text heuristic). func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) { if p == nil || setup == nil { return } if v, ok := setup["remove_header_footer"].(bool); ok { p.RemoveHeaderFooter = v } if v, ok := setup["remove_toc"].(bool); ok { p.RemoveTOC = v } } // ParseWithResult emits one item per block-level HTML element // (headings, paragraphs, lists, pre blocks). The walker is a // pure-Go replacement for the previous `fmt.Printf` debug output: // it descends the html.Parse tree, joins the leaf text of each // block-level element, and emits the python-compatible // `{text, doc_type_kwd:"text"}` shape. // // Phase 2.5 (Slice 1) of port-rag-flow-pipeline-to-go.md makes // HTMLParser a ParseResultProducer so the dispatch seam routes // the html family through the structured path. Inline formatting // (bold / links / images) is intentionally NOT surfaced as a // separate ck_type — the python HtmlParser collapses inline // formatting into the parent block's text. func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult { // remove_header_footer: pre-parse strip of
/