mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-08 00:18:12 +08:00
Fix(parser): keep real line breaks when merging HTML fragments and PDF boxes (#17856)
Net effect: inline prose stays on one line (`Hello World`), real `<br>` boundaries survive (including before tags and repeated breaks), and source formatting whitespace no longer over-splits.
This commit is contained in:
Jack
@@ -105,27 +105,31 @@ class RAGFlowHtmlParser:
|
||||
return table_str_list
|
||||
|
||||
@classmethod
|
||||
def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None):
|
||||
def read_text_recursively(cls, element, parser_result, chunk_token_num=512, parent_name=None, block_id=None, newline_before=0):
|
||||
if isinstance(element, NavigableString):
|
||||
content = element.strip()
|
||||
# Keep the text verbatim (do not strip): source whitespace is
|
||||
# folded later by merge_block_text, which preserves <br> breaks.
|
||||
content = str(element)
|
||||
|
||||
def is_valid_html(content):
|
||||
def is_valid_html(text):
|
||||
try:
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
soup = BeautifulSoup(text, "html.parser")
|
||||
return bool(soup.find())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return_info = []
|
||||
if content:
|
||||
if content.strip():
|
||||
if is_valid_html(content):
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id)
|
||||
child_info = cls.read_text_recursively(soup, parser_result, chunk_token_num, element.name, block_id, newline_before)
|
||||
parser_result.extend(child_info)
|
||||
else:
|
||||
info = {"content": element.strip(), "tag_name": "inner_text", "metadata": {"block_id": block_id}}
|
||||
info = {"content": content, "tag_name": "inner_text", "metadata": {"block_id": block_id}}
|
||||
if parent_name:
|
||||
info["tag_name"] = parent_name
|
||||
if newline_before:
|
||||
info["metadata"]["newline_before"] = newline_before
|
||||
return_info.append(info)
|
||||
return return_info
|
||||
elif isinstance(element, Tag):
|
||||
@@ -139,22 +143,55 @@ class RAGFlowHtmlParser:
|
||||
else:
|
||||
if str.lower(element.name) in BLOCK_TAGS:
|
||||
block_id = str(uuid.uuid1())
|
||||
# A <br> is a hard line break. Thread a pending break count
|
||||
# through so it survives nesting (e.g. "<p>A<br><span>B</span></p>")
|
||||
# and repeated breaks ("<br><br>"). The count resets only after
|
||||
# a descendant emits text, mirroring the Go walker.
|
||||
pending_newlines = int(newline_before)
|
||||
for child in element.children:
|
||||
child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id)
|
||||
if isinstance(child, Tag) and str.lower(child.name) == "br":
|
||||
pending_newlines += 1
|
||||
continue
|
||||
result_count = len(parser_result)
|
||||
child_info = cls.read_text_recursively(child, parser_result, chunk_token_num, element.name, block_id, pending_newlines)
|
||||
parser_result.extend(child_info)
|
||||
if child_info or len(parser_result) > result_count:
|
||||
pending_newlines = 0
|
||||
return []
|
||||
|
||||
# Hard line-break sentinel used between fragments so CSS whitespace
|
||||
# folding (which collapses real whitespace runs) does not eat explicit
|
||||
# <br> breaks. Replaced with "\n" after folding.
|
||||
_HARD_BREAK = "\x0b"
|
||||
_WS_RE = re.compile(r"[ \t\r\n\f]+")
|
||||
_PRE_TAGS = ("pre", "textarea")
|
||||
|
||||
@classmethod
|
||||
def _fold_block(cls, raw, preserve):
|
||||
if preserve:
|
||||
return raw.replace(cls._HARD_BREAK, "\n")
|
||||
# Collapse each line between hard breaks: runs of collapsible
|
||||
# whitespace become one space, and line edges are trimmed. This
|
||||
# matches the Go walker's CSS whitespace folding.
|
||||
lines = raw.split(cls._HARD_BREAK)
|
||||
folded = [cls._WS_RE.sub(" ", ln).strip() for ln in lines]
|
||||
return "\n".join(folded)
|
||||
|
||||
@classmethod
|
||||
def merge_block_text(cls, parser_result):
|
||||
block_content = []
|
||||
current_content = ""
|
||||
current_is_pre = False
|
||||
table_info_list = []
|
||||
last_block_id = None
|
||||
for item in parser_result:
|
||||
content = item.get("content")
|
||||
content = item.get("content") or ""
|
||||
tag_name = item.get("tag_name")
|
||||
title_flag = tag_name in TITLE_TAGS
|
||||
block_id = item.get("metadata", {}).get("block_id")
|
||||
# newline_before is a break count: 0 -> join verbatim (no
|
||||
# separator), >0 -> insert that many hard breaks (one per <br>).
|
||||
newline_before = item.get("metadata", {}).get("newline_before", 0)
|
||||
if block_id:
|
||||
if title_flag:
|
||||
content = f"{TITLE_TAGS[tag_name]} {content}"
|
||||
@@ -163,18 +200,21 @@ class RAGFlowHtmlParser:
|
||||
# precedes the first block (last_block_id is None there),
|
||||
# which was otherwise overwritten and lost.
|
||||
if current_content:
|
||||
block_content.append(current_content)
|
||||
block_content.append(cls._fold_block(current_content, current_is_pre))
|
||||
current_content = content
|
||||
current_is_pre = tag_name in cls._PRE_TAGS
|
||||
last_block_id = block_id
|
||||
else:
|
||||
current_content += (" " if current_content else "") + content
|
||||
sep = cls._HARD_BREAK * newline_before
|
||||
current_content += sep + content
|
||||
else:
|
||||
if tag_name == "table":
|
||||
table_info_list.append(item)
|
||||
else:
|
||||
current_content += (" " if current_content else "") + content
|
||||
sep = cls._HARD_BREAK * newline_before
|
||||
current_content += sep + content
|
||||
if current_content:
|
||||
block_content.append(current_content)
|
||||
block_content.append(cls._fold_block(current_content, current_is_pre))
|
||||
return block_content, table_info_list
|
||||
|
||||
# Characters from scripts written without spaces between words (CJK, kana,
|
||||
|
||||
@@ -110,7 +110,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
|
||||
for child := root.FirstChild; child != nil; child = child.NextSibling {
|
||||
if child.Type == html.TextNode {
|
||||
if emitsLooseHTMLText(root) {
|
||||
appendHTMLTextItem(out, child.Data, "text")
|
||||
appendHTMLTextItem(out, child.Data, "text", true)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -131,7 +131,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
|
||||
continue
|
||||
}
|
||||
text := htmlLeafText(child)
|
||||
appendHTMLTextItem(out, text, htmlTagToCkType(tag))
|
||||
appendHTMLTextItem(out, text, htmlTagToCkType(tag), tag != "pre" && tag != "textarea")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,8 +139,10 @@ func emitsLooseHTMLText(root *html.Node) bool {
|
||||
return root.Type == html.ElementNode && root.Data == "body"
|
||||
}
|
||||
|
||||
func appendHTMLTextItem(out *[]map[string]any, text, ckType string) {
|
||||
text = strings.TrimSpace(text)
|
||||
func appendHTMLTextItem(out *[]map[string]any, text, ckType string, trim bool) {
|
||||
if trim {
|
||||
text = strings.TrimSpace(text)
|
||||
}
|
||||
if text == "" {
|
||||
return
|
||||
}
|
||||
@@ -174,38 +176,116 @@ func htmlTagToCkType(tag string) string {
|
||||
return "text"
|
||||
}
|
||||
|
||||
// leafWriter accumulates the visible text of an HTML subtree while applying
|
||||
// CSS whitespace folding (the default white-space: normal rules):
|
||||
// - collapsible whitespace runs collapse to a single space;
|
||||
// - leading/trailing whitespace of a line is dropped;
|
||||
// - a <br> forces a hard line break (and resets the leading-whitespace state);
|
||||
// - <pre>/<textarea> are emitted verbatim (no folding, no injected breaks).
|
||||
type leafWriter struct {
|
||||
b *bytes.Buffer
|
||||
lastSpace bool // last written rune was a collapsed single space
|
||||
lineStart bool // at the start of a line, so leading whitespace is dropped
|
||||
endsNL bool // builder currently ends with a hard line break
|
||||
pre bool // inside <pre>/<textarea>: emit verbatim
|
||||
}
|
||||
|
||||
func isCollapsibleWS(r rune) bool {
|
||||
return r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == '\f'
|
||||
}
|
||||
|
||||
// writeText appends s, folding collapsible whitespace unless in pre mode.
|
||||
func (w *leafWriter) writeText(s string) {
|
||||
if w.pre {
|
||||
for _, r := range s {
|
||||
w.b.WriteRune(r)
|
||||
w.endsNL = r == '\n'
|
||||
}
|
||||
w.lastSpace = false
|
||||
w.lineStart = false
|
||||
return
|
||||
}
|
||||
for _, r := range s {
|
||||
if isCollapsibleWS(r) {
|
||||
if w.lineStart || w.lastSpace {
|
||||
continue
|
||||
}
|
||||
w.b.WriteRune(' ')
|
||||
w.lastSpace = true
|
||||
w.lineStart = false
|
||||
w.endsNL = false
|
||||
continue
|
||||
}
|
||||
w.b.WriteRune(r)
|
||||
w.lastSpace = false
|
||||
w.lineStart = false
|
||||
w.endsNL = false
|
||||
}
|
||||
}
|
||||
|
||||
// hardBreak inserts a forced line break (a <br> or block boundary). Per CSS,
|
||||
// whitespace immediately before a break is dropped (so "Hello <br>" yields
|
||||
// "Hello\n", not "Hello \n"). Inside <pre>/<textarea> whitespace is preserved,
|
||||
// so the preceding space is kept.
|
||||
func (w *leafWriter) hardBreak() {
|
||||
if !w.pre && w.lastSpace && w.b.Len() > 0 {
|
||||
w.b.Truncate(w.b.Len() - 1)
|
||||
}
|
||||
w.b.WriteByte('\n')
|
||||
w.lastSpace = false
|
||||
w.lineStart = true
|
||||
w.endsNL = true
|
||||
}
|
||||
|
||||
// htmlLeafText joins the visible text of an HTML node and its
|
||||
// descendants. <script>/<style>/<noscript> subtrees are skipped.
|
||||
// The output preserves whitespace runs so headings like
|
||||
// "<h1>Hello world</h1>" round-trip with their spacing intact.
|
||||
// descendants. <script>/<style>/<noscript> subtrees are skipped. Whitespace
|
||||
// is folded per CSS rules (so "<h1>Hello world</h1>" becomes "Hello world"
|
||||
// and "<br>" survives as a real line break), while <pre>/<textarea> keep
|
||||
// their source formatting verbatim.
|
||||
func htmlLeafText(n *html.Node) string {
|
||||
var b strings.Builder
|
||||
walkHTMLLeaf(n, &b)
|
||||
var b bytes.Buffer
|
||||
w := &leafWriter{b: &b}
|
||||
walkHTMLLeaf(n, w)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func walkHTMLLeaf(n *html.Node, b *strings.Builder) {
|
||||
func walkHTMLLeaf(n *html.Node, w *leafWriter) {
|
||||
switch n.Type {
|
||||
case html.TextNode:
|
||||
b.WriteString(n.Data)
|
||||
w.writeText(n.Data)
|
||||
case html.ElementNode:
|
||||
if n.Data == "script" || n.Data == "style" || n.Data == "noscript" {
|
||||
return
|
||||
}
|
||||
// Add a line break between block children so headings,
|
||||
// paragraphs, and list items don't run together.
|
||||
switch n.Data {
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
|
||||
"tr", "blockquote":
|
||||
if b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
|
||||
b.WriteString("\n")
|
||||
if n.Data == "br" {
|
||||
w.hardBreak()
|
||||
return
|
||||
}
|
||||
if n.Data == "pre" || n.Data == "textarea" {
|
||||
// Verbatim: no folding, no injected block breaks.
|
||||
w.pre = true
|
||||
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
||||
walkHTMLLeaf(child, w)
|
||||
}
|
||||
w.pre = false
|
||||
return
|
||||
}
|
||||
// Add a line break between block children so headings, paragraphs,
|
||||
// and list items don't run together.
|
||||
if !w.pre {
|
||||
switch n.Data {
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
|
||||
"tr", "blockquote":
|
||||
if w.b.Len() > 0 && !w.endsNL {
|
||||
w.hardBreak()
|
||||
}
|
||||
}
|
||||
}
|
||||
for child := n.FirstChild; child != nil; child = child.NextSibling {
|
||||
walkHTMLLeaf(child, b)
|
||||
walkHTMLLeaf(child, w)
|
||||
}
|
||||
if isBlockTag(n.Data) && b.Len() > 0 && !strings.HasSuffix(b.String(), "\n") {
|
||||
b.WriteString("\n")
|
||||
if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
|
||||
w.hardBreak()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
114
internal/parser/parser/html_parser_unified_test.go
Normal file
114
internal/parser/parser/html_parser_unified_test.go
Normal file
@@ -0,0 +1,114 @@
|
||||
// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// unifiedHTMLCasesJSON is the single source of truth for the browser-faithful
|
||||
// HTML parsing semantics that BOTH the Go and Python parsers must converge on.
|
||||
// It is embedded from testdata/unified_html_cases.json, which the Python mirror
|
||||
// (test/unit_test/deepdoc/parser/test_html_parser.py) also loads — so the two
|
||||
// engines share one fixture and can no longer drift.
|
||||
//
|
||||
// Each case wraps its content in a single block element so that
|
||||
// ParseWithResult emits exactly one item and the Python merge_block_text emits
|
||||
// exactly one block string, enabling a 1:1 byte comparison between engines.
|
||||
//
|
||||
//go:embed testdata/unified_html_cases.json
|
||||
var unifiedHTMLCasesJSON []byte
|
||||
|
||||
type unifiedHTMLCase struct {
|
||||
Name string `json:"name"`
|
||||
HTML string `json:"html"`
|
||||
Want string `json:"want"`
|
||||
}
|
||||
|
||||
func loadUnifiedHTMLCases(t *testing.T) []unifiedHTMLCase {
|
||||
t.Helper()
|
||||
var cases []unifiedHTMLCase
|
||||
if err := json.Unmarshal(unifiedHTMLCasesJSON, &cases); err != nil {
|
||||
t.Fatalf("unmarshal unified html cases: %v", err)
|
||||
}
|
||||
return cases
|
||||
}
|
||||
|
||||
// TestHTMLParser_ParseWithResult_UnifiedSemantics asserts the browser-faithful
|
||||
// semantics on the Go engine. The cases are loaded from the shared embedded
|
||||
// fixture, so this test and its Python mirror stay in lockstep.
|
||||
func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {
|
||||
for _, tc := range loadUnifiedHTMLCases(t) {
|
||||
t.Run(tc.Name, func(t *testing.T) {
|
||||
p := NewHTMLParser()
|
||||
res := p.ParseWithResult(t.Context(), "doc.html", []byte(tc.HTML))
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
if len(res.JSON) != 1 {
|
||||
t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)
|
||||
}
|
||||
if got := res.JSON[0]["text"].(string); got != tc.Want {
|
||||
t.Errorf("got %q, want %q", got, tc.Want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHTMLParser_ParseWithResult_RealisticSmoke exercises the Go HTML walker on
|
||||
// a realistic multi-block document: a heading, a paragraph with an inline
|
||||
// <b> and a <br> line break, a CJK paragraph with an inline element, and a
|
||||
// verbatim <pre> block. It guards the leafWriter CSS-folding rewrite against
|
||||
// hidden regressions specific to the Go reimplementation:
|
||||
// - <br> becomes a hard line break;
|
||||
// - inline boundaries join verbatim, with NO inserted space even for CJK;
|
||||
// - block-internal whitespace collapses to a single space and is trimmed;
|
||||
// - <pre> keeps its source whitespace verbatim (leading/trailing included).
|
||||
func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {
|
||||
const html = `<h1>产品说明 Product Guide</h1>
|
||||
<p>第一步：打开应用<br>第二步：点击<b>设置</b>按钮完成配置。</p>
|
||||
<p>欢迎使用我们的<b>智能助手</b>，它能帮您快速处理任务。</p>
|
||||
<pre> code
|
||||
block</pre>`
|
||||
p := NewHTMLParser()
|
||||
res := p.ParseWithResult(t.Context(), "doc.html", []byte(html))
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
var texts []string
|
||||
for _, item := range res.JSON {
|
||||
texts = append(texts, item["text"].(string))
|
||||
}
|
||||
want := []string{
|
||||
// Heading: whitespace folded, no injected break.
|
||||
"产品说明 Product Guide",
|
||||
// <br> => hard break; inline <b> joined verbatim (无空格).
|
||||
"第一步：打开应用\n第二步：点击设置按钮完成配置。",
|
||||
// CJK inline joined verbatim: 我们的 + 智能助手, no space.
|
||||
"欢迎使用我们的智能助手，它能帮您快速处理任务。",
|
||||
// <pre> preserved verbatim, leading/trailing whitespace intact.
|
||||
" code\n block",
|
||||
}
|
||||
if len(texts) != len(want) {
|
||||
t.Fatalf("block count = %d, want %d; got %#v", len(texts), len(want), texts)
|
||||
}
|
||||
for i := range want {
|
||||
if texts[i] != want[i] {
|
||||
t.Errorf("block %d: got %q, want %q", i, texts[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
12
internal/parser/parser/testdata/unified_html_cases.json
vendored
Normal file
12
internal/parser/parser/testdata/unified_html_cases.json
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{"name": "br_basic", "html": "<p>line1<br>line2</p>", "want": "line1\nline2"},
|
||||
{"name": "br_surrounding_space", "html": "<p>Hello <br> World</p>", "want": "Hello\nWorld"},
|
||||
{"name": "br_double", "html": "<p>A<br><br>B</p>", "want": "A\n\nB"},
|
||||
{"name": "br_before_inline", "html": "<p>Line1<br><span>Line2</span></p>", "want": "Line1\nLine2"},
|
||||
{"name": "inline_no_space_latin", "html": "<p>Hello<b>World</b></p>", "want": "HelloWorld"},
|
||||
{"name": "inline_no_space_cjk", "html": "<p>你好<b>世界</b></p>", "want": "你好世界"},
|
||||
{"name": "inline_with_space", "html": "<p>Hello <b>World</b></p>", "want": "Hello World"},
|
||||
{"name": "inline_three", "html": "<p>First<b>Second</b>Third</p>", "want": "FirstSecondThird"},
|
||||
{"name": "whitespace_collapse", "html": "<p>\n Hello\n <b>World</b>\n</p>", "want": "Hello World"},
|
||||
{"name": "pre_preserved", "html": "<pre> code\n block</pre>", "want": " code\n block"}
|
||||
]
|
||||
@@ -23,10 +23,15 @@ scripts that have no whitespace word boundaries (e.g. Chinese).
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Load html_parser by file path so we don't trigger deepdoc/parser/__init__.py
|
||||
# (which pulls in heavy parsers) or the real rag.nlp tokenizer. The heavy
|
||||
# optional modules are stubbed; rag.nlp is stubbed so the module imports, and
|
||||
@@ -170,3 +175,59 @@ def test_parser_txt_keeps_loose_text_between_and_after_blocks():
|
||||
assert "loose" in between
|
||||
trailing = "\n".join(RAGFlowHtmlParser.parser_txt("<p>Only.</p>tail", chunk_token_num=512))
|
||||
assert "tail" in trailing
|
||||
|
||||
|
||||
# Unified HTML semantics: the browser-faithful rules that BOTH the Python and
|
||||
# Go parsers must converge on. The cases are the single source of truth shared
|
||||
# with the Go engine, loaded from the JSON fixture below (so the two engines
|
||||
# can no longer drift). They must all currently PASS on both engines.
|
||||
#
|
||||
# The fixture lives in the Go package's testdata because //go:embed requires
|
||||
# the file to sit inside the package directory tree; the Python mirror reads
|
||||
# the same file by absolute repo-root path.
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
_UNIFIED_HTML_FIXTURE = _REPO_ROOT / "internal/parser/parser/testdata/unified_html_cases.json"
|
||||
_UNIFIED_HTML_CASES = [(c["name"], c["html"], c["want"]) for c in json.loads(_UNIFIED_HTML_FIXTURE.read_text())]
|
||||
|
||||
|
||||
def _merge_one_block(html):
|
||||
# Use read_text_recursively + merge_block_text (not parser_txt) so block
|
||||
# boundaries survive as separate list entries and no "#" heading prefix or
|
||||
# chunk-merge join is applied — enabling a 1:1 comparison with the Go
|
||||
# per-block output.
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
temp = []
|
||||
RAGFlowHtmlParser.read_text_recursively(soup, temp)
|
||||
blocks, _ = RAGFlowHtmlParser.merge_block_text(temp)
|
||||
return blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,html,want", _UNIFIED_HTML_CASES)
|
||||
def test_unified_html_semantics_parity(name, html, want):
|
||||
assert _merge_one_block(html) == [want]
|
||||
|
||||
|
||||
def test_merge_block_text_loose_text_newline_join():
|
||||
# The block_id is None branch (loose text, e.g. text directly under
|
||||
# <body>) must join fragments with hard breaks, not silently
|
||||
# concatenate. Regression guard for the inline-verbatim / <br> rewrite in
|
||||
# merge_block_text (this branch is not exercised by the block_id cases
|
||||
# above, which all carry a block_id).
|
||||
# Single hard break between two loose fragments.
|
||||
blocks, _ = RAGFlowHtmlParser.merge_block_text(
|
||||
[
|
||||
{"content": "Loose one", "tag_name": "inner_text", "metadata": {}},
|
||||
{"content": "Loose two", "tag_name": "inner_text", "metadata": {"newline_before": 1}},
|
||||
]
|
||||
)
|
||||
assert blocks == ["Loose one\nLoose two"]
|
||||
|
||||
# Consecutive breaks between loose fragments are preserved (1:1 with the
|
||||
# <br><br> rule for blocks).
|
||||
blocks2, _ = RAGFlowHtmlParser.merge_block_text(
|
||||
[
|
||||
{"content": "A", "tag_name": "inner_text", "metadata": {}},
|
||||
{"content": "B", "tag_name": "inner_text", "metadata": {"newline_before": 2}},
|
||||
]
|
||||
)
|
||||
assert blocks2 == ["A\n\nB"]
|
||||
|
||||
Reference in New Issue
Block a user