From bb3b99f0a5b5501b50e1e2d0325f51f92e0df035 Mon Sep 17 00:00:00 2001
From: Magicbook1108 <newyorkupperbay@gmail.com>
Date: Thu, 30 Apr 2026 12:30:41 +0800
Subject: [PATCH] Feat: add button for remove header & footer in pipeline
 (#14486)

### What problem does this PR solve?

Feat: add button for remove header & footer in pipeline

### Type of change


- [x] New Feature (non-breaking change which adds functionality)
---
 deepdoc/parser/html_parser.py                 |  3 +-
 rag/flow/parser/parser.py                     | 99 +++++--------------
 rag/flow/parser/pdf_chunk_metadata.py         |  5 +-
 rag/flow/parser/utils.py                      | 43 ++++++++
 web/src/locales/en.ts                         |  1 +
 web/src/locales/zh.ts                         |  1 +
 web/src/pages/agent/constant/pipeline.tsx     |  5 +
 .../form/parser-form/common-form-fields.tsx   | 22 +++++
 .../pages/agent/form/parser-form/index.tsx    |  3 +
 .../form/parser-form/pdf-form-fields.tsx      |  2 +
 .../parser-form/text-html-form-fields.tsx     |  8 +-
 .../form/parser-form/word-form-fields.tsx     |  2 +
 web/src/pages/agent/utils.ts                  | 23 +++++
 13 files changed, 135 insertions(+), 82 deletions(-)

diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py
index f4d360c641..7462ad99e9 100644
--- a/deepdoc/parser/html_parser.py
+++ b/deepdoc/parser/html_parser.py
@@ -52,7 +52,7 @@ class RAGFlowHtmlParser:
             raise TypeError("txt type should be string!")
 
         temp_sections = []
-        soup = BeautifulSoup(txt, "html5lib")
+        soup = BeautifulSoup(txt, "html.parser")
         # delete <style> tag
         for style_tag in soup.find_all(["style", "script"]):
             style_tag.decompose()
@@ -210,4 +210,3 @@ class RAGFlowHtmlParser:
             chunks.append(current_block)
 
         return chunks
-
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index 069ac9b826..e4f66a554f 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -46,6 +46,9 @@ from rag.flow.parser.schema import ParserFromUpstream
 from rag.flow.parser.utils import (
     enhance_media_sections_with_vision,
     extract_word_outlines,
+    extract_docx_header_footer_texts,
+    remove_header_footer_docx_sections,
+    remove_header_footer_html_blob,
     remove_toc,
     remove_toc_pdf,
     remove_toc_word,
@@ -113,6 +116,7 @@ class ParserParam(ProcessParamBase):
                 "lang": "Chinese",
                 "flatten_media_to_text": False,
                 "remove_toc": False,
+                "remove_header_footer": False,
                 "suffix": [
                     "pdf",
                 ],
@@ -130,6 +134,7 @@ class ParserParam(ProcessParamBase):
             },
             "doc": {
                 "remove_toc": False,
+                "remove_header_footer": False,
                 "suffix": [
                     "doc",
                 ],
@@ -138,6 +143,7 @@ class ParserParam(ProcessParamBase):
             "docx": {
                 "flatten_media_to_text": False,
                 "remove_toc": False,
+                "remove_header_footer": False,
                 "suffix": [
                     "docx",
                 ],
@@ -170,7 +176,8 @@ class ParserParam(ProcessParamBase):
             },
             "html": {
                 "suffix": ["htm", "html"],
-                "remove_toc": "false",
+                "remove_toc": False,
+                "remove_header_footer": False,
                 "output_format": "json",
             },
             "slides": {
@@ -321,10 +328,6 @@ class Parser(ProcessBase):
         flatten_media_to_text = conf.get("flatten_media_to_text")
         pdf_parser = None
 
-        # Optional PDF post-processing flags applied after parsing.
-        abstract_enabled = "abstract" in conf.get("preprocess", [])
-        author_enabled = "author" in conf.get("preprocess", [])
-
         # Normalize parser selection and optional provider-specific model name.
         raw_parse_method = conf.get("parse_method", "")
         parser_model_name = None
@@ -587,7 +590,6 @@ class Parser(ProcessBase):
                 if image is not None:
                     box["image"] = image
                 bboxes.append(box)
-
         # Vision parser treats each page as a large image block.
         else:
             if conf.get("parse_method"):
@@ -634,19 +636,15 @@ class Parser(ProcessBase):
                 toc_bboxes, _ = remove_toc(bboxes[:split_at])
                 bboxes = toc_bboxes + bboxes[split_at:]
 
+        normalize_bboxes = []
         # Normalize shared bbox fields for downstream consumers.
-        layout_counters = {}
         for b in bboxes:
             raw_layout = str(b.get("layout_type") or "").strip()
             has_layout = bool(raw_layout)
             layout = re.sub(r"\s+", " ", raw_layout) if has_layout else "text"
             b["layout_type"] = layout
-
-            if not b.get("layoutno"):
-                seq = layout_counters.get(layout, 0)
-                layout_counters[layout] = seq + 1
-                b["layoutno"] = f"{layout}-{seq}"
-
+            if conf.get("remove_header_footer") and re.search(r"(header|footer|number)", raw_layout, re.I):
+                continue 
             if flatten_media_to_text:
                 b["doc_type_kwd"] = "text"
             elif layout == "table":
@@ -657,67 +655,8 @@ class Parser(ProcessBase):
                 b["doc_type_kwd"] = "image"
             else:
                 b["doc_type_kwd"] = "text"
-
-        # Mark likely author blocks near the title when enabled.
-        if author_enabled:
-            def _begin(txt):
-                if not isinstance(txt, str):
-                    return False
-                return re.match(
-                    r"[0-9. 一、i]*(introduction|abstract|摘要|引言|keywords|key words|关键词|background|背景|目录|前言|contents)",
-                    txt.lower().strip(),
-                )
-
-            i = 0
-            while i < min(32, len(bboxes) - 1):
-                b = bboxes[i]
-                i += 1
-                layout_type = b.get("layout_type", "")
-                layoutno = b.get("layoutno", "")
-                is_title = "title" in str(layout_type).lower() or "title" in str(layoutno).lower()
-                if not is_title:
-                    continue
-
-                title_txt = b.get("text", "")
-                if _begin(title_txt):
-                    break
-
-                for j in range(3):
-                    next_idx = i + j
-                    if next_idx >= len(bboxes):
-                        break
-                    candidate = bboxes[next_idx].get("text", "")
-                    if _begin(candidate):
-                        break
-                    if isinstance(candidate, str) and "@" in candidate:
-                        break
-                    bboxes[next_idx]["author"] = True
-                break
-
-        # Mark the abstract block when enabled.
-        if abstract_enabled:
-            i = 0
-            abstract_idx = None
-            while i + 1 < min(32, len(bboxes)):
-                b = bboxes[i]
-                i += 1
-                txt = b.get("text", "")
-                if not isinstance(txt, str):
-                    continue
-                txt = txt.lower().strip()
-                if re.match(r"(abstract|摘要)", txt):
-                    if len(txt.split()) > 32 or len(txt) > 64:
-                        abstract_idx = i - 1
-                        break
-                    next_txt = bboxes[i].get("text", "") if i < len(bboxes) else ""
-                    if isinstance(next_txt, str):
-                        next_txt = next_txt.lower().strip()
-                        if len(next_txt.split()) > 32 or len(next_txt) > 64:
-                            abstract_idx = i
-                    i += 1
-                    break
-            if abstract_idx is not None:
-                bboxes[abstract_idx]["abstract"] = True
+            normalize_bboxes.append(b)
+        bboxes = normalize_bboxes
 
         enhance_media_sections_with_vision(
             bboxes,
@@ -900,6 +839,9 @@ class Parser(ProcessBase):
         # JSON output keeps text/image blocks and appends table HTML as table items.
         if conf.get("output_format") == "json":
             main_sections = docx_parser(name, binary=blob)
+            if conf.get("remove_header_footer"):
+                header_footer_texts = extract_docx_header_footer_texts(binary=blob)
+                main_sections = remove_header_footer_docx_sections(main_sections, header_footer_texts)
             if conf.get("remove_toc"):
                 main_sections = remove_toc_word(main_sections, outlines)
             sections = []
@@ -931,6 +873,10 @@ class Parser(ProcessBase):
         # Markdown output removes TOC on plain markdown lines before writing back.
         elif conf.get("output_format") == "markdown":
             markdown_text = docx_parser.to_markdown(name, binary=blob)
+            if conf.get("remove_header_footer"):
+                header_footer_texts = extract_docx_header_footer_texts(binary=blob)
+                markdown_lines = remove_header_footer_docx_sections(markdown_text.split("\n"), header_footer_texts)
+                markdown_text = "\n".join(markdown_lines)
             if conf.get("remove_toc"):
                 markdown_text = "\n".join(remove_toc_word(markdown_text.split("\n"), outlines))
 
@@ -1091,8 +1037,11 @@ class Parser(ProcessBase):
         conf = self._param.setups["html"]
         self.set_output("output_format", conf["output_format"])
 
+        if conf.get("remove_header_footer"):
+            blob = remove_header_footer_html_blob(blob)
+
         sections = HtmlParser()(name, blob, int(conf.get("chunk_token_num", 512)))
-        if conf.get("remove_toc") == "true":
+        if conf.get("remove_toc"):
             sections, _ = remove_toc(sections)
         if conf.get("output_format") == "json":
             self.set_output("json", [{"text": section, "doc_type_kwd": "text"} for section in sections if section])
diff --git a/rag/flow/parser/pdf_chunk_metadata.py b/rag/flow/parser/pdf_chunk_metadata.py
index 74921b201b..175ac3772e 100644
--- a/rag/flow/parser/pdf_chunk_metadata.py
+++ b/rag/flow/parser/pdf_chunk_metadata.py
@@ -72,7 +72,6 @@ def extract_pdf_positions(item):
         return []
 
     positions = _extract_raw_positions(item)
-    uses_position_tag = isinstance(item.get("position_tag"), str) and bool(item.get("position_tag"))
     ref_page_number = item.get("page_number")
     ref_page_number = int(ref_page_number) if isinstance(ref_page_number, (int, float)) else None
     if ref_page_number is not None and ref_page_number <= 0:
@@ -86,9 +85,7 @@ def extract_pdf_positions(item):
         page_number = pos[0][-1] if isinstance(pos[0], list) else pos[0]
         try:
             page_number = int(page_number)
-            if uses_position_tag:
-                page_number += 1
-            elif ref_page_number is not None and page_number == ref_page_number - 1:
+            if ref_page_number is not None and page_number == ref_page_number - 1:
                 page_number = ref_page_number
             elif page_number <= 0:
                 page_number += 1
diff --git a/rag/flow/parser/utils.py b/rag/flow/parser/utils.py
index 96bfaff29a..93bc1652a8 100644
--- a/rag/flow/parser/utils.py
+++ b/rag/flow/parser/utils.py
@@ -16,6 +16,7 @@
 import re
 from io import BytesIO
 
+from bs4 import BeautifulSoup
 from docx import Document
 from api.db.services.llm_service import LLMBundle
 from api.db.joint_services.tenant_model_service import (
@@ -34,6 +35,48 @@ def remove_toc(items):
     return [items[i] for i in kept_indices], kept_indices
 
 
+def extract_docx_header_footer_texts(filename=None, binary=None):
+    doc = Document(filename) if binary is None else Document(BytesIO(binary))
+    texts = set()
+    for section in doc.sections:
+        for container in (section.header, section.footer):
+            for paragraph in container.paragraphs:
+                normalized = re.sub(r"\s+", " ", paragraph.text).strip()
+                if normalized:
+                    texts.add(normalized)
+            for table in container.tables:
+                for row in table.rows:
+                    for cell in row.cells:
+                        normalized = re.sub(r"\s+", " ", cell.text).strip()
+                        if normalized:
+                            texts.add(normalized)
+    return texts
+
+
+def remove_header_footer_docx_sections(items, header_footer_texts):
+    if not header_footer_texts:
+        return items
+
+    filtered = []
+    for item in items:
+        text = _item_text(item)
+        normalized = re.sub(r"\s+", " ", text).strip() if isinstance(text, str) else ""
+        if normalized and normalized in header_footer_texts:
+            continue
+        filtered.append(item)
+    return filtered
+
+
+def remove_header_footer_html_blob(blob):
+    soup = BeautifulSoup(blob, "html.parser")
+    for element in soup.find_all(
+        lambda tag: tag.name in {"header", "footer"}
+        or tag.get("role") in {"banner", "contentinfo"}
+    ):
+        element.decompose()
+    return str(soup).encode("utf-8")
+
+
 def extract_word_outlines(filename, binary=None):
     doc = Document(filename) if binary is None else Document(BytesIO(binary))
     outlines = []
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index f8713ac3f5..fa8a4c25f1 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1527,6 +1527,7 @@ Best for: Documents with flowing, contextually connected content — such as boo
       removeToc: 'Remove original table of contents',
       removeTocTip:
         'Remove the table of contents included in the original PDF, so it is not parsed as regular content or chunked for retrieval.',
+      removeHeaderFooter: 'Remove header and footer',
       autoPlay: 'Auto play audio',
       downloadFileTypeTip: 'The file type to download',
       downloadFileType: 'Download file type',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 75f1a4e755..eb2047e355 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1278,6 +1278,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       removeToc: '移除原始目录',
       removeTocTip:
         '移除原始PDF中包含的目录，这样它就不会被解析为常规内容或作为检索块。',
+      removeHeaderFooter: '移除页眉页脚',
       autoPlay: '自动播放',
       downloadFileTypeTip: '文件下载的类型',
       downloadFileType: '文件类型',
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 8271838f18..4da1c2aa1d 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -198,6 +198,7 @@ export const initialParserValues = {
       parse_method: ParseDocumentType.DeepDOC,
       preprocess: PreprocessValue.main_content,
       flatten_media_to_text: false,
+      remove_header_footer: false,
     },
     {
       fileFormat: FileType.Spreadsheet,
@@ -234,17 +235,21 @@ export const initialParserValues = {
       fileFormat: FileType.Html,
       output_format: TextJsonOutputFormat.Json,
       preprocess: PreprocessValue.main_content,
+      remove_header_footer: false,
     },
     {
       fileFormat: FileType.Doc,
       output_format: DocxOutputFormat.Json,
       preprocess: PreprocessValue.main_content,
+      flatten_media_to_text: false,
+      remove_header_footer: false,
     },
     {
       fileFormat: FileType.Docx,
       output_format: DocxOutputFormat.Json,
       preprocess: PreprocessValue.main_content,
       flatten_media_to_text: false,
+      remove_header_footer: false,
     },
     {
       fileFormat: FileType.PowerPoint,
diff --git a/web/src/pages/agent/form/parser-form/common-form-fields.tsx b/web/src/pages/agent/form/parser-form/common-form-fields.tsx
index de4757573b..c857f4733f 100644
--- a/web/src/pages/agent/form/parser-form/common-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/common-form-fields.tsx
@@ -157,6 +157,28 @@ export function RmdirFormField({ prefix }: CommonProps) {
   );
 }
 
+export function RemoveHeaderFooterFormField({ prefix }: CommonProps) {
+  const { t } = useTranslation();
+  return (
+    <RAGFlowFormItem
+      name={buildFieldNameWithPrefix(`remove_header_footer`, prefix)}
+      label={t('flow.removeHeaderFooter')}
+      horizontal={true}
+      labelClassName="w-full"
+      valueClassName="w-8"
+    >
+      {(field) => (
+        <Switch
+          checked={field.value}
+          onCheckedChange={(checked) => {
+            field.onChange?.(checked);
+          }}
+        />
+      )}
+    </RAGFlowFormItem>
+  );
+}
+
 export function LanguageFormField({ prefix }: CommonProps) {
   const { t } = useTranslation();
 
diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsx
index 1aa32a83e9..20488beefa 100644
--- a/web/src/pages/agent/form/parser-form/index.tsx
+++ b/web/src/pages/agent/form/parser-form/index.tsx
@@ -127,6 +127,7 @@ const FileFormatWidgetMap = {
   [FileType.PDF]: PdfFormFields,
   [FileType.Spreadsheet]: SpreadsheetFormFields,
   [FileType.PowerPoint]: PptFormFields,
+  [FileType.Doc]: WordFormFields,
   [FileType.Docx]: WordFormFields,
   [FileType.Video]: VideoFormFields,
   [FileType.Audio]: AudioFormFields,
@@ -160,6 +161,7 @@ export const FormSchema = z.object({
       markdown_image_response_type: z.string().optional(),
       enable_multi_column: z.boolean().optional(),
       remove_toc: z.boolean().optional(),
+      remove_header_footer: z.boolean().optional(),
     }),
   ),
 });
@@ -352,6 +354,7 @@ const ParserForm = ({ node }: INextOperatorForm) => {
       vlm: { llm_id: '' },
       table_result_type: '',
       markdown_image_response_type: '',
+      remove_header_footer: false,
       // preprocess: [],
     });
   }, [append]);
diff --git a/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx b/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
index 94eb516e54..f00930d24a 100644
--- a/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
@@ -15,6 +15,7 @@ import {
   LanguageFormField,
   LargeModelFormField,
   ParserMethodFormField,
+  RemoveHeaderFooterFormField,
   RmdirFormField,
   TwoColumnCheckFormField,
 } from './common-form-fields';
@@ -104,6 +105,7 @@ export function PdfFormFields({ prefix }: CommonProps) {
     <>
       <TwoColumnCheckFormField prefix={prefix} />
       <RmdirFormField prefix={prefix} />
+      <RemoveHeaderFooterFormField prefix={prefix} />
       <ParserMethodFormField prefix={prefix}></ParserMethodFormField>
       <FlattenMediaToTextFormField prefix={prefix} />
       {!flattenMediaToText && (
diff --git a/web/src/pages/agent/form/parser-form/text-html-form-fields.tsx b/web/src/pages/agent/form/parser-form/text-html-form-fields.tsx
index 6ec348e80b..1bd500466f 100644
--- a/web/src/pages/agent/form/parser-form/text-html-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/text-html-form-fields.tsx
@@ -4,6 +4,7 @@ import { useWatch } from 'react-hook-form';
 import {
   FlattenMediaToTextFormField,
   LargeModelFormField,
+  RemoveHeaderFooterFormField,
   RmdirFormField,
 } from './common-form-fields';
 import { CommonProps } from './interface';
@@ -32,5 +33,10 @@ export function TextMarkdownFormFields({ prefix }: CommonProps) {
 }
 
 export function HtmlFormFields({ prefix }: CommonProps) {
-  return <RmdirFormField prefix={prefix} />;
+  return (
+    <>
+      <RmdirFormField prefix={prefix} />
+      <RemoveHeaderFooterFormField prefix={prefix} />
+    </>
+  );
 }
diff --git a/web/src/pages/agent/form/parser-form/word-form-fields.tsx b/web/src/pages/agent/form/parser-form/word-form-fields.tsx
index a2808d7ba0..a6bd900305 100644
--- a/web/src/pages/agent/form/parser-form/word-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/word-form-fields.tsx
@@ -5,6 +5,7 @@ import {
   FlattenMediaToTextFormField,
   LargeModelFormField,
   OutputFormatFormFieldProps,
+  RemoveHeaderFooterFormField,
   RmdirFormField,
 } from './common-form-fields';
 import { buildFieldNameWithPrefix } from './utils';
@@ -20,6 +21,7 @@ export function WordFormFields({ prefix }: OutputFormatFormFieldProps) {
   return (
     <>
       <RmdirFormField prefix={prefix} />
+      <RemoveHeaderFooterFormField prefix={prefix} />
       <FlattenMediaToTextFormField prefix={prefix} />
       {!flattenMediaToText && (
         <LargeModelFormField
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index d77948d93a..15463f4ff4 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -231,6 +231,7 @@ function transformParserParams(params: ParserFormSchemaType) {
             flatten_media_to_text: cur.flatten_media_to_text,
             enable_multi_column: cur.enable_multi_column,
             remove_toc: cur.remove_toc,
+            remove_header_footer: cur.remove_header_footer || false,
           };
           // Only include TCADP parameters if TCADP Parser is selected
           if (cur.parse_method?.toLowerCase() === 'tcadp parser') {
@@ -279,7 +280,29 @@ function transformParserParams(params: ParserFormSchemaType) {
             fields: cur.fields,
           };
           break;
+        case FileType.Doc:
+          filteredSetup = {
+            ...filteredSetup,
+            vlm: { llm_id: cur.vlm?.llm_id },
+            flatten_media_to_text: cur.flatten_media_to_text,
+            remove_header_footer: cur.remove_header_footer || false,
+          };
+          break;
         case FileType.Docx:
+          filteredSetup = {
+            ...filteredSetup,
+            vlm: { llm_id: cur.vlm?.llm_id },
+            flatten_media_to_text: cur.flatten_media_to_text,
+            remove_header_footer: cur.remove_header_footer || false,
+          };
+          break;
+        case FileType.Html:
+          filteredSetup = {
+            ...filteredSetup,
+            remove_toc: cur.remove_toc,
+            remove_header_footer: cur.remove_header_footer || false,
+          };
+          break;
         case FileType.TextMarkdown:
           filteredSetup = {
             ...filteredSetup,