feat(raptor): add Psi tree builder with original-space ranking and safe migration (#14679)

### What problem does this PR solve?

Closes #14674.

This PR improves RAPTOR configuration and tree construction while
preserving the existing RAPTOR behavior as the default.

RAPTOR currently builds summary layers with the original UMAP + GMM
clustering path. This PR keeps that default path, and adds:

- A hidden backend tree-builder option:
  - `tree_builder="raptor"`: default, existing RAPTOR behavior.
- `tree_builder="psi"`: rank-aware Psi-style tree builder using original
embedding-space cosine ranking.
- A user-facing clustering method option for the default RAPTOR builder:
  - `clustering_method="gmm"`: existing default.
- `clustering_method="ahc"`: agglomerative hierarchical clustering path.
- A RAPTOR UI setting for `Clustering method` and `Max cluster`.

### What changed

#### Backend

- Added `tree_builder` support for RAPTOR/Psi.
- Added `clustering_method` support for GMM/AHC.
- Kept existing RAPTOR + GMM as the default.
- Added Psi tree building from original-space cosine similarity.
- Added bucketed Psi building controls for large inputs:
  - `raptor.ext.psi_exact_max_leaves`
  - `raptor.ext.psi_bucket_size`
- Added method-aware RAPTOR summary metadata using existing
`extra.raptor_method`.
- Avoided adding a dedicated DB schema field for experimental method
tracking.
- Added cleanup/migration logic to avoid mixing stale RAPTOR summary
trees.
- Added defensive checks for Psi tree construction and summary failures.

#### Frontend/UI

- Added `Clustering method` in RAPTOR settings with `GMM` and `AHC`.
- Added/kept `Max cluster` in RAPTOR settings.
- Enlarged max cluster UI limit to `1024`, matching backend validation.
- Kept AHC editable even when a RAPTOR task has already finished.
- Fixed the UI save payload so `clustering_method` and `tree_builder`
are serialized through `parser_config.raptor.ext`, avoiding backend
validation errors for extra top-level RAPTOR fields.

Example saved RAPTOR config:

```json
{
  "raptor": {
    "max_cluster": 317,
    "ext": {
      "clustering_method": "ahc",
      "tree_builder": "raptor"
    }
  }
}

Co-authored-by: CaptainTimon <CaptainTimon@users.noreply.github.com>

rag/utils/ob_conn.py

@@ -46,6 +46,8 @@ column_order_id = Column("_order_id", Integer, nullable=True, comment="chunk ord
 column_group_id = Column("group_id", String(256), nullable=True, comment="group id for external retrieval")
 column_mom_id = Column("mom_id", String(256), nullable=True, comment="parent chunk id")
 column_chunk_data = Column("chunk_data", JSON, nullable=True, comment="table parser row data")
+column_raptor_kwd = Column("raptor_kwd", String(256), nullable=True, comment="RAPTOR summary marker")
+column_raptor_layer_int = Column("raptor_layer_int", Integer, nullable=True, comment="RAPTOR summary layer")
 
 column_definitions: list[Column] = [
     Column("id", String(256), primary_key=True, comment="chunk id"),
@@ -86,6 +88,8 @@ column_definitions: list[Column] = [
     Column("rank_flt", Double, nullable=True, comment="rank of this entity"),
     Column("removed_kwd", String(256), nullable=True, index=True, server_default="'N'",
            comment="whether it has been deleted"),
+    column_raptor_kwd,
+    column_raptor_layer_int,
     column_chunk_data,
     Column("metadata", JSON, nullable=True, comment="metadata for this chunk"),
     Column("extra", JSON, nullable=True, comment="extra information of non-general chunk"),
@@ -127,7 +131,14 @@ FTS_COLUMNS_TKS: list[str] = [
 ]
 
 # Extra columns to add after table creation (for migration)
-EXTRA_COLUMNS: list[Column] = [column_order_id, column_group_id, column_mom_id, column_chunk_data]
+EXTRA_COLUMNS: list[Column] = [
+    column_order_id,
+    column_group_id,
+    column_mom_id,
+    column_chunk_data,
+    column_raptor_kwd,
+    column_raptor_layer_int,
+]
 
 
 class SearchResult(BaseModel):

rag/utils/raptor_utils.py

@@ -18,15 +18,111 @@
 Utility functions for Raptor processing decisions.
 """
 
+import json
 import logging
 from typing import Optional
 
+import xxhash
+
+RAPTOR_TREE_BUILDER = "raptor"
+PSI_TREE_BUILDER = "psi"
+SUPPORTED_TREE_BUILDERS = {RAPTOR_TREE_BUILDER, PSI_TREE_BUILDER}
+GMM_CLUSTERING_METHOD = "gmm"
+AHC_CLUSTERING_METHOD = "ahc"
+SUPPORTED_CLUSTERING_METHODS = {GMM_CLUSTERING_METHOD, AHC_CLUSTERING_METHOD}
+
 # File extensions for structured data types
 EXCEL_EXTENSIONS = {".xls", ".xlsx", ".xlsm", ".xlsb"}
 CSV_EXTENSIONS = {".csv", ".tsv"}
 STRUCTURED_EXTENSIONS = EXCEL_EXTENSIONS | CSV_EXTENSIONS
 
 
+def get_raptor_tree_builder(raptor_config: dict | None) -> str:
+    """Return the configured RAPTOR tree builder with legacy ext fallback."""
+    raptor_config = raptor_config or {}
+    ext = raptor_config.get("ext") or {}
+    tree_builder = ext.get("tree_builder") or raptor_config.get("tree_builder") or RAPTOR_TREE_BUILDER
+    if tree_builder not in SUPPORTED_TREE_BUILDERS:
+        raise ValueError(f"Unsupported RAPTOR tree builder: {tree_builder}")
+    return tree_builder
+
+
+def get_raptor_clustering_method(raptor_config: dict | None) -> str:
+    """Return the configured RAPTOR clustering method with legacy ext fallback."""
+    raptor_config = raptor_config or {}
+    ext = raptor_config.get("ext") or {}
+    clustering_method = ext.get("clustering_method") or raptor_config.get("clustering_method") or GMM_CLUSTERING_METHOD
+    if clustering_method not in SUPPORTED_CLUSTERING_METHODS:
+        raise ValueError(f"Unsupported RAPTOR clustering method: {clustering_method}")
+    return clustering_method
+
+
+def _as_extra_dict(extra) -> dict:
+    """Normalize a chunk extra payload into a dictionary."""
+    if isinstance(extra, dict):
+        return extra
+    if isinstance(extra, str) and extra:
+        try:
+            parsed = json.loads(extra)
+        except json.JSONDecodeError:
+            logging.warning(
+                "Ignoring malformed RAPTOR extra payload while collecting chunk metadata: %s",
+                extra[:200],
+                exc_info=True,
+            )
+            return {}
+        return parsed if isinstance(parsed, dict) else {}
+    return {}
+
+
+def _has_raptor_marker(marker) -> bool:
+    """Return whether a chunk marker identifies a RAPTOR summary chunk."""
+    if isinstance(marker, list):
+        return any(str(item) == RAPTOR_TREE_BUILDER for item in marker)
+    return str(marker) == RAPTOR_TREE_BUILDER
+
+
+def _raptor_methods_from_fields(fields: dict, extra: dict | None = None) -> set[str]:
+    """Read RAPTOR builder methods from stored chunk fields."""
+    extra = extra if extra is not None else _as_extra_dict(fields.get("extra"))
+    method = extra.get("raptor_method") or RAPTOR_TREE_BUILDER
+    if isinstance(method, list):
+        return {str(item) for item in method if item}
+    return {str(method)} if method else set()
+
+
+def collect_raptor_methods(field_map: dict) -> set[str]:
+    """Collect tree-builder methods from RAPTOR summary chunk fields."""
+    methods = set()
+    for fields in field_map.values():
+        extra = _as_extra_dict(fields.get("extra"))
+        marker = fields.get("raptor_kwd") or extra.get("raptor_kwd")
+        if not _has_raptor_marker(marker):
+            continue
+
+        methods.update(_raptor_methods_from_fields(fields, extra))
+    return methods
+
+
+def collect_raptor_chunk_ids(field_map: dict, exclude_methods: set[str] | None = None) -> set[str]:
+    """Collect RAPTOR summary chunk IDs, optionally excluding some methods."""
+    chunk_ids = set()
+    exclude_methods = exclude_methods or set()
+    for chunk_id, fields in field_map.items():
+        extra = _as_extra_dict(fields.get("extra"))
+        marker = fields.get("raptor_kwd") or extra.get("raptor_kwd")
+        if _has_raptor_marker(marker):
+            if _raptor_methods_from_fields(fields, extra).issubset(exclude_methods):
+                continue
+            chunk_ids.add(chunk_id)
+    return chunk_ids
+
+
+def make_raptor_summary_chunk_id(content: str, doc_id: str) -> str:
+    """Build the stable ID used for generated RAPTOR summary chunks."""
+    return xxhash.xxh64((content + str(doc_id)).encode("utf-8")).hexdigest()
+
+
 def is_structured_file_type(file_type: Optional[str]) -> bool:
     """
     Check if a file type is structured data (Excel, CSV, etc.)