Refine handling of POST /api/v1/datasets/search in GO (#15583)

### What problem does this PR solve?

Refine handling of POST /api/v1/datasets/search in GO

### Type of change

- [x] Refactoring

api/apps/services/dataset_api_service.py

@@ -1058,9 +1058,8 @@ async def search(dataset_id: str, tenant_id: str, req: dict):
                 ranks["chunks"].insert(0, ck)
         except Exception:
             logging.warning("search KG retrieval failed: dataset=%s tenant=%s", dataset_id, tenant_id, exc_info=True)
-    total = ranks.get("total", 0)
     ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], tenant_ids)
-    ranks["total"] = total
+    ranks["total"] = len(ranks["chunks"])
 
     for c in ranks["chunks"]:
         c.pop("vector", None)
@@ -1430,9 +1429,8 @@ async def search_datasets(tenant_id: str, req: dict):
                 ranks["chunks"].insert(0, ck)
         except Exception:
             logging.warning("search_datasets KG retrieval failed: datasets=%s tenant=%s", kb_ids, tenant_id, exc_info=True)
-    total = ranks.get("total", 0)
     ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], tenant_ids)
-    ranks["total"] = total
+    ranks["total"] = len(ranks["chunks"])
 
     for c in ranks["chunks"]:
         c.pop("vector", None)

api/db/services/doc_metadata_service.py

@@ -24,7 +24,7 @@ import json
 import logging
 import re
 from copy import deepcopy
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from api.db.db_models import DB, Document
 from common import settings
@@ -33,6 +33,36 @@ from api.db.db_models import Knowledgebase
 from common.doc_store.doc_store_base import OrderByExpr
 
 
+def _es_response_total(response: Any) -> Optional[int]:
+    """Extract the exact total hit count from an ES search response.
+
+    Returns ``None`` when the field is missing or in an unexpected shape
+    — callers should treat that as "cannot verify" rather than "no
+    overflow".
+    """
+    if not isinstance(response, dict):
+        try:
+            response = dict(response)
+        except Exception:
+            return None
+    hits = response.get("hits")
+    if not isinstance(hits, dict):
+        return None
+    total = hits.get("total")
+    if isinstance(total, dict):
+        value = total.get("value")
+        if isinstance(value, bool):
+            return None
+        if isinstance(value, int):
+            return value
+        if isinstance(value, float):
+            return int(value)
+    elif isinstance(total, int):
+        # Legacy shape: some clients return the count directly.
+        return total
+    return None
+
+
 class DocMetadataService:
     """Service for managing document metadata in ES/Infinity"""
 
@@ -876,6 +906,10 @@ class DocMetadataService:
             **query_body,
             "size": limit,
             "_source": ["id"],
+            # Make hits.total.value exact. ES otherwise caps the tracked
+            # total at 10,000 with relation="gte", which would let
+            # overflow slip through undetected.
+            "track_total_hits": True,
         }
 
         try:
@@ -898,6 +932,22 @@ class DocMetadataService:
                 f"ES metadata filter hit limit {limit} for KBs {kb_ids}"
             )
 
+        # Detect silent truncation: the push-down is a fast path, not
+        # the system of record. When the query matched more than
+        # ``limit`` docs, the slice we built here is necessarily a
+        # strict subset of the truth, and the caller treats any
+        # non-None result as definitive. Bail out and let the caller
+        # fall back to the in-memory ``meta_filter`` (correct, just
+        # slower for very large result sets) instead of silently
+        # dropping docs.
+        total = _es_response_total(response)
+        if total is not None and total > limit:
+            logging.warning(
+                f"ES metadata filter result exceeds push-down cap, falling back to in-memory: "
+                f"total={total}, cap={limit}, kb_ids={kb_ids}"
+            )
+            return None
+
         logging.debug(f"ES metadata filter returned {len(unique)} matches for KBs {kb_ids}")
         return unique