From bda703b588682c9c0190e6f8d90abb3fe77465e3 Mon Sep 17 00:00:00 2001
From: SYED ALI ABBAS RAHIL <76390003+RahilOp@users.noreply.github.com>
Date: Thu, 16 Jul 2026 10:33:48 +0900
Subject: [PATCH] test: add regression coverage for metadata filter pagination
 beyond push-down cap (#16932)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

### Summary

#16524 reports that a manual metadata filter matching more documents
than the ES push-down cap (`filter_doc_ids_by_meta_pushdown`'s default
`limit=10000`) drops documents once the request falls back to the
in-memory path — e.g. a `canon Not in ["0"]` filter over a
39,573-document KB where ~38,500 matching documents never come back.

I traced through the current code path for this exact scenario:
- `_filter_doc_ids_by_metadata_es` correctly detects when the match
total exceeds the push-down cap and bails to the in-memory fallback
instead of returning a truncated slice.
- `get_flatted_meta_by_kbs` (fixed by #16095) now fully paginates
through every document in the KB rather than stopping after the first
page.
- `es_conn.py`'s `search()` already switches to `search_after`-based
pagination once `offset + limit` would exceed ES's `max_result_window`
(10,000), so the outer pagination loop doesn't get cut off by that
ceiling either.
- `meta_filter()` then aggregates over the complete flattened metadata
with no additional cap.

I couldn't reproduce the drop against current `main` following that
path. This PR adds a test that simulates the exact reported scenario
(12,000 synthetic documents, `canon Not in ["0"]` matching all but 30 of
them) against a fake, paginated `docStoreConn` standing in for
Elasticsearch — both assertions pass on current `main`.

To make sure this is a meaningful regression test and not a false
positive, I temporarily reverted `get_flatted_meta_by_kbs` to stop after
the first page (the pre-#16095 behavior) and confirmed the test
correctly fails (970 of the expected 11,970 documents), then restored
the original code before committing.

Given all of that, it looks like #16524 may already be fixed by the
combination of #16095 and the existing `search_after` handling in
`es_conn.py`, but I could be missing something about the reporter's
specific deployment or a scenario I haven't considered (e.g. a
downstream cap once matched doc_ids feed into the content-chunk
retrieval query). I've left a comment on the issue with this same
analysis so a maintainer familiar with the history here can confirm or
point me at what I'm missing. Either way, this test is a useful
regression guard for the pagination behavior going forward.
---
 .../test_doc_metadata_service_pagination.py   | 93 +++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 test/unit_test/api/db/services/test_doc_metadata_service_pagination.py

diff --git a/test/unit_test/api/db/services/test_doc_metadata_service_pagination.py b/test/unit_test/api/db/services/test_doc_metadata_service_pagination.py
new file mode 100644
index 0000000000..014a2d03e5
--- /dev/null
+++ b/test/unit_test/api/db/services/test_doc_metadata_service_pagination.py
@@ -0,0 +1,93 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+"""Regression test for #16524: a manual metadata filter over a knowledge base
+with more documents than the ES push-down cap (``filter_doc_ids_by_meta_pushdown``'s
+default ``limit=10000``) must still see every document once the request falls
+back to the in-memory path, not just the first page.
+
+Exercises ``DocMetadataService.get_flatted_meta_by_kbs`` end-to-end against a
+fake, paginated ``docStoreConn`` standing in for Elasticsearch, then feeds the
+result into ``meta_filter`` with the same ``not in`` condition from the
+original report.
+"""
+
+from types import SimpleNamespace
+
+import pytest
+
+from common import settings
+from common.metadata_utils import meta_filter
+from api.db.services.doc_metadata_service import DocMetadataService
+from api.db.db_models import DB
+
+pytestmark = pytest.mark.p2
+
+TOTAL_DOCS = 12000
+CANON_ZERO_COUNT = 30  # a small minority tagged "0"; the rest are "1"
+
+
+class _FakeDocStoreConn:
+    """Stands in for the ES connection's paginated ``search``.
+
+    Mirrors the shape ``DocMetadataService._iter_search_results`` expects
+    (``{"hits": {"hits": [{"_id": ..., "_source": {...}}]}}``) and actually
+    honors ``offset``/``limit`` so a caller that stops paginating too early
+    provably sees a truncated result, the way the reported bug did.
+    """
+
+    def __init__(self, total: int, canon_zero_count: int):
+        self._docs = []
+        for i in range(total):
+            canon = "0" if i < canon_zero_count else "1"
+            self._docs.append({"_id": f"doc-{i}", "_source": {"meta_fields": {"canon": canon}}})
+
+    def index_exist(self, index_name, kb_id):
+        return True
+
+    def search(self, select_fields, highlight_fields, condition, match_expressions, order_by, offset, limit, index_names, knowledgebase_ids, agg_fields=None, rank_feature=None):
+        page = self._docs[offset : offset + limit]
+        return {"hits": {"hits": page}}
+
+
+def test_get_flatted_meta_by_kbs_returns_every_document_beyond_pushdown_cap(monkeypatch):
+    monkeypatch.setattr(DB, "connect", lambda *args, **kwargs: None)
+    monkeypatch.setattr(DB, "close", lambda *args, **kwargs: None)
+    monkeypatch.setattr(settings, "docStoreConn", _FakeDocStoreConn(TOTAL_DOCS, CANON_ZERO_COUNT))
+    monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+    fake_kb = SimpleNamespace(tenant_id="tenant-1")
+    monkeypatch.setattr("api.db.services.doc_metadata_service.Knowledgebase.get_by_id", lambda kb_id: fake_kb)
+
+    metas = DocMetadataService.get_flatted_meta_by_kbs(["kb-1"])
+
+    assert len(metas["canon"]["1"]) == TOTAL_DOCS - CANON_ZERO_COUNT
+    assert len(metas["canon"]["0"]) == CANON_ZERO_COUNT
+
+
+def test_manual_not_in_filter_matches_every_document_beyond_pushdown_cap(monkeypatch):
+    # Same scenario as the #16524 report: a "canon Not in ['0']" manual filter
+    # over a KB whose match set (TOTAL_DOCS - CANON_ZERO_COUNT) exceeds the
+    # push-down cap, so this exercises the in-memory fallback exclusively.
+    monkeypatch.setattr(DB, "connect", lambda *args, **kwargs: None)
+    monkeypatch.setattr(DB, "close", lambda *args, **kwargs: None)
+    monkeypatch.setattr(settings, "docStoreConn", _FakeDocStoreConn(TOTAL_DOCS, CANON_ZERO_COUNT))
+    monkeypatch.setattr(settings, "DOC_ENGINE_INFINITY", False)
+    fake_kb = SimpleNamespace(tenant_id="tenant-1")
+    monkeypatch.setattr("api.db.services.doc_metadata_service.Knowledgebase.get_by_id", lambda kb_id: fake_kb)
+
+    metas = DocMetadataService.get_flatted_meta_by_kbs(["kb-1"])
+    doc_ids = meta_filter(metas, [{"key": "canon", "op": "not in", "value": ["0"]}])
+
+    assert len(doc_ids) == TOTAL_DOCS - CANON_ZERO_COUNT