From f1e6b22c4ba5002c352d95c68f412e1f52852ab0 Mon Sep 17 00:00:00 2001 From: Sevenzuo <104006262+Sevenzuo@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:05:49 +0800 Subject: [PATCH] fix: honor minimum_should_match in GaussDB search (#18456) ### Summary GaussDB DocEngine could return no chunks for conversational queries even when relevant content was available. `Dealer.search()` supplies `minimum_should_match` (30%, then 10% on retry), but the GaussDB adapter discarded it and built a single `plainto_tsquery` from every token. This effectively required all conversational filler terms to match. --- common/doc_store/gaussdb_conn_base.py | 78 +++++++++++++++++-- rag/utils/gaussdb_conn.py | 5 ++ test/unit_test/rag/utils/test_gaussdb_conn.py | 4 +- .../rag/utils/test_gaussdb_search_sql.py | 22 ++++++ 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/common/doc_store/gaussdb_conn_base.py b/common/doc_store/gaussdb_conn_base.py index 684248b6d0..95bbe2a90e 100644 --- a/common/doc_store/gaussdb_conn_base.py +++ b/common/doc_store/gaussdb_conn_base.py @@ -1041,6 +1041,7 @@ class GaussDBSearchBuilder: offset: int, limit: int, similarity_threshold: float | None = None, + minimum_should_match: float | str | None = None, topn: int | None = None, highlight_fields: list[str] | None = None, order_by: OrderByExpr | None = None, @@ -1064,6 +1065,7 @@ class GaussDBSearchBuilder: vector_dim=vector_dim, vector_weight=vector_weight, similarity_threshold=similarity_threshold, + minimum_should_match=minimum_should_match, candidate_limit=candidate_limit, offset=effective_offset, limit=page_limit, @@ -1089,6 +1091,7 @@ class GaussDBSearchBuilder: select_fields=select_fields, condition=condition, keywords=query_keywords, + minimum_should_match=minimum_should_match, offset=effective_offset, limit=page_limit, highlight_fields=highlight_fields, @@ -1158,10 +1161,15 @@ class GaussDBSearchBuilder: params.append(value) return " AND ".join(fragments), params - def build_text_score_expr(self, keywords: list[str]) -> tuple[str, list[Any]]: + def build_text_score_expr( + self, + keywords: list[str], + minimum_should_match: float | str | None = None, + ) -> tuple[str, list[Any]]: score_exprs: list[str] = [] params: list[Any] = [] - for config, query_text in self._text_queries(keywords): + text_queries = self._text_queries(keywords) if minimum_should_match is None else self._text_query_terms(keywords) + for config, query_text in text_queries: weighted_score = " + ".join( f"{weight} * COALESCE(ts_rank(to_tsvector('{config}', coalesce({column}, ' ')), plainto_tsquery('{config}', %s)), 0)" for column, weight in self.FTS_WEIGHTS.items() ) @@ -1273,6 +1281,7 @@ class GaussDBSearchBuilder: select_fields: list[str], condition: dict, keywords: list[str], + minimum_should_match: float | str | None, offset: int, limit: int, highlight_fields: list[str] | None, @@ -1280,9 +1289,9 @@ class GaussDBSearchBuilder: ) -> tuple[str, list[Any]]: table_name = self.ddl.qualified_name(table) columns = self.normalize_select_fields(select_fields) - score_expr, score_params = self.build_text_score_expr(keywords) + score_expr, score_params = self.build_text_score_expr(keywords, minimum_should_match) score_expr, pagerank_params = self._score_with_pagerank(score_expr, pagerank_weight) - match_expr, match_params = self._build_text_match_expr(keywords) + match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match) where_sql, where_params = self.build_condition_where(condition) where_parts = [part for part in (where_sql, match_expr) if part] select_exprs = [*self._select_exprs(columns), f"{score_expr} AS _score", "COUNT(*) OVER() AS __total"] @@ -1344,6 +1353,7 @@ class GaussDBSearchBuilder: vector_dim: int, vector_weight: float, similarity_threshold: float | None, + minimum_should_match: float | str | None, candidate_limit: int, offset: int, limit: int, @@ -1356,8 +1366,8 @@ class GaussDBSearchBuilder: dim = self.ddl.validate_vector_dim(vector_dim) vector_col = self.ddl.vector_column_name(dim) valid_col = self.ddl.vector_valid_column_name(dim) - text_score_expr, text_score_params = self.build_text_score_expr(keywords) - match_expr, match_params = self._build_text_match_expr(keywords) + text_score_expr, text_score_params = self.build_text_score_expr(keywords, minimum_should_match) + match_expr, match_params = self._build_text_match_expr(keywords, minimum_should_match) where_sql, where_params = self.build_condition_where(condition) base_where = where_sql or "TRUE" fts_where = " AND ".join([base_where, match_expr]) @@ -1413,7 +1423,26 @@ class GaussDBSearchBuilder: offset, ] - def _build_text_match_expr(self, keywords: list[str]) -> tuple[str, list[Any]]: + def _build_text_match_expr( + self, + keywords: list[str], + minimum_should_match: float | str | None = None, + ) -> tuple[str, list[Any]]: + if minimum_should_match is not None: + text_query_terms = self._text_query_terms(keywords) + predicates = [f"{self.build_fts_vector_expr(config)} @@ plainto_tsquery('{config}', %s)" for config, _query_text in text_query_terms] + params = [query_text for _config, query_text in text_query_terms] + required = self._minimum_should_match_count(minimum_should_match, len(predicates)) + if not predicates: + return "FALSE", [] + if required <= 1: + return "(" + " OR ".join(predicates) + ")", params + if required >= len(predicates): + return "(" + " AND ".join(predicates) + ")", params + any_match = " OR ".join(predicates) + match_count = " + ".join(f"CASE WHEN {predicate} THEN 1 ELSE 0 END" for predicate in predicates) + return f"(({any_match}) AND ({match_count}) >= {required})", [*params, *params] + match_exprs: list[str] = [] params: list[Any] = [] for config, query_text in self._text_queries(keywords): @@ -1528,6 +1557,41 @@ class GaussDBSearchBuilder: queries.append(("ngram", self._text_query_param(ngram_terms))) return queries + def _text_query_terms(self, keywords: list[str]) -> list[tuple[str, str]]: + simple_terms, ngram_terms = self.split_text_query_terms(keywords) + terms: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for config, values in (("simple", simple_terms), ("ngram", ngram_terms)): + for value in values: + key = (config, value.casefold()) + if key in seen: + continue + seen.add(key) + terms.append((config, value)) + return terms + + @staticmethod + def _minimum_should_match_count(minimum_should_match: float | str, term_count: int) -> int: + if term_count <= 0: + return 0 + + value: float | str = minimum_should_match + if isinstance(value, str): + text = value.strip() + try: + if text.endswith("%"): + value = float(text[:-1]) / 100.0 + else: + value = float(text) + except ValueError: + return 1 + + if isinstance(value, float) and 0.0 <= value <= 1.0: + required = int(term_count * value) + else: + required = int(value) + return min(term_count, max(1, required)) + def _vector_param(self, vector: list[float] | tuple[float, ...], vector_dim: int | None) -> str: if vector_dim is None: raise ValueError("vector_dim is required for vector search") diff --git a/rag/utils/gaussdb_conn.py b/rag/utils/gaussdb_conn.py index 0ce2958dec..91f73a0b44 100644 --- a/rag/utils/gaussdb_conn.py +++ b/rag/utils/gaussdb_conn.py @@ -824,6 +824,7 @@ class GaussDBConnection(GaussDBConnectionBase): vector_dim=parsed["vector_dim"], vector_weight=parsed["vector_weight"], similarity_threshold=parsed["similarity_threshold"], + minimum_should_match=parsed["minimum_should_match"], topn=parsed["topn"], offset=offset, limit=limit, @@ -845,6 +846,7 @@ class GaussDBConnection(GaussDBConnectionBase): vector_dim=parsed["vector_dim"], vector_weight=parsed["vector_weight"], similarity_threshold=parsed["similarity_threshold"], + minimum_should_match=parsed["minimum_should_match"], topn=parsed["topn"], offset=0, limit=1, @@ -897,12 +899,14 @@ class GaussDBConnection(GaussDBConnectionBase): topn = None vector_weight = None similarity_threshold = None + minimum_should_match = None pagerank_weight = 10.0 for expr in match_expressions or []: if isinstance(expr, MatchTextExpr): query_text = (expr.extra_options or {}).get("original_query") or expr.matching_text or "" keywords = _tokenize_query_terms(query_text) + minimum_should_match = (expr.extra_options or {}).get("minimum_should_match") topn = expr.topn if topn is None else min(topn, expr.topn) elif isinstance(expr, MatchDenseExpr): if expr.embedding_data_type != "float": @@ -931,6 +935,7 @@ class GaussDBConnection(GaussDBConnectionBase): "vector_dim": vector_dim, "vector_weight": vector_weight, "similarity_threshold": similarity_threshold, + "minimum_should_match": minimum_should_match, "topn": topn, "pagerank_weight": pagerank_weight, } diff --git a/test/unit_test/rag/utils/test_gaussdb_conn.py b/test/unit_test/rag/utils/test_gaussdb_conn.py index 03f885b2c1..dc39a29bf5 100644 --- a/test/unit_test/rag/utils/test_gaussdb_conn.py +++ b/test/unit_test/rag/utils/test_gaussdb_conn.py @@ -2822,7 +2822,7 @@ def test_tc_ret_805_parse_fusion_vector_weight_returns_none_for_missing_or_bad_w def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaults(monkeypatch): monkeypatch.setattr(gaussdb_conn_module, "_tokenize_query_terms", lambda query: str(query).split()) conn = make_conn(RecordingCursor()) - text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello"}) + text = MatchTextExpr(["content_with_weight"], "hello", 10, {"original_query": "hello", "minimum_should_match": 0.3}) dense = MatchDenseExpr("q_4_vec", [0.1, 0.2, 0.3, 0.4], "float", "cosine", 10, {"similarity": 0.0}) fusion = FusionExpr("weighted_sum", 6, {"weights": "0.7,0.3"}) @@ -2832,7 +2832,9 @@ def test_tc_ret_806_parse_match_expressions_applies_gaussdb_vector_weight_defaul hybrid_explicit = conn._parse_match_expressions([text, dense, fusion], rank_feature={"pagerank_fea": 7}) assert text_only["vector_weight"] == 0.0 + assert text_only["minimum_should_match"] == 0.3 assert vector_only["vector_weight"] == 1.0 + assert vector_only["minimum_should_match"] is None assert hybrid_default["vector_weight"] == 0.5 assert {text_only["pagerank_weight"], vector_only["pagerank_weight"], hybrid_default["pagerank_weight"]} == {10.0} assert hybrid_explicit["vector_weight"] == 0.3 diff --git a/test/unit_test/rag/utils/test_gaussdb_search_sql.py b/test/unit_test/rag/utils/test_gaussdb_search_sql.py index 8c27422d1a..b4308cb6e6 100644 --- a/test/unit_test/rag/utils/test_gaussdb_search_sql.py +++ b/test/unit_test/rag/utils/test_gaussdb_search_sql.py @@ -176,6 +176,28 @@ def test_tc_ret_109_chinese_ngram_query_preserves_english_semantics_and_highligh assert conn.get_highlight(result, ["深圳数据", "深圳"], "content_with_weight") == {"overlap": "深圳数据库审计"} +def test_tc_ret_111_minimum_should_match_uses_partial_unique_terms_and_term_scores(): + builder = GaussDBSearchBuilder(schema="public") + keywords = ["你好", "一下", "王楠", "简历", "大概", "介绍", "一下"] + + match_sql, match_params = builder._build_text_match_expr(keywords, minimum_should_match=0.3) + score_sql, score_params = builder.build_text_score_expr(keywords, minimum_should_match=0.3) + + unique_terms = ["你好", "一下", "王楠", "简历", "大概", "介绍"] + assert " OR " in match_sql + assert match_params == unique_terms + assert " / 6.0)" in score_sql + assert score_params == [term for term in unique_terms for _field in builder.FTS_WEIGHTS] + assert builder._minimum_should_match_count(0.3, 6) == 1 + assert builder._minimum_should_match_count("50%", 6) == 3 + assert builder._minimum_should_match_count(2, 6) == 2 + + two_match_sql, two_match_params = builder._build_text_match_expr(keywords, minimum_should_match=2) + assert "CASE WHEN" in two_match_sql + assert ">= 2" in two_match_sql + assert two_match_params == unique_terms * 2 + + def test_tc_ret_203_vector_search_filters_invalid_placeholder_vectors(): builder = GaussDBSearchBuilder(schema="public") sql, params = builder.build_search_sql(