mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-13 04:13:35 +08:00
f1641228e2
## Summary This PR improves the RAGFlow agentic-search path in three areas: it stops the outer agent from re-looping over the same rag call, lets the medium thinking mode discover and follow new sub-claims mid-loop, and strengthens retrieval by having the LLM emit synonym-rich queries with time/date/number terms boosted. 1. Avoid the outer re-loop — keep all multi-hop cycles inside agentic RAG 2. Dynamic claims in medium mode — keep querying newly discovered sub-questions medium now enables allows_dynamic_claims. During orchestration, when claim analysis discovers a new required sub-question (discovered_claims), the loop spawns it as a new ClaimTarget and continues searching it in subsequent cycles (bounded by the dynamic-claim budget) instead of stopping. Also added: 3. Stronger query strategy — synonym-rich queries + time/date/number weighting LLM-generated synonyms: the claim-analysis prompt now instructs the model to write each next_queries entry as a retrieval-boosted query that actively folds in entity aliases, DATE/TIME synonyms (e.g. 1994 → 1994, 66th Academy Awards), and number/unit variants (e.g. 1.95 m → 6 ft 5 in). Time/date/number boosting: query.py boosts numeric/date tokens to a high weight (_NUM_DATE_TOKEN_RE).
255 lines
11 KiB
Python
255 lines
11 KiB
Python
#
|
||
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
#
|
||
|
||
import logging
|
||
import json
|
||
import re
|
||
from collections import defaultdict
|
||
|
||
from common.query_base import QueryBase
|
||
from common.doc_store.doc_store_base import MatchTextExpr
|
||
from rag.nlp import rag_tokenizer, term_weight, synonym
|
||
from rag.utils.redis_conn import REDIS_CONN
|
||
|
||
|
||
# Tokens that are hard time/date/number constraints (a year, a date, a measurement).
|
||
# They get a high BM25 weight so chunks carrying the exact value rank above passages
|
||
# that merely mention the surrounding entity. Matches:
|
||
# pure numbers / decimals: 1994, 2001, 1.95, 3.68
|
||
# dates: 2011-02-05, 1994-06-23, 02/05/2011
|
||
# numbers with a unit: 50m, 1.95m, 6ft5in, 94kg
|
||
_NUM_DATE_TOKEN_RE = re.compile(
|
||
r"^\d[\d.,/:\-]*$"
|
||
r"|^\d+(?:\.\d+)?\s*(?:m|km|cm|mm|kg|g|lb|ft|in|yd|s|ms|min|h|hr|sec|y|yr|yo|yrs|k|m|b|th|nd|rd|st|%)$",
|
||
re.IGNORECASE,
|
||
)
|
||
|
||
|
||
class FulltextQueryer(QueryBase):
|
||
def __init__(self):
|
||
self.tw = term_weight.Dealer()
|
||
self.syn = synonym.Dealer(redis=REDIS_CONN.REDIS if REDIS_CONN.is_alive() else None)
|
||
self.query_fields = [
|
||
"title_tks^10",
|
||
"title_sm_tks^5",
|
||
"important_kwd^30",
|
||
"important_tks^20",
|
||
"question_tks^20",
|
||
"content_ltks^2",
|
||
"content_sm_ltks",
|
||
]
|
||
|
||
def question(self, txt, tbl="qa", min_match: float = 0.6):
|
||
original_query = txt
|
||
txt = self.add_space_between_eng_zh(txt)
|
||
|
||
# Strip Infinity ESCAPABLE characters from the query.
|
||
#
|
||
# Infinity's search_lexer.l defines ESCAPABLE characters [\x20()^"'~*?:\\]
|
||
# If these characters appear unescaped in a query, Infinity's lexer will
|
||
# interpret them as special tokens, causing parsing errors.
|
||
txt = re.sub(
|
||
r"[ :|\r\n\t,，。？?/`!！&^%%()\[\]{}<>*~'\"\\]+",
|
||
" ",
|
||
rag_tokenizer.tradi2simp(rag_tokenizer.strQ2B(txt.lower())),
|
||
).strip()
|
||
otxt = txt
|
||
txt = self.rmWWW(txt)
|
||
|
||
if not self.is_chinese(txt):
|
||
txt = self.rmWWW(txt)
|
||
tks = rag_tokenizer.tokenize(txt).split()
|
||
keywords = [t for t in tks if t]
|
||
tks_w = self.tw.weights(tks, preprocess=False)
|
||
tks_w = [(re.sub(r"[ \\\"'^]", "", tk), w) for tk, w in tks_w]
|
||
tks_w = [(re.sub(r"^[\+-]", "", tk), w) for tk, w in tks_w if tk]
|
||
tks_w = [(tk.strip(), w) for tk, w in tks_w if tk.strip()]
|
||
# Time/date/number terms are hard constraints in retrieval (e.g. a year,
|
||
# a date, a measurement). Boost them so chunks carrying the exact value
|
||
# rank well above passages that merely mention the surrounding entity.
|
||
#
|
||
# NOTE on decimals/dates: ``rag_tokenizer`` splits "1.95" into "1" "95"
|
||
# and "2011-02-05" into "2011" "02" "05". Each integer fragment is still
|
||
# boosted to 10, because the query_string builder below pairs every two
|
||
# adjacent tokens into a bigram phrase (``"1 95"^max*2``). That bigram
|
||
# re-connects the fragments and matches the exact value "1.95" in the
|
||
# index, so weighting the fragments is exactly what makes the decimal
|
||
# match — they are NOT independent numbers in the query.
|
||
tks_w = [(tk, 10.0 if _NUM_DATE_TOKEN_RE.match(tk) else w) for tk, w in tks_w]
|
||
syns = []
|
||
for tk, w in tks_w[:256]:
|
||
# Strip single quotes from synonym terms to avoid Infinity lexer TokenError
|
||
# (e.g. WordNet returns "cat-o'-nine-tails" for "cat")
|
||
syn = [rag_tokenizer.tokenize(s).replace("'", "") for s in self.syn.lookup(tk)]
|
||
keywords.extend(syn)
|
||
syn = ['"{}"^{:.4f}'.format(s, w / 4.0) for s in syn if s.strip()]
|
||
syns.append(" ".join(syn))
|
||
|
||
q = ["({}^{:.4f}".format(tk, w) + " {})".format(syn) for (tk, w), syn in zip(tks_w, syns) if tk and not re.match(r"[.^+\(\)-]", tk)]
|
||
for i in range(1, len(tks_w)):
|
||
left, right = tks_w[i - 1][0].strip(), tks_w[i][0].strip()
|
||
if not left or not right:
|
||
continue
|
||
q.append(
|
||
'"%s %s"^%.4f'
|
||
% (
|
||
tks_w[i - 1][0],
|
||
tks_w[i][0],
|
||
max(tks_w[i - 1][1], tks_w[i][1]) * 2,
|
||
)
|
||
)
|
||
if not q:
|
||
q.append(txt)
|
||
query = " ".join(q)
|
||
return MatchTextExpr(self.query_fields, query, 100, {"original_query": original_query}), keywords
|
||
|
||
def need_fine_grained_tokenize(tk):
|
||
if len(tk) < 3:
|
||
return False
|
||
if re.match(r"[0-9a-z\.\+#_\*-]+$", tk):
|
||
return False
|
||
return True
|
||
|
||
txt = self.rmWWW(txt)
|
||
qs, keywords = [], []
|
||
for tt in self.tw.split(txt)[:256]: # .split():
|
||
if not tt:
|
||
continue
|
||
keywords.append(tt)
|
||
twts = self.tw.weights([tt])
|
||
syns = self.syn.lookup(tt)
|
||
if syns and len(keywords) < 32:
|
||
keywords.extend(syns)
|
||
logging.debug(json.dumps(twts, ensure_ascii=False))
|
||
tms = []
|
||
for tk, w in sorted(twts, key=lambda x: x[1] * -1):
|
||
sm = rag_tokenizer.fine_grained_tokenize(tk).split() if need_fine_grained_tokenize(tk) else []
|
||
sm = [
|
||
re.sub(
|
||
r"[ ,\./;'\[\]\\`~!@#$%\^&\*\(\)=\+_<>\?:\"\{\}\|，。；‘’【】、！￥……（）——《》？：“”-]+",
|
||
"",
|
||
m,
|
||
)
|
||
for m in sm
|
||
]
|
||
sm = [self.sub_special_char(m) for m in sm if len(m) > 1]
|
||
sm = [m for m in sm if len(m) > 1]
|
||
|
||
if len(keywords) < 32:
|
||
keywords.append(re.sub(r"[ \\\"']+", "", tk))
|
||
keywords.extend(sm)
|
||
|
||
tk_syns = self.syn.lookup(tk)
|
||
tk_syns = [self.sub_special_char(s) for s in tk_syns]
|
||
if len(keywords) < 32:
|
||
keywords.extend([s for s in tk_syns if s])
|
||
tk_syns = [rag_tokenizer.fine_grained_tokenize(s) for s in tk_syns if s]
|
||
tk_syns = [f'"{s}"' if s.find(" ") > 0 else s for s in tk_syns]
|
||
|
||
if len(keywords) >= 32:
|
||
break
|
||
|
||
tk = self.sub_special_char(tk)
|
||
if tk.find(" ") > 0:
|
||
tk = '"%s"' % tk
|
||
if tk_syns:
|
||
tk = f"({tk} OR (%s)^0.2)" % " ".join(tk_syns)
|
||
if sm:
|
||
tk = f'{tk} OR "%s" OR ("%s"~2)^0.5' % (" ".join(sm), " ".join(sm))
|
||
if tk.strip():
|
||
tms.append((tk, w))
|
||
|
||
tms = " ".join([f"({t})^{w}" for t, w in tms])
|
||
|
||
if len(twts) > 1:
|
||
tms += ' ("%s"~2)^1.5' % rag_tokenizer.tokenize(tt)
|
||
|
||
syns = " OR ".join(['"%s"' % rag_tokenizer.tokenize(self.sub_special_char(s)) for s in syns])
|
||
if syns and tms:
|
||
tms = f"({tms})^5 OR ({syns})^0.7"
|
||
|
||
qs.append(tms)
|
||
|
||
if qs:
|
||
query = " OR ".join([f"({t})" for t in qs if t])
|
||
if not query:
|
||
query = otxt
|
||
return MatchTextExpr(self.query_fields, query, 100, {"minimum_should_match": min_match, "original_query": original_query}), keywords
|
||
return None, keywords
|
||
|
||
def hybrid_similarity(self, avec, bvecs, atks, btkss, tkweight=0.3, vtweight=0.7):
|
||
from sklearn.metrics.pairwise import cosine_similarity
|
||
import numpy as np
|
||
|
||
sims = cosine_similarity([avec], bvecs)
|
||
tksim = self.token_similarity(atks, btkss)
|
||
if np.sum(sims[0]) == 0:
|
||
return np.array(tksim), tksim, sims[0]
|
||
return np.array(sims[0]) * vtweight + np.array(tksim) * tkweight, tksim, sims[0]
|
||
|
||
def token_similarity(self, atks, btkss):
|
||
def to_dict(tks):
|
||
if isinstance(tks, str):
|
||
tks = tks.split()
|
||
d = defaultdict(int)
|
||
wts = self.tw.weights(tks, preprocess=False)
|
||
for i, (t, c) in enumerate(wts):
|
||
d[t] += c * 0.4
|
||
if i + 1 < len(wts):
|
||
_t, _c = wts[i + 1]
|
||
d[t + _t] += max(c, _c) * 0.6
|
||
return d
|
||
|
||
atks = to_dict(atks)
|
||
btkss = [to_dict(tks) for tks in btkss]
|
||
return [self.similarity(atks, btks) for btks in btkss]
|
||
|
||
def similarity(self, qtwt, dtwt):
|
||
if isinstance(dtwt, type("")):
|
||
dtwt = {t: w for t, w in self.tw.weights(self.tw.split(dtwt), preprocess=False)}
|
||
if isinstance(qtwt, type("")):
|
||
qtwt = {t: w for t, w in self.tw.weights(self.tw.split(qtwt), preprocess=False)}
|
||
s = 1e-9
|
||
for k, v in qtwt.items():
|
||
if k in dtwt:
|
||
s += v # * dtwt[k]
|
||
q = 1e-9
|
||
for k, v in qtwt.items():
|
||
q += v # * v
|
||
return s / q # math.sqrt(3. * (s / q / math.log10( len(dtwt.keys()) + 512 )))
|
||
|
||
def paragraph(self, content_tks: str, keywords: list = [], keywords_topn=30):
|
||
if isinstance(content_tks, str):
|
||
content_tks = [c.strip() for c in content_tks.split() if c.strip()]
|
||
tks_w = self.tw.weights(content_tks, preprocess=False)
|
||
|
||
origin_keywords = keywords.copy()
|
||
keywords = [f'"{k.strip()}"' for k in keywords]
|
||
for tk, w in sorted(tks_w, key=lambda x: x[1] * -1)[:keywords_topn]:
|
||
tk_syns = self.syn.lookup(tk)
|
||
tk_syns = [self.sub_special_char(s) for s in tk_syns]
|
||
tk_syns = [rag_tokenizer.fine_grained_tokenize(s) for s in tk_syns if s]
|
||
tk_syns = [f'"{s}"' if s.find(" ") > 0 else s for s in tk_syns]
|
||
tk = self.sub_special_char(tk)
|
||
if tk.find(" ") > 0:
|
||
tk = '"%s"' % tk
|
||
if tk_syns:
|
||
tk = f"({tk} OR (%s)^0.2)" % " ".join(tk_syns)
|
||
if tk:
|
||
keywords.append(f"{tk}^{w}")
|
||
|
||
return MatchTextExpr(self.query_fields, " ".join(keywords), 100, {"minimum_should_match": min(3, round(len(keywords) / 10)), "original_query": " ".join(origin_keywords)})
|