From 4cc2dbc06746ee2cd60fad6aacfe7147aed96386 Mon Sep 17 00:00:00 2001 From: Yingfeng Date: Fri, 7 Aug 2026 22:10:22 +0800 Subject: [PATCH] More stable sufficient check for agentic search (#17962) --- pyproject.toml | 7 + rag/advanced_rag/agentic_rag.py | 105 ++- rag/advanced_rag/agentic_rag_graph.py | 49 +- rag/advanced_rag/harness/agent.py | 46 +- rag/advanced_rag/harness/config.py | 16 +- .../harness/orchestrator/agentic.py | 106 ++- .../harness/orchestrator/decompose.py | 77 +- .../harness/orchestrator/sufficiency_llm.py | 264 ++++++ .../harness/prompts/decompose_prompts.py | 60 +- .../harness/prompts/report_prompt.py | 10 + .../harness/prompts/research_agent_prompt.py | 40 + .../harness/prompts/sufficiency_prompt.py | 31 - rag/advanced_rag/harness/route.py | 24 +- rag/advanced_rag/harness/sufficiency.py | 819 ++++++++++++++++-- .../harness/sufficiency_ladder.py | 176 ++++ rag/advanced_rag/harness/tools/navigation.py | 96 +- rag/advanced_rag/harness/tools/registry.py | 17 +- rag/advanced_rag/harness/types.py | 43 +- rag/prompts/sufficiency_select.md | 32 +- uv.lock | 149 +++- 20 files changed, 1980 insertions(+), 187 deletions(-) create mode 100644 rag/advanced_rag/harness/orchestrator/sufficiency_llm.py delete mode 100644 rag/advanced_rag/harness/prompts/sufficiency_prompt.py create mode 100644 rag/advanced_rag/harness/sufficiency_ladder.py diff --git a/pyproject.toml b/pyproject.toml index 02f1b8f452..13ababa913 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,7 +114,14 @@ dependencies = [ "scholarly==1.7.11", "selenium-wire==5.1.0", "spacy==3.8.14", + "langdetect==1.0.9", "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl", + "zh-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl", + "de-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl", + "fr-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl", + "es-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl", + "pt-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl", + "ja-core-news-sm @ https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl", "slack-sdk==3.37.0", "socksio==1.0.0", "agentrun-sdk>=0.0.51,<1.0.0", diff --git a/rag/advanced_rag/agentic_rag.py b/rag/advanced_rag/agentic_rag.py index 2fff48f5c0..8914d5f68f 100644 --- a/rag/advanced_rag/agentic_rag.py +++ b/rag/advanced_rag/agentic_rag.py @@ -67,6 +67,73 @@ from rag.utils.web_search_conn import WebSearchProvider # lets us trim the evidence up front instead. _EVIDENCE_PROMPT_RESERVE_TOKENS = 1024 +# Fixed evidence budget for ``_fit_evidence`` (sufficiency judge / follow-ups / +# formalize-answer evidence trimming). Kept far below the model context so a +# large retrieval pool never fills the window with evidence; each call stays +# cheap. ~8000 tokens ≈ 32K chars is enough to support a grounded answer. +_EVIDENCE_BUDGET_TOKENS = 8000 + +_LOG = logging.getLogger(__name__) + +# P0: significant-keyword overlap above which a new `rag` question reuses a +# cached answer. Overlap = |shared| / min(|a|, |b|) over the question's +# significant words (stopwords dropped). 0.6 + ">=2 shared words" collapses the +# re-ask pattern in the logs ("legal population" → "estimated population of +# Paris in 2019", both overlap 0.75) while leaving genuinely different questions +# (Paris vs. Brown County = 0.25) untouched. +_RAG_CACHE_MIN_OVERLAP = 0.6 +_RAG_CACHE_MIN_SHARED = 2 + +# Lightweight stopwords for the cross-`rag`-call dedup only. Never reused for +# retrieval/answer quality. +_RAG_CACHE_STOPWORDS = frozenset( + "the a an is was were what which when where who how of in to for and or but on at by be as it that this" + " about with their its have has had been being from over under do does did not no yes can could should would" + " also only very much more most some any".split() +) + + +def _question_keywords(question: str) -> tuple[set[str], set[str]]: + """(significant words, numeric tokens) of a question. + + For English (the observed re-ask pattern) plain tokenisation suffices; CJK + text falls back to the whole-token as a single significant unit. Numeric + tokens (years, figures) are returned separately so ``_cache_similar`` can + refuse to collapse questions that differ in the number being asked about + (e.g. "Paris population in 2019" vs "in 2015"). + """ + tokens = re.findall(r"[a-zA-Z0-9\u4e00-\u9fff]+", (question or "").lower()) + numbers = {t for t in tokens if t.isdigit()} + sig = {t for t in tokens if t not in _RAG_CACHE_STOPWORDS and len(t) > 1 and not t.isdigit()} + if not sig: + sig = {t for t in tokens if len(t) > 1 and not t.isdigit()} + return sig, numbers + + +def _cache_similar( + a: tuple[set[str], set[str]], + b: tuple[set[str], set[str]], +) -> bool: + """True when a new question's significant words mostly overlap a cached one. + + Uses word overlap (shared / min cardinality) so "legal population of Paris + 2019" (5 sig words) is caught by "population of Paris 2019" (3 sig words) + and vice-versa, while requiring >= 2 shared words. The numeric sets must + either be both empty or identical: if the two questions name different + years/figures they are NOT the same question and must not share an answer. + """ + aw, an = a + bw, bn = b + if not aw or not bw: + return False + if an or bn: # a question with an explicit number must match on it exactly + if an != bn: + return False + shared = len(aw & bw) + if shared < _RAG_CACHE_MIN_SHARED: + return False + return shared / min(len(aw), len(bw)) >= _RAG_CACHE_MIN_OVERLAP + class RAGTools: def __init__( @@ -127,6 +194,15 @@ class RAGTools: # ``[ID:n]`` markers index), so the caller can resolve references. self.kbinfos: dict[str, list] = {"chunks": [], "doc_aggs": []} + # P0: cross-`rag`-call result cache keyed by a character n-gram digest of + # the question. When a new sub-question is near-identical to one already + # researched this turn (e.g. the user re-asks a Paris figure as + # "legal population" then "commune inhabitants"), we reuse the cached + # answer instead of re-running the whole agentic graph. Conservative + # threshold (≥0.85 n-gram Jaccard) so near-identical phrasing is caught + # without confusing genuinely different questions. + self._rag_cache: dict[str, tuple[str, set[str]]] = {} + # Per-request retrieval cache keyed by the effective query + scope, so # the same question is never retrieved twice within one turn (e.g. # pre_search vs. an identical claim search in orchestrator_loop). @@ -474,15 +550,18 @@ class RAGTools: def _fit_evidence(self, question: str, evidence_md: str) -> str: """Trim ``evidence_md`` so ``question`` + evidence + the prompt template - stay inside the model's context window. + stay inside a *bounded* budget. ``message_fit_in`` keeps the small side (the question) whole and trims - the large side (the evidence); we shrink the budget by a reserve so the - template skeleton and JSON output rules still fit afterwards. + the large side (the evidence). We use a FIXED budget (not the full model + context) so a large retrieval pool can never fill the whole context + window with evidence — each sufficiency/answer call stays cheap. Callers + that want snippet-based evidence (``_narrow_by_keywords``) do so at the + source; this is the final safety cap. """ if not evidence_md: return evidence_md - budget = max(256, self.chat_mdl.max_length - _EVIDENCE_PROMPT_RESERVE_TOKENS) + budget = _EVIDENCE_BUDGET_TOKENS _, msg = message_fit_in(form_message(question, evidence_md), budget) return msg[-1]["content"] @@ -574,6 +653,20 @@ class RAGTools: if self.tool_started_sink is not None: self.tool_started_sink() + # P0: reuse a near-identical question's cached answer instead of re-running + # the whole agentic graph. Significant-keyword overlap (>= min_overlap AND + # >=2 shared words, and matching numbers) collapses the re-ask pattern + # while leaving genuinely different questions untouched. Attachments bypass + # the cache (their content is appended to the question message below). + if question and not self.text_attachments_content: + qk = _question_keywords(question) + if self._rag_cache: + for cached_q, (cached_answer, cached_gram) in list(self._rag_cache.items()): + if cached_gram and _cache_similar(qk, cached_gram): + shared = len(qk[0] & cached_gram[0]) + _LOG.info("[rag] Reusing cached answer for near-identical question %r (%d shared words); skipping re-research.", question, shared) + return cached_answer + messages = [{"role": "user", "content": question}] if question else [] if self.text_attachments_content and messages: messages[-1]["content"] += self.text_attachments_content @@ -585,6 +678,10 @@ class RAGTools: self.answer_sink(delta, kind == "think") for p, r in [(r"\(\**(ID:\d)\**\)", "[\1]")]: final = re.sub(p, r, final) + + # Cache the freshly produced answer for later near-identical questions. + if question and final and not self.text_attachments_content: + self._rag_cache[question] = (final, _question_keywords(question)) return final @tool diff --git a/rag/advanced_rag/agentic_rag_graph.py b/rag/advanced_rag/agentic_rag_graph.py index 37c27d4c5b..d9ac823445 100644 --- a/rag/advanced_rag/agentic_rag_graph.py +++ b/rag/advanced_rag/agentic_rag_graph.py @@ -360,8 +360,49 @@ def build_agentic_graph(tools, token_queue: asyncio.Queue, gen_conf: dict | None token_queue.put_nowait(tools.empty_response) return {"final_answer": tools.empty_response} - # Build evidence - evidence_blocks = kb_prompt(kbinfos, tools.chat_mdl.max_length) + # Build evidence — narrow the gathered chunks to keyword-bearing + # snippets BEFORE feeding the answer model, instead of dumping every + # full chunk into the prompt. The retriever already narrows per-query, + # but multi-search accumulation and the "keep-all-on-no-match" fallback + # can still leave many full passages here (one run produced a 46-passage + # / 70K-token call). Re-narrowing on the final question keeps each + # passage a snippet and bounds the prompt, while preserving originals if + # nothing matches (so we never answer with empty evidence). + # + # IMPORTANT: narrowing runs on a COPY of the chunk list — we never mutate + # ``kbinfos["chunks"]`` because that pool is also the main-agent citation + # reference; in-place narrowing there would shrink what the agent can cite + # and make it re-ask (more sub-questions). Also, a chunk whose original + # carried a number but whose narrowed snippet lost every digit is kept + # whole (a numeric answer sentence often lacks the keyword itself). + kw = state.get("keywords") or "" + from rag.advanced_rag.harness.orchestrator.sufficiency_llm import _narrow_snippet_safe + + kw_list = [k for k in re.split(r"[,\s]+", kw or "") if k] + evidence_chunks = [] + for c in kbinfos.get("chunks") or []: + if not kw_list: + evidence_chunks.append(c) + continue + raw = c.get("content_with_weight") or c.get("text") or "" + # General informative-sentence guard (same policy as `_evidence_md`): + # keep the narrowed snippet, but fall back to the whole chunk when + # narrowing would drop every fact-bearing sentence (numbers / proper + # nouns / quotes) — the answer may be far from any keyword. Bounds + # the prompt while never losing critical evidence. + narrowed = _narrow_snippet_safe(raw, kw_list) + if narrowed: + evidence_chunks.append({**c, "content_with_weight": narrowed}) + else: + evidence_chunks.append(c) + evidence_kbinfos = dict(kbinfos, chunks=evidence_chunks) + # Bounded evidence budget: the raw model context (``max_length``) is far + # too large — with a big pool it lets evidence fill the whole window + # (observed 141K tokens in one call). Use a fixed, modest budget so the + # answer model sees only a compact evidence set. + from rag.advanced_rag.agentic_rag import _EVIDENCE_BUDGET_TOKENS + + evidence_blocks = kb_prompt(evidence_kbinfos, min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS)) evidence = "\n".join(evidence_blocks) if isinstance(evidence_blocks, list) else str(evidence_blocks) parts = [f"Question:\n{question}\n"] @@ -391,7 +432,9 @@ def build_agentic_graph(tools, token_queue: asyncio.Queue, gen_conf: dict | None parts.append(f"Evidence:\n{evidence}") user_content = "\n".join(parts) - _, msg = message_fit_in(form_message(system, user_content), tools.chat_mdl.max_length) + # Same bounded budget for the final message fit — never fill the whole + # model context with evidence. + _, msg = message_fit_in(form_message(system, user_content), min(tools.chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS)) try: async for tok in tools.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], answer_conf): token_queue.put_nowait(tok) diff --git a/rag/advanced_rag/harness/agent.py b/rag/advanced_rag/harness/agent.py index d1e5a3f377..9aa43b8215 100644 --- a/rag/advanced_rag/harness/agent.py +++ b/rag/advanced_rag/harness/agent.py @@ -145,8 +145,18 @@ async def research_agent_loop( context, mode: ExecutionStrategy, compilation_map: dict, + followups: list[str] | None = None, ) -> dict: - """Inner loop for a single claim — native tool-calling with a text fallback.""" + """Inner loop for a single claim — native tool-calling with a text fallback. + + ``followups`` (Phase-2 LLM missing-pieces feedback from the previous + sufficiency round) is passed in explicitly by the orchestrator rather than + read from the shared ``context`` here. The orchestrator consumes and clears + ``context.pending_followups`` ONCE per round so every claim researched in a + parallel batch receives the SAME follow-up guidance (reading the shared list + per-claim would race: the first claim to execute would clear it, starving + the rest). + """ phase = determine_current_phase(context) phase_config = SEARCH_PHASES.get(phase, {}) gated_defs = get_gated_tools( @@ -161,10 +171,10 @@ async def research_agent_loop( # Clone so binding tools never leaks onto the shared chat model. agent_mdl = tools.chat_mdl.clone() if getattr(agent_mdl, "is_tools", False): - return await _research_native(claim, agent_mdl, pipeline, phase, phase_config, gated_defs, mode) + return await _research_native(claim, agent_mdl, pipeline, phase, phase_config, gated_defs, mode, followups) _LOG.info("research_agent: model lacks native tool support; falling back to text-based tool selection") - return await _research_text(claim, tools, pipeline, phase, phase_config, gated_defs, mode) + return await _research_text(claim, tools, pipeline, phase, phase_config, gated_defs, mode, followups) async def _research_native( @@ -175,6 +185,7 @@ async def _research_native( phase_config: dict, gated_defs: list[dict], mode: ExecutionStrategy, + followups: list[str] | None = None, ) -> dict: """Bind tools onto ``agent_mdl`` and let its native tool loop drive research.""" schemas = _build_tool_schemas(gated_defs) @@ -191,6 +202,15 @@ async def _research_native( max_cycles=mode.max_agent_cycles, ) history = [{"role": "user", "content": f"Research task: {claim.description}\nBegin."}] + # Phase-2 missing-pieces guidance: focus this round on the specific gaps the + # Sufficient Context AutoRater flagged, rather than re-searching broadly. + if followups: + history.append( + { + "role": "user", + "content": "Previous evidence was incomplete. Run targeted searches specifically for the following missing pieces:\n- " + "\n- ".join(followups), + } + ) final_text = "" try: @@ -224,6 +244,7 @@ async def _research_text( phase_config: dict, gated_defs: list[dict], mode: ExecutionStrategy, + followups: list[str] | None = None, ) -> dict: """Fallback: prompt-based tool selection for models without native tools.""" system = RESEARCH_AGENT_TEXT_PROMPT.format( @@ -235,6 +256,13 @@ async def _research_text( ) history: list[dict] = [] + if followups: + history.append( + { + "role": "user", + "content": "Previous evidence was incomplete. Run targeted searches specifically for the following missing pieces:\n- " + "\n- ".join(followups), + } + ) for cycle in range(mode.max_agent_cycles): try: @@ -377,7 +405,17 @@ def _fmt_tool_result(result: ToolResult) -> str: answer = (result.metadata or {}).get("answer") if isinstance(result.metadata, dict) else "" if answer: parts.append(f"Answer: {answer}") - parts.extend(c.get("content_with_weight", c.get("text", ""))[:300] for c in result.chunks[:3]) + # Show more chunks (6 vs the old 3) and order them by retrieval similarity, + # so the agent actually sees the passages that best match its query. The old + # "first 3 chunks, 300 chars each" made the agent miss the key evidence when + # it sat in chunk 4+ (5.log/6.log: it retrieved 48m/27m but reported + # "unrelated (Barack Obama)" and re-searched endlessly). + chunks = list(result.chunks) + chunks.sort(key=lambda c: float(c.get("similarity", 0.0) or 0.0), reverse=True) + for c in chunks[:6]: + text = c.get("content_with_weight") or c.get("text") or "" + if text: + parts.append(text[:300]) if not parts: return "[no results found]" return "\n\n".join(parts) diff --git a/rag/advanced_rag/harness/config.py b/rag/advanced_rag/harness/config.py index c7af23a0b4..962ab2918e 100644 --- a/rag/advanced_rag/harness/config.py +++ b/rag/advanced_rag/harness/config.py @@ -17,7 +17,6 @@ THINKING_MODES: dict[str, ExecutionStrategy] = { max_parallel_agents=1, available_tools=["hybrid_search", "web_search", "bm25_search"], sufficiency_threshold=0.85, - partial_threshold=0.50, fallback_to_direct_llm=False, ), "medium": ExecutionStrategy( @@ -34,8 +33,11 @@ THINKING_MODES: dict[str, ExecutionStrategy] = { max_parallel_agents=1, available_tools=["hybrid_search", "web_search", "bm25_search"], sufficiency_threshold=0.75, - partial_threshold=0.40, fallback_to_direct_llm=False, + c_high=0.75, + c_low=0.45, + llm_floor=0.55, + allows_reconcile=False, ), "high": ExecutionStrategy( label="high", @@ -60,8 +62,11 @@ THINKING_MODES: dict[str, ExecutionStrategy] = { "inspector_compare", ], sufficiency_threshold=0.65, - partial_threshold=0.30, fallback_to_direct_llm=False, + c_high=0.70, + c_low=0.40, + llm_floor=0.50, + allows_reconcile=True, ), "ultra": ExecutionStrategy( label="ultra", @@ -91,8 +96,11 @@ THINKING_MODES: dict[str, ExecutionStrategy] = { "inspector_request_adjacent", ], sufficiency_threshold=0.55, - partial_threshold=0.20, fallback_to_direct_llm=True, + c_high=0.65, + c_low=0.35, + llm_floor=0.45, + allows_reconcile=True, ), } diff --git a/rag/advanced_rag/harness/orchestrator/agentic.py b/rag/advanced_rag/harness/orchestrator/agentic.py index 18970a1219..621b0058b0 100644 --- a/rag/advanced_rag/harness/orchestrator/agentic.py +++ b/rag/advanced_rag/harness/orchestrator/agentic.py @@ -16,6 +16,7 @@ from rag.advanced_rag.harness.sufficiency import ( compute_fusion_score, route_sufficiency_verdict, ) +from rag.advanced_rag.harness.orchestrator.sufficiency_llm import llm_sufficiency_boost _LOG = logging.getLogger(__name__) CLAIM_RESEARCH_TIMEOUT_SECONDS = 180 @@ -65,6 +66,16 @@ async def agentic_research(state: dict, tools) -> dict: ctx = OrchestratorContext(question=question, claims=claims, mode=mode_label) pipeline = Pipeline(tools, compilation_map) + # Stagnation guard: if the fusion score stops improving across consecutive + # rounds, further searching is unlikely to help (e.g. the corpus simply lacks + # the data, and follow-ups keep returning nothing). Without this, a + # persistently INSUFFICIENT verdict burns every remaining cycle and, in the + # worst case, feeds a long empty loop (see check.log Q4: AutoRater said + # "not in corpus", follow-ups found nothing, yet the loop kept spinning). + prev_score: float | None = None + _STAGNATION_CYCLES = 2 # rounds with no meaningful gain before giving up + _STAGNATION_GAIN = 0.05 # minimum fusion-score improvement to count + for cycle in range(mode.max_orchestrator_cycles): ctx.iteration = cycle _LOG.info("[Agentic research] Research round %d of %d — %d step(s) still unanswered.", cycle + 1, mode.max_orchestrator_cycles, sum(1 for c in ctx.claims if not c.is_verified)) @@ -73,6 +84,23 @@ async def agentic_research(state: dict, tools) -> dict: unverified = [c for c in ctx.claims if not c.is_verified] if unverified: + # Consume Phase-2 follow-up queries (missing-pieces feedback) ONCE for + # this round and hand the SAME list to every claim in the batch. + # Reading the shared ``ctx.pending_followups`` inside + # research_agent_loop would race under ``asyncio.gather``: the first + # claim to execute would clear it, starving the parallel siblings. + # We only consume/clear here — inside ``if unverified`` — because a + # research task must actually dispatch to use them. When everything + # is already verified no task runs, so we retain the queries for the + # next cycle instead of discarding them. + followups: list[str] = [] + if ctx.pending_followups: + followups = [str(q.get("query") or q.get("question") or "") for q in ctx.pending_followups if q] + followups = [q for q in followups if q.strip()] + ctx.pending_followups = [] + if followups: + _LOG.info("[Agentic research] Round %d: injecting %d follow-up query(ies) to all claims: %s", cycle + 1, len(followups), followups) + # Process in batches of max_parallel_agents batch_size = mode.max_parallel_agents for i in range(0, len(unverified), batch_size): @@ -83,7 +111,7 @@ async def agentic_research(state: dict, tools) -> dict: len(batch), "; ".join(f'"{c.description}"' for c in batch), ) - tasks = [_run_claim_research(c, tools, pipeline, ctx, mode, compilation_map) for c in batch] + tasks = [_run_claim_research(c, tools, pipeline, ctx, mode, compilation_map, followups=followups) for c in batch] agent_results = await asyncio.gather(*tasks) _LOG.info( "[Agentic research] Round %d: finished researching %d step(s).", @@ -95,6 +123,15 @@ async def agentic_research(state: dict, tools) -> dict: is_verified = result.get("is_verified", False) c.is_verified = is_verified c.confidence = result.get("confidence", 0.0) + grounded = result.get("grounded", []) + numbers = result.get("numbers", []) + if "grounded" not in result or "numbers" not in result: + _LOG.warning( + "[Agentic research] claim=%s report omitted the schema-required grounded/numbers fields (grounded=%r numbers=%r) — verification for it is degraded", + c.claim_id, + grounded, + numbers, + ) c.agent_result = AgentResult( claim_id=c.claim_id, report=result.get("report", ""), @@ -103,6 +140,8 @@ async def agentic_research(state: dict, tools) -> dict: evidence_ids=result.get("evidence_ids", []), gaps=result.get("gaps", []), discovered_claims=result.get("discovered_claims", []), + grounded=grounded, + numbers=numbers, ) # Ultra: dynamic claim expansion @@ -124,17 +163,75 @@ async def agentic_research(state: dict, tools) -> dict: # ── Step B: Sufficiency Check ── all_chunks = {i: c for i, c in enumerate(tools.kbinfos.get("chunks", []))} agent_results_list = [c.agent_result for c in ctx.claims if c.agent_result] + _LOG.info( + "[Sufficiency] Round %d: evidence pool=%d chunk(s), %d claim(s) with agent results: %s", + cycle + 1, + len(all_chunks), + len(agent_results_list), + [ + { + "claim_id": r.claim_id, + "self_verified": r.is_verified, + "self_confidence": round(r.confidence, 3), + "evidence_ids": len(r.evidence_ids), + } + for r in agent_results_list + ], + ) cross_results = [cross_check_claim(r, all_chunks) for r in agent_results_list] - verdict = compute_fusion_score(agent_results_list, cross_results, mode) + verdict = compute_fusion_score( + agent_results_list, + cross_results, + mode, + question=ctx.question, + claims=ctx.claims, + all_chunks=all_chunks, + ) ctx.verdict = verdict - action, should_continue = route_sufficiency_verdict( + # Decision ladder: the LLM Sufficient Context AutoRater is the primary + # sufficiency judge (invoked every round in high/ultra). Its verdict is + # combined with the code-level signals (hard vetoes + agent confidence) + # inside ``route_sufficiency_verdict`` → ``sufficiency_ladder``. The + # AutoRater's missing-pieces feedback is saved for the next round. + cited_ids: list[str] = [] + for r in agent_results_list: + cited_ids.extend(r.evidence_ids or []) + boost = await llm_sufficiency_boost(tools, ctx.question, verdict, evidence_ids=cited_ids) + if boost and boost.get("followups"): + # Missing pieces → targeted follow-up searches for the next round. + ctx.pending_followups = boost.get("followups", []) + _LOG.info("[Agentic research] Stored %d follow-up query(ies) for next round.", len(ctx.pending_followups)) + if boost: + _LOG.info("[Agentic research] Round %d: AutoRater is_sufficient=%s confidence=%.2f", cycle + 1, boost.get("is_sufficient"), boost.get("confidence", 1.0)) + + action, should_continue, caveat = route_sufficiency_verdict( verdict, mode_label, cycle, mode.max_orchestrator_cycles, + auto=boost, ) + if caveat: + _LOG.info("[Agentic research] Round %d: caveat=%s", cycle + 1, caveat) + + # Stagnation guard: when the verdict is not (yet) sufficient and the + # fusion score has not meaningfully improved for a couple of rounds, + # stop instead of burning the remaining cycle budget on unproductive + # re-searches. Override the CONTINUE decision with a partial answer. + if should_continue and verdict.status in ("INSUFFICIENT", "USEFUL_BUT_INCOMPLETE"): + if prev_score is not None and cycle >= _STAGNATION_CYCLES and verdict.score - prev_score < _STAGNATION_GAIN: + _LOG.info( + "[Agentic research] Round %d: score stagnant (%.3f → %.3f) — early-stopping to partial answer", + cycle + 1, + prev_score, + verdict.score, + ) + action = "ANSWER_PARTIAL" + should_continue = False + else: + prev_score = verdict.score _LOG.info("[Agentic research] Round %d: evidence looks %s (confidence %.0f%%) — next: %s", cycle + 1, verdict.status, verdict.score * 100, action) @@ -179,11 +276,12 @@ async def _run_claim_research( ctx: OrchestratorContext, mode, compilation_map: dict, + followups: list[str] | None = None, ) -> dict: _LOG.info('[Agentic research] Researching: "%s"', _snip(claim.description)) try: result = await asyncio.wait_for( - research_agent_loop(claim, tools, pipeline, ctx, mode, compilation_map), + research_agent_loop(claim, tools, pipeline, ctx, mode, compilation_map, followups=followups), timeout=CLAIM_RESEARCH_TIMEOUT_SECONDS, ) except asyncio.CancelledError: diff --git a/rag/advanced_rag/harness/orchestrator/decompose.py b/rag/advanced_rag/harness/orchestrator/decompose.py index 0a5aaa23a6..90646fdc68 100644 --- a/rag/advanced_rag/harness/orchestrator/decompose.py +++ b/rag/advanced_rag/harness/orchestrator/decompose.py @@ -12,6 +12,7 @@ from rag.advanced_rag.harness.sufficiency import ( compute_fusion_score, route_sufficiency_verdict, ) +from rag.advanced_rag.harness.orchestrator.sufficiency_llm import llm_sufficiency_boost from rag.advanced_rag.harness.tools.search import hybrid_search _LOG = logging.getLogger(__name__) @@ -48,8 +49,11 @@ Return JSON: "confidence": 0.0, "report": "Short evidence-backed finding, or what was learned so far.", "gaps": ["specific missing fact or relationship"], - "next_queries": ["standalone follow-up search query"] -}}""" + "next_queries": ["standalone follow-up search query"], + "grounded": ["key asserted facts that ARE directly supported by the cited evidence, atomically and verbatim enough to match"], + "numbers": ["for numerical/multi-hop answers: each figure used + its source, e.g. '2,161,000 from Wikipedia Demographics of Paris'; list ALL conflicting figures if several sources disagree"] +}} +Only list in grounded the facts you actually SAW in the evidence; prior-knowledge guesses go in gaps. If the claim is numerical or multi-hop and the evidence has multiple close-but-different figures, disclose all of them in numbers rather than silently picking one.""" async def decompose_and_search(state: dict, tools) -> dict: @@ -68,6 +72,13 @@ async def decompose_and_search(state: dict, tools) -> dict: pending_queries: dict[str, list[str]] = {c.claim_id: [] for c in ctx.claims} completed_cycles = 0 + # Stagnation guard: stop when the fusion score stops improving across + # consecutive rounds (corpus lacks the data, follow-ups return nothing) + # instead of burning the remaining cycle budget unproductively. + prev_score: float | None = None + _STAGNATION_CYCLES = 2 + _STAGNATION_GAIN = 0.05 + for cycle in range(max_cycles): ctx.iteration = cycle unverified = [c for c in ctx.claims if not c.is_verified] @@ -145,6 +156,8 @@ async def decompose_and_search(state: dict, tools) -> dict: evidence_ids=evidence_ids, gaps=analysis["gaps"], discovered_claims=[], + grounded=analysis.get("grounded", []), + numbers=analysis.get("numbers", []), ) next_queries = _new_queries( @@ -172,15 +185,60 @@ async def decompose_and_search(state: dict, tools) -> dict: agent_results = [c.agent_result for c in ctx.claims if c.agent_result] cross_results = [cross_check_claim(r, all_chunks) for r in agent_results] - verdict = compute_fusion_score(agent_results, cross_results, mode) + # HEAD sufficiency enhancements: pass the question + claims + evidence so + # the fusion activates required-entity AND-semantics veto, grounded-fact + # verification, and numeric multi-source conflict detection (not just the + # baseline ratio/mean that upstream's 3-arg call skipped). + verdict = compute_fusion_score( + agent_results, + cross_results, + mode, + question=ctx.question, + claims=ctx.claims, + all_chunks=all_chunks, + ) ctx.verdict = verdict - action, should_continue = route_sufficiency_verdict( + # LLM Sufficient Context AutoRater (primary sufficiency judge). Medium + # keeps it gated to the borderline band for cost control; its verdict + # (`auto=boost`) is fed to the decision ladder inside + # ``route_sufficiency_verdict``, which replaces the old manual + # SUFFICIENT upgrade. Missing-piece follow-ups feed the next hop. + boost: dict = {} + if verdict.status in ("USEFUL_BUT_INCOMPLETE", "INSUFFICIENT", "CONFLICTING"): + boost = await llm_sufficiency_boost(tools, ctx.question, verdict, evidence_ids=_global_evidence_ids(tools, {"chunks": tools.kbinfos.get("chunks", [])})) + followups = boost.get("followups") or [] + if followups: + ctx.pending_followups = followups + _LOG.info("[Decompose] Stored %d follow-up query(ies) for next round.", len(ctx.pending_followups)) + if boost: + _LOG.info("[Decompose] AutoRater is_sufficient=%s confidence=%.2f", boost.get("is_sufficient"), boost.get("confidence", 1.0)) + + action, should_continue, caveat = route_sufficiency_verdict( verdict, mode_label, cycle, max_cycles, + auto=boost, ) + if caveat: + _LOG.info("[Decompose] caveat=%s", caveat) + + # Stagnation guard: if the verdict is not (yet) sufficient and the score + # has not meaningfully improved across consecutive rounds, stop instead + # of burning the remaining cycle budget on unproductive re-searches. + if should_continue and verdict.status in ("INSUFFICIENT", "USEFUL_BUT_INCOMPLETE"): + if prev_score is not None and cycle >= _STAGNATION_CYCLES and verdict.score - prev_score < _STAGNATION_GAIN: + _LOG.info( + "[Decompose] Round %d: score stagnant (%.3f → %.3f) — early-stopping to partial answer", + cycle + 1, + prev_score, + verdict.score, + ) + action = "ANSWER_PARTIAL" + should_continue = False + else: + prev_score = verdict.score if action in ("ANSWER", "ANSWER_PARTIAL"): return _finalize(ctx, tools, partial=action == "ANSWER_PARTIAL", loop=completed_cycles) @@ -285,11 +343,16 @@ def _normalize_analysis( report = str(parsed.get("report") or "").strip() or _summarize(result) gaps = _string_list(parsed.get("gaps")) next_queries = _string_list(parsed.get("next_queries"))[:_MAX_NEXT_QUERIES] + grounded = _string_list(parsed.get("grounded")) + numbers = _string_list(parsed.get("numbers")) if is_verified: gaps = [] next_queries = [] - confidence = max(confidence, 0.65) + # NOTE: no optimistic floor here. The old ``max(confidence, 0.65)`` + # inflated medium's agent confidence and distorted the decision-ladder + # gate (agent_confidence >= c_high/c_low). Keep the LLM's raw confidence + # so the ladder's thresholds behave as designed. elif not next_queries and cycle + 1 < max_cycles: next_queries = _fallback_queries(question, claim) @@ -299,6 +362,8 @@ def _normalize_analysis( "report": report, "gaps": gaps, "next_queries": next_queries, + "grounded": grounded, + "numbers": numbers, } @@ -319,6 +384,8 @@ def _fallback_analysis( "report": _summarize(result), "gaps": [] if is_verified else ["need more specific evidence"], "next_queries": next_queries, + "grounded": [], + "numbers": [], } diff --git a/rag/advanced_rag/harness/orchestrator/sufficiency_llm.py b/rag/advanced_rag/harness/orchestrator/sufficiency_llm.py new file mode 100644 index 0000000000..70bdb74fe8 --- /dev/null +++ b/rag/advanced_rag/harness/orchestrator/sufficiency_llm.py @@ -0,0 +1,264 @@ +"""LLM Sufficient Context AutoRater for the orchestrator. + +This is the *primary* sufficiency judge in the decision-ladder design +(Sufficient Context paper, arXiv 2411.06037). We delegate to the LLM +AutoRater (``tools.judge_sufficiency``, backing prompt ``sufficiency_select``) +to decide whether the retrieved evidence actually supports a plausible answer. + + - AutoRater judges query + evidence semantically, not just keyword matching. + - On "insufficient" it returns concrete ``missing_information`` which we turn + into follow-up search queries (missing-pieces feedback) for the next round. + - It also returns ``confidence`` / ``contradictions`` / ``required_entities`` + consumed by the decision ladder (``sufficiency_ladder``). + +Call frequency: high/ultra invoke it every round; medium keeps it gated to the +critical band (token cost stays bounded for the default mode). The +``verdict.status in (...)"`` check below implements that medium gate. +""" + +from __future__ import annotations + +import logging +import re + +from rag.advanced_rag.harness.types import SufficiencyVerdict + +_LOG = logging.getLogger(__name__) + + +# Cap the AutoRater's evidence so a single judge call does not balloon to +# hundreds of KB (6.log showed 271KB from 80 accumulated passages). The judge +# only needs the *cited* evidence plus a bounded prefix — not the whole pool. +_MAX_EVIDENCE_CHUNKS = 24 +_MAX_CHUNK_CHARS = 800 +_MAX_EVIDENCE_CHARS = 24_000 + + +def _narrow_keywords(question: str) -> list[str]: + """Language-agnostic keywords of ``question`` for snippet narrowing. + + No language-specific stopword list (RAGFlow supports en/zh/de/fr/es/pt/ja and + any hard-coded English stopwords would be wrong for the rest). Instead we use + a length heuristic that generalises across scripts: + - numeric tokens (years / figures) — universal, high-signal; + - latin tokens of length >= 4 (keeps content words like "population" / + "paris" / "statistics" while the short function words "the"/"of"/"was" + fall below the cut — a length rule, not a stopword list); + - CJK runs are converted to character bigrams (巴黎 → 巴黎; 人口 → 人口), + which is tokeniser-free and works for zh/ja without any word segmenter. + + Keeping a few extra tokens is safe: narrowing matches more sentences, so it + retains more of the chunk and is less likely to drop the answer. + """ + tokens = re.findall(r"[a-zA-Z0-9]+|[\u4e00-\u9fff]+", (question or "").lower()) + kw: list[str] = [] + for t in tokens: + if t.isdigit(): + kw.append(t) + elif re.search(r"[a-zA-Z]", t): + if len(t) >= 4: + kw.append(t) + else: # CJK run → character bigrams + if len(t) >= 2: + kw.extend(t[i : i + 2] for i in range(len(t) - 1)) + return kw + + +def _clamp(value) -> float: + """Clamp the AutoRater's confidence field to [0, 1], defaulting to 1.0.""" + try: + return max(0.0, min(1.0, float(value))) + except (TypeError, ValueError): + return 1.0 + + +# A chunk with this many sentences or fewer is never narrowed — there is little +# to save and a real risk of losing the answer sentence. +_NARROW_MIN_SENTENCES = 3 +# Number of neighbours kept around each keyword-bearing sentence. +_NARROW_NEIGHBOURS = 1 + + +def _narrow_snippet_safe(content: str, kw_list: list[str]) -> str | None: + """Self-contained keyword narrowing of ``content`` to a compact snippet. + + This is a purpose-built replacement for ``_narrow_content`` (which returns + a re-formatted string full of ``...`` and ``*kw*`` markers that, when split + again downstream, corrupts sentence counts). Here we work directly on the + original sentences and re-emit plain text, so there is no intermediate + format to misinterpret. + + The only rule — a deliberately small one, because no rule set can cover + every phrasing: **narrow only when the keywords actually cover a meaningful + share of the chunk.** Concretely, we keep the keyword-bearing sentences plus + ``_NARROW_NEIGHBOURS`` around each, and refuse to narrow (return ``None``, + caller keeps the whole chunk) when: + * the chunk is short (<= ``_NARROW_MIN_SENTENCES`` sentences), or + * no sentence mentions a keyword, or + * keywords hit too few sentences for the narrowing to be safe + (< 2 hits, or fewer than half the sentences). + + That last case is the whole point: if the answer sentence is phrased without + the exact keyword (a bare figure, a synonym, a date), keyword hits will be + sparse and we keep the whole chunk so the answer is never dropped. + + Returns the narrowed plain text, or ``None`` to keep the chunk whole. + """ + from rag.advanced_rag.harness.tools.search import _split_sentences + + sents = _split_sentences(content or "") + if len(sents) <= _NARROW_MIN_SENTENCES: + return None + + hit_idx = [i for i, s in enumerate(sents) if any(k in s.lower() for k in kw_list)] + # Refuse to narrow when keywords hit fewer than 2 sentences: with a single hit + # the narrowed snippet would keep just that sentence (+neighbours) and could + # drop most of the chunk, including the answer sentence. Two or more hits + # means the keywords genuinely relate to the passage, so narrowing is safe. + if len(hit_idx) < 2: + return None + + keep_idx: set[int] = set() + for i in hit_idx: + for j in range(max(0, i - _NARROW_NEIGHBOURS), min(len(sents), i + _NARROW_NEIGHBOURS + 1)): + keep_idx.add(j) + return " ".join(sents[i] for i in sorted(keep_idx)).strip() + + +def _evidence_md(tools, evidence_ids=None, keywords: str | None = None) -> str: + """Render cited evidence chunks with ``ID: n`` markers for the AutoRater. + + Prefers the chunks referenced by ``evidence_ids`` (the union of all claims' + evidence); falls back to a bounded prefix of the pool when no ids are + given. When ``keywords`` is provided each chunk is first narrowed to the + keyword-bearing sentences via ``_narrow_by_keywords`` (snippet evidence), + so the AutoRater sees compact relevant snippets instead of full passages. + Either way the output is capped (see module constants). + """ + kb = getattr(tools, "kbinfos", None) + chunks = (kb or {}).get("chunks", []) + if not chunks: + return "" + + picked: list[tuple[int, dict]] = [] + if evidence_ids: + seen = set() + for eid in evidence_ids: + try: + idx = int(eid) + except (TypeError, ValueError): + continue + if 0 <= idx < len(chunks) and idx not in seen: + picked.append((idx, chunks[idx])) + seen.add(idx) + if len(picked) >= _MAX_EVIDENCE_CHUNKS: + break + if not picked: + picked = [(i, c) for i, c in enumerate(chunks[:_MAX_EVIDENCE_CHUNKS])] + + # Narrow each kept chunk to its keyword-bearing sentences when keywords are + # available, preserving the original ``ID: n`` index (the answer cites these). + # If narrowing empties a chunk it is skipped; if everything narrows away we + # fall back to the original text so the AutoRater never sees nothing. + texts: list[tuple[int, str, str]] = [] + # Normalise keywords: accept a comma/space-separated string OR an iterable of + # words. Narrowing operates on the raw text, independent of chunk structure. + if isinstance(keywords, str): + kw_list = [k.strip() for k in re.split(r"[,\s]+", keywords) if k.strip()] + else: + kw_list = [str(k) for k in (keywords or []) if str(k).strip()] + for idx, c in picked: + raw = c.get("content_with_weight") or c.get("text") or "" + title = c.get("docnm_kwd", "") or "" + if kw_list: + narrowed = _narrow_snippet_safe(raw, kw_list) + if narrowed: + texts.append((idx, title, narrowed)) + continue + texts.append((idx, title, raw)) + + blocks: list[str] = [] + used = 0 + for idx, title, text in texts: + text = text[:_MAX_CHUNK_CHARS] + if used + len(text) > _MAX_EVIDENCE_CHARS: + break + blocks.append(f"ID: {idx} | {title}\n{text}") + used += len(text) + 8 + return "\n\n".join(blocks) + + +async def llm_sufficiency_boost( + tools, + question: str, + verdict: SufficiencyVerdict, + evidence_ids=None, +) -> dict: + """Phase-2 LLM fallback, gated to the critical verdict band. + + Returns ``{}`` when no LLM boost is applicable (verdict already clear, or + the tools object lacks an LLM judge). Otherwise returns:: + + { + "is_sufficient": bool, # LLM's call + "missing": [str, ...], # concrete gaps (Google missing pieces) + "followups": [ {question, query}, ... ], # next-round search queries + } + """ + # Only boost ambiguous verdicts — clear SUFFICIENT/UNANSWERABLE need no LLM. + if verdict.status not in ("USEFUL_BUT_INCOMPLETE", "INSUFFICIENT", "CONFLICTING"): + return {} + if not hasattr(tools, "judge_sufficiency"): + return {} + chat_mdl = getattr(tools, "chat_mdl", None) + if chat_mdl is None: + return {} + + evidence_md = _evidence_md(tools, evidence_ids, keywords=_narrow_keywords(question)) + if not evidence_md: + return {} + + _LOG.info("[LLM-sufficiency] verdict=%s → triggering LLM Sufficient Context AutoRater (evidence %d chars)", verdict.status, len(evidence_md)) + try: + llm_result = await tools.judge_sufficiency(question, evidence_md) or {} + except Exception as exc: + _LOG.info("[LLM-sufficiency] judge_sufficiency failed: %s", exc) + return {} + + is_suff = bool(llm_result.get("is_sufficient") or llm_result.get("Sufficient Context")) + missing = [str(m) for m in (llm_result.get("missing_information") or llm_result.get("missing") or []) if str(m).strip()] + reasoning = (llm_result.get("reasoning") or "").strip() + # New Sufficient Context paper fields consumed by the decision ladder. + auto_confidence = _clamp(llm_result.get("confidence")) + contradictions = [str(c) for c in (llm_result.get("contradictions") or []) if str(c).strip()] + required_entities = [str(e) for e in (llm_result.get("required_entities") or []) if str(e).strip()] + coverage = llm_result.get("coverage") or {} + _LOG.info( + "[LLM-sufficiency] is_sufficient=%s confidence=%.2f contradictions=%s missing=%s reasoning=%s", + is_suff, + auto_confidence, + contradictions, + missing, + reasoning[:200], + ) + + followups: list[dict] = [] + if missing and hasattr(tools, "gen_followups"): + try: + followups = await tools.gen_followups(question, question, missing, evidence_md) or [] + except Exception as exc: + _LOG.info("[LLM-sufficiency] gen_followups failed: %s", exc) + + if followups: + _LOG.info("[LLM-sufficiency] %d follow-up query(ie)s generated for next round: %s", len(followups), [q.get("question", "") for q in followups]) + + return { + "is_sufficient": is_suff, + "confidence": auto_confidence, + "missing": missing, + "contradictions": contradictions, + "required_entities": required_entities, + "coverage": coverage, + "reasoning": reasoning, + "followups": followups, + } diff --git a/rag/advanced_rag/harness/prompts/decompose_prompts.py b/rag/advanced_rag/harness/prompts/decompose_prompts.py index db5e15548d..a4dbda1cd8 100644 --- a/rag/advanced_rag/harness/prompts/decompose_prompts.py +++ b/rag/advanced_rag/harness/prompts/decompose_prompts.py @@ -9,6 +9,22 @@ DECOMPOSE_FACTUAL = """This is a factual question. List all atomic facts that ne If there are multiple facts, list them one by one. If there is only one fact, output exactly one item. Base the decomposition on BOTH the question and the preliminary retrieved context below. +Relevance rules (strict): +- Every claim MUST be directly necessary to answer the question. Do NOT add + background, biographical, or tangential facts about entities merely mentioned + in the corpus (e.g. an unrelated person's birthplace) unless the question + explicitly asks for them. +- Prefer fewer, atomic claims over many overlapping ones: merge facts about the + same entity/measure into one claim when they can be researched together. +- Each claim must be verifiable from the corpus; do not invent facts. +- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target + (what/which/when/how many/what is X), NOT a pre-answered assertion. Never bake a + guessed value into the description. WRONG: "Dustin Brown's hometown is Ithaca". + RIGHT: "What is Dustin Brown's hometown?". The researcher verifies the OPEN target; + a pre-answered assertion biases it to merely confirm the guess and skips the real + answer. When a fact is genuinely unknown (e.g. a person's hometown), phrase the + claim as a question, never as "X is ". + Question: {question} Maximum number of claims: {max_claims} Detail level: {detail_level} @@ -21,7 +37,7 @@ Output format (JSON): "claims": [ {{ "claim_id": "c1", - "description": "The year Apple acquired Beats", + "description": "What year did Apple acquire Beats?", "priority": 1 }} ] @@ -35,6 +51,17 @@ DECOMPOSE_COMPARATIVE = """This is a comparative question. It needs to be decomp 3. Optional information that directly compares the two entities. Base the decomposition on BOTH the question and the preliminary retrieved context below. +Relevance rules (strict): +- Every claim MUST be directly needed to answer the comparison. Do NOT add + tangential background about the entities (e.g. a third party's unrelated + details) unless the question explicitly asks for them. +- Merge overlapping facts about the same entity into one claim. +- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target + (what/which/when/how many/what is X), NOT a pre-answered assertion. Never bake a + guessed value into the description. RIGHT: "What is the distance from Hangzhou to + Beijing?". WRONG: "The distance from Hangzhou to Beijing is 1,200 km". The + researcher verifies the OPEN target; an assertion only biases it to confirm the guess. + Question: {question} Maximum number of claims: {max_claims} Detail level: {detail_level} @@ -47,17 +74,17 @@ Output format (JSON): "claims": [ {{ "claim_id": "c1", - "description": "The distance from Hangzhou to Beijing", + "description": "What is the distance from Hangzhou to Beijing?", "priority": 1 }}, {{ "claim_id": "c2", - "description": "The distance from Shanghai to Beijing", + "description": "What is the distance from Shanghai to Beijing?", "priority": 1 }}, {{ "claim_id": "c3", - "description": "Which city is closer to Beijing", + "description": "Which city is closer to Beijing?", "priority": 2 }} ] @@ -68,6 +95,14 @@ Output format (JSON): DECOMPOSE_PROCEDURAL = """This is a procedural question. Decompose it into the information needed for each step required to complete the operation. Base the decomposition on BOTH the question and the preliminary retrieved context below. +Relevance rules (strict): +- Every claim MUST be information directly needed by one of the procedure's + steps. Skip any fact that is not required to carry out the operation. +- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target + (what/which/when/how/what is X), NOT a pre-answered assertion. Never bake a + guessed value into the description. RIGHT: "What is the input format for step 2?". + WRONG: "The input format for step 2 is CSV". + Question: {question} Maximum number of claims: {max_claims} Detail level: {detail_level} @@ -80,12 +115,12 @@ Output format (JSON): "claims": [ {{ "claim_id": "c1", - "description": "Information needed for the first step", + "description": "What information is needed for the first step?", "priority": 1 }}, {{ "claim_id": "c2", - "description": "Information needed for the second step", + "description": "What information is needed for the second step?", "priority": 2 }} ] @@ -96,6 +131,15 @@ Output format (JSON): DECOMPOSE_EXPLORATORY = """This is an analytical or exploratory question. Decompose it into the main aspects or dimensions that need to be researched. Base the decomposition on BOTH the question and the preliminary retrieved context below. +Relevance rules (strict): +- Every aspect/claim MUST be directly relevant to the question's core. Do NOT + list tangential or background aspects just because the corpus mentions them. +- Prefer fewer, well-scoped aspects over many overlapping ones. +- OPEN QUERY (CRITICAL): Each claim's description must be an OPEN research target + (what/which/when/how/what is X), NOT a pre-answered assertion. Never bake a + guessed value into the description. RIGHT: "What were the main factors behind X?". + WRONG: "The main factor behind X was Y". + Question: {question} Maximum number of claims: {max_claims} Detail level: {detail_level} @@ -108,12 +152,12 @@ Output format (JSON): "claims": [ {{ "claim_id": "c1", - "description": "The first aspect that needs to be researched", + "description": "What is the first aspect that needs to be researched?", "priority": 1 }}, {{ "claim_id": "c2", - "description": "The second aspect that needs to be researched", + "description": "What is the second aspect that needs to be researched?", "priority": 2 }} ] diff --git a/rag/advanced_rag/harness/prompts/report_prompt.py b/rag/advanced_rag/harness/prompts/report_prompt.py index e10b63549b..11a2ee0f90 100644 --- a/rag/advanced_rag/harness/prompts/report_prompt.py +++ b/rag/advanced_rag/harness/prompts/report_prompt.py @@ -12,6 +12,16 @@ obey it over any research-summary wording. # Citation rules {cite_rules} +# Attribute fidelity (CRITICAL) +Answer the EXACT attribute/relation the question asks for. Do NOT substitute a similar but +different attribute, even when it is semantically related. For example: +- HOMETOWN ≠ BIRTHPLACE (place of birth): if asked for someone's hometown, do not answer with + where they were born unless the evidence equates the two. +- FIRST ≠ LARGEST, AGE AT DEATH ≠ BIRTH YEAR, etc. +Answer the question's own attribute using the evidence for THAT attribute. If the evidence only +supports a different attribute, say that you could only find the related (different) attribute and +do not present it as the answer to the requested one. + # Language Answer in the SAME language as the question. Translate retrieved evidence into that language as part of composing the answer; only verbatim quoted snippets may stay in their source language. diff --git a/rag/advanced_rag/harness/prompts/research_agent_prompt.py b/rag/advanced_rag/harness/prompts/research_agent_prompt.py index 67bec12038..6bd81eca3c 100644 --- a/rag/advanced_rag/harness/prompts/research_agent_prompt.py +++ b/rag/advanced_rag/harness/prompts/research_agent_prompt.py @@ -26,6 +26,29 @@ Rules: 5. When you have gathered enough evidence, call generate_report with your findings (report, is_verified, confidence, evidence_ids, gaps, discovered_claims). +ATTRIBUTE FIDELITY (CRITICAL): +Answer the EXACT attribute/relation the research task asks for. Do NOT substitute a +similar but different attribute, even when the substitution is semantically related and +tempting. For example, if the task asks for a person's HOMETOWN, do not report their +BIRTHPLACE (born in) as the answer — these are different facts. Likewise do not silently +swap "first" for "largest", "age at death" for "birth year", etc. +- In SEARCH queries you MAY use synonymous, translated, or corpus-specific terms (e.g. + "hometown", "place of residence", "ville natale") to find the evidence — retrieval + benefits from flexible wording. What matters is that the terms still TARGET the exact + attribute the task asks for; do not steer the search at a different fact. +- The REPORT must state the exact attribute the task asked for. Never present a different + attribute's value as the answer. +- If the exact attribute cannot be found in the evidence, mark the claim unverified and + list it in gaps — do NOT answer with a different attribute's value. + +SOURCE ANCHORING (CRITICAL): +If the research task names a specific source (e.g. "In the Wikipedia article 'Demographics +of Paris', ..."), treat THAT named source as authoritative and retrieve the value from it. +Do NOT substitute another source's value (e.g. a statistical-agency estimate, a news site, +or a different webpage) as the answer. If the named source is found, its value wins. Only if +the named source cannot be located at all may you fall back, but then say so in the report +and set confidence accordingly. + You have at most {max_cycles} tool-calling rounds. Call exactly one tool per round, and do not write a plain-text answer until you call generate_report. """ @@ -49,6 +72,23 @@ Rules: 4. Use think_tool to analyze results after each step. 5. When you are confident enough to answer the research task, call generate_report. +ATTRIBUTE FIDELITY (CRITICAL): +Answer the EXACT attribute/relation the research task asks for. Do NOT substitute a similar +but different attribute (e.g. do not report HOMETOWN as BIRTHPLACE, do not swap "first" for +"largest", "age at death" for "birth year"). In SEARCH queries you MAY use synonymous, +translated, or corpus-specific terms (e.g. "hometown", "place of residence") as long as they +still TARGET the exact requested attribute — retrieval benefits from flexible wording. The +REPORT must state the exact attribute asked for and never present a different attribute's +value as the answer. If the exact attribute cannot be found in evidence, mark the claim +unverified and list it in gaps — do NOT answer with a different attribute's value. + +SOURCE ANCHORING (CRITICAL): +If the research task names a specific source (e.g. "In the Wikipedia article 'Demographics of +Paris', ..."), treat THAT named source as authoritative and retrieve the value from it. Do +NOT substitute another source's value (statistical-agency estimate, news site, other webpage) +as the answer. If the named source is found, its value wins. Only if it cannot be located may +you fall back, and then say so in the report and lower confidence. + Tool call format: output exactly one JSON tool call per round: {{"name": "tool_name", "arguments": {{"parameter_name": "value"}} }} diff --git a/rag/advanced_rag/harness/prompts/sufficiency_prompt.py b/rag/advanced_rag/harness/prompts/sufficiency_prompt.py deleted file mode 100644 index a941e62c47..0000000000 --- a/rag/advanced_rag/harness/prompts/sufficiency_prompt.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Sufficiency judge prompt: verdict with claim-level assessment.""" - -SUFFICIENCY_JUDGE_PROMPT = """You are an expert judge of information retrieval sufficiency. Decide whether the currently collected evidence is sufficient to answer the question. - -Question: {question} - -Claim-level evidence: -{evidence_summary} - -Judgment tasks: -1. Evaluate each claim one by one and decide whether it has been sufficiently verified. -2. Make an overall judgment about whether the evidence is sufficient to answer the user's question. -3. If it is not sufficient, provide targeted feedback. - -Output format (JSON): -{{ - "status": "SUFFICIENT" | "USEFUL_BUT_INCOMPLETE" | "INSUFFICIENT" | "UNANSWERABLE", - "score": 0.85, - "claim_assessments": [ - {{ - "claim_id": "c1", - "is_verified": true, - "confidence": 0.95, - "reason": "Consistent data was found in three chunks." - }} - ], - "missing": ["Some data for c2 was not found."], - "feedback": "Use web_search for c2 to supplement the latest data.", - "overall_reason": "The main facts are covered, but some details still need supplementation." -}} -""" diff --git a/rag/advanced_rag/harness/route.py b/rag/advanced_rag/harness/route.py index d3d4cbb7bf..641c83b5ab 100644 --- a/rag/advanced_rag/harness/route.py +++ b/rag/advanced_rag/harness/route.py @@ -12,19 +12,31 @@ _LOG = logging.getLogger(__name__) def _extract_json(text: str) -> dict: - """Extract JSON from LLM response, handling markdown fences and think tags.""" + """Extract a JSON *object* from the LLM response. + + Handles markdown fences and think tags, then attempts ``json_repair`` + followed by ``json.loads``. Robustness: the LLM can return a bare string, + a list, or a JSON primitive — anything that is not a ``dict`` is coerced to + ``{}`` (with a warning) so callers can safely use ``result.get(...)``. This + guards against the crash seen in check.log where ``route_node`` received a + string and ``result.get("question_type")`` raised AttributeError. + """ text = re.sub(r"^.*", "", text, flags=re.DOTALL).strip() text = re.sub(r"```(?:json)?\s*|\s*```", "", text).strip() try: import json_repair - return json_repair.loads(text) + parsed = json_repair.loads(text) except Exception: try: - return json.loads(text) + parsed = json.loads(text) except Exception: _LOG.warning("route: failed to parse LLM output: %s", text[:200]) return {} + if not isinstance(parsed, dict): + _LOG.warning("route: LLM returned non-object JSON (%s), coercing to {}: %s", type(parsed).__name__, str(parsed)[:200]) + return {} + return parsed async def route_node(state: dict, tools) -> dict: @@ -47,6 +59,12 @@ async def route_node(state: dict, tools) -> dict: _LOG.exception("route_node failed") result = {} + # Belt-and-suspenders: _extract_json already coerces to dict, but guard + # here too so a future code path can never crash on result.get(). + if not isinstance(result, dict): + _LOG.warning("route_node: result not a dict (%s); falling back to defaults", type(result).__name__) + result = {} + question_type = result.get("question_type", "factual") requires_decomp = result.get("requires_decomposition", True) suggests_comp = result.get("suggests_compilation") diff --git a/rag/advanced_rag/harness/sufficiency.py b/rag/advanced_rag/harness/sufficiency.py index 15b9462a43..54931c8b02 100644 --- a/rag/advanced_rag/harness/sufficiency.py +++ b/rag/advanced_rag/harness/sufficiency.py @@ -9,6 +9,11 @@ from rag.advanced_rag.harness.types import ( ExecutionStrategy, ) from rag.advanced_rag.harness.config import get_mode +from rag.advanced_rag.harness.sufficiency_ladder import ( + ANSWER_WITH_CAVEAT, + RECONCILE, + UNANSWERABLE, +) _LOG = logging.getLogger(__name__) @@ -25,18 +30,342 @@ def extract_numbers(text: str) -> list[float]: return [float(m) for m in re.findall(r"\d+\.?\d*", text)] +def _filter_relevant_numbers(numbers: list[float]) -> list[float]: + """Drop numbers that carry no factual-claim signal. + + - Values in ``[0, 1]`` are overwhelmingly ratios / probabilities / + confidence scores the agent sprinkled into its prose, not facts to + verify against the evidence. + - Drop duplicates: "48 m" appearing three times should be checked once. + """ + kept: list[float] = [] + for n in numbers: + if 0.0 < n < 1.0: + continue + if n in kept: + continue + kept.append(n) + return kept + + +# ── Multilingual named-entity extraction ────────────────────────────── +# Cross-check previously only recognized English capitalized sequences, so +# Chinese/Japanese/Korean reports yielded zero entities and could never be +# verified. We now detect the report's language and route: +# - en/zh/de/fr/es/pt/ja: spaCy NER (models pre-loaded at build time; see +# pyproject.toml). +# - Any other language: langdetect returns the code; if no spaCy model is +# mapped the report still degrades gracefully to a no-op (never crashes). +# Language is detected with ``langdetect`` (Bayesian N-gram profiles, pure +# Python, 55+ languages — a product-grade alternative to hand-rolled +# heuristics), with a small Unicode-range heuristic as offline fallback. +# The spaCy pipeline is lazy-loaded once per process (singleton cache, like +# lightgraph) and degrades gracefully if a model is unavailable, so +# sufficiency never crashes on a missing model. + +# RAGFlow language label / ISO code → spaCy model (mirrors lightgraph). +_LANG_TO_SPACY_MODEL = { + "en": "en_core_web_sm", + "english": "en_core_web_sm", + "zh": "zh_core_web_sm", + "chinese": "zh_core_web_sm", + "zh-cn": "zh_core_web_sm", + "de": "de_core_news_sm", + "german": "de_core_news_sm", + "fr": "fr_core_news_sm", + "french": "fr_core_news_sm", + "es": "es_core_news_sm", + "spanish": "es_core_news_sm", + "pt": "pt_core_news_sm", + "portuguese": "pt_core_news_sm", + "ja": "ja_core_news_sm", + "japanese": "ja_core_news_sm", +} + +# spaCy NER labels that are not "evidence" (numbers, time, percentages). +_SPACY_SKIP_LABELS = {"ORDINAL", "CARDINAL", "DATE", "TIME", "PERCENT", "MONEY", "QUANTITY"} + +_spacy_nlp_cache: dict = {} + + +def _is_cjk(text: str) -> bool: + """True if ``text`` contains Chinese/Japanese characters.""" + return any("\u4e00" <= ch <= "\u9fff" for ch in text) + + +def _detect_language(text: str) -> str: + """Detect the report's language and return an ISO 639-1 code. + + Primary path uses ``langdetect`` (Bayesian N-gram profile classifier, + pure Python, ~55 languages). If langdetect is unavailable (or returns + nothing reliable for a very short / punctuation-only string), we fall + back to a small Unicode-range heuristic so the pipeline still works + offline without the extra dependency. + """ + if not text: + return "en" + try: + from langdetect import detect, DetectorFactory + + # langdetect's global DetectorFactory is randomly seeded by default, so + # ambiguous/short inputs can flip between runs. Pin the seed once so + # detection (and therefore the spaCy NER model selected) is deterministic. + DetectorFactory.seed = 0 + lang = detect(text[:500]) # long reports don't need the full body + if lang and lang != "unknown": + return lang + except Exception as exc: # langdetect not installed / detection error + _LOG.info( + "[multilingual-ner] langdetect unavailable/failed, falling back to Unicode heuristic: %s", + exc, + ) + # Offline heuristic fallback. + if any("\u4e00" <= ch <= "\u9fff" for ch in text): + # Distinguish zh vs ja by presence of hiragana/katakana. + if any("\u3040" <= ch <= "\u30ff" for ch in text): + return "ja" + return "zh" + if any("\uac00" <= ch <= "\ud7af" for ch in text): + return "ko" + return "en" + + +def _resolve_spacy_model(language: str) -> str: + key = (language or "en").strip().lower() + return _LANG_TO_SPACY_MODEL.get(key, "en_core_web_sm") + + +def _spacy_ner_entities(text: str, language: str) -> list[str]: + """Extract named entities via spaCy NER, with per-process model caching. + + Returns [] on any failure (missing model, spacy import error) so the caller + can fall back to the regex path without crashing. + """ + model_name = _resolve_spacy_model(language) + if model_name in _spacy_nlp_cache: + nlp = _spacy_nlp_cache[model_name] + else: + try: + import spacy + + nlp = spacy.load(model_name) + _spacy_nlp_cache[model_name] = nlp + except Exception as exc: # model not installed / spacy unavailable + _LOG.info("[multilingual-ner] spaCy model %s unavailable, falling back to regex: %s", model_name, exc) + return [] + if nlp is None: + return [] + try: + doc = nlp(text) + except Exception as exc: + _LOG.info("[multilingual-ner] spaCy inference failed for %s: %s", model_name, exc) + return [] + seen: set[str] = set() + out: list[str] = [] + for ent in doc.ents: + label = ent.label_ or "" + if label in _SPACY_SKIP_LABELS: + continue + name = ent.text.strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out + + def extract_named_entities(text: str) -> list[str]: - """Simple entity extraction — looks for capitalized multi-word sequences.""" - entities = re.findall(r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b", text) - return list(set(entities)) + """Extract named entities via spaCy NER for any configured language. + + All languages (en/zh/de/fr/es/pt/ja) go through the spaCy NER pipeline — + no hand-rolled regexes or stopword lists. spaCy only returns recognized + named entities (PER/LOC/ORG/NORP/MISC/...), so connective tissue never + appears. Number/time/percentage labels are filtered (they are verified + separately via ``extract_numbers``). Returns [] if spaCy is unavailable. + """ + if not text: + return [] + lang = _detect_language(text) + return _spacy_ner_entities(text, lang) + + +def _detect_numeric_conflict(disclosed: list[str]) -> list[str]: + """Detect close-but-different figures among the agent's disclosed numbers. + + Extracts the leading number from each disclosure entry (e.g. "2,161,000 from + Wikipedia ..." → 2161000) and flags pairs that are numerically *close but not + equal* — the signature of a multi-source口径 conflict (same quantity, different + figure), which Q754 hit (population 2,161,000 vs 2,145,906 → 228 vs 227). + Numbers that are far apart are likely different quantities and not flagged. + """ + import re as _re + + figures: list[tuple[float, str]] = [] + for entry in disclosed: + m = _re.search(r"([\d][\d,]*(?:\.\d+)?)", entry) + if not m: + continue + try: + fig = float(m.group(1).replace(",", "")) + except ValueError: + continue + figures.append((fig, entry[:80])) + + conflicts: list[str] = [] + for i in range(len(figures)): + for j in range(i + 1, len(figures)): + a, b = figures[i][0], figures[j][0] + if a <= 0 or b <= 0: + continue + ratio = max(a, b) / min(a, b) + # Close (within 30%) but not equal → same quantity, conflicting value. + if 1 < ratio <= 1.3: + conflicts.append(f"{figures[i][1]} vs {figures[j][1]}") + return conflicts + + +# Attribute/relationship descriptors: content words in a grounded fact that are +# neither named entities nor numbers, e.g. "hometown", "captain", "born", "age". +# Their presence in the evidence is what makes a grounded assertion authoritative +# (the fact's *relationship*, not just its entities, must be supported). +_PREDICATE_STOP = { + "the", + "a", + "an", + "of", + "in", + "on", + "at", + "to", + "for", + "and", + "or", + "is", + "was", + "are", + "were", + "by", + "with", + "from", + "his", + "her", + "their", + "its", + "that", + "this", + "he", + "she", + "they", + "it", + "also", + "as", + "when", + "who", + "what", + "which", + "there", + "have", + "has", + "had", + "be", + "been", + "being", +} + + +def _predicate_terms(fact: str, excluded: list[str]) -> list[str]: + """Return the predicate/attribute descriptors of ``fact`` (lowercased, ≤4 tokens). + + These are the content words that carry the relationship being asserted, + excluding proper entities (in ``excluded``) and common stop words. For + "hometown is Ithaca" → ["hometown"]; for "Dustin Brown captain of Los Angeles + Kings" → ["captain"]. Used to verify the fact's relationship is evidenced. + """ + excluded_lower = {e.lower() for e in excluded} + tokens = re.findall(r"[A-Za-z][A-Za-z'-]{1,19}", fact.lower()) + out: list[str] = [] + for tok in tokens: + if tok in excluded_lower or tok in _PREDICATE_STOP or tok in out: + continue + out.append(tok) + return out[:4] + + +def _entity_present(ent: str, chunk_texts: list[str]) -> bool: + """Whether ``ent`` (lowercased) appears in any of the evidence chunk texts. + + CJK entities have no word boundaries (every Han char is ``\\w``) and are + commonly followed by function words ("的/是"), so they use a substring + match. Non-CJK entities use a bounded word/phrase match (Ann must not match + Annual). Mirrors the matching inside ``cross_check_claim``. + """ + if _is_cjk(ent): + return any(ent.lower() in t for t in chunk_texts) + return any(re.search(rf"(? dict[str, list[str]]: + """Find per-claim "required entities" missing from the evidence. + + Aligns with the Sufficient Context paper (arXiv 2411.06037): sufficiency is + anchored on *what the question needs*, not on what the agent *claims* in its + report. Extracting entities from the report is vulnerable to the agent + back-filling facts from prior knowledge (check1.log Q2: the agent injected + Tyson Fury's "age 35" from memory while the corpus had no Fury data, so the + report-based cross-check passed on Mike Tyson's numbers that padded the + score). Instead we extract entities from the *claim description + question* + (what the answer actually requires) and check each against the evidence. + + Returns {claim_id: [missing entities]}. A claim whose required entity is + absent from every cited/evidence chunk is flagged — under AND semantics one + missing required entity means that part of the question is unsupported. + """ + # Union of all evidence chunk texts (independent of any claim's citation). + chunk_texts: list[str] = [] + for chunk in (all_chunks or {}).values(): + text = chunk.get("content_with_weight") or chunk.get("text") or "" + if text: + chunk_texts.append(text.lower()) + + # Pre-extract question entities once (shared across claims) + seed per-claim. + q_entities = extract_named_entities(question or "") + gaps: dict[str, list[str]] = {} + for claim in claims or []: + cid = getattr(claim, "claim_id", None) + desc = getattr(claim, "description", None) or "" + if not cid or not desc: + continue + # Required entities = those in the claim description, plus any + # question-level entities also mentioned by the claim (so a composite + # question like "Mike Tyson AND Tyson Fury" keeps each named entity + # individually accountable). + desc_entities = extract_named_entities(desc) + desc_lower = {e.lower() for e in desc_entities} + required = [e for e in desc_entities] + [e for e in q_entities if e.lower() in desc_lower] + missing = [e for e in required if not _entity_present(e, chunk_texts)] + if missing: + gaps[cid] = missing + return gaps def cross_check_claim(agent_result: AgentResult, all_chunks: dict) -> ClaimCrossCheckResult: """Code-level cross-check: number matching + entity presence.""" report = agent_result.report claimed = agent_result.is_verified + _LOG.info( + "[Cross-check] claim=%s entering — self_verified=%s, report_len=%d, evidence_ids=%s", + agent_result.claim_id, + claimed, + len(report or ""), + agent_result.evidence_ids, + ) if not claimed: + _LOG.info("[Cross-check] claim=%s → FAILED (agent self-reported as unverified, score=0.0)", agent_result.claim_id) return ClaimCrossCheckResult( claim_id=agent_result.claim_id, cross_check_passed=False, @@ -44,44 +373,167 @@ def cross_check_claim(agent_result: AgentResult, all_chunks: dict) -> ClaimCross mismatches=["agent self-reported as unverified"], ) - numbers = extract_numbers(report) + raw_numbers = extract_numbers(report) + numbers = _filter_relevant_numbers(raw_numbers) entities = extract_named_entities(report) + _LOG.info( + "[Cross-check] claim=%s extracted %d raw number(s) → %d relevant (noise filtered/deduped): %s, %d entity(ies)=%s from report", + agent_result.claim_id, + len(raw_numbers), + len(numbers), + numbers[:8], + len(entities), + entities[:8], + ) - mismatches = [] - matches = [] - + # Existence check across the *union* of evidence chunks, not per-chunk. + # Verifying "does this fact appear anywhere in the cited evidence" is the + # right semantic — a number/entity supported by one chunk is verified. + # The old per-chunk loop demanded a fact appear in EVERY evidence chunk, + # so a fact confirmed in chunk A was recorded as a mismatch in chunks B..N + # and the score was diluted to near-zero even when the answer was correct + # (see benchmark/3.log: c4's 48/157/27/89 all matched chunk 0 yet scored + # 0.286; c5's 21/27/48/89 matched chunk 3 yet scored 0.168). + chunk_texts: list[str] = [] + missing_ids: list[str] = [] for eid in agent_result.evidence_ids or []: chunk = all_chunks.get(eid) if not chunk: - mismatches.append(f"evidence_id={eid}: chunk not found") + missing_ids.append(str(eid)) continue - text = chunk.get("content_with_weight", chunk.get("text", "")) - text_lower = text.lower() + chunk_texts.append((chunk.get("content_with_weight") or chunk.get("text") or "").lower()) + if missing_ids: + _LOG.info("[Cross-check] claim=%s %d evidence_id(s) MISSING from pool (index drift?): %s", agent_result.claim_id, len(missing_ids), missing_ids[:5]) - for num in numbers: - # Numbers are extracted as floats ("1976" -> 1976.0) while chunk - # text spells them "1976" — match both the raw and integral forms. - # Bounded match: a number must not sit adjacent to other digit - # characters, so 1976 does not match inside 19760. - forms = {str(num), str(int(num))} if float(num).is_integer() else {str(num)} - if any(re.search(rf"(? bool: + f = fact.lower() + if _is_cjk(f): + return any(f in t for t in chunk_texts) + # Key tokens = named entities + numbers mentioned in the fact. + key_tokens = [tok for tok in extract_named_entities(fact)] + key_tokens += [f"{int(n)}" for n in _filter_relevant_numbers(extract_numbers(fact))] + if not key_tokens: + # No extractable key content — fall back to whole-phrase match. + return any(re.search(rf"(?= 0.5 + + grounded_facts = [str(g) for g in (agent_result.grounded or []) if str(g).strip()] + if grounded_facts: + ungounded = [g for g in grounded_facts if not _grounded_hit(g)] + if ungounded: + _LOG.warning( + "[Cross-check] claim=%s → GROUNDED-FACT VIOLATION: %d key fact(s) agent marked as evidence-backed are ABSENT from evidence: %s — hard-failing claim", + agent_result.claim_id, + len(ungounded), + ungounded[:6], + ) + return ClaimCrossCheckResult( + claim_id=agent_result.claim_id, + cross_check_passed=False, + cross_check_score=0.0, + mismatches=[f"grounded fact not in evidence: {g}" for g in ungounded], + ) + + # ── Numeric multi-source conflict detection (Q754: 225 vs 228口径) ── + # The agent may have used one figure while the evidence holds another value + # for the same quantity (e.g. Paris population from INSEE vs Wikipedia vs a + # news estimate). A ratio/mean cross-check passes as long as the used number + # matches somewhere, hiding the fact that a DIFFERENT authoritative figure + # exists. When the report discloses several distinct figures that are close + # but not equal (the classic "multiple sources, pick one" trap), surface them + # as conflicts and cap the claim below the pass floor so the caller does not + # blindly accept one口径. + disclosed = [str(n) for n in (agent_result.numbers or []) if str(n).strip()] + if disclosed: + conflict = _detect_numeric_conflict(disclosed) + if conflict: + _LOG.warning( + "[Cross-check] claim=%s → NUMERIC CONFLICT: multiple close-but-different figures for the same quantity: %s — capping below pass", + agent_result.claim_id, + conflict, + ) + return ClaimCrossCheckResult( + claim_id=agent_result.claim_id, + cross_check_passed=False, + cross_check_score=0.0, + mismatches=[f"numeric source conflict: {c}" for c in conflict], + ) + + def _anywhere(needle: str) -> bool: + return any(re.search(needle, t) for t in chunk_texts) + + matches: list[str] = [] + mismatches: list[str] = [] + for num in numbers: + # Numbers are extracted as floats ("1976" -> 1976.0) while chunk text + # spells them "1976" — match both raw and integral forms. Bounded so a + # number does not match inside a longer digit run (1976 vs 19760). + forms = {str(num), str(int(num))} if float(num).is_integer() else {str(num)} + found = any(_anywhere(rf"(? ClaimCross mismatches=["no evidence"], ) # Evidence IDs exist but nothing extractable to verify against (e.g. - # Chinese reports yield no capitalized entities and no digits) — the - # cross-check cannot falsify the claim, so pass neutrally. + # Chinese reports yield no capitalized entities and no digits). We + # cannot confirm OR falsify — score it neutral (0.5) and do NOT mark it + # passed. The old "pass neutrally with score=1.0" treated unverifiable + # claims as fully verified, which let any entity-free, digit-free report + # sail through as SUFFICIENT. + _LOG.info( + "[Cross-check] claim=%s → NEUTRAL (evidence ids exist but nothing extractable to verify, score=0.5, not passed)", + agent_result.claim_id, + ) return ClaimCrossCheckResult( claim_id=agent_result.claim_id, - cross_check_passed=True, - cross_check_score=1.0, + cross_check_passed=False, + cross_check_score=0.5, + mismatches=["nothing extractable to cross-check"], ) cross_score = len(matches) / total - cross_passed = len(mismatches) < len(matches) * 0.5 + # Pass when at least half the checked facts (numbers + entities) are + # confirmed in the evidence. The old ``mismatch < match*0.5`` required a + # 2/3 match rate and treated a single spurious number as fatal, which + # systematically failed otherwise-correct claims (see benchmark/2.log). + cross_passed = cross_score >= 0.5 + _LOG.info( + "[Cross-check] claim=%s → %s (%d/%d matched, score=%.3f, pass>=0.50). matches=%s mismatches=%s", + agent_result.claim_id, + "PASSED" if cross_passed else "FAILED", + len(matches), + total, + cross_score, + matches[:5], + mismatches[:5], + ) return ClaimCrossCheckResult( claim_id=agent_result.claim_id, @@ -117,52 +591,178 @@ def compute_fusion_score( agent_results: list[AgentResult], cross_check_results: list[ClaimCrossCheckResult], mode: ExecutionStrategy, + question: str = "", + claims: list | None = None, + all_chunks: dict | None = None, ) -> SufficiencyVerdict: - """Dual-signal fusion: agent confidence + cross-check pass rate.""" - # Signal A: agent self-assessment - verified_count = sum(1 for r in agent_results if r.is_verified) - agent_score = verified_count / max(len(agent_results), 1) + """Extract sufficiency *signals* — hard vetoes, agent confidence, conflicts. - # Signal B: cross-check - passed_count = sum(1 for r in cross_check_results if r.cross_check_passed) - cross_score = passed_count / max(len(cross_check_results), 1) + This is no longer a weighted fusion. The LLM AutoRater (invoked by the + orchestrator via ``llm_sufficiency_boost``) is the primary sufficiency + judge; this function only produces the *code-level* inputs the decision + ladder consumes: + - hard_violations: claims with a proven evidence gap (required entity + missing / grounded absent / numeric conflict) that must veto "good + enough" even if the AutoRater says sufficient; + - agent_confidence: mean self-confidence over the trusted subset; + - has_conflicts / missing_claims: surfaced for the ladder / caveat. - # Fusion strategy by mode - fusion = { - "ultra": lambda a, c: min(a, c), - "high": lambda a, c: (a + c) / 2, - "medium": lambda a, c: max(a, c), - "low": lambda a, c: max(a, c), - }.get(mode.label, lambda a, c: max(a, c)) - fusion_score = fusion(agent_score, cross_score) + ``question`` / ``claims`` / ``all_chunks`` are optional; when provided they + drive the *required-entity* AND-semantics veto from the Sufficient Context + paper (anchored on what the question needs, not what the agent claims), plus + suppression of self-confidence for claims whose required entities are + missing from the evidence. + """ + # ── Required-entity gaps (Sufficient Context paper, AND semantics) ── + # Anchored on *what the question needs* rather than what the agent claims + # (see required_entity_gaps). Any claim whose required entity is absent from + # the evidence flags a localized gap: under AND semantics the whole verdict + # must not be SUFFICIENT, and the agent's self-confidence for that claim is + # suppressed (selective generation: confidence must not override missing + # evidence). Computed once here so it drives both Signal A and the veto. + required_gaps: dict[str, list[str]] = {} + if question or claims: + required_gaps = required_entity_gaps(question, claims, all_chunks) + if required_gaps: + _LOG.info( + "[Sufficiency] Required-entity gaps (AND semantics): %s", + {k: v for k, v in required_gaps.items()}, + ) + gapped_ids = set(required_gaps.keys()) - # Conflict detection - has_conflicts = any(len(r.mismatches) > 0 for r in cross_check_results) + # Signal A: agent self-assessed confidence (continuous, per design doc). + # Only self-verified claims count toward "agent is confident" — an + # unverified claim's confidence is not trustworthy. This replaces the old + # boolean pass-rate (verified_count / n) which inflated the score to 1.0 + # whenever the agent merely said "verified" (benchmark/2.log showed + # confidence 0.5-0.7 being reported as agent_score=1.0). + # + # A claim whose REQUIRED entity is missing from the evidence must not lend + # its self-confidence to Signal A — the agent is confident about a fact the + # corpus cannot support (prior-knowledge back-fill, see check1.log Q2). Its + # confidence is zeroed for the mean, so self-assessed confidence can never + # mask an evidence gap. + verified = [r for r in agent_results if r.is_verified and r.claim_id not in gapped_ids] + suppressed = [r.claim_id for r in agent_results if r.is_verified and r.claim_id in gapped_ids] + verified_count = len(verified) + agent_score = sum(r.confidence for r in verified) / verified_count if verified_count else 0.0 + if suppressed: + _LOG.info( + "[Sufficiency] Signal A: suppressed %d self-verified claim(s) with missing required entities (confidence zeroed): %s", + len(suppressed), + suppressed, + ) + _LOG.info( + "[Sufficiency] Signal A (self): %d/%d claims self-verified (and evidence-backed), mean confidence → agent_score=%.3f (raw confidence values=%s)", + verified_count, + len(agent_results), + agent_score, + [round(r.confidence, 3) for r in agent_results], + ) - # 5-way verdict - if has_conflicts and fusion_score < mode.partial_threshold: - status = "CONFLICTING" - elif fusion_score >= mode.sufficiency_threshold: - status = "SUFFICIENT" - elif fusion_score >= mode.partial_threshold: - status = "USEFUL_BUT_INCOMPLETE" - elif not any(r.cross_check_passed for r in cross_check_results): + # Signal B: cross-check score (continuous match rate), per design doc. + # Uses each claim's actual cross_check_score rather than a boolean + # pass/fail count, so partial-but-real evidence (e.g. 0.6) contributes + # proportionally instead of being zeroed. + # + # Unrelated-claim pollution (see 6.log/7.log): the planner occasionally + # invents a claim with no bearing on the question (e.g. "Suharto was born + # in Kemusuk" while the question asks about a mosque's heights). The agent + # finds no evidence for it (cross_check_score≈0), fails the cross-check, + # AND self-reports it as unverified. Such a claim drags Signal B down and + # pushes an otherwise sufficient fusion into the critical band that + # needlessly triggers the LLM fallback. We exclude "unanswerable + agent + # self-unverified" claims from the mean — they neither help nor should + # punish the verdict. We key on the agent's is_verified flag (not the + # confidence threshold) because a claim the agent itself calls unverified + # is the strongest unrelated/ungrounded signal (7.log's c2 reported + # confidence 0.35 but self-flagged unverified, which a confidence<0.2 + # threshold would have missed). + cross_results = list(cross_check_results) + noise_threshold = 0.2 + agent_verified = {r.claim_id: r.is_verified for r in agent_results} + noise_ids = [r.claim_id for r in cross_results if r.cross_check_score < noise_threshold and not r.cross_check_passed and not agent_verified.get(r.claim_id, False)] + kept = [r for r in cross_results if r.claim_id not in noise_ids] + if noise_ids and kept: + _LOG.info( + "[Sufficiency] Excluding %d unrelated/unverifiable claim(s) from Signal B: %s (cross<%.2f AND agent self-unverified)", + len(noise_ids), + noise_ids, + noise_threshold, + ) + cross_results = kept + + # ── Hard-veto floor: a *localized* evidence gap must veto "good enough" ── + # Multi-claim questions (e.g. "Mike Tyson AND Tyson Fury") can average a + # genuinely weak claim up to the sufficient band. Q2 (check.log): c4/c5's + # "Tyson Fury" / "Usyk" entities matched 0 chunks, yet the 5-claim mean + # cross_check_score=0.822 crossed the threshold and produced an answer that + # could not back the Fury half. These claims become ``hard_violations`` that + # force the decision ladder to a caveated answer even if the LLM AutoRater + # says sufficient (code-proven evidence gap beats "roughly good enough"). + min_cross_floor = getattr(mode, "fusion_min_cross", 0.5) or 0.5 + self_verified_ids = {r.claim_id for r in agent_results if r.is_verified} + weak = [r.claim_id for r in cross_results if r.claim_id in self_verified_ids and r.cross_check_score < min_cross_floor] + # AND-semantics required-entity veto: a claim missing a required entity + # (even if its report-based cross-check passed — the numbers matched on + # unrelated padding) must veto. This catches the Q2 case that min-cross + # alone missed: c2/c3 self-verified with score 0.83/0.80 (padded by Mike + # Tyson's digits) yet Tyson Fury's entities were absent from every chunk. + weak += [cid for cid in gapped_ids if cid not in weak] + if weak: + _LOG.info( + "[Sufficiency] Hard-veto: %d self-verified claim(s) below floor %.2f OR missing a required entity: %s", + len(weak), + min_cross_floor, + weak, + ) + + # Conflict detection — based on the kept (non-noisy) claims so an + # unrelated claim's mismatches don't manufacture a conflict. + has_conflicts = any(len(r.mismatches) > 0 for r in cross_results) + _LOG.info("[Sufficiency] Conflict detection: has_conflicts=%s", has_conflicts) + + # Cross-check status (code-only view, no AutoRater). This is a *preliminary* + # label used by the orchestrator to decide whether to call the LLM AutoRater + # (medium triggers it only in the borderline band); the final decision comes + # from the decision ladder with the AutoRater's verdict. + if not any(r.cross_check_passed for r in cross_results): status = "UNANSWERABLE" - else: + elif has_conflicts: + status = "CONFLICTING" + elif weak: status = "INSUFFICIENT" + elif agent_score >= mode.sufficiency_threshold: + status = "SUFFICIENT" + else: + status = "USEFUL_BUT_INCOMPLETE" + _LOG.info( + "[Sufficiency] Code-level status=%s (agent_conf=%.3f, conflicts=%s, hard_veto=%s)", + status, + agent_score, + has_conflicts, + bool(weak), + ) - missing = [r.claim_id for r in cross_check_results if not r.cross_check_passed] + missing = [r.claim_id for r in cross_results if not r.cross_check_passed] + # Excluded unrelated claims are surfaced (not silently dropped) so the + # caller knows the planner invented unanswerable claims. + missing += noise_ids + # Thin-evidence claims that vetoed are surfaced too, so the caller sees + # exactly which part of the question still lacks support. + missing += [c for c in weak if c not in missing] return SufficiencyVerdict( status=status, - score=fusion_score, - agent_score=agent_score, - cross_score=cross_score, - claim_assessments=[{"claim_id": r.claim_id, "is_verified": r.cross_check_passed, "score": r.cross_check_score, "mismatches": r.mismatches} for r in cross_check_results], + # Reference score for logging/monitoring only — the final decision comes + # from the decision ladder, not from this scalar. + score=agent_score, + claim_assessments=[{"claim_id": r.claim_id, "is_verified": r.cross_check_passed, "score": r.cross_check_score, "mismatches": r.mismatches} for r in cross_results], has_conflicts=has_conflicts, missing_claims=missing, - feedback=_build_feedback(missing, cross_check_results), - overall_reason=_format_reason(status, fusion_score, missing), + feedback=_build_feedback(missing, cross_results), + hard_violations=weak, + agent_confidence=agent_score, ) @@ -181,38 +781,71 @@ def _build_feedback(missing: list[str], results: list[ClaimCrossCheckResult]) -> return "missing: " + "; ".join(hints) -def _format_reason(status: str, score: float, missing: list[str]) -> str: - return f"{status} score={score:.2f} missing={missing}" +def route_sufficiency_verdict( + verdict: SufficiencyVerdict, + mode_label: str, + cycle: int, + max_cycles: int, + auto: dict | None = None, +) -> tuple: + """Decision-ladder routing → (action, should_continue, caveat). - -def route_sufficiency_verdict(verdict: SufficiencyVerdict, mode_label: str, cycle: int, max_cycles: int) -> tuple: - """Return (action, should_continue).""" + The LLM AutoRater (``auto``) is the primary sufficiency judge; the agent + confidence (``verdict.agent_confidence``) is the risk gate. ``auto`` is the + dict returned by ``llm_sufficiency_boost`` (``is_sufficient`` / + ``confidence`` / ``missing`` / ``contradictions``). When absent (e.g. the + medium mode did not trigger the AutoRater, or the tools lack an LLM judge), + we fall back to a code-only decision so the loop still terminates sensibly. + """ mode = get_mode(mode_label) + hard_violations = getattr(verdict, "hard_violations", []) or [] + agent_confidence = getattr(verdict, "agent_confidence", getattr(verdict, "score", 0.0)) - if verdict.status == "SUFFICIENT": - return ("ANSWER", False) + # AutoRater signals, with sane defaults when it was not invoked. + auto_sufficient = bool(auto.get("is_sufficient")) if auto else (verdict.status == "SUFFICIENT") + auto_confidence = float(auto.get("confidence") or 1.0) if auto else 1.0 + missing = list(auto.get("missing") or []) if auto else verdict.missing_claims + contradictions = list(auto.get("contradictions") or []) if auto else ([verdict.feedback] if verdict.has_conflicts else []) - if verdict.status == "USEFUL_BUT_INCOMPLETE": - if mode.requires_selective_gen: - return ("ANSWER_PARTIAL", False) - return ("CONTINUE", False) + from rag.advanced_rag.harness.sufficiency_ladder import sufficiency_ladder - if verdict.status == "INSUFFICIENT": - # ``cycle`` is 0-based, so the last cycle is ``max_cycles - 1`` — the - # old ``max_cycles * 0.8`` threshold was never reached for the 3/3/4 - # cycle budgets, making this branch dead code. - if cycle >= max_cycles - 1: - return ("ANSWER_PARTIAL", False) - return ("CONTINUE", True) + out = sufficiency_ladder( + auto_sufficient=auto_sufficient, + auto_confidence=auto_confidence, + missing=missing, + contradictions=contradictions, + agent_confidence=agent_confidence, + c_high=mode.c_high, + c_low=mode.c_low, + llm_floor=mode.llm_floor, + allows_reconcile=mode.allows_reconcile, + cycle=cycle, + max_cycles=max_cycles, + hard_violations=hard_violations, + ) + _LOG.info( + "[Sufficiency ladder] auto_sufficient=%s auto_conf=%.2f agent_conf=%.2f hard_violations=%s → %s", + auto_sufficient, + auto_confidence, + agent_confidence, + hard_violations, + out.action, + ) - if verdict.status == "CONFLICTING": - if mode.allows_replan and cycle < max_cycles * 0.5: - return ("REPLAN", True) - return ("ANSWER_PARTIAL", False) - - if verdict.status == "UNANSWERABLE": + # Map ladder action onto orchestrator actions. + action = out.action + if action == ANSWER_WITH_CAVEAT: + action = "ANSWER_PARTIAL" + elif action == RECONCILE: + # medium has no reconcile loop → degrade to CONTINUE (keep searching). + if mode.allows_reconcile: + return ("CONTINUE", True, out.caveat) + return ("CONTINUE", True, out.caveat) + elif action == UNANSWERABLE: if mode.fallback_to_direct_llm: - return ("FALLBACK_LLM", False) - return ("ABSTAIN", False) - - return ("CONTINUE", True) + action = "FALLBACK_LLM" + else: + action = "ABSTAIN" + return (action, False, out.caveat) + # ANSWER / GAP + return (action, out.should_continue, out.caveat) diff --git a/rag/advanced_rag/harness/sufficiency_ladder.py b/rag/advanced_rag/harness/sufficiency_ladder.py new file mode 100644 index 0000000000..cb38fb5aaf --- /dev/null +++ b/rag/advanced_rag/harness/sufficiency_ladder.py @@ -0,0 +1,176 @@ +"""Decision ladder for sufficiency — LLM AutoRater as primary judge, agent +confidence as a risk gate (Sufficient Context paper, arXiv 2411.06037). + +This replaces the old "code cross-check + dual-signal weighted fusion + 5-way +verdict" pipeline. Because the operator cannot train/calibrate fusion weights +(the paper's logistic regression), we use a **monotonic threshold ladder**: the +LLM AutoRater decides *whether the evidence is sufficient to infer a plausible +answer*; the agent's self-assessed confidence only modulates *how the verdict is +presented* (full answer vs. caveated answer vs. reconcile). The only continuous +degree of freedom is conservativeness thresholds (``c_high`` / ``c_low`` / +``llm_floor``), which are product policy knobs, not learned parameters. + +Semantics (paper §3 / §5.1): + - Sufficiency = "a plausible answer A' can be inferred from the context", + independent of whether that answer is correct, and without a ground truth. + - Selective generation = confidence does not re-score the LLM's sufficiency + verdict; it only decides whether to answer fully, caveat, or investigate. + +Actions: + - ``ANSWER`` full answer (sufficient + high confidence) + - ``ANSWER_WITH_CAVEAT`` answer with an explicit caveat + - ``GAP`` follow-up search on the concrete missing pieces + - ``RECONCILE`` re-investigate a low-confidence / contradictory point + - ``UNANSWERABLE`` cannot answer (or fall back to a direct LLM) +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +# Action constants (shared with callers / orchestrators). +ANSWER = "ANSWER" +ANSWER_WITH_CAVEAT = "ANSWER_WITH_CAVEAT" +GAP = "GAP" +RECONCILE = "RECONCILE" +UNANSWERABLE = "UNANSWERABLE" + + +@dataclass +class LadderOutput: + """Result of the decision ladder.""" + + action: str + should_continue: bool + caveat: str = "" + missing: list[str] = field(default_factory=list) + + +def aggregate_agent_confidence( + agent_results: list[Any], + hard_violations: set[str] | dict[str, list[str]] | None = None, +) -> float: + """Mean self-confidence over the trusted (self-verified, no hard veto) claims. + + No weights, no normalization — we only pick the trustworthy subset and take + the mean. Hard vetoes (required-entity gaps, grounded-fact violations, + numeric conflicts) suppress a claim's confidence entirely, so a confidence + signal can never mask a missing-evidence gap (selective generation). + """ + violations: set[str] = set() + if hard_violations: + if isinstance(hard_violations, dict): + violations = set(hard_violations.keys()) + else: + violations = set(hard_violations) + + trusted = [r for r in agent_results if getattr(r, "is_verified", False) and getattr(r, "claim_id", "") not in violations] + if not trusted: + return 0.0 + total = 0.0 + for r in trusted: + try: + total += float(getattr(r, "confidence", 0.0) or 0.0) + except (TypeError, ValueError): + continue + return total / len(trusted) + + +def sufficiency_ladder( + *, + auto_sufficient: bool, + auto_confidence: float, + missing: list[str], + contradictions: list[str], + agent_confidence: float, + c_high: float, + c_low: float, + llm_floor: float, + allows_reconcile: bool, + cycle: int, + max_cycles: int, + hard_violations: set[str] | dict[str, list[str]] | None = None, +) -> LadderOutput: + """Evaluate the decision ladder and return the action. + + Parameters + ---------- + auto_sufficient : bool + LLM AutoRater's verdict (a plausible answer can be inferred). + auto_confidence : float + AutoRater's own confidence in its sufficiency judgment. + missing : list[str] + Concrete gaps when AutoRater says insufficient. + contradictions : list[str] + Evidence-internal contradictions (even if sufficient, caveat). + agent_confidence : float + Aggregated agent self-confidence (0..1). + c_high / c_low : float + Agent-confidence gates for full vs. caveated answer. + llm_floor : float + If AutoRater confidence < this, re-investigate regardless of verdict. + allows_reconcile : bool + Whether the mode can force a re-investigation (medium=False). + cycle / max_cycles : int + 0-based current cycle and the mode's budget. + hard_violations : set | dict + Claim IDs (or {id: gaps}) with a proven evidence gap (required entity + missing / grounded absent / numeric conflict). Any non-empty value + forces a caveated answer even if AutoRater says sufficient. + """ + violations: set[str] = set() + if hard_violations: + violations = set(hard_violations.keys()) if isinstance(hard_violations, dict) else set(hard_violations) + + # 1. Hard veto floor: code-proven evidence gap beats the LLM's "good enough". + if violations: + return LadderOutput( + action=ANSWER_WITH_CAVEAT, + should_continue=False, + caveat=f"hard evidence gap in claim(s): {sorted(violations)[:6]}", + missing=missing, + ) + + # 2. AutoRater says insufficient. + if not auto_sufficient: + if missing: + return LadderOutput(action=GAP, should_continue=True, missing=missing) + return LadderOutput(action=UNANSWERABLE, should_continue=False, missing=missing) + + # 3. AutoRater is not confident in its own sufficiency call. + if auto_confidence < llm_floor: + if allows_reconcile and cycle < max_cycles - 1: + return LadderOutput( + action=RECONCILE, + should_continue=True, + caveat="AutoRater itself is unsure; re-investigating", + ) + return LadderOutput( + action=ANSWER_WITH_CAVEAT, + should_continue=False, + caveat="AutoRater sufficiency judgment is low-confidence", + ) + + # 4. Sufficient + confident: agent confidence sets the presentation. + caveat = "" + should_continue = False + if agent_confidence >= c_high: + action = ANSWER + elif agent_confidence >= c_low: + action = ANSWER_WITH_CAVEAT + caveat = "evidence partially supports the answer" + elif allows_reconcile and cycle < max_cycles - 1: + action = RECONCILE + should_continue = True + else: + action = ANSWER_WITH_CAVEAT + caveat = "evidence partially supports the answer" + + if contradictions: + # Never silently pick one side of a contradiction; surface it. + caveat = "evidence contains conflicting figures" + action = ANSWER_WITH_CAVEAT + should_continue = False + + return LadderOutput(action=action, should_continue=should_continue, caveat=caveat, missing=missing) diff --git a/rag/advanced_rag/harness/tools/navigation.py b/rag/advanced_rag/harness/tools/navigation.py index 60f480f2a7..1c03be3cdc 100644 --- a/rag/advanced_rag/harness/tools/navigation.py +++ b/rag/advanced_rag/harness/tools/navigation.py @@ -354,6 +354,15 @@ _NAV_CHILDREN_PAGE_SIZE = 1000 # children fetched per node _NAV_TREE_MAX_DEPTH = 6 # BFS depth cap when descending sub-clusters to leaves _NAV_TREE_MAX_LEAVES = 300 # document leaves rendered to the doc-select LLM +# Chunk-level content recall (fallback) tunables. +# The nav tree routes by *cluster summaries*; a question that matches a detail +# only present in a document's body (not its one-line summary) can fall through +# the tree. We therefore back the tree result with a plain chunk retrieval and +# fold the documents it hits back in as a recall fallback — reusing the existing +# chunk index, so no new compilation artifact is required. +_NAV_RECALL_TOP_N = 40 # chunk candidates fetched before doc aggregation +_NAV_RECALL_MAX_DOCS = 4 # extra docs the content recall may add on top of the tree + _NAV_SELECT_SYSTEM = """You are routing a question through a dataset's navigation tree. You are given a QUESTION and a numbered list of {noun}, each with a name and a short description. @@ -466,6 +475,60 @@ def _nav_cluster_names(clusters: list[dict]) -> str: return ", ".join(n for n in names if n) or "none" +async def _content_recall_docs(tools, query: str, doc_scope: list[str] | None = None) -> list[str]: + """Fallback doc discovery: recall by chunk *content*, aggregated to docs. + + Runs a plain hybrid retrieval over the bound KBs' chunk index and returns the + doc_ids of the hit documents, most-hit-first. This is the safety net for the + nav-tree router: a question that matches detail living only in a document's + body (not its cluster summary) never appears in the tree, so this retrieval — + which reads real chunk text — is what catches it. + + ``doc_scope`` (the already-effective routed doc scope) is forwarded as + ``doc_ids`` to the retrieval so content recall stays restricted to the same + documents the tree routed to, instead of re-opening the whole KB. + + Returns ``[]`` on failure or when nothing hits. + """ + if not query: + return [] + from common import settings + + target_ids = getattr(tools, "kb_ids", None) or [] + if not target_ids: + return [] + # Hybrid weight: blend vector + term recall when an embedder exists, and + # fall back to pure term matching otherwise (mirrors ``hybrid_search``). + embd_mdl = getattr(tools, "embed_mdl", None) + vector_weight = 0.3 if embd_mdl else 0 + try: + kbinfos = await settings.retriever.retrieval( + query, + embd_mdl, + getattr(tools, "tenant_ids", None), + target_ids, + 1, + _NAV_RECALL_TOP_N, + 0.2, + vector_similarity_weight=vector_weight, + doc_ids=doc_scope, + aggs=True, + highlight=False, + ) + except Exception: + _LOG.exception("[Dataset navigation] content-recall retrieval failed") + return [] + doc_ids: list[str] = [] + seen: set[str] = set() + for agg in kbinfos.get("doc_aggs") or []: + did = str(agg.get("doc_id") or "").strip() + if did and did not in seen: + seen.add(did) + doc_ids.append(did) + _LOG.info("[Dataset navigation] Content recall found %d candidate doc(s).", len(doc_ids)) + return doc_ids + + async def dataset_navigation_by_tree(tools, topic: str, keywords: str = "", doc_scope: list[str] | None = None) -> list[str]: """Return the ``doc_id``s most relevant to the question / keywords by walking the dataset nav tree with the chat model. @@ -510,27 +573,27 @@ async def dataset_navigation_by_tree(tools, topic: str, keywords: str = "", doc_ clusters.append({**item, "kb": kb}) if not clusters: - _LOG.info("[Dataset navigation] no cluster there.") - return [] + _LOG.info("[Dataset navigation] no cluster there — falling back to content recall.") + return (await _content_recall_docs(tools, query, doc_scope))[:_NAV_MAX_DOCS] # 2. Ask the model which clusters are relevant. Nothing relevant → []. selected_clusters = await _ask_nav_select(tools, query, clusters, "clusters", _NAV_MAX_CLUSTERS) if not selected_clusters: - _LOG.info("[Dataset navigation] no cluster found.") - return [] + _LOG.info("[Dataset navigation] no cluster found — falling back to content recall.") + return (await _content_recall_docs(tools, query, doc_scope))[:_NAV_MAX_DOCS] _LOG.info("[Dataset navigation] %d/%d cluster(s) selected.", len(selected_clusters), len(clusters)) # 3. Descend the selected clusters to their document leaves. leaves = await _collect_nav_leaves(dataset_api_service, selected_clusters, doc_scope) if not leaves: - _LOG.info("[Dataset navigation] no leaf under selected cluster %s.", _nav_cluster_names(selected_clusters)) - return [] + _LOG.info("[Dataset navigation] no leaf under selected cluster %s — falling back to content recall.", _nav_cluster_names(selected_clusters)) + return (await _content_recall_docs(tools, query, doc_scope))[:_NAV_MAX_DOCS] # 4. Ask the model which documents to look into. Nothing relevant → []. selected_docs = await _ask_nav_select(tools, query, leaves, "documents", _NAV_TREE_MAX_LEAVES) if not selected_docs: - _LOG.info("[Dataset navigation] no doc selected under cluster %s.", _nav_cluster_names(selected_clusters)) - return [] + _LOG.info("[Dataset navigation] no doc selected under cluster %s — falling back to content recall.", _nav_cluster_names(selected_clusters)) + return (await _content_recall_docs(tools, query, doc_scope))[:_NAV_MAX_DOCS] routed: list[str] = [] seen_docs: set[str] = set() @@ -540,6 +603,23 @@ async def dataset_navigation_by_tree(tools, topic: str, keywords: str = "", doc_ seen_docs.add(did) routed.append(did) _LOG.info("[Dataset navigation] Routed to %d document(s).", len(routed)) + + # 5. Content-recall fallback. The tree routes only by cluster summaries, so + # a question matching a detail present only in a document's body can fall + # through it. Back the routed set with a plain chunk retrieval: docs that + # actually hit by content are folded back in (deduped, tree docs first) so + # a missed document still reaches the caller. Skip the extra retrieval + # when the tree already filled the whole cap — nothing would be added. + if len(routed) < _NAV_MAX_DOCS: + fallback = await _content_recall_docs(tools, query, doc_scope) + added = [d for d in fallback if d not in seen_docs] + if added: + routed.extend(added[:_NAV_RECALL_MAX_DOCS]) + _LOG.info( + "[Dataset navigation] Content recall added %d fallback doc(s) on top of the %d tree-routed one(s).", + len(added[:_NAV_RECALL_MAX_DOCS]), + len(routed) - len(added[:_NAV_RECALL_MAX_DOCS]), + ) return routed[:_NAV_MAX_DOCS] diff --git a/rag/advanced_rag/harness/tools/registry.py b/rag/advanced_rag/harness/tools/registry.py index 60cf8b5f3f..b4b4a4c746 100644 --- a/rag/advanced_rag/harness/tools/registry.py +++ b/rag/advanced_rag/harness/tools/registry.py @@ -132,13 +132,28 @@ def _generate_report_schema() -> dict: "items": {"type": "string"}, "description": "Information that was not found.", }, + "grounded": { + "type": "array", + "items": {"type": "string"}, + "description": "The KEY factual assertions in the report that ARE directly supported by the cited evidence chunks (evidence_ids). List each atomic fact you saw in the evidence, verbatim enough to be matched. This must include the answer-critical facts — e.g. 'hometown is Ithaca', 'first solo album was 1970'. Facts you inferred from prior knowledge but did NOT see in the evidence must be omitted from this list (and instead placed in gaps). Leaving a key asserted fact out of grounded, or listing a fact not actually in the evidence, will be caught and the claim marked unverified.", + }, + "numbers": { + "type": "array", + "items": {"type": "string"}, + "description": "For numerical / multi-hop questions, list each number you used in the final answer together with its exact figure and source. Format each as '
from ', e.g. '2,161,000 from Wikipedia Demographics of Paris 2019 table', '9,508 from Brown County 2020 census'. This makes the numeric口径 transparent and verifiable. If the evidence contained multiple conflicting values for the same quantity, list ALL of them and say which one you chose and why. Do NOT silently pick one of several conflicting figures; disclose the conflict.", + }, "discovered_claims": { "type": "array", "items": {"type": "string"}, "description": "New research directions discovered during research.", }, }, - "required": ["report", "is_verified", "confidence"], + # grounded and numbers are required: the agent must explicitly + # declare which facts are evidence-backed (grounded) and which + # figures+source it used (numbers, [] for non-numeric). These feed + # the cross-check's grounded-fact and numeric-conflict detection, + # so silently omitting them would bypass verification. + "required": ["report", "is_verified", "confidence", "grounded", "numbers"], }, }, } diff --git a/rag/advanced_rag/harness/types.py b/rag/advanced_rag/harness/types.py index 30b9bee55c..2245f8865e 100644 --- a/rag/advanced_rag/harness/types.py +++ b/rag/advanced_rag/harness/types.py @@ -42,8 +42,21 @@ class ExecutionStrategy: max_parallel_agents: int available_tools: list[str] sufficiency_threshold: float - partial_threshold: float fallback_to_direct_llm: bool + # Min cross-check floor for a self-verified claim. Any claim scoring below + # this becomes a hard veto (its localized evidence gap must not be averaged + # away by stronger sibling claims). Default 0.5 == the cross-check pass bar. + fusion_min_cross: float = 0.5 + # ── Decision-ladder operating points (Sufficient Context redesign) ── + # These are monotonic product-policy thresholds — NOT trained weights. They + # express "how conservative to be" (higher = investigate more before + # answering), replacing the old weighted fusion. + c_high: float = 0.75 # agent confidence >= this and AutoRater sufficient → ANSWER + c_low: float = 0.45 # agent confidence >= this → ANSWER_WITH_CAVEAT; below → reconcile + llm_floor: float = 0.55 # AutoRater confidence < this → re-investigate regardless of verdict + # Whether the mode can force a re-investigation (RECONCILE). medium=False so + # RECONCILE degrades to CONTINUE; high/ultra=True. + allows_reconcile: bool = False # ═══════════════════════════════════════════════════════════════ @@ -83,6 +96,17 @@ class AgentResult: evidence_ids: list[int] = field(default_factory=list) gaps: list[str] = field(default_factory=list) discovered_claims: list[str] = field(default_factory=list) + # Key factual assertions the agent claims are directly supported by the + # cited evidence. Used by cross-check as the *ground-truth* list to verify: + # if a grounded fact is absent from the evidence, the claim is a + # prior-knowledge-injection risk (see check1.log Q203 hometown=Ithaca, + # Q665 first-solo-album=1970) and must not count as sufficient. + grounded: list[str] = field(default_factory=list) + # For numerical / multi-hop questions: the figures used in the answer with + # their sources ("2,161,000 from Wikipedia Demographics of Paris 2019"). + # Used to detect multi-source numeric conflicts that a ratio/mean cross-check + # would otherwise paper over (see check.log Q754: 225 vs 228 population口径). + numbers: list[str] = field(default_factory=list) # ═══════════════════════════════════════════════════════════════ @@ -106,15 +130,19 @@ class ClaimCrossCheckResult: @dataclass class SufficiencyVerdict: - status: str # SUFFICIENT | USEFUL_BUT_INCOMPLETE | INSUFFICIENT | CONFLICTING | UNANSWERABLE + status: str # code-level preliminary label (decision ladder decides final action) score: float - agent_score: float - cross_score: float claim_assessments: list[dict] has_conflicts: bool missing_claims: list[str] feedback: str - overall_reason: str + # Decision-ladder inputs (Sufficient Context redesign): + # - hard_violations: claim IDs with a code-proven evidence gap (required + # entity missing / grounded absent / numeric conflict) that must veto a + # full answer even if the LLM AutoRater says sufficient. + # - agent_confidence: mean self-confidence over the trusted subset. + hard_violations: list[str] = field(default_factory=list) + agent_confidence: float = 0.0 # ═══════════════════════════════════════════════════════════════ @@ -146,6 +174,11 @@ class OrchestratorContext: current_phase: str = "locate" verdict: SufficiencyVerdict | None = None history: list[dict] = field(default_factory=list) + # Follow-up search queries produced by the Phase-2 LLM Sufficient Context + # AutoRater when evidence was deemed insufficient. They are consumed by the + # next research round to guide targeted follow-up search (Google's + # missing-pieces feedback), then cleared. + pending_followups: list[dict] = field(default_factory=list) _last_entity: str | None = None @property diff --git a/rag/prompts/sufficiency_select.md b/rag/prompts/sufficiency_select.md index c0435b9190..b4fbfc43d3 100644 --- a/rag/prompts/sufficiency_select.md +++ b/rag/prompts/sufficiency_select.md @@ -1,4 +1,6 @@ -You are an information retrieval evaluation expert. Assess whether the currently retrieved content is sufficient to answer the user's question(s), and identify exactly which retrieved chunks are useful. +You are an information retrieval evaluation expert. Determine whether the retrieved content is sufficient to answer the user's question(s), following the "Sufficient Context" criterion: + +The CONTEXT is sufficient to answer the question if and only if a PLAUSIBLE answer can be inferred from it — that is, the retrieved content either directly contains or logically entails an answer to the question. The answer does NOT need to be proven correct; it only needs to be a reasonable, supportable answer. If the context cannot be used to infer any plausible answer, it is INSUFFICIENT. Each retrieved chunk is labeled with an integer ID on a line like `ID: 3`. @@ -8,21 +10,35 @@ User question(s): Retrieved content: {{ retrieved_docs }} -Determine whether these contents are sufficient to answer the user's question(s), and list the IDs of the chunks that actually contribute useful information toward answering them. +Reasoning procedure (do this step-by-step before answering): +1. Identify the REQUIRED ENTITIES or key facts that a plausible answer to the question must involve. +2. For each required entity, check whether the retrieved content provides evidence about it. Record this in "coverage". +3. Check for multi-hop inference: if answering requires combining facts not present in the context, or inferring a connection the context does not state, that is NOT inferable from the context. +4. Check whether the context is ambiguous: if it could support multiple mutually exclusive plausible answers and nothing in the context lets you distinguish them, mark it insufficient. +5. Note any internally conflicting figures/statements in the context ("contradictions"). +6. Decide whether a plausible answer can be inferred; give your confidence in that decision. Output format (JSON): ```json { + "Sufficient Context": true/false, "is_sufficient": true/false, - "reasoning": "Your reasoning for the judgment", + "required_entities": ["Entity 1", "Entity 2"], + "coverage": {"Entity 1": true, "Entity 2": false}, "missing_information": ["Missing information 1", "Missing information 2"], + "contradictions": ["conflicting figures/statements if any"], + "confidence": 0.0, + "reasoning": "Step-by-step reasoning for the judgment", "useful_chunk_ids": [0, 3, 7] } ``` Requirements: -1. If the retrieved content contains the key information needed to answer the question(s), judge as sufficient (true). -2. If key information is missing, judge as insufficient (false), and list the missing information. -3. `useful_chunk_ids` must contain ONLY the integer IDs (taken from the `ID:` labels above) of chunks that provide information useful for answering the question(s). Exclude irrelevant or redundant chunks. Use an empty array when none are useful. -4. The `missing_information` should only be filled when insufficient, otherwise an empty array. -5. The `reasoning` should be concise and clear. +1. `Sufficient Context` / `is_sufficient` must be true if and only if a plausible answer can be inferred from the context (per the definition above). A missing detail that a reasonable answer would still require makes it false. +2. If not sufficient, list the concrete `missing_information`. +3. `coverage` must mark, for each required entity, whether the context provides evidence about it. Missing required entities belong in `missing_information`. +4. `confidence` (0-1): how confident you are in your sufficiency decision. 0.9-1.0 if the context clearly supports or clearly fails a plausible answer; 0.5-0.7 if evidence is partial or ambiguous; below 0.5 if you cannot tell. +5. `contradictions`: list any internally conflicting figures/statements that would make a single answer ambiguous. Empty array when none. +6. `useful_chunk_ids` must contain ONLY the integer IDs (taken from the `ID:` labels above) of chunks that provide information useful for answering the question(s). Exclude irrelevant or redundant chunks. Use an empty array when none are useful. +7. The `missing_information` should only be filled when insufficient, otherwise an empty array. +8. The `reasoning` should be concise and clear. diff --git a/uv.lock b/uv.lock index 6651d5369d..e0bc3b096a 100644 --- a/uv.lock +++ b/uv.lock @@ -1657,6 +1657,14 @@ version = "0.8.3" source = { registry = "https://mirrors.aliyun.com/pypi/simple" } sdist = { url = "https://mirrors.aliyun.com/pypi/packages/51/0b/c0f53a14317b304e2e93b29a831b0c83306caae9af7f0e2e037d17c4f63f/datrie-0.8.3.tar.gz", hash = "sha256:ea021ad4c8a8bf14e08a71c7872a622aa399a510f981296825091c7ca0436e80" } +[[package]] +name = "de-core-news-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl" } +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl", hash = "sha256:fec69fec52b1780f2d269d5af7582a5e28028738bd3190532459aeb473bfa3e7" }, +] + [[package]] name = "debugpy" version = "1.8.21" @@ -1874,6 +1882,14 @@ wheels = [ { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl", hash = "sha256:1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85" }, ] +[[package]] +name = "es-core-news-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl" } +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl", hash = "sha256:e451a83d6df79b87e9eed0cb553f03e99e36a3bab18a7b79f0dcfd1fdf875e12" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -2148,6 +2164,14 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/c7/4e/ce75a57ff3aebf6fc1f4e9d508b8e5810618a33d900ad6c19eb30b290b97/fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371" }, ] +[[package]] +name = "fr-core-news-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl" } +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl", hash = "sha256:7d6ad14cd5078e53147bfbf70fb9d433c6a3865b695fda2657140bbc59a27e29" }, +] + [[package]] name = "free-proxy" version = "1.1.3" @@ -3038,6 +3062,24 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef" }, ] +[[package]] +name = "ja-core-news-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl" } +dependencies = [ + { name = "sudachidict-core" }, + { name = "sudachipy" }, +] +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl", hash = "sha256:65693016b7cf33ffa845af3523a9c9b8d9aabae85dfc2e46731513a5e03def04" }, +] + +[package.metadata] +requires-dist = [ + { name = "sudachidict-core", specifier = ">=20211220" }, + { name = "sudachipy", specifier = ">=0.5.2,!=0.6.1" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -3269,6 +3311,15 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/99/2e/d82db9eec13ad0f72e7aaad5c4bc730ab111934fdc83c85523206eb9b0a0/langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a" }, ] +[[package]] +name = "langdetect" +version = "1.0.9" +source = { registry = "https://mirrors.aliyun.com/pypi/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz", hash = "sha256:cbc1fef89f8d062739774bd51eda3da3274006b3661d199c2655f6b3f6d605a0" } + [[package]] name = "langfuse" version = "4.7.1" @@ -4076,12 +4127,12 @@ name = "onnxruntime-gpu" version = "1.23.2" source = { registry = "https://mirrors.aliyun.com/pypi/simple" } dependencies = [ - { name = "coloredlogs" }, - { name = "flatbuffers" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "sympy" }, + { name = "coloredlogs", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "flatbuffers", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "numpy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "packaging", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "protobuf", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "sympy", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')" }, ] wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/03/05/40d561636e4114b54aa06d2371bfbca2d03e12cfdf5d4b85814802f18a75/onnxruntime_gpu-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1e8f75af5da07329d0c3a5006087f4051d8abd133b4be7c9bae8cdab7bea4c26" }, @@ -4838,6 +4889,14 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/80/2d/1bb683f64737bbb1f86c82b7359db1eb2be4e2c0c13b947f80efefa7d3e5/psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa" }, ] +[[package]] +name = "pt-core-news-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl" } +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl", hash = "sha256:c304fa04db3af73cd08a250feacf560506e15a2ec2469bd1b09f06847f6b455c" }, +] + [[package]] name = "py" version = "1.11.0" @@ -8065,6 +8124,7 @@ dependencies = [ { name = "cohere" }, { name = "crawl4ai" }, { name = "dashscope" }, + { name = "de-core-news-sm" }, { name = "debugpy" }, { name = "deepl" }, { name = "demjson3" }, @@ -8074,6 +8134,7 @@ dependencies = [ { name = "editdistance" }, { name = "elasticsearch-dsl" }, { name = "en-core-web-sm" }, + { name = "es-core-news-sm" }, { name = "exceptiongroup" }, { name = "extract-msg" }, { name = "feedparser" }, @@ -8083,6 +8144,7 @@ dependencies = [ { name = "flask-login" }, { name = "flask-mail" }, { name = "flask-session" }, + { name = "fr-core-news-sm" }, { name = "google-api-python-client" }, { name = "google-auth-oauthlib" }, { name = "google-cloud-bigquery" }, @@ -8095,8 +8157,10 @@ dependencies = [ { name = "html-text" }, { name = "infinity-emb" }, { name = "infinity-sdk" }, + { name = "ja-core-news-sm" }, { name = "jira" }, { name = "json-repair" }, + { name = "langdetect" }, { name = "langfuse" }, { name = "langgraph" }, { name = "lark-oapi" }, @@ -8128,6 +8192,7 @@ dependencies = [ { name = "peewee" }, { name = "pluginlib" }, { name = "psycopg2-binary" }, + { name = "pt-core-news-sm" }, { name = "pyairtable" }, { name = "pyclipper" }, { name = "pycryptodomex" }, @@ -8175,6 +8240,7 @@ dependencies = [ { name = "xpinyin" }, { name = "yfinance" }, { name = "zai-sdk" }, + { name = "zh-core-web-sm" }, ] [package.dev-dependencies] @@ -8221,6 +8287,7 @@ requires-dist = [ { name = "cohere", specifier = "==5.6.2" }, { name = "crawl4ai", specifier = ">=0.9.2,<0.9.3" }, { name = "dashscope", specifier = "==1.25.11" }, + { name = "de-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl" }, { name = "debugpy", specifier = ">=1.8.13" }, { name = "deepl", specifier = "==1.18.0" }, { name = "demjson3", specifier = "==3.0.6" }, @@ -8231,6 +8298,7 @@ requires-dist = [ { name = "editdistance", specifier = "==0.8.1" }, { name = "elasticsearch-dsl", specifier = "==8.12.0" }, { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, + { name = "es-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl" }, { name = "exceptiongroup", specifier = ">=1.3.0,<2.0.0" }, { name = "extract-msg", specifier = ">=0.39.0" }, { name = "feedparser", specifier = ">=6.0.11,<7.0.0" }, @@ -8240,6 +8308,7 @@ requires-dist = [ { name = "flask-login", specifier = "==0.6.3" }, { name = "flask-mail", specifier = ">=0.10.0" }, { name = "flask-session", specifier = "==0.8.0" }, + { name = "fr-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl" }, { name = "google-api-python-client", specifier = ">=2.190.0,<3.0.0" }, { name = "google-auth-oauthlib", specifier = ">=1.2.0,<2.0.0" }, { name = "google-cloud-bigquery", specifier = ">=3.25.0,<4.0.0" }, @@ -8252,8 +8321,10 @@ requires-dist = [ { name = "html-text", specifier = "==0.6.2" }, { name = "infinity-emb", specifier = ">=0.0.66,<0.0.67" }, { name = "infinity-sdk", specifier = "==0.7.2" }, + { name = "ja-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/ja_core_news_sm-3.8.0/ja_core_news_sm-3.8.0-py3-none-any.whl" }, { name = "jira", specifier = "==3.10.5" }, { name = "json-repair", specifier = "==0.60.1" }, + { name = "langdetect", specifier = "==1.0.9" }, { name = "langfuse", specifier = ">=4.0.1" }, { name = "langgraph", specifier = "==1.2.0" }, { name = "lark-oapi", specifier = ">=1.2.0" }, @@ -8285,6 +8356,7 @@ requires-dist = [ { name = "peewee", specifier = ">=3.17.1,<4.0.0" }, { name = "pluginlib", specifier = ">=0.10.0" }, { name = "psycopg2-binary", specifier = ">=2.9.11,<3.0.0" }, + { name = "pt-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.8.0/pt_core_news_sm-3.8.0-py3-none-any.whl" }, { name = "pyairtable", specifier = ">=3.3.0" }, { name = "pyclipper", specifier = ">=1.4.0,<2.0.0" }, { name = "pycryptodomex", specifier = "==3.20.0" }, @@ -8332,6 +8404,7 @@ requires-dist = [ { name = "xpinyin", specifier = "==0.7.6" }, { name = "yfinance", specifier = "==0.2.65" }, { name = "zai-sdk", specifier = ">=0.2.3,<1.0.0" }, + { name = "zh-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl" }, ] [package.metadata.requires-dev] @@ -9056,6 +9129,25 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/33/78/d1a1a026ef3af911159398c939b1509d5c36fe524c7b644f34a5146c4e16/spacy_loggers-1.0.5-py3-none-any.whl", hash = "sha256:196284c9c446cc0cdb944005384270d775fdeaf4f494d8e269466cfa497ef645" }, ] +[[package]] +name = "spacy-pkuseg" +version = "1.0.1" +source = { registry = "https://mirrors.aliyun.com/pypi/simple" } +dependencies = [ + { name = "numpy" }, + { name = "srsly" }, +] +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/75/14/c21fc3a5a9cee55c675d864674df0409fd5653b564e968e4ebbf15b461ad/spacy_pkuseg-1.0.1.tar.gz", hash = "sha256:b48078775afff34914375344d56f70a37ec044188ccaece3f70806fd322a47eb" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/15/32/75d7145677db3b8dd4d099f76a47ae01e97af245d7408b0e8c01f8f067de/spacy_pkuseg-1.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c3dfe0d4398b8fd5d462747daea67929873ca40df69e39cd86f400fbfc2af52" }, + { url = "https://mirrors.aliyun.com/pypi/packages/8f/a2/3ecb492d16861720332ab6289c57801cf82fe9cf64c6f14b2cd7a39773dd/spacy_pkuseg-1.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b242a559343302fe2077122a699a0fa0d92d5ac66640bf88a6e306a988632f62" }, + { url = "https://mirrors.aliyun.com/pypi/packages/d7/32/07dc131c20242a0e4324e60fb1a9bfbdf9420037a3549ec58f7221c770b8/spacy_pkuseg-1.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8402b879ccd482a00850133ff563d6f182b8274e0b633b2e066f2d4ae3b6824" }, + { url = "https://mirrors.aliyun.com/pypi/packages/f2/46/d3ce46d59c599dd6267355d788c5af8a28a12cdc53928328698a51bf0a5b/spacy_pkuseg-1.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6329d66a96b806b88081a733329a355659239826c4f36758fea51d25974cbf" }, + { url = "https://mirrors.aliyun.com/pypi/packages/b7/a4/1a799ee08fd8a037014332250912bae3a5e6777c3f1c1146b26427a9d7e1/spacy_pkuseg-1.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae70691212406b7d160e8b635fe90e68ac97a4f8c85485230debc646754a4df5" }, + { url = "https://mirrors.aliyun.com/pypi/packages/74/8f/035978f90ead9bd55a8b9d890f12e2b07d2b1a25048b261ec6c6df7a9f57/spacy_pkuseg-1.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fae8721e91bb6944a61d9102a4d25162e8f159e7b8a91222f39621afa4eb0bcd" }, + { url = "https://mirrors.aliyun.com/pypi/packages/87/62/da5358e96ba359b977b0ed9293a8569dfc7ede329c5632c57f3d892f688f/spacy_pkuseg-1.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:36f89cd2bd8757b003f96dc7622e1af3f21abda8dd86fb8a2513ce926e722265" }, +] + [[package]] name = "sphinx" version = "8.2.3" @@ -9295,6 +9387,37 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/5c/92/d0c83f63d3518e5f0b8a311937c31347349ec9a47b209ddc17f7566f58fc/stone-3.3.1-py3-none-any.whl", hash = "sha256:e15866fad249c11a963cce3bdbed37758f2e88c8ff4898616bc0caeb1e216047" }, ] +[[package]] +name = "sudachidict-core" +version = "20260723" +source = { registry = "https://mirrors.aliyun.com/pypi/simple" } +dependencies = [ + { name = "sudachipy" }, +] +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/2f/69/e8df4402744319080f9db9f20dc40a0cc3f2042db82ab7c040b785becf14/sudachidict_core-20260723.tar.gz", hash = "sha256:d50c302f907c0bb9b048b9d224abc3b24cdbeb8416a73942de00b74235798fe7" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/46/fe/68a146fced55319af40d25a4fe19b94c3a988406ce4674a8b2f0237fbc9f/sudachidict_core-20260723-py3-none-any.whl", hash = "sha256:b3869ce6b12b4bfa09575dc19030703bb669ab41bac12a74cafcbb28c6be2498" }, +] + +[[package]] +name = "sudachipy" +version = "0.6.11" +source = { registry = "https://mirrors.aliyun.com/pypi/simple" } +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/13/34/a52dc020e1ac67ffe2469d22771a850589a1184e0d91d48addd364ee02ba/sudachipy-0.6.11.tar.gz", hash = "sha256:4f03310fd3fc779b3000f49395c939d18a30081632d3cb14488426f7c07cc526" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/09/71/4d4a6b771fef1571d2f6e73d25ffcb3e8de462af674464c1fd41c9787899/sudachipy-0.6.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f38624fc01c2bb087c875a32750e3735b67b6b66487ac30a8e3f2d3599c4e3ea" }, + { url = "https://mirrors.aliyun.com/pypi/packages/3c/6f/cc4b530725321c098dbaf57fdc74306e22224b8b5c5fd88feaf8a501aa20/sudachipy-0.6.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a07a9a9232137353bb96d1ae2ce7e33def1ac48b571e338b3433b90581a9bbb" }, + { url = "https://mirrors.aliyun.com/pypi/packages/ec/a8/94cbedeef4179536ee716504bdf5772060944dce7fa6cf0d3648c50a46de/sudachipy-0.6.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d074154c652ed6637a6a3581c67346ce642e47173e849267deb8c711b74157c4" }, + { url = "https://mirrors.aliyun.com/pypi/packages/98/52/447115dc0a638081fd3f48a54f4ac8f455847366504fc68029ab1b7bd59b/sudachipy-0.6.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b87831c5f98d634da04629bb6e3cace2a9aee275e83c1b62c8fdd7da4aca2876" }, + { url = "https://mirrors.aliyun.com/pypi/packages/2a/7e/1b2cbdc8f1a889b76db9aad5599a86ed08d894a197b41dcdab265b1f8440/sudachipy-0.6.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc86a6832448a56bcc6200b078a9bebd6059d925328214b96ee6b8830b8d34ab" }, + { url = "https://mirrors.aliyun.com/pypi/packages/69/4a/d5c1021335edfe8d0e78befb591fb11f4f8543bc0db1acf0ffc4ba135650/sudachipy-0.6.11-cp313-cp313-win_amd64.whl", hash = "sha256:c56d8308e799bff8f0edc0a9921f0bacf9b0b65c5c5df031022a048402f1d9ac" }, + { url = "https://mirrors.aliyun.com/pypi/packages/db/20/c1ed0b8ab7a9f21d8954659de8113c5ae2f3a1667f21260b408038a94d78/sudachipy-0.6.11-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:ca04ecca9530d7b663f34f32309ad0961b3f85a8921908ecf39274e00522b598" }, + { url = "https://mirrors.aliyun.com/pypi/packages/67/a3/064353cbc163866aaf42f7735c9b85ea1ff8376ae1ad1b0a60f5a4e97a35/sudachipy-0.6.11-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:445cd4f8596d308582743cc48425b2c0aebcda04b37b7f18e00ad927d86194b1" }, + { url = "https://mirrors.aliyun.com/pypi/packages/d0/2c/022b1c2e491bddfd2daa488314973f864941992e20d602e8713672b37765/sudachipy-0.6.11-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b49d244423c39acb6ffa44a32c0296464f7454213f872d8738e9c43db4bc33d4" }, + { url = "https://mirrors.aliyun.com/pypi/packages/02/51/42de4a2cb82e428b3f2d04cc5f40370477724429f6dcb5acadb3af488128/sudachipy-0.6.11-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5f7d234e77577ac62fb207e65bb925bdec08fb8a3ac8df52302d3903fd581e26" }, + { url = "https://mirrors.aliyun.com/pypi/packages/24/e9/b6fb38b6788fdbdeb5e08c26200468a636df4e0ed8fb85b44572e184f6e5/sudachipy-0.6.11-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2e5316b6279c93eca1b325a0f6eda16c284d647daaa1215ee994dd9c543decd4" }, +] + [[package]] name = "sympy" version = "1.14.0" @@ -10163,6 +10286,20 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/4b/18/2b2c012421936ed370436d844e250e9dbdefa20e845d6aeea7152c26a403/zai_sdk-0.2.3-py3-none-any.whl", hash = "sha256:f8d6417f8cff58d5b6cba5c27577ee55aa817ae825c6c88fe883f6f6f9265d90" }, ] +[[package]] +name = "zh-core-web-sm" +version = "3.8.0" +source = { url = "https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl" } +dependencies = [ + { name = "spacy-pkuseg" }, +] +wheels = [ + { url = "https://github.com/explosion/spacy-models/releases/download/zh_core_web_sm-3.8.0/zh_core_web_sm-3.8.0-py3-none-any.whl", hash = "sha256:7de3bd267176b9b2a8defb6997c1cd296da16c57b5e712f72ea44a51755421c8" }, +] + +[package.metadata] +requires-dist = [{ name = "spacy-pkuseg", specifier = ">=1.0.0,<2.0.0" }] + [[package]] name = "zipp" version = "3.23.0"