mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-31 21:13:49 +08:00
75ac8cec2e
Port RAPTOR knowledge compilation to match Python: remove Go-only capacity guardrails, fix prompts, add token truncation, response post-processing, retries, replace AHC with watershed.
1964 lines
51 KiB
Go
1964 lines
51 KiB
Go
// Package wiki implements the "wiki" variant of KnowledgeCompiler: a
|
|
// document artifact pipeline MAP -> REDUCE -> PLAN -> REFINE that produces a
|
|
// wiki-style page (and supporting section products) with in-memory state only.
|
|
// The stage semantics are aligned with the Python wiki.py design, but the Go
|
|
// port keeps all intermediate artifacts in memory instead of persisting them to
|
|
// ES between stages.
|
|
package wiki
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"net/url"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"ragflow/internal/ingestion/component/knowledge_compiler/common"
|
|
"ragflow/internal/ingestion/component/knowledge_compiler/structure"
|
|
)
|
|
|
|
type wikiPipeline struct {
|
|
ctx context.Context
|
|
deps common.Deps
|
|
param common.Param
|
|
inputs common.Inputs
|
|
docID string
|
|
tenantID string
|
|
datasetID string
|
|
llmID string
|
|
|
|
mapExtracts []wikiExtract
|
|
reduced wikiExtract
|
|
plan wikiPlan
|
|
pages []wikiPageResult
|
|
}
|
|
|
|
type wikiExtract struct {
|
|
Entities []wikiEntity `json:"entities"`
|
|
Concepts []wikiConcept `json:"concepts"`
|
|
Claims []wikiClaim `json:"claims"`
|
|
Relations []wikiRelation `json:"relations"`
|
|
Topics []string `json:"topics"`
|
|
}
|
|
|
|
type wikiEntity struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
Aliases []string `json:"aliases,omitempty"`
|
|
SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
|
|
}
|
|
|
|
type wikiConcept struct {
|
|
Term string `json:"term"`
|
|
Definition string `json:"definition_excerpt"`
|
|
SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
|
|
}
|
|
|
|
type wikiClaim struct {
|
|
Statement string `json:"statement"`
|
|
Subject string `json:"subject"`
|
|
Confidence string `json:"confidence"`
|
|
SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
|
|
}
|
|
|
|
type wikiRelation struct {
|
|
From string `json:"from"`
|
|
To string `json:"to"`
|
|
Type string `json:"type"`
|
|
SourceChunkIDs []string `json:"source_chunk_ids,omitempty"`
|
|
}
|
|
|
|
type wikiPlan struct {
|
|
Title string `json:"title"`
|
|
Slug string `json:"slug"`
|
|
Lead string `json:"lead"`
|
|
Sections []wikiPlanSection `json:"sections"`
|
|
PageType string `json:"page_type,omitempty"`
|
|
Topic string `json:"topic,omitempty"`
|
|
Entities []string `json:"entity_names,omitempty"`
|
|
Related []string `json:"related_kb_pages,omitempty"`
|
|
Pages []wikiPlanPage `json:"pages,omitempty"`
|
|
}
|
|
|
|
type wikiPlanSection struct {
|
|
Heading string `json:"heading"`
|
|
Points []string `json:"points"`
|
|
}
|
|
|
|
type wikiPlanPage struct {
|
|
Action string `json:"action"`
|
|
Slug string `json:"slug"`
|
|
Title string `json:"title"`
|
|
PageType string `json:"page_type"`
|
|
Topic string `json:"topic"`
|
|
EntityNames []string `json:"entity_names"`
|
|
RelatedKB []string `json:"related_kb_pages"`
|
|
Priority int `json:"priority"`
|
|
Lead string `json:"lead"`
|
|
Sections []wikiPlanSection `json:"sections"`
|
|
}
|
|
|
|
const (
|
|
wikiPlanUpdateThreshold = 0.92
|
|
wikiPlanMaybeThreshold = 0.78
|
|
wikiPlanSearchTopK = 3
|
|
wikiMergeShrinkRatio = 0.7
|
|
)
|
|
|
|
// Run executes the wiki variant.
|
|
func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {
|
|
docID := inputs.DocID
|
|
llmID := firstNonEmpty(inputs.LLMID, param.LLMID)
|
|
if docID == "" {
|
|
docID = "unknown"
|
|
}
|
|
p := &wikiPipeline{
|
|
ctx: ctx,
|
|
deps: deps,
|
|
param: param,
|
|
inputs: inputs,
|
|
docID: docID,
|
|
tenantID: deps.TenantID,
|
|
datasetID: deps.DatasetID,
|
|
llmID: llmID,
|
|
}
|
|
if err := p.run(); err != nil {
|
|
return common.Outputs{}, err
|
|
}
|
|
|
|
products := buildWikiPageProducts(p.tenantID, p.docID, p.pages)
|
|
for i := range products {
|
|
if products[i].Meta == nil {
|
|
products[i].Meta = map[string]any{}
|
|
}
|
|
}
|
|
if len(products) > 0 {
|
|
if deps.Chat == nil {
|
|
return common.Outputs{}, fmt.Errorf("wiki: chat model required for page generation")
|
|
}
|
|
if deps.Embed == nil {
|
|
return common.Outputs{}, fmt.Errorf("wiki: embedder required (page + sections must carry vectors)")
|
|
}
|
|
texts := make([]string, len(products))
|
|
for i, p := range products {
|
|
texts[i] = p.Content
|
|
}
|
|
vectors, err := deps.Embed.Encode(ctx, texts)
|
|
if err != nil {
|
|
return common.Outputs{}, err
|
|
}
|
|
for i := range products {
|
|
if i < len(vectors) {
|
|
products[i].Vector = vectors[i]
|
|
}
|
|
}
|
|
}
|
|
|
|
store := common.NewMemStore()
|
|
decider := structure.CosineDecider{Threshold: param.SimilarityThreshold}
|
|
stats := structure.MergeStats{}
|
|
for _, p := range products {
|
|
s, err := structure.MergeIntoStore(ctx, store, decider, []common.Product{p})
|
|
if err != nil {
|
|
return common.Outputs{}, err
|
|
}
|
|
stats.Inserted += s.Inserted
|
|
stats.Updated += s.Updated
|
|
stats.DuplicatesDropped += s.DuplicatesDropped
|
|
}
|
|
|
|
prod := store.Snapshot()
|
|
if param.EnableHistoricalDedup || len(inputs.HistoricalCandidates) > 0 {
|
|
survivors, dropped, err := dedupHistorical(ctx, deps, param, p.tenantID, p.datasetID, inputs.HistoricalCandidates, prod)
|
|
if err != nil {
|
|
return common.Outputs{}, err
|
|
}
|
|
prod = survivors
|
|
stats.DuplicatesDropped += dropped
|
|
}
|
|
|
|
// Buffer every product in one slice; the component merges them into the
|
|
// upstream chunk stream (matching Python, which appends compiled units onto
|
|
// the chunk list).
|
|
out := common.Outputs{
|
|
Products: prod,
|
|
DuplicatesDropped: stats.DuplicatesDropped,
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (p *wikiPipeline) run() error {
|
|
if len(p.inputs.Chunks) == 0 {
|
|
return nil
|
|
}
|
|
if err := p.runMap(); err != nil {
|
|
return err
|
|
}
|
|
p.reduced = reduceExtracts(p.mapExtracts)
|
|
plan, err := p.runPlan()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.plan = plan
|
|
pages, err := p.runRefine()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.pages = pages
|
|
return nil
|
|
}
|
|
|
|
func (p *wikiPipeline) runMap() error {
|
|
batches := common.PackBatches(p.inputs.Chunks, 4096, p.deps.Tokenizer)
|
|
for _, batch := range batches {
|
|
if err := p.ctx.Err(); err != nil {
|
|
return err
|
|
}
|
|
extract, err := p.mapBatch(batch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.mapExtracts = append(p.mapExtracts, extract)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) {
|
|
parserConfig, _ := p.inputs.VariantSpecific["parser_config"].(map[string]any)
|
|
user, _ := buildWikiMapPrompt(p.docID, batch, parserConfig, p.param.Language)
|
|
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: wikiMapSystem,
|
|
UserPrompt: user,
|
|
})
|
|
if err != nil {
|
|
return wikiExtract{}, err
|
|
}
|
|
return parseWikiExtract(raw), nil
|
|
}
|
|
|
|
func (p *wikiPipeline) runPlan() (wikiPlan, error) {
|
|
batches := packWikiPlanBatches(p.reduced, wikiPlanTokenBudget)
|
|
if len(batches) == 0 {
|
|
batches = []wikiExtract{p.reduced}
|
|
}
|
|
plans := make([]wikiPlan, 0, len(batches))
|
|
for i, batch := range batches {
|
|
plan, err := p.runPlanBatch(batch, i+1, len(batches))
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
plans = append(plans, plan)
|
|
}
|
|
if len(plans) == 1 {
|
|
plan := normalizeWikiPlan(plans[0], p.docID, p.reduced)
|
|
return p.reconcilePlan(plan)
|
|
}
|
|
plan, err := p.mergePlanCandidates(plans)
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
return p.reconcilePlan(plan)
|
|
}
|
|
|
|
func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {
|
|
pages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)
|
|
if len(pages) == 0 {
|
|
return nil, nil
|
|
}
|
|
pageTitles := map[string]string{}
|
|
allPlanSlugs := make([]string, 0, len(pages))
|
|
for _, page := range pages {
|
|
if page.Slug == "" {
|
|
continue
|
|
}
|
|
allPlanSlugs = append(allPlanSlugs, page.Slug)
|
|
pageTitles[page.Slug] = page.Title
|
|
}
|
|
entityLookup := buildWikiEntityLookup(p.reduced.Entities)
|
|
conceptLookup := buildWikiConceptLookup(p.reduced.Concepts)
|
|
results := make([]wikiPageResult, 0, len(pages))
|
|
for _, planItem := range pages {
|
|
if p.ctx.Err() != nil {
|
|
return nil, p.ctx.Err()
|
|
}
|
|
evidence := assembleWikiPageEvidence(planItem, p.reduced.Claims, entityLookup, conceptLookup)
|
|
sourceChunkIDs := collectWikiEvidenceChunkIDs(evidence)
|
|
sourceContext := buildSourceContext(p.inputs.Chunks, sourceChunkIDs)
|
|
if strings.TrimSpace(sourceContext) == "" {
|
|
sourceContext = buildSourceContext(p.inputs.Chunks, p.reduced.sourceChunkIDs())
|
|
}
|
|
available := make([]string, 0, len(allPlanSlugs))
|
|
for _, slug := range allPlanSlugs {
|
|
if slug != planItem.Slug {
|
|
available = append(available, "- [["+slug+"]]")
|
|
}
|
|
}
|
|
if len(available) == 0 {
|
|
available = []string{"(none — this is the only page)"}
|
|
}
|
|
var existing *common.WikiPageCandidate
|
|
var err error
|
|
if strings.EqualFold(planItem.Action, "UPDATE") && p.deps.WikiPages != nil {
|
|
existing, err = p.deps.WikiPages.GetPageBySlug(p.ctx, p.tenantID, p.datasetID, planItem.Slug)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
existingSection := ""
|
|
existingRaw := ""
|
|
if existing != nil {
|
|
existingRaw = firstNonEmpty(existing.ContentMDRaw, existing.ContentMD)
|
|
if strings.TrimSpace(existingRaw) != "" {
|
|
existingSection = "## Existing page content (UPDATE — integrate new evidence into this)\n\n" + existingRaw + "\n"
|
|
}
|
|
}
|
|
user := renderWikiTemplate(wikiRefineWriterUserTemplate, map[string]string{
|
|
"action": firstNonEmpty(planItem.Action, "CREATE"),
|
|
"slug": planItem.Slug,
|
|
"title": firstNonEmpty(planItem.Title, planItem.Slug),
|
|
"page_type": firstNonEmpty(planItem.PageType, "concept"),
|
|
"all_plan_slugs": strings.Join(available, "\n"),
|
|
"existing_section": existingSection,
|
|
"source_context": sourceContext,
|
|
"evidence_count": fmt.Sprintf("%d", len(evidence)),
|
|
"evidence_blocks": formatWikiEvidenceBlocks(evidence),
|
|
})
|
|
resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: buildWikiRefineWriterSystem(""),
|
|
UserPrompt: user,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if resp == nil {
|
|
return nil, fmt.Errorf("knowledge_compiler: wiki refine returned no response")
|
|
}
|
|
contentRaw := strings.TrimSpace(firstNonEmpty(resp.Content))
|
|
if contentRaw == "" {
|
|
contentRaw = "# " + firstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"
|
|
}
|
|
if strings.TrimSpace(existingRaw) != "" {
|
|
contentRaw, err = p.mergeWikiPageContent(existingRaw, contentRaw, planItem.Slug)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles)
|
|
sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)
|
|
summary := firstParagraph(contentRendered)
|
|
if summary == "" {
|
|
summary = firstNonEmpty(planItem.Title, planItem.Slug)
|
|
}
|
|
topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)
|
|
results = append(results, wikiPageResult{
|
|
Slug: planItem.Slug,
|
|
Title: firstNonEmpty(planItem.Title, planItem.Slug),
|
|
PageType: firstNonEmpty(planItem.PageType, "concept"),
|
|
Topic: topic,
|
|
Action: firstNonEmpty(planItem.Action, "CREATE"),
|
|
EntityNames: uniqueStrings(planItem.EntityNames),
|
|
RelatedKBPages: uniqueStrings(planItem.RelatedKB),
|
|
ContentRaw: contentRaw,
|
|
Content: contentRendered,
|
|
Summary: summary,
|
|
Outlinks: outlinks,
|
|
SourceChunkIDs: sourceChunkIDs,
|
|
SourceDocIDs: sourceDocIDs,
|
|
})
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal int) (wikiPlan, error) {
|
|
user := renderWikiTemplate(wikiPlanBatchUserTemplate, map[string]string{
|
|
"doc_id": p.docID,
|
|
"batch_index": fmt.Sprintf("%d", batchIndex),
|
|
"batch_total": fmt.Sprintf("%d", batchTotal),
|
|
"entities": mustJSON(batch.Entities),
|
|
"concepts": mustJSON(batch.Concepts),
|
|
"claims": mustJSON(batch.Claims),
|
|
"relations": mustJSON(batch.Relations),
|
|
"topics": mustJSON(batch.Topics),
|
|
})
|
|
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: wikiPlanSystem,
|
|
UserPrompt: user,
|
|
})
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
return parseWikiPlan(raw, p.docID, batch), nil
|
|
}
|
|
|
|
func (p *wikiPipeline) mergePlanCandidates(plans []wikiPlan) (wikiPlan, error) {
|
|
user := renderWikiTemplate(wikiPlanMergeUserTemplate, map[string]string{
|
|
"doc_id": p.docID,
|
|
"candidates": mustPrettyJSON(plans),
|
|
})
|
|
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: wikiPlanSystem,
|
|
UserPrompt: user,
|
|
})
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
return normalizeWikiPlan(parseWikiPlan(raw, p.docID, p.reduced), p.docID, p.reduced), nil
|
|
}
|
|
|
|
func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {
|
|
if p.deps.WikiPages == nil || p.deps.Embed == nil || strings.TrimSpace(p.datasetID) == "" || len(plan.Pages) == 0 {
|
|
return plan, nil
|
|
}
|
|
queryTexts := make([]string, 0, len(plan.Pages))
|
|
pageIdx := make([]int, 0, len(plan.Pages))
|
|
for i, page := range plan.Pages {
|
|
page = normalizeWikiPlanPage(page)
|
|
plan.Pages[i] = page
|
|
query := buildWikiPageQueryText(page)
|
|
if strings.TrimSpace(query) == "" {
|
|
continue
|
|
}
|
|
queryTexts = append(queryTexts, query)
|
|
pageIdx = append(pageIdx, i)
|
|
}
|
|
if len(queryTexts) == 0 {
|
|
return plan, nil
|
|
}
|
|
vectors, err := p.deps.Embed.Encode(p.ctx, queryTexts)
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
for i, idx := range pageIdx {
|
|
if idx >= len(plan.Pages) || i >= len(vectors) {
|
|
continue
|
|
}
|
|
page := plan.Pages[idx]
|
|
existing, err := p.reconcilePlanPage(page, vectors[i])
|
|
if err != nil {
|
|
return wikiPlan{}, err
|
|
}
|
|
if existing == nil {
|
|
continue
|
|
}
|
|
page.Action = "UPDATE"
|
|
page.Slug = firstNonEmpty(existing.Slug, page.Slug)
|
|
page.Title = firstNonEmpty(existing.Title, page.Title)
|
|
page.PageType = firstNonEmpty(existing.PageType, page.PageType)
|
|
page.Topic = firstNonEmpty(existing.Topic, page.Topic)
|
|
page.RelatedKB = mergeStrings(page.RelatedKB, existing.RelatedKBPages)
|
|
page.EntityNames = mergeStrings(page.EntityNames, existing.EntityNames)
|
|
plan.Pages[idx] = page
|
|
}
|
|
plan.Pages = normalizeWikiPlanPages(plan.Pages, p.reduced)
|
|
return plan, nil
|
|
}
|
|
|
|
func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32) (*common.WikiPageCandidate, error) {
|
|
if p.deps.WikiPages == nil {
|
|
return nil, nil
|
|
}
|
|
if hit, err := p.deps.WikiPages.GetPageBySlug(p.ctx, p.tenantID, p.datasetID, page.Slug); err != nil {
|
|
return nil, err
|
|
} else if hit != nil {
|
|
return hit, nil
|
|
}
|
|
cands, err := p.deps.WikiPages.FindSimilarPages(p.ctx, p.tenantID, p.datasetID, queryVec, wikiPlanSearchTopK)
|
|
if err != nil || len(cands) == 0 {
|
|
return nil, err
|
|
}
|
|
cands = rerankWikiPlanCandidates(page, cands)
|
|
targetNames := normalizedStringSet(page.EntityNames)
|
|
pageTitle := normKey(page.Title)
|
|
pageTopic := normKey(page.Topic)
|
|
maybe := make([]common.WikiPageCandidate, 0, len(cands))
|
|
for i := range cands {
|
|
cand := cands[i]
|
|
if cand.Score >= wikiPlanUpdateThreshold {
|
|
return &cand, nil
|
|
}
|
|
if cand.Score < wikiPlanMaybeThreshold {
|
|
continue
|
|
}
|
|
if normKey(cand.Title) == pageTitle || normKey(cand.Topic) == pageTopic {
|
|
return &cand, nil
|
|
}
|
|
if intersectsNormalized(targetNames, cand.EntityNames) {
|
|
return &cand, nil
|
|
}
|
|
maybe = append(maybe, cand)
|
|
}
|
|
if len(maybe) > 0 {
|
|
return p.resolveMaybePlanPage(page, maybe)
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func rerankWikiPlanCandidates(page wikiPlanPage, candidates []common.WikiPageCandidate) []common.WikiPageCandidate {
|
|
if len(candidates) <= 1 {
|
|
return candidates
|
|
}
|
|
type scored struct {
|
|
candidate common.WikiPageCandidate
|
|
boost float64
|
|
}
|
|
targetNames := normalizedStringSet(page.EntityNames)
|
|
pageType := normKey(page.PageType)
|
|
pageTitle := normKey(page.Title)
|
|
pageTopic := normKey(page.Topic)
|
|
scoredCands := make([]scored, 0, len(candidates))
|
|
for _, cand := range candidates {
|
|
boost := cand.Score
|
|
if pageType != "" && normKey(cand.PageType) == pageType {
|
|
boost += 0.08
|
|
}
|
|
if pageTitle != "" && normKey(cand.Title) == pageTitle {
|
|
boost += 0.12
|
|
}
|
|
if pageTopic != "" && normKey(cand.Topic) == pageTopic {
|
|
boost += 0.08
|
|
}
|
|
if overlap := normalizedOverlapCount(targetNames, cand.EntityNames); overlap > 0 {
|
|
boost += 0.05 * float64(overlap)
|
|
}
|
|
scoredCands = append(scoredCands, scored{candidate: cand, boost: boost})
|
|
}
|
|
sort.SliceStable(scoredCands, func(i, j int) bool {
|
|
if scoredCands[i].boost == scoredCands[j].boost {
|
|
return scoredCands[i].candidate.Slug < scoredCands[j].candidate.Slug
|
|
}
|
|
return scoredCands[i].boost > scoredCands[j].boost
|
|
})
|
|
out := make([]common.WikiPageCandidate, 0, len(scoredCands))
|
|
for _, item := range scoredCands {
|
|
out = append(out, item.candidate)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (p *wikiPipeline) resolveMaybePlanPage(page wikiPlanPage, candidates []common.WikiPageCandidate) (*common.WikiPageCandidate, error) {
|
|
if len(candidates) == 0 {
|
|
return nil, nil
|
|
}
|
|
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: wikiPlanReconcileSystem,
|
|
UserPrompt: renderWikiTemplate(wikiPlanReconcileUserTemplate, map[string]string{
|
|
"planned_page": mustPrettyJSON(page),
|
|
"candidates": mustPrettyJSON(candidates),
|
|
}),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
action := strings.ToUpper(strings.TrimSpace(firstString(raw["action"])))
|
|
if action != "UPDATE" {
|
|
return nil, nil
|
|
}
|
|
slug := strings.TrimSpace(firstString(raw["slug"]))
|
|
if slug == "" {
|
|
return nil, nil
|
|
}
|
|
for i := range candidates {
|
|
if candidates[i].Slug == slug {
|
|
return &candidates[i], nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func buildWikiPageQueryText(page wikiPlanPage) string {
|
|
parts := []string{page.Title, page.Topic, strings.Join(page.EntityNames, " ")}
|
|
var out []string
|
|
for _, part := range parts {
|
|
if s := strings.TrimSpace(part); s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return strings.Join(out, "\n")
|
|
}
|
|
|
|
func (p *wikiPipeline) mergeWikiPageContent(existingMD, newMD, slug string) (string, error) {
|
|
if strings.TrimSpace(existingMD) == "" {
|
|
return newMD, nil
|
|
}
|
|
if strings.TrimSpace(newMD) == "" {
|
|
return existingMD, nil
|
|
}
|
|
if strings.TrimSpace(existingMD) == strings.TrimSpace(newMD) {
|
|
return newMD, nil
|
|
}
|
|
fallback := conservativeMergeWikiMarkdown(existingMD, newMD)
|
|
user := fmt.Sprintf("Merge these two versions of wiki page `%s`:\n\n## EXISTING VERSION\n\n%s\n\n---\n\n## INCOMING VERSION\n\n%s\n\n---\n\nProduce the merged page now. Return only the markdown content.", slug, existingMD, newMD)
|
|
resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{
|
|
LLMID: p.llmID,
|
|
SystemPrompt: wikiRefineMergeSystem,
|
|
UserPrompt: user,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if resp == nil {
|
|
return fallback, nil
|
|
}
|
|
merged := strings.TrimSpace(firstNonEmpty(resp.Content))
|
|
if merged == "" {
|
|
return fallback, nil
|
|
}
|
|
minAcceptable := int(float64(maxInt(len(existingMD), len(newMD))) * wikiMergeShrinkRatio)
|
|
if len(merged) < minAcceptable {
|
|
return fallback, nil
|
|
}
|
|
if wikiMarkdownDropsContent(merged, existingMD) || wikiMarkdownDropsContent(merged, newMD) {
|
|
return fallback, nil
|
|
}
|
|
return merged, nil
|
|
}
|
|
|
|
func conservativeMergeWikiMarkdown(existingMD, newMD string) string {
|
|
if strings.Contains(existingMD, newMD) {
|
|
return strings.TrimSpace(existingMD)
|
|
}
|
|
if strings.Contains(newMD, existingMD) {
|
|
return strings.TrimSpace(newMD)
|
|
}
|
|
title := firstNonEmpty(wikiMarkdownTitle(existingMD), wikiMarkdownTitle(newMD))
|
|
blocks := make([]string, 0, 8)
|
|
seen := map[string]bool{}
|
|
appendBlocks := func(body string) {
|
|
for _, block := range splitWikiMarkdownBlocks(body) {
|
|
key := normKey(block)
|
|
if key == "" || seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
blocks = append(blocks, block)
|
|
}
|
|
}
|
|
appendBlocks(wikiMarkdownBody(existingMD))
|
|
appendBlocks(wikiMarkdownBody(newMD))
|
|
if title != "" {
|
|
if len(blocks) == 0 {
|
|
return "# " + title
|
|
}
|
|
return "# " + title + "\n\n" + strings.Join(blocks, "\n\n")
|
|
}
|
|
return strings.Join(blocks, "\n\n")
|
|
}
|
|
|
|
func wikiMarkdownDropsContent(mergedMD, sourceMD string) bool {
|
|
mergedNorm := normKey(mergedMD)
|
|
for _, block := range splitWikiMarkdownBlocks(wikiMarkdownBody(sourceMD)) {
|
|
if !wikiMarkdownContentBlock(block) {
|
|
continue
|
|
}
|
|
if !strings.Contains(mergedNorm, normKey(block)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func wikiMarkdownTitle(md string) string {
|
|
for _, line := range strings.Split(md, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if strings.HasPrefix(line, "# ") {
|
|
return strings.TrimSpace(strings.TrimPrefix(line, "# "))
|
|
}
|
|
if line != "" {
|
|
break
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func wikiMarkdownBody(md string) string {
|
|
lines := strings.Split(md, "\n")
|
|
for i, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if strings.HasPrefix(line, "# ") {
|
|
return strings.TrimSpace(strings.Join(lines[i+1:], "\n"))
|
|
}
|
|
if line != "" {
|
|
break
|
|
}
|
|
}
|
|
return strings.TrimSpace(md)
|
|
}
|
|
|
|
func splitWikiMarkdownBlocks(md string) []string {
|
|
parts := strings.Split(strings.TrimSpace(md), "\n\n")
|
|
out := make([]string, 0, len(parts))
|
|
for _, part := range parts {
|
|
part = strings.TrimSpace(part)
|
|
if part != "" {
|
|
out = append(out, part)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func wikiMarkdownContentBlock(block string) bool {
|
|
block = strings.TrimSpace(block)
|
|
if block == "" {
|
|
return false
|
|
}
|
|
if strings.HasPrefix(block, "#") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func reduceExtracts(extracts []wikiExtract) wikiExtract {
|
|
out := wikiExtract{}
|
|
type entityKey struct {
|
|
Name string
|
|
Type string
|
|
}
|
|
type conceptKey struct {
|
|
Term string
|
|
}
|
|
entities := map[entityKey]*wikiEntity{}
|
|
concepts := map[conceptKey]*wikiConcept{}
|
|
seenRelations := map[string]bool{}
|
|
seenTopics := map[string]bool{}
|
|
|
|
for _, ex := range extracts {
|
|
for _, e := range ex.Entities {
|
|
key := entityKey{Name: normKey(e.Name), Type: normKey(e.Type)}
|
|
if cur, ok := entities[key]; ok {
|
|
cur.Aliases = mergeStrings(cur.Aliases, e.Aliases)
|
|
cur.SourceChunkIDs = mergeStrings(cur.SourceChunkIDs, e.SourceChunkIDs)
|
|
if cur.Type == "" {
|
|
cur.Type = e.Type
|
|
}
|
|
continue
|
|
}
|
|
item := e
|
|
item.Name = strings.TrimSpace(item.Name)
|
|
item.Type = strings.TrimSpace(item.Type)
|
|
item.Aliases = uniqueStrings(item.Aliases)
|
|
item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)
|
|
entities[key] = &item
|
|
}
|
|
for _, c := range ex.Concepts {
|
|
key := conceptKey{Term: normKey(c.Term)}
|
|
if cur, ok := concepts[key]; ok {
|
|
if cur.Definition == "" {
|
|
cur.Definition = c.Definition
|
|
}
|
|
cur.SourceChunkIDs = mergeStrings(cur.SourceChunkIDs, c.SourceChunkIDs)
|
|
continue
|
|
}
|
|
item := c
|
|
item.Term = strings.TrimSpace(item.Term)
|
|
item.Definition = strings.TrimSpace(item.Definition)
|
|
item.SourceChunkIDs = uniqueStrings(item.SourceChunkIDs)
|
|
concepts[key] = &item
|
|
}
|
|
for _, c := range ex.Claims {
|
|
c.SourceChunkIDs = uniqueStrings(c.SourceChunkIDs)
|
|
out.Claims = append(out.Claims, c)
|
|
}
|
|
for _, r := range ex.Relations {
|
|
key := normKey(r.From) + "\x00" + normKey(r.Type) + "\x00" + normKey(r.To)
|
|
if seenRelations[key] {
|
|
continue
|
|
}
|
|
r.SourceChunkIDs = uniqueStrings(r.SourceChunkIDs)
|
|
seenRelations[key] = true
|
|
out.Relations = append(out.Relations, r)
|
|
}
|
|
for _, t := range ex.Topics {
|
|
t = strings.TrimSpace(t)
|
|
if t == "" {
|
|
continue
|
|
}
|
|
key := normKey(t)
|
|
if seenTopics[key] {
|
|
continue
|
|
}
|
|
seenTopics[key] = true
|
|
out.Topics = append(out.Topics, t)
|
|
}
|
|
}
|
|
|
|
out.Entities = make([]wikiEntity, 0, len(entities))
|
|
keys := make([]entityKey, 0, len(entities))
|
|
for k := range entities {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
if keys[i].Name == keys[j].Name {
|
|
return keys[i].Type < keys[j].Type
|
|
}
|
|
return keys[i].Name < keys[j].Name
|
|
})
|
|
for _, k := range keys {
|
|
out.Entities = append(out.Entities, *entities[k])
|
|
}
|
|
out.Concepts = make([]wikiConcept, 0, len(concepts))
|
|
ckeys := make([]conceptKey, 0, len(concepts))
|
|
for k := range concepts {
|
|
ckeys = append(ckeys, k)
|
|
}
|
|
sort.Slice(ckeys, func(i, j int) bool { return ckeys[i].Term < ckeys[j].Term })
|
|
for _, k := range ckeys {
|
|
out.Concepts = append(out.Concepts, *concepts[k])
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiExtract(raw map[string]any) wikiExtract {
|
|
out := wikiExtract{}
|
|
out.Entities = parseWikiEntities(raw["entities"])
|
|
out.Concepts = parseWikiConcepts(raw["concepts"])
|
|
out.Claims = parseWikiClaims(raw["claims"])
|
|
out.Relations = parseWikiRelations(raw["relations"])
|
|
out.Topics = parseWikiStrings(raw["topics"])
|
|
return out
|
|
}
|
|
|
|
func parseWikiPlan(raw map[string]any, docID string, reduced wikiExtract) wikiPlan {
|
|
plan := wikiPlan{
|
|
Title: firstString(raw["title"]),
|
|
Slug: firstString(raw["slug"]),
|
|
Lead: firstString(raw["lead"]),
|
|
Sections: parseWikiPlanSections(raw["sections"]),
|
|
PageType: firstString(raw["page_type"]),
|
|
Topic: firstString(raw["topic"]),
|
|
Entities: parseWikiStrings(raw["entity_names"]),
|
|
Related: parseWikiStrings(raw["related_kb_pages"]),
|
|
Pages: parseWikiPlanPages(raw["pages"]),
|
|
}
|
|
if len(plan.Pages) > 0 {
|
|
first := plan.Pages[0]
|
|
if plan.Title == "" {
|
|
plan.Title = first.Title
|
|
}
|
|
if plan.Slug == "" {
|
|
plan.Slug = first.Slug
|
|
}
|
|
if plan.Lead == "" {
|
|
plan.Lead = first.Lead
|
|
}
|
|
if plan.PageType == "" {
|
|
plan.PageType = first.PageType
|
|
}
|
|
if plan.Topic == "" {
|
|
plan.Topic = first.Topic
|
|
}
|
|
if len(plan.Entities) == 0 {
|
|
plan.Entities = first.EntityNames
|
|
}
|
|
if len(plan.Related) == 0 {
|
|
plan.Related = first.RelatedKB
|
|
}
|
|
if len(plan.Sections) == 0 {
|
|
plan.Sections = first.Sections
|
|
}
|
|
}
|
|
if plan.Title == "" {
|
|
if len(reduced.Topics) > 0 {
|
|
plan.Title = reduced.Topics[0]
|
|
} else if len(reduced.Entities) > 0 {
|
|
plan.Title = reduced.Entities[0].Name
|
|
} else if len(reduced.Concepts) > 0 {
|
|
plan.Title = reduced.Concepts[0].Term
|
|
} else {
|
|
plan.Title = docID
|
|
}
|
|
}
|
|
if plan.Slug == "" {
|
|
plan.Slug = slugify(plan.Title)
|
|
}
|
|
if plan.Lead == "" {
|
|
plan.Lead = plan.Title
|
|
}
|
|
if len(plan.Sections) == 0 {
|
|
plan.Sections = []wikiPlanSection{{Heading: "Overview", Points: []string{plan.Lead}}}
|
|
}
|
|
return plan
|
|
}
|
|
|
|
func normalizeWikiPlan(plan wikiPlan, docID string, reduced wikiExtract) wikiPlan {
|
|
plan.Pages = normalizeWikiPlanPages(plan.Pages, reduced)
|
|
if plan.Title == "" {
|
|
plan.Title = firstPlanTitle(plan.Pages, reduced, docID)
|
|
}
|
|
if plan.Slug == "" {
|
|
plan.Slug = slugify(plan.Title)
|
|
}
|
|
if plan.Lead == "" {
|
|
plan.Lead = plan.Title
|
|
}
|
|
if len(plan.Sections) == 0 {
|
|
plan.Sections = []wikiPlanSection{{Heading: "Overview", Points: []string{plan.Lead}}}
|
|
}
|
|
return plan
|
|
}
|
|
|
|
func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlanPage {
|
|
existing := map[string]wikiPlanPage{}
|
|
out := make([]wikiPlanPage, 0, len(pages))
|
|
for _, page := range pages {
|
|
page = normalizeWikiPlanPage(page)
|
|
if page.Slug == "" {
|
|
continue
|
|
}
|
|
if _, ok := existing[page.Slug]; ok {
|
|
continue
|
|
}
|
|
existing[page.Slug] = page
|
|
out = append(out, page)
|
|
}
|
|
fallback := buildWikiFallbackPages(reduced)
|
|
if len(out) == 0 {
|
|
for _, page := range fallback {
|
|
if page.Slug == "" {
|
|
continue
|
|
}
|
|
existing[page.Slug] = page
|
|
out = append(out, page)
|
|
}
|
|
}
|
|
sort.SliceStable(out, func(i, j int) bool {
|
|
if out[i].Priority == out[j].Priority {
|
|
return out[i].Slug < out[j].Slug
|
|
}
|
|
return out[i].Priority < out[j].Priority
|
|
})
|
|
for i := range out {
|
|
if out[i].Priority <= 0 {
|
|
out[i].Priority = i + 1
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {
|
|
page.Action = strings.ToUpper(strings.TrimSpace(page.Action))
|
|
if page.Action == "" {
|
|
page.Action = "CREATE"
|
|
}
|
|
page.Slug = strings.TrimSpace(page.Slug)
|
|
if page.Slug == "" {
|
|
page.Slug = slugify(page.Title)
|
|
}
|
|
page.Title = strings.TrimSpace(page.Title)
|
|
if page.Title == "" {
|
|
page.Title = page.Slug
|
|
}
|
|
page.PageType = strings.TrimSpace(page.PageType)
|
|
if page.PageType == "" {
|
|
page.PageType = "concept"
|
|
}
|
|
page.Topic = strings.TrimSpace(page.Topic)
|
|
if page.Topic == "" {
|
|
page.Topic = page.Title
|
|
}
|
|
page.EntityNames = uniqueStrings(page.EntityNames)
|
|
page.RelatedKB = uniqueStrings(page.RelatedKB)
|
|
page.Sections = normalizeWikiPlanSections(page.Sections)
|
|
if page.Priority <= 0 {
|
|
page.Priority = 99
|
|
}
|
|
return page
|
|
}
|
|
|
|
func normalizeWikiPlanSections(sections []wikiPlanSection) []wikiPlanSection {
|
|
if len(sections) == 0 {
|
|
return []wikiPlanSection{{Heading: "Overview", Points: nil}}
|
|
}
|
|
out := make([]wikiPlanSection, 0, len(sections))
|
|
for _, sec := range sections {
|
|
sec.Heading = strings.TrimSpace(sec.Heading)
|
|
sec.Points = uniqueStrings(sec.Points)
|
|
if sec.Heading == "" {
|
|
continue
|
|
}
|
|
out = append(out, sec)
|
|
}
|
|
if len(out) == 0 {
|
|
return []wikiPlanSection{{Heading: "Overview", Points: nil}}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {
|
|
var out []wikiPlanPage
|
|
seen := map[string]bool{}
|
|
appendPage := func(page wikiPlanPage) {
|
|
page = normalizeWikiPlanPage(page)
|
|
if page.Slug == "" || seen[page.Slug] {
|
|
return
|
|
}
|
|
seen[page.Slug] = true
|
|
out = append(out, page)
|
|
}
|
|
for _, e := range reduced.Entities {
|
|
appendPage(wikiPlanPage{
|
|
Action: "CREATE",
|
|
Slug: "entity/" + slugify(e.Name),
|
|
Title: e.Name,
|
|
PageType: "entity",
|
|
Topic: e.Name,
|
|
EntityNames: append([]string{e.Name}, e.Aliases...),
|
|
Priority: len(out) + 1,
|
|
Lead: e.Name,
|
|
Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{e.Name}}},
|
|
})
|
|
}
|
|
for _, c := range reduced.Concepts {
|
|
appendPage(wikiPlanPage{
|
|
Action: "CREATE",
|
|
Slug: "concept/" + slugify(c.Term),
|
|
Title: c.Term,
|
|
PageType: "concept",
|
|
Topic: c.Term,
|
|
EntityNames: []string{c.Term},
|
|
Priority: len(out) + 1,
|
|
Lead: c.Definition,
|
|
Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{c.Term}}},
|
|
})
|
|
}
|
|
if len(out) == 0 {
|
|
for _, t := range reduced.Topics {
|
|
appendPage(wikiPlanPage{
|
|
Action: "CREATE",
|
|
Slug: "topic/" + slugify(t),
|
|
Title: t,
|
|
PageType: "topic",
|
|
Topic: t,
|
|
Priority: len(out) + 1,
|
|
Lead: t,
|
|
Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{t}}},
|
|
})
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
appendPage(wikiPlanPage{
|
|
Action: "CREATE",
|
|
Slug: slugify(firstPlanTitle(nil, reduced, "")),
|
|
Title: firstPlanTitle(nil, reduced, ""),
|
|
PageType: "concept",
|
|
Topic: firstPlanTitle(nil, reduced, ""),
|
|
Priority: 1,
|
|
Lead: firstPlanTitle(nil, reduced, ""),
|
|
Sections: []wikiPlanSection{{Heading: "Overview", Points: []string{firstPlanTitle(nil, reduced, "")}}},
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func firstPlanTitle(pages []wikiPlanPage, reduced wikiExtract, docID string) string {
|
|
for _, page := range pages {
|
|
if s := strings.TrimSpace(page.Title); s != "" {
|
|
return s
|
|
}
|
|
}
|
|
if len(reduced.Topics) > 0 {
|
|
return reduced.Topics[0]
|
|
}
|
|
if len(reduced.Entities) > 0 {
|
|
return reduced.Entities[0].Name
|
|
}
|
|
if len(reduced.Concepts) > 0 {
|
|
return reduced.Concepts[0].Term
|
|
}
|
|
if docID != "" {
|
|
return docID
|
|
}
|
|
return "wiki"
|
|
}
|
|
|
|
func (p wikiPlan) firstPageCandidate() wikiPlanPage {
|
|
if len(p.Pages) > 0 {
|
|
return p.Pages[0]
|
|
}
|
|
return wikiPlanPage{
|
|
Action: "CREATE",
|
|
Slug: p.Slug,
|
|
Title: p.Title,
|
|
PageType: p.PageType,
|
|
Topic: p.Topic,
|
|
EntityNames: uniqueStrings(p.Entities),
|
|
RelatedKB: uniqueStrings(p.Related),
|
|
Priority: 1,
|
|
Lead: p.Lead,
|
|
Sections: p.Sections,
|
|
}
|
|
}
|
|
|
|
func packWikiPlanBatches(reduced wikiExtract, budget int) []wikiExtract {
|
|
if budget <= 0 {
|
|
budget = wikiPlanTokenBudget
|
|
}
|
|
var batches []wikiExtract
|
|
cur := wikiExtract{}
|
|
curTokens := 0
|
|
flush := func() {
|
|
if len(cur.Entities)+len(cur.Concepts)+len(cur.Claims)+len(cur.Relations)+len(cur.Topics) > 0 {
|
|
batches = append(batches, cur)
|
|
cur = wikiExtract{}
|
|
curTokens = 0
|
|
}
|
|
}
|
|
add := func(itemTokens int, addFn func()) {
|
|
if curTokens > 0 && curTokens+itemTokens > budget {
|
|
flush()
|
|
}
|
|
addFn()
|
|
curTokens += itemTokens
|
|
}
|
|
for _, e := range reduced.Entities {
|
|
add(common.EstimateTokens(mustJSON(e)), func() { cur.Entities = append(cur.Entities, e) })
|
|
}
|
|
for _, c := range reduced.Concepts {
|
|
add(common.EstimateTokens(mustJSON(c)), func() { cur.Concepts = append(cur.Concepts, c) })
|
|
}
|
|
for _, c := range reduced.Claims {
|
|
add(common.EstimateTokens(mustJSON(c)), func() { cur.Claims = append(cur.Claims, c) })
|
|
}
|
|
for _, r := range reduced.Relations {
|
|
add(common.EstimateTokens(mustJSON(r)), func() { cur.Relations = append(cur.Relations, r) })
|
|
}
|
|
for _, t := range reduced.Topics {
|
|
add(common.EstimateTokens(mustJSON(t)), func() { cur.Topics = append(cur.Topics, t) })
|
|
}
|
|
flush()
|
|
return batches
|
|
}
|
|
|
|
func parseWikiPlanPages(raw any) []wikiPlanPage {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiPlanPage, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
page := wikiPlanPage{
|
|
Action: strings.TrimSpace(firstString(m["action"])),
|
|
Slug: strings.TrimSpace(firstString(m["slug"])),
|
|
Title: strings.TrimSpace(firstString(m["title"])),
|
|
PageType: strings.TrimSpace(firstString(m["page_type"])),
|
|
Topic: strings.TrimSpace(firstString(m["topic"])),
|
|
EntityNames: parseWikiStrings(m["entity_names"]),
|
|
RelatedKB: parseWikiStrings(m["related_kb_pages"]),
|
|
Priority: int(firstNumber(m["priority"])),
|
|
Lead: strings.TrimSpace(firstString(m["lead"])),
|
|
Sections: parseWikiPlanSections(m["sections"]),
|
|
}
|
|
if page.Slug != "" || page.Title != "" {
|
|
out = append(out, page)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiEntities(raw any) []wikiEntity {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiEntity, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
e := wikiEntity{
|
|
Name: strings.TrimSpace(firstString(m["name"])),
|
|
Type: strings.TrimSpace(firstString(m["type"])),
|
|
Aliases: parseWikiStrings(m["aliases"]),
|
|
SourceChunkIDs: parseWikiStrings(m["source_chunk_ids"]),
|
|
}
|
|
if len(e.SourceChunkIDs) == 0 {
|
|
if s := strings.TrimSpace(firstString(m["source_chunk_id"])); s != "" {
|
|
e.SourceChunkIDs = []string{s}
|
|
}
|
|
}
|
|
if e.Name != "" {
|
|
out = append(out, e)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiConcepts(raw any) []wikiConcept {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiConcept, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
c := wikiConcept{
|
|
Term: strings.TrimSpace(firstString(m["term"])),
|
|
Definition: strings.TrimSpace(firstString(m["definition_excerpt"])),
|
|
SourceChunkIDs: parseWikiStrings(m["source_chunk_ids"]),
|
|
}
|
|
if len(c.SourceChunkIDs) == 0 {
|
|
if s := strings.TrimSpace(firstString(m["source_chunk_id"])); s != "" {
|
|
c.SourceChunkIDs = []string{s}
|
|
}
|
|
}
|
|
if c.Term != "" {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiClaims(raw any) []wikiClaim {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiClaim, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
c := wikiClaim{
|
|
Statement: strings.TrimSpace(firstString(m["statement"])),
|
|
Subject: strings.TrimSpace(firstString(m["subject"])),
|
|
Confidence: strings.TrimSpace(firstString(m["confidence"])),
|
|
SourceChunkIDs: parseWikiStrings(m["source_chunk_ids"]),
|
|
}
|
|
if len(c.SourceChunkIDs) == 0 {
|
|
if s := strings.TrimSpace(firstString(m["source_chunk_id"])); s != "" {
|
|
c.SourceChunkIDs = []string{s}
|
|
}
|
|
}
|
|
if c.Statement != "" {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiRelations(raw any) []wikiRelation {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiRelation, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
r := wikiRelation{
|
|
From: strings.TrimSpace(firstString(m["from"])),
|
|
To: strings.TrimSpace(firstString(m["to"])),
|
|
Type: strings.TrimSpace(firstString(m["type"])),
|
|
SourceChunkIDs: parseWikiStrings(m["source_chunk_ids"]),
|
|
}
|
|
if len(r.SourceChunkIDs) == 0 {
|
|
if s := strings.TrimSpace(firstString(m["source_chunk_id"])); s != "" {
|
|
r.SourceChunkIDs = []string{s}
|
|
}
|
|
}
|
|
if r.From != "" && r.To != "" {
|
|
out = append(out, r)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func parseWikiPlanSections(raw any) []wikiPlanSection {
|
|
arr, ok := raw.([]any)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
out := make([]wikiPlanSection, 0, len(arr))
|
|
for _, item := range arr {
|
|
m, ok := item.(map[string]any)
|
|
if !ok {
|
|
continue
|
|
}
|
|
sec := wikiPlanSection{
|
|
Heading: strings.TrimSpace(firstString(m["heading"])),
|
|
Points: parseWikiStrings(m["points"]),
|
|
}
|
|
if sec.Heading != "" {
|
|
out = append(out, sec)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func firstNumber(v any) float64 {
|
|
switch x := v.(type) {
|
|
case int:
|
|
return float64(x)
|
|
case int64:
|
|
return float64(x)
|
|
case float32:
|
|
return float64(x)
|
|
case float64:
|
|
return x
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
func parseWikiStrings(raw any) []string {
|
|
switch v := raw.(type) {
|
|
case []string:
|
|
return uniqueStrings(v)
|
|
case []any:
|
|
out := make([]string, 0, len(v))
|
|
for _, item := range v {
|
|
if s := strings.TrimSpace(firstString(item)); s != "" {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return uniqueStrings(out)
|
|
case string:
|
|
if s := strings.TrimSpace(v); s != "" {
|
|
return []string{s}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func buildSourceContext(chunks []common.Chunk, sourceChunkIDs []string) string {
|
|
if len(sourceChunkIDs) == 0 {
|
|
var b strings.Builder
|
|
for _, ch := range chunks {
|
|
text := firstNonEmpty(ch.Text, ch.Content)
|
|
if strings.TrimSpace(text) == "" {
|
|
continue
|
|
}
|
|
b.WriteString(text)
|
|
b.WriteString("\n\n")
|
|
}
|
|
return b.String()
|
|
}
|
|
lookup := map[string]string{}
|
|
for i, ch := range chunks {
|
|
id := strings.TrimSpace(ch.ID)
|
|
if id == "" {
|
|
id = fmt.Sprintf("chunk-%d", i+1)
|
|
}
|
|
lookup[id] = firstNonEmpty(ch.Text, ch.Content)
|
|
}
|
|
var b strings.Builder
|
|
for _, id := range sourceChunkIDs {
|
|
if text := strings.TrimSpace(lookup[id]); text != "" {
|
|
b.WriteString("[CHUNK_ID ")
|
|
b.WriteString(id)
|
|
b.WriteString("]\n")
|
|
b.WriteString(text)
|
|
b.WriteString("\n\n")
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func buildEvidenceChecklist(reduced wikiExtract) string {
|
|
var lines []string
|
|
for _, e := range reduced.Entities {
|
|
lines = append(lines, fmt.Sprintf("- entity: %s (%s)", e.Name, e.Type))
|
|
}
|
|
for _, c := range reduced.Concepts {
|
|
lines = append(lines, fmt.Sprintf("- concept: %s", c.Term))
|
|
}
|
|
for _, c := range reduced.Claims {
|
|
lines = append(lines, fmt.Sprintf("- claim: %s", c.Statement))
|
|
}
|
|
for _, r := range reduced.Relations {
|
|
lines = append(lines, fmt.Sprintf("- relation: %s -> %s (%s)", r.From, r.To, r.Type))
|
|
}
|
|
for _, t := range reduced.Topics {
|
|
lines = append(lines, fmt.Sprintf("- topic: %s", t))
|
|
}
|
|
if len(lines) == 0 {
|
|
return "- (none)"
|
|
}
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
type wikiEvidenceItem struct {
|
|
Statement string
|
|
Subject string
|
|
Confidence string
|
|
ChunkIDs []string
|
|
Synthetic bool
|
|
}
|
|
|
|
func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityByName, conceptByTerm map[string]wikiExtractItem) []wikiEvidenceItem {
|
|
rawNames := uniqueStrings(append([]string{}, planItem.EntityNames...))
|
|
if len(rawNames) == 0 {
|
|
if t := strings.TrimSpace(planItem.Title); t != "" {
|
|
rawNames = []string{t}
|
|
} else if s := strings.TrimSpace(planItem.Slug); s != "" {
|
|
rawNames = []string{s}
|
|
}
|
|
}
|
|
if len(rawNames) == 0 {
|
|
return nil
|
|
}
|
|
|
|
namesLower := make([]string, 0, len(rawNames))
|
|
patterns := make([]*regexp.Regexp, 0, len(rawNames))
|
|
for _, n := range rawNames {
|
|
namesLower = append(namesLower, strings.ToLower(n))
|
|
patterns = append(patterns, regexp.MustCompile(`(?i)\b`+regexp.QuoteMeta(n)+`\b`))
|
|
}
|
|
|
|
var evidence []wikiEvidenceItem
|
|
for _, claim := range claims {
|
|
subjRaw := strings.TrimSpace(claim.Subject)
|
|
if subjRaw == "" {
|
|
continue
|
|
}
|
|
subjLower := strings.ToLower(subjRaw)
|
|
matched := false
|
|
for _, n := range namesLower {
|
|
if subjLower == n {
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
if !matched {
|
|
for _, re := range patterns {
|
|
if re.MatchString(subjRaw) {
|
|
matched = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if !matched {
|
|
continue
|
|
}
|
|
evidence = append(evidence, wikiEvidenceItem{
|
|
Statement: claim.Statement,
|
|
Subject: claim.Subject,
|
|
Confidence: firstNonEmpty(claim.Confidence, "explicit"),
|
|
ChunkIDs: uniqueStrings(claim.SourceChunkIDs),
|
|
})
|
|
}
|
|
if len(evidence) > 0 {
|
|
return evidence
|
|
}
|
|
|
|
var fallbackChunkIDs []string
|
|
matchedNames := make([]string, 0, len(rawNames))
|
|
for _, name := range rawNames {
|
|
key := strings.ToLower(strings.TrimSpace(name))
|
|
var hit wikiExtractItem
|
|
var ok bool
|
|
if entityByName != nil {
|
|
hit, ok = entityByName[key]
|
|
}
|
|
if !ok && conceptByTerm != nil {
|
|
hit, ok = conceptByTerm[key]
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
fallbackChunkIDs = mergeStrings(fallbackChunkIDs, hit.SourceChunkIDs)
|
|
matchedNames = append(matchedNames, name)
|
|
}
|
|
if len(fallbackChunkIDs) == 0 {
|
|
return nil
|
|
}
|
|
subject := rawNames[0]
|
|
if len(matchedNames) > 0 {
|
|
subject = matchedNames[0]
|
|
}
|
|
return []wikiEvidenceItem{{
|
|
Statement: "",
|
|
Subject: subject,
|
|
Confidence: "inferred",
|
|
ChunkIDs: fallbackChunkIDs,
|
|
Synthetic: true,
|
|
}}
|
|
}
|
|
|
|
type wikiExtractItem struct {
|
|
SourceChunkIDs []string
|
|
}
|
|
|
|
func buildWikiEntityLookup(items []wikiEntity) map[string]wikiExtractItem {
|
|
out := map[string]wikiExtractItem{}
|
|
for _, item := range items {
|
|
hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}
|
|
name := strings.TrimSpace(item.Name)
|
|
if name != "" {
|
|
out[strings.ToLower(name)] = hit
|
|
}
|
|
for _, alias := range item.Aliases {
|
|
if alias = strings.TrimSpace(alias); alias != "" {
|
|
out[strings.ToLower(alias)] = hit
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func buildWikiConceptLookup(items []wikiConcept) map[string]wikiExtractItem {
|
|
out := map[string]wikiExtractItem{}
|
|
for _, item := range items {
|
|
hit := wikiExtractItem{SourceChunkIDs: uniqueStrings(item.SourceChunkIDs)}
|
|
term := strings.TrimSpace(item.Term)
|
|
if term != "" {
|
|
out[strings.ToLower(term)] = hit
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func formatWikiEvidenceBlocks(evidence []wikiEvidenceItem) string {
|
|
var lines []string
|
|
for _, ev := range evidence {
|
|
if ev.Synthetic {
|
|
continue
|
|
}
|
|
confidence := strings.ToUpper(firstNonEmpty(ev.Confidence, "explicit"))
|
|
lines = append(lines, fmt.Sprintf("%d. [%s] %s\n %s", len(lines)+1, confidence, ev.Subject, ev.Statement))
|
|
}
|
|
if len(lines) == 0 {
|
|
return "(no pre-extracted evidence — extract facts directly from the source document text above)"
|
|
}
|
|
return strings.Join(lines, "\n\n")
|
|
}
|
|
|
|
func collectWikiEvidenceChunkIDs(evidence []wikiEvidenceItem) []string {
|
|
var out []string
|
|
for _, ev := range evidence {
|
|
out = mergeStrings(out, ev.ChunkIDs)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func collectWikiSourceDocIDs(chunks []common.Chunk, chunkIDs []string, fallback string) []string {
|
|
if len(chunkIDs) == 0 {
|
|
if fallback != "" {
|
|
return []string{fallback}
|
|
}
|
|
return nil
|
|
}
|
|
lookup := map[string][]string{}
|
|
for _, ch := range chunks {
|
|
id := strings.TrimSpace(ch.ID)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
docID := strings.TrimSpace(firstString(ch.Meta["doc_id"]))
|
|
if docID == "" {
|
|
docID = strings.TrimSpace(firstString(ch.Meta["doc_id_kwd"]))
|
|
}
|
|
if docID != "" {
|
|
lookup[id] = []string{docID}
|
|
}
|
|
}
|
|
out := make([]string, 0, len(chunkIDs))
|
|
for _, cid := range chunkIDs {
|
|
if ids := lookup[strings.TrimSpace(cid)]; len(ids) > 0 {
|
|
out = mergeStrings(out, ids)
|
|
}
|
|
}
|
|
if len(out) == 0 && fallback != "" {
|
|
return []string{fallback}
|
|
}
|
|
return out
|
|
}
|
|
|
|
var (
|
|
wikiWikilinkPipeRe = regexp.MustCompile(`\[\[([^\[\]\|]+?)\|([^\[\]]+?)\]\]`)
|
|
wikiWikilinkSimpleRe = regexp.MustCompile(`\[\[([^\[\]\|]+?)\]\]`)
|
|
wikiArtifactMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\(([^)]+)\)`)
|
|
)
|
|
|
|
func transformWikiLinks(content, kbID string, pageTitles map[string]string) (string, []string) {
|
|
kbID = strings.TrimSpace(kbID)
|
|
seen := map[string]bool{}
|
|
var outlinks []string
|
|
track := func(slug string) {
|
|
slug = strings.TrimSpace(slug)
|
|
if slug == "" || seen[slug] {
|
|
return
|
|
}
|
|
seen[slug] = true
|
|
outlinks = append(outlinks, slug)
|
|
}
|
|
displayText := func(label, slug string) string {
|
|
label = strings.TrimSpace(label)
|
|
if label != slug && label != slugify(lastPathSlug(slug)) {
|
|
return label
|
|
}
|
|
if title := strings.TrimSpace(pageTitles[slug]); title != "" {
|
|
return title
|
|
}
|
|
readable := strings.ReplaceAll(lastPathSlug(slug), "-", " ")
|
|
readable = strings.ReplaceAll(readable, "_", " ")
|
|
readable = strings.TrimSpace(readable)
|
|
if readable == "" {
|
|
return label
|
|
}
|
|
return strings.Title(readable)
|
|
}
|
|
artifactSlug := func(href string) string {
|
|
parsed, err := url.Parse(href)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
path := parsed.Path
|
|
if path == "" && parsed.Opaque != "" {
|
|
path = parsed.Opaque
|
|
}
|
|
parts := strings.Split(strings.Trim(path, "/"), "/")
|
|
if len(parts) >= 2 && parts[0] == "artifact" && parts[1] == kbID {
|
|
return strings.Join(parts[2:], "/")
|
|
}
|
|
if len(parts) >= 2 && parts[0] == kbID {
|
|
return strings.Join(parts[1:], "/")
|
|
}
|
|
return ""
|
|
}
|
|
rewriteMD := func(label, href string) string {
|
|
slug := artifactSlug(href)
|
|
if slug == "" {
|
|
return "[" + label + "](" + href + ")"
|
|
}
|
|
track(slug)
|
|
return "[" + displayText(label, slug) + "](artifact/" + kbID + "/" + slug + ")"
|
|
}
|
|
out := wikiArtifactMarkdownLink.ReplaceAllStringFunc(content, func(match string) string {
|
|
sub := wikiArtifactMarkdownLink.FindStringSubmatch(match)
|
|
if len(sub) != 3 {
|
|
return match
|
|
}
|
|
return rewriteMD(sub[1], sub[2])
|
|
})
|
|
out = wikiWikilinkPipeRe.ReplaceAllStringFunc(out, func(match string) string {
|
|
sub := wikiWikilinkPipeRe.FindStringSubmatch(match)
|
|
if len(sub) != 3 {
|
|
return match
|
|
}
|
|
slug := strings.TrimSpace(sub[1])
|
|
track(slug)
|
|
return "[" + sub[2] + "](artifact/" + kbID + "/" + slug + ")"
|
|
})
|
|
out = wikiWikilinkSimpleRe.ReplaceAllStringFunc(out, func(match string) string {
|
|
sub := wikiWikilinkSimpleRe.FindStringSubmatch(match)
|
|
if len(sub) != 2 {
|
|
return match
|
|
}
|
|
slug := strings.TrimSpace(sub[1])
|
|
track(slug)
|
|
return "[" + displayText(slug, slug) + "](artifact/" + kbID + "/" + slug + ")"
|
|
})
|
|
return out, outlinks
|
|
}
|
|
|
|
func lastPathSlug(slug string) string {
|
|
slug = strings.TrimSpace(slug)
|
|
if idx := strings.LastIndex(slug, "/"); idx >= 0 && idx < len(slug)-1 {
|
|
return slug[idx+1:]
|
|
}
|
|
return slug
|
|
}
|
|
|
|
func mustJSON(v any) string {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return "[]"
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func mustPrettyJSON(v any) string {
|
|
b, err := json.MarshalIndent(v, "", " ")
|
|
if err != nil {
|
|
return "{}"
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func renderWikiTemplate(tmpl string, values map[string]string) string {
|
|
out := tmpl
|
|
keys := make([]string, 0, len(values))
|
|
for k := range values {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
out = strings.ReplaceAll(out, "{"+k+"}", values[k])
|
|
}
|
|
return out
|
|
}
|
|
|
|
func prefixWithDash(items []string) []string {
|
|
out := make([]string, 0, len(items))
|
|
for _, item := range items {
|
|
out = append(out, "- "+item)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func firstNonEmpty(vals ...string) string {
|
|
for _, v := range vals {
|
|
if s := strings.TrimSpace(v); s != "" {
|
|
return s
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func uniqueStrings(in []string) []string {
|
|
if len(in) == 0 {
|
|
return nil
|
|
}
|
|
seen := map[string]bool{}
|
|
out := make([]string, 0, len(in))
|
|
for _, s := range in {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" || seen[s] {
|
|
continue
|
|
}
|
|
seen[s] = true
|
|
out = append(out, s)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func mergeStrings(a, b []string) []string {
|
|
return uniqueStrings(append(append([]string{}, a...), b...))
|
|
}
|
|
|
|
func normalizedStringSet(items []string) map[string]struct{} {
|
|
out := make(map[string]struct{}, len(items))
|
|
for _, item := range items {
|
|
if key := normKey(item); key != "" {
|
|
out[key] = struct{}{}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func intersectsNormalized(needle map[string]struct{}, haystack []string) bool {
|
|
if len(needle) == 0 {
|
|
return false
|
|
}
|
|
for _, item := range haystack {
|
|
if _, ok := needle[normKey(item)]; ok {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func normalizedOverlapCount(needle map[string]struct{}, haystack []string) int {
|
|
if len(needle) == 0 {
|
|
return 0
|
|
}
|
|
count := 0
|
|
seen := map[string]struct{}{}
|
|
for _, item := range haystack {
|
|
key := normKey(item)
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, dup := seen[key]; dup {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
if _, ok := needle[key]; ok {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|
|
|
|
func normKey(s string) string {
|
|
return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(s)), " "))
|
|
}
|
|
|
|
func firstString(v any) string {
|
|
switch x := v.(type) {
|
|
case string:
|
|
return x
|
|
case fmt.Stringer:
|
|
return x.String()
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func (e wikiExtract) sourceChunkIDs() []string {
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
add := func(s string) {
|
|
s = strings.TrimSpace(s)
|
|
if s == "" || seen[s] {
|
|
return
|
|
}
|
|
seen[s] = true
|
|
out = append(out, s)
|
|
}
|
|
for _, item := range e.Entities {
|
|
for _, id := range item.SourceChunkIDs {
|
|
add(id)
|
|
}
|
|
}
|
|
for _, item := range e.Concepts {
|
|
for _, id := range item.SourceChunkIDs {
|
|
add(id)
|
|
}
|
|
}
|
|
for _, item := range e.Claims {
|
|
for _, id := range item.SourceChunkIDs {
|
|
add(id)
|
|
}
|
|
}
|
|
for _, item := range e.Relations {
|
|
for _, id := range item.SourceChunkIDs {
|
|
add(id)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (e wikiExtract) asPlanPayload() map[string]any {
|
|
return map[string]any{
|
|
"entities": e.Entities,
|
|
"concepts": e.Concepts,
|
|
"claims": e.Claims,
|
|
"relations": e.Relations,
|
|
"topics": e.Topics,
|
|
}
|
|
}
|
|
|
|
func compactJSON(v any) string {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return "{}"
|
|
}
|
|
return string(bytes.TrimSpace(b))
|
|
}
|
|
|
|
func (p *wikiPipeline) maybeSourceIDs() []string {
|
|
if ids := p.reduced.sourceChunkIDs(); len(ids) > 0 {
|
|
return ids
|
|
}
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, ch := range p.inputs.Chunks {
|
|
id := strings.TrimSpace(ch.ID)
|
|
if id == "" || seen[id] {
|
|
continue
|
|
}
|
|
seen[id] = true
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (p *wikiPipeline) runMapBatch(batch []common.Chunk) error {
|
|
extract, err := p.mapBatch(batch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
p.mapExtracts = append(p.mapExtracts, extract)
|
|
return nil
|
|
}
|
|
|
|
// dedupHistorical drops products that are near-duplicates of existing historical
|
|
// artifacts, implementing cross-run dedup for the wiki variant. This remains a
|
|
// read-only historical lookup and does not store any wiki intermediate state.
|
|
func dedupHistorical(ctx context.Context, deps common.Deps, param common.Param, tenantID, datasetID string, override []common.Candidate, products []common.Product) (survivors []common.Product, dropped int, err error) {
|
|
if len(products) == 0 {
|
|
return products, 0, nil
|
|
}
|
|
threshold := param.SimilarityThreshold
|
|
|
|
if len(override) > 0 {
|
|
for _, p := range products {
|
|
if nearDuplicateOf(p.Vector, override, threshold) {
|
|
dropped++
|
|
continue
|
|
}
|
|
survivors = append(survivors, p)
|
|
}
|
|
return survivors, dropped, nil
|
|
}
|
|
|
|
if deps.HistoricalKNN == nil {
|
|
return products, 0, nil
|
|
}
|
|
for _, p := range products {
|
|
hits, e := deps.HistoricalKNN.TopKHistory(ctx, tenantID, datasetID, string(common.VariantWiki), p.Vector, wikiHistoricalK, threshold)
|
|
if e != nil {
|
|
return survivors, dropped, e
|
|
}
|
|
if len(hits) > 0 {
|
|
dropped++
|
|
continue
|
|
}
|
|
survivors = append(survivors, p)
|
|
}
|
|
return survivors, dropped, nil
|
|
}
|
|
|
|
func nearDuplicateOf(vec []float32, cands []common.Candidate, threshold float64) bool {
|
|
qn := l2Norm32(vec)
|
|
if qn == 0 {
|
|
qn = 1
|
|
}
|
|
for _, c := range cands {
|
|
cn := l2Norm32(c.Vector)
|
|
if cn == 0 {
|
|
cn = 1
|
|
}
|
|
var dot float64
|
|
for i := 0; i < len(vec) && i < len(c.Vector); i++ {
|
|
dot += float64(vec[i]) * float64(c.Vector[i])
|
|
}
|
|
if dot/(qn*cn) >= threshold {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func l2Norm32(v []float32) float64 {
|
|
var s float64
|
|
for _, x := range v {
|
|
s += float64(x) * float64(x)
|
|
}
|
|
return math.Sqrt(s)
|
|
}
|
|
|
|
func maxInt(a, b int) int {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
// wikiHistoricalK is the K used for the historical-dedup KNN lookup.
|
|
const wikiHistoricalK = 5
|
|
|
|
// wikiPlanTokenBudget caps one planning round's approximate token load.
|
|
const wikiPlanTokenBudget = 3500
|