mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-14 20:54:30 +08:00
Go: move logger to common module (#14545)
### What problem does this PR solve? As title ### Type of change - [x] Refactoring Signed-off-by: Jin Hai <haijin.chn@gmail.com>
This commit is contained in:
@@ -19,14 +19,15 @@ package infinity
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/common"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
"ragflow/internal/server"
|
||||
"ragflow/internal/logger"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
)
|
||||
|
||||
// infinityClient Infinity SDK client wrapper
|
||||
@@ -52,7 +53,7 @@ func NewInfinityClient(cfg *server.InfinityConfig) (*infinityClient, error) {
|
||||
}
|
||||
|
||||
// Retry connecting for up to 120 seconds (24 attempts * 5 seconds)
|
||||
logger.Info("Connecting to Infinity")
|
||||
common.Info("Connecting to Infinity")
|
||||
var conn *infinity.InfinityConnection
|
||||
var err error
|
||||
for i := 0; i < 24; i++ {
|
||||
@@ -78,7 +79,7 @@ func NewInfinityClient(cfg *server.InfinityConfig) (*infinityClient, error) {
|
||||
|
||||
// WaitForHealthy blocks until Infinity is healthy or timeout
|
||||
func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Duration) error {
|
||||
logger.Info("Waiting for Infinity to be healthy")
|
||||
common.Info("Waiting for Infinity to be healthy")
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
@@ -110,7 +111,7 @@ func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Durati
|
||||
if errorCode.Int() == 0 {
|
||||
status := serverStatus.String()
|
||||
if status == "started" || status == "alive" {
|
||||
logger.Info("Infinity is healthy")
|
||||
common.Info("Infinity is healthy")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -121,7 +122,7 @@ func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Durati
|
||||
|
||||
// Engine Infinity engine implementation using Go SDK
|
||||
type infinityEngine struct {
|
||||
config *server.InfinityConfig
|
||||
config *server.InfinityConfig
|
||||
client *infinityClient
|
||||
mappingFileName string
|
||||
docMetaMappingFileName string
|
||||
@@ -155,9 +156,9 @@ func NewEngine(cfg interface{}) (*infinityEngine, error) {
|
||||
}
|
||||
|
||||
engine := &infinityEngine{
|
||||
config: infConfig,
|
||||
client: client,
|
||||
mappingFileName: mappingFileName,
|
||||
config: infConfig,
|
||||
client: client,
|
||||
mappingFileName: mappingFileName,
|
||||
docMetaMappingFileName: docMetaMappingFileName,
|
||||
}
|
||||
|
||||
|
||||
@@ -21,10 +21,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"ragflow/internal/common"
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/logger"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
)
|
||||
|
||||
@@ -46,7 +45,7 @@ func (e *infinityEngine) Delete(ctx context.Context, condition map[string]interf
|
||||
|
||||
table, err := db.GetTable(tableName)
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName))
|
||||
common.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName))
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
@@ -199,7 +198,7 @@ func existsCondition(field string, tableColumns map[string]struct {
|
||||
}) string {
|
||||
col, colOk := tableColumns[field]
|
||||
if !colOk {
|
||||
logger.Warn(fmt.Sprintf("Column '%s' not found in table columns", field))
|
||||
common.Warn(fmt.Sprintf("Column '%s' not found in table columns", field))
|
||||
return fmt.Sprintf("%s!=null", field)
|
||||
}
|
||||
if strings.Contains(strings.ToLower(col.Type), "char") {
|
||||
|
||||
@@ -22,14 +22,15 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
"ragflow/internal/logger"
|
||||
"ragflow/internal/utility"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@@ -48,12 +49,12 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
// Skill index: table name is just indexName (e.g., "skill_abc123_def456")
|
||||
tableName = indexName
|
||||
mappingFile = "skill_infinity_mapping.json"
|
||||
logger.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
|
||||
common.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
|
||||
} else {
|
||||
// Regular document index: table name is {indexName}_{datasetID}
|
||||
tableName = fmt.Sprintf("%s_%s", indexName, datasetID)
|
||||
mappingFile = e.mappingFileName
|
||||
logger.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
|
||||
common.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
|
||||
}
|
||||
|
||||
// Use configured schema
|
||||
@@ -87,7 +88,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
var table *infinity.Table
|
||||
if exists {
|
||||
// Table exists, open it and check if vector column needs to be added
|
||||
logger.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
|
||||
common.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
|
||||
table, err = db.GetTable(tableName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to open existing table %s: %w", tableName, err)
|
||||
@@ -96,12 +97,12 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
// Check if vector column exists (for embedding model changes)
|
||||
colExists, err := e.columnExists(table, vectorColName)
|
||||
if err != nil {
|
||||
logger.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err))
|
||||
common.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err))
|
||||
}
|
||||
|
||||
// Add new vector column if it doesn't exist (handles embedding model change)
|
||||
if !colExists {
|
||||
logger.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize))
|
||||
common.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize))
|
||||
addColSchema := infinity.TableSchema{
|
||||
&infinity.ColumnDefinition{
|
||||
Name: vectorColName,
|
||||
@@ -109,14 +110,14 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
},
|
||||
}
|
||||
if _, err := table.AddColumns(addColSchema); err != nil {
|
||||
logger.Error("Failed to add vector column "+vectorColName, err)
|
||||
common.Error("Failed to add vector column "+vectorColName, err)
|
||||
return fmt.Errorf("Failed to add vector column %s: %w", vectorColName, err)
|
||||
}
|
||||
logger.Info("Successfully added vector column", zap.String("column", vectorColName))
|
||||
common.Info("Successfully added vector column", zap.String("column", vectorColName))
|
||||
}
|
||||
} else {
|
||||
// Table doesn't exist, create it with vector column in the initial schema
|
||||
logger.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize))
|
||||
common.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize))
|
||||
|
||||
// Build column definitions (preserving JSON order)
|
||||
var columns infinity.TableSchema
|
||||
@@ -151,7 +152,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create table: %w", err)
|
||||
}
|
||||
logger.Debug("Infinity created table", zap.String("tableName", tableName))
|
||||
common.Debug("Infinity created table", zap.String("tableName", tableName))
|
||||
}
|
||||
|
||||
// Create HNSW index on vector column with unique name based on vector size
|
||||
@@ -171,7 +172,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create HNSW index %s: %w", vectorIndexName, err)
|
||||
}
|
||||
logger.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
|
||||
common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
|
||||
|
||||
// Create full-text indexes for varchar fields with analyzers
|
||||
for _, fieldName := range schema.Keys {
|
||||
@@ -255,7 +256,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
|
||||
// Delete existing rows with matching IDs before insert
|
||||
func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string]interface{}, tableNamePrefix string, knowledgebaseID string) ([]string, error) {
|
||||
tableName := fmt.Sprintf("%s_%s", tableNamePrefix, knowledgebaseID)
|
||||
logger.Info("InfinityConnection.InsertDataset called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))
|
||||
common.Info("InfinityConnection.InsertDataset called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))
|
||||
|
||||
db, err := e.client.conn.GetDatabase(e.client.dbName)
|
||||
if err != nil {
|
||||
@@ -346,12 +347,12 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string]
|
||||
idList[i] = fmt.Sprintf("'%v'", chunk["id"])
|
||||
}
|
||||
filter := fmt.Sprintf("id IN (%s)", strings.Join(idList, ", "))
|
||||
logger.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter))
|
||||
common.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter))
|
||||
delResp, delErr := table.Delete(filter)
|
||||
if delErr != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr))
|
||||
common.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr))
|
||||
} else {
|
||||
logger.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows))
|
||||
common.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,7 +362,7 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string]
|
||||
return nil, fmt.Errorf("Failed to insert chunks to dataset: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("InfinityConnection.InsertDataset result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks)))
|
||||
common.Info("InfinityConnection.InsertDataset result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks)))
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
@@ -369,7 +370,7 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string]
|
||||
// Table name format: {tableNamePrefix}_{knowledgebaseID}
|
||||
func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, tableNamePrefix string, knowledgebaseID string) error {
|
||||
tableName := fmt.Sprintf("%s_%s", tableNamePrefix, knowledgebaseID)
|
||||
logger.Info("InfinityConnection.UpdateDataset called", zap.String("tableName", tableName), zap.Any("condition", condition))
|
||||
common.Info("InfinityConnection.UpdateDataset called", zap.String("tableName", tableName), zap.Any("condition", condition))
|
||||
|
||||
db, err := e.client.conn.GetDatabase(e.client.dbName)
|
||||
if err != nil {
|
||||
@@ -448,7 +449,7 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string
|
||||
// Query rows to be updated
|
||||
queryResult, err := table.Output(colToRemove).Filter(filter).ToResult()
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err))
|
||||
common.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err))
|
||||
} else {
|
||||
qr, ok := queryResult.(*infinity.QueryResult)
|
||||
if ok && len(qr.Data) > 0 {
|
||||
@@ -491,10 +492,10 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string
|
||||
for colName, valueToIDs := range removeOpt {
|
||||
for newVal, ids := range valueToIDs {
|
||||
idFilter := filter + " AND id IN (" + strings.Join(ids, ", ") + ")"
|
||||
logger.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal))
|
||||
common.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal))
|
||||
_, err := table.Update(idFilter, map[string]interface{}{colName: newVal})
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err))
|
||||
common.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,13 +504,13 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string
|
||||
}
|
||||
|
||||
// Execute the main update
|
||||
logger.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))
|
||||
common.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))
|
||||
_, err = table.Update(filter, newValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to update chunks: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("InfinityConnection.UpdateDataset completes", zap.String("tableName", tableName))
|
||||
common.Info("InfinityConnection.UpdateDataset completes", zap.String("tableName", tableName))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -19,10 +19,10 @@ package infinity
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/common"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"ragflow/internal/logger"
|
||||
)
|
||||
|
||||
// IndexDocument indexes a single document
|
||||
@@ -77,9 +77,9 @@ func (e *infinityEngine) InsertSkill(ctx context.Context, tableName, docID strin
|
||||
filter := fmt.Sprintf("skill_id = '%s'", docIDEscaped)
|
||||
delResp, delErr := table.Delete(filter)
|
||||
if delErr != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to delete existing skill document: %v", delErr))
|
||||
common.Warn(fmt.Sprintf("Failed to delete existing skill document: %v", delErr))
|
||||
} else if delResp.DeletedRows > 0 {
|
||||
logger.Debug(fmt.Sprintf("Deleted %d existing skill document(s)", delResp.DeletedRows))
|
||||
common.Debug(fmt.Sprintf("Deleted %d existing skill document(s)", delResp.DeletedRows))
|
||||
}
|
||||
|
||||
// Insert the document
|
||||
@@ -124,7 +124,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string,
|
||||
for _, doc := range docs {
|
||||
docMap, ok := doc.(map[string]interface{})
|
||||
if !ok {
|
||||
logger.Warn("Invalid doc type in bulk insert, expected map[string]interface{}")
|
||||
common.Warn("Invalid doc type in bulk insert, expected map[string]interface{}")
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -150,7 +150,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string,
|
||||
}
|
||||
|
||||
if len(insertDocs) == 0 {
|
||||
logger.Warn("No valid documents to bulk insert", zap.String("tableName", tableName))
|
||||
common.Warn("No valid documents to bulk insert", zap.String("tableName", tableName))
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
@@ -161,12 +161,12 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string,
|
||||
filter := fmt.Sprintf("skill_id = '%s'", docIDEscaped)
|
||||
delResp, delErr := table.Delete(filter)
|
||||
if delErr != nil {
|
||||
logger.Warn("Failed to delete existing skill document before bulk insert",
|
||||
common.Warn("Failed to delete existing skill document before bulk insert",
|
||||
zap.String("tableName", tableName),
|
||||
zap.String("skill_id", skillID),
|
||||
zap.Error(delErr))
|
||||
} else if delResp.DeletedRows > 0 {
|
||||
logger.Debug("Deleted existing skill document before bulk insert",
|
||||
common.Debug("Deleted existing skill document before bulk insert",
|
||||
zap.String("tableName", tableName),
|
||||
zap.String("skill_id", skillID),
|
||||
zap.Int64("deletedRows", delResp.DeletedRows))
|
||||
@@ -179,7 +179,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string,
|
||||
return 0, fmt.Errorf("failed to bulk insert skill documents: %w", err)
|
||||
}
|
||||
|
||||
logger.Debug("Bulk upserted skill documents",
|
||||
common.Debug("Bulk upserted skill documents",
|
||||
zap.String("tableName", tableName),
|
||||
zap.Int("count", len(insertDocs)),
|
||||
zap.Int("skillIDs", len(skillIDs)))
|
||||
@@ -229,7 +229,7 @@ func (e *infinityEngine) DeleteDocument(ctx context.Context, tableName, docID st
|
||||
return fmt.Errorf("failed to delete document: %w", err)
|
||||
}
|
||||
|
||||
logger.Debug("Deleted document from Infinity",
|
||||
common.Debug("Deleted document from Infinity",
|
||||
zap.String("tableName", tableName),
|
||||
zap.String("docID", docID),
|
||||
zap.String("idField", idField),
|
||||
|
||||
@@ -19,9 +19,9 @@ package infinity
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/common"
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/logger"
|
||||
"ragflow/internal/utility"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
@@ -115,7 +115,7 @@ func (e *infinityEngine) GetChunk(ctx context.Context, tableName, chunkID string
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
logger.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))
|
||||
common.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))
|
||||
|
||||
// Apply field mappings (same as in GetFields)
|
||||
// docnm -> docnm_kwd, title_tks, title_sm_tks
|
||||
|
||||
@@ -22,12 +22,13 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"ragflow/internal/common"
|
||||
"strings"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
"ragflow/internal/logger"
|
||||
"ragflow/internal/utility"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@@ -65,9 +66,9 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e
|
||||
var columns infinity.TableSchema
|
||||
for fieldName, fieldInfo := range schema {
|
||||
col := infinity.ColumnDefinition{
|
||||
Name: fieldName,
|
||||
Name: fieldName,
|
||||
DataType: fieldInfo.Type,
|
||||
Default: fieldInfo.Default,
|
||||
Default: fieldInfo.Default,
|
||||
// Comment: fieldInfo.Comment,
|
||||
}
|
||||
columns = append(columns, &col)
|
||||
@@ -78,7 +79,7 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create doc meta table: %w", err)
|
||||
}
|
||||
logger.Debug("Infinity created doc meta table", zap.String("tableName", indexName))
|
||||
common.Debug("Infinity created doc meta table", zap.String("tableName", indexName))
|
||||
|
||||
// Get table for creating indexes
|
||||
table, err := db.GetTable(indexName)
|
||||
@@ -117,7 +118,7 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e
|
||||
// Replace existing metadata with same id and kb_id
|
||||
func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) {
|
||||
tableName := fmt.Sprintf("ragflow_doc_meta_%s", tenantID)
|
||||
logger.Info("InfinityConnection.InsertMetadata called", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata)))
|
||||
common.Info("InfinityConnection.InsertMetadata called", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata)))
|
||||
|
||||
db, err := e.client.conn.GetDatabase(e.client.dbName)
|
||||
if err != nil {
|
||||
@@ -167,12 +168,12 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
|
||||
idList[i] = fmt.Sprintf("(id = %s AND kb_id = %s)", docID, kbID)
|
||||
}
|
||||
filter := strings.Join(idList, " OR ")
|
||||
logger.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter))
|
||||
common.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter))
|
||||
delResp, delErr := table.Delete(filter)
|
||||
if delErr != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to delete existing metadata: %v", delErr))
|
||||
common.Warn(fmt.Sprintf("Failed to delete existing metadata: %v", delErr))
|
||||
} else if delResp.DeletedRows > 0 {
|
||||
logger.Info(fmt.Sprintf("Deleted %d existing metadata entries", delResp.DeletedRows))
|
||||
common.Info(fmt.Sprintf("Deleted %d existing metadata entries", delResp.DeletedRows))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +183,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
|
||||
return nil, fmt.Errorf("Failed to insert metadata: %w", err)
|
||||
}
|
||||
|
||||
logger.Info("InfinityConnection.InsertMetadata result", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata)))
|
||||
common.Info("InfinityConnection.InsertMetadata result", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata)))
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
@@ -192,7 +193,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri
|
||||
// Table name format: ragflow_doc_meta_{tenant_id}
|
||||
func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID string, metaFields map[string]interface{}, tenantID string) error {
|
||||
tableName := fmt.Sprintf("ragflow_doc_meta_%s", tenantID)
|
||||
logger.Info("InfinityConnection.UpdateMetadata called", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("kbID", kbID))
|
||||
common.Info("InfinityConnection.UpdateMetadata called", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("kbID", kbID))
|
||||
|
||||
db, err := e.client.conn.GetDatabase(e.client.dbName)
|
||||
if err != nil {
|
||||
@@ -216,7 +217,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID
|
||||
result, err := queryTable.ToResult()
|
||||
rowExists := false
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to query existing metadata: %v", err))
|
||||
common.Warn(fmt.Sprintf("Failed to query existing metadata: %v", err))
|
||||
// If query fails, treat as not exists and insert
|
||||
} else {
|
||||
// Get results - ToResult returns *infinity.QueryResult
|
||||
@@ -234,7 +235,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID
|
||||
switch v := existingMetaFieldsVal.(type) {
|
||||
case string:
|
||||
if err := json.Unmarshal([]byte(v), &existingMetaFields); err != nil {
|
||||
logger.Warn(fmt.Sprintf("Failed to parse existing meta_fields: %v", err))
|
||||
common.Warn(fmt.Sprintf("Failed to parse existing meta_fields: %v", err))
|
||||
existingMetaFields = make(map[string]interface{})
|
||||
}
|
||||
case map[string]interface{}:
|
||||
@@ -261,7 +262,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID
|
||||
|
||||
if rowExists {
|
||||
// Row exists: update it with merged metadata
|
||||
logger.Info(fmt.Sprintf("UpdateMetadata: updating existing row, table=%s, filter=%s, newValue=%v", tableName, filter, updatedFields))
|
||||
common.Info(fmt.Sprintf("UpdateMetadata: updating existing row, table=%s, filter=%s, newValue=%v", tableName, filter, updatedFields))
|
||||
_, err = table.Update(filter, updatedFields)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update metadata: %w", err)
|
||||
@@ -273,14 +274,13 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID
|
||||
"kb_id": kbID,
|
||||
"meta_fields": utility.ConvertMapToJSONString(metaFields),
|
||||
}
|
||||
logger.Info(fmt.Sprintf("UpdateMetadata: inserting new row, table=%s, newValue=%v", tableName, insertFields))
|
||||
common.Info(fmt.Sprintf("UpdateMetadata: inserting new row, table=%s, newValue=%v", tableName, insertFields))
|
||||
_, err = table.Insert(insertFields)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert metadata: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("InfinityConnection.UpdateMetadata completes", zap.String("tableName", tableName), zap.String("docID", docID))
|
||||
common.Info("InfinityConnection.UpdateMetadata completes", zap.String("tableName", tableName), zap.String("docID", docID))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -30,8 +30,6 @@ import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"ragflow/internal/logger"
|
||||
|
||||
infinity "github.com/infiniflow/infinity-go-sdk"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@@ -40,8 +38,8 @@ import (
|
||||
// It supports three matching types: MatchTextExpr (full-text), MatchDenseExpr (vector), and FusionExpr (combined).
|
||||
// If no match expressions are provided, Search relies solely on filter (e.g., doc_id, available_int) to find results.
|
||||
func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
|
||||
logger.Debug("Search in Infinity started", zap.Any("indexNames", req.IndexNames))
|
||||
if logger.IsDebugEnabled() {
|
||||
common.Debug("Search in Infinity started", zap.Any("indexNames", req.IndexNames))
|
||||
if common.IsDebugEnabled() {
|
||||
// Format match expressions for logging
|
||||
var matchExprsStr string
|
||||
for i, expr := range req.MatchExprs {
|
||||
@@ -56,7 +54,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
matchExprsStr += fmt.Sprintf(" [%d] unknown type\n", i)
|
||||
}
|
||||
}
|
||||
logger.Debug(fmt.Sprintf("Search request:\n"+
|
||||
common.Debug(fmt.Sprintf("Search request:\n"+
|
||||
" indexNames=%v\n"+
|
||||
" KbIDs=%v\n"+
|
||||
" offset=%d, limit=%d\n"+
|
||||
@@ -298,12 +296,12 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
if matchText != nil && len(matchText.Fields) > 0 {
|
||||
textFields = matchText.Fields
|
||||
} else if isSkillIndex {
|
||||
textFields = []string{
|
||||
"name^10",
|
||||
"tags^5",
|
||||
"description^3",
|
||||
"content^1",
|
||||
}
|
||||
textFields = []string{
|
||||
"name^10",
|
||||
"tags^5",
|
||||
"description^3",
|
||||
"content^1",
|
||||
}
|
||||
} else {
|
||||
textFields = []string{
|
||||
"title_tks^10",
|
||||
@@ -352,7 +350,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
|
||||
table = table.MatchText(fields, questionText, textTopN, extraOptions)
|
||||
|
||||
logger.Debug(fmt.Sprintf(
|
||||
common.Debug(fmt.Sprintf(
|
||||
"MatchTextExpr:\n"+
|
||||
" fields=%s\n"+
|
||||
" matching_text=%s\n"+
|
||||
@@ -386,14 +384,14 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
vectorTopN = int(matchDense.TopN)
|
||||
}
|
||||
|
||||
denseFilterStr := filterStr
|
||||
if denseFilterStr == "" {
|
||||
if isSkillIndex {
|
||||
denseFilterStr = "status='1'"
|
||||
} else {
|
||||
denseFilterStr = "available_int=1"
|
||||
denseFilterStr := filterStr
|
||||
if denseFilterStr == "" {
|
||||
if isSkillIndex {
|
||||
denseFilterStr = "status='1'"
|
||||
} else {
|
||||
denseFilterStr = "available_int=1"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasTextMatch && fusionExpr == nil {
|
||||
fieldsStr := strings.Join(convertedFields, ",")
|
||||
@@ -405,7 +403,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
"filter": denseFilterStr,
|
||||
}
|
||||
|
||||
logger.Debug("MatchDense for hybrid search",
|
||||
common.Debug("MatchDense for hybrid search",
|
||||
zap.String("fieldName", fieldName),
|
||||
zap.String("distanceType", distanceType),
|
||||
zap.Int("topN", vectorTopN),
|
||||
@@ -430,7 +428,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug("Applying Fusion for hybrid search",
|
||||
common.Debug("Applying Fusion for hybrid search",
|
||||
zap.String("method", fusionMethod),
|
||||
zap.Int("topN", fusionTopK),
|
||||
zap.Any("params", fusionParams))
|
||||
@@ -453,7 +451,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
|
||||
// Add filter when there's no text/vector match (like metadata queries)
|
||||
if !hasTextMatch && !hasVectorMatch && filterStr != "" {
|
||||
logger.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr))
|
||||
common.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr))
|
||||
table = table.Filter(filterStr)
|
||||
}
|
||||
|
||||
@@ -469,7 +467,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
// Execute query
|
||||
df, err := table.ToDataFrame()
|
||||
if err != nil {
|
||||
logger.Warn("Infinity query failed",
|
||||
common.Warn("Infinity query failed",
|
||||
zap.String("tableName", tableName),
|
||||
zap.Bool("hasTextMatch", hasTextMatch),
|
||||
zap.Bool("hasVectorMatch", hasVectorMatch),
|
||||
@@ -547,7 +545,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (
|
||||
allResults = allResults[:pageSize]
|
||||
}
|
||||
|
||||
logger.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
|
||||
common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
|
||||
|
||||
return &types.SearchResult{
|
||||
Chunks: allResults,
|
||||
@@ -647,10 +645,10 @@ func convertMatchingField(fieldWeightStr string) string {
|
||||
"authors_sm_tks": "authors@ft_authors_rag_fine",
|
||||
"tag_kwd": "tag_kwd@ft_tag_kwd_whitespace__",
|
||||
// Skill index fields
|
||||
"name": "name@ft_name_rag_coarse",
|
||||
"tags": "tags@ft_tags_rag_coarse",
|
||||
"description": "description@ft_description_rag_coarse",
|
||||
"content": "content@ft_content_rag_coarse",
|
||||
"name": "name@ft_name_rag_coarse",
|
||||
"tags": "tags@ft_tags_rag_coarse",
|
||||
"description": "description@ft_description_rag_coarse",
|
||||
"content": "content@ft_content_rag_coarse",
|
||||
}
|
||||
|
||||
if newField, ok := fieldMapping[field]; ok {
|
||||
@@ -1099,4 +1097,4 @@ func (e *infinityEngine) GetHighlight(chunks []map[string]interface{}, keywords
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user