Feat: add skills space to context engine (#13908)

### What problem does this PR solve? issue #13714 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
2026-09-08 10:14:35 +08:00 · 2026-04-30 12:36:03 +08:00
parent bb3b99f0a5
commit 4ee0702aed
101 changed files with 19161 additions and 633 deletions
@@ -36,14 +36,28 @@ import (
 // CreateDataset creates a table in Infinity
 // indexName is the table name prefix (e.g., "ragflow_<tenant_id>")
 // The full table name is built as "{indexName}_{datasetID}"
+// For skill index (datasetID="skill"), tableName is just indexName and uses skill_infinity_mapping.json
 func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID string, vectorSize int, parserID string) error {
 	vecSize := vectorSize

-	// Build full table name: {indexName}_{datasetID}
-	tableName := fmt.Sprintf("%s_%s", indexName, datasetID)
+	// Determine table name and mapping file based on index type
+	var tableName string
+	var mappingFile string
+
+	if datasetID == "skill" {
+		// Skill index: table name is just indexName (e.g., "skill_abc123_def456")
+		tableName = indexName
+		mappingFile = "skill_infinity_mapping.json"
+		logger.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
+	} else {
+		// Regular document index: table name is {indexName}_{datasetID}
+		tableName = fmt.Sprintf("%s_%s", indexName, datasetID)
+		mappingFile = e.mappingFileName
+		logger.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile))
+	}

 	// Use configured schema
-	fpMapping := filepath.Join(utility.GetProjectRoot(), "conf", e.mappingFileName)
+	fpMapping := filepath.Join(utility.GetProjectRoot(), "conf", mappingFile)

 	schemaData, err := os.ReadFile(fpMapping)
 	if err != nil {
@@ -61,54 +75,90 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
 		return fmt.Errorf("Failed to get database: %w", err)
 	}

+	// Determine vector column name
+	vectorColName := fmt.Sprintf("q_%d_vec", vecSize)
+
 	// Check if table already exists
 	exists, err := e.TableExists(ctx, tableName)
 	if err != nil {
 		return fmt.Errorf("Failed to check if table exists: %w", err)
 	}
+
+	var table *infinity.Table
 	if exists {
-		return fmt.Errorf("table '%s' already exists", tableName)
-	}
-
-	// Build column definitions (preserving JSON order)
-	var columns infinity.TableSchema
-	for _, fieldName := range schema.Keys {
-		fieldInfo := schema.Fields[fieldName]
-		col := infinity.ColumnDefinition{
-			Name:     fieldName,
-			DataType: fieldInfo.Type,
-			Default:  fieldInfo.Default,
-			// Comment:  fieldInfo.Comment,
+		// Table exists, open it and check if vector column needs to be added
+		logger.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
+		table, err = db.GetTable(tableName)
+		if err != nil {
+			return fmt.Errorf("Failed to open existing table %s: %w", tableName, err)
 		}
-		columns = append(columns, &col)
-	}

-	// Add vector column
-	vectorColName := fmt.Sprintf("q_%d_vec", vecSize)
-	columns = append(columns, &infinity.ColumnDefinition{
-		Name:     vectorColName,
-		DataType: fmt.Sprintf("vector,%d,float", vecSize),
-	})
+		// Check if vector column exists (for embedding model changes)
+		colExists, err := e.columnExists(table, vectorColName)
+		if err != nil {
+			logger.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err))
+		}

-	// Add chunk_data column for table parser
-	if parserID == "table" {
+		// Add new vector column if it doesn't exist (handles embedding model change)
+		if !colExists {
+			logger.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize))
+			addColSchema := infinity.TableSchema{
+				&infinity.ColumnDefinition{
+					Name:     vectorColName,
+					DataType: fmt.Sprintf("vector,%d,float", vecSize),
+				},
+			}
+			if _, err := table.AddColumns(addColSchema); err != nil {
+				logger.Error("Failed to add vector column "+vectorColName, err)
+				return fmt.Errorf("Failed to add vector column %s: %w", vectorColName, err)
+			}
+			logger.Info("Successfully added vector column", zap.String("column", vectorColName))
+		}
+	} else {
+		// Table doesn't exist, create it with vector column in the initial schema
+		logger.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize))
+
+		// Build column definitions (preserving JSON order)
+		var columns infinity.TableSchema
+		for _, fieldName := range schema.Keys {
+			fieldInfo := schema.Fields[fieldName]
+			col := infinity.ColumnDefinition{
+				Name:     fieldName,
+				DataType: fieldInfo.Type,
+				Default:  fieldInfo.Default,
+				// Comment:  fieldInfo.Comment,
+			}
+			columns = append(columns, &col)
+		}
+
+		// Add vector column
 		columns = append(columns, &infinity.ColumnDefinition{
-			Name:     "chunk_data",
-			DataType: "json",
-			Default:  "{}",
+			Name:     vectorColName,
+			DataType: fmt.Sprintf("vector,%d,float", vecSize),
 		})
+
+		// Add chunk_data column for table parser
+		if parserID == "table" {
+			columns = append(columns, &infinity.ColumnDefinition{
+				Name:     "chunk_data",
+				DataType: "json",
+				Default:  "{}",
+			})
+		}
+
+		// Create table
+		table, err = db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
+		if err != nil {
+			return fmt.Errorf("Failed to create table: %w", err)
+		}
+		logger.Debug("Infinity created table", zap.String("tableName", tableName))
 	}

-	// Create table
-	table, err := db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
-	if err != nil {
-		return fmt.Errorf("Failed to create table: %w", err)
-	}
-	logger.Debug("Infinity created table", zap.String("tableName", tableName))
-
-	// Create HNSW index on vector column
+	// Create HNSW index on vector column with unique name based on vector size
+	// Use unique index name to avoid conflict when embedding model changes
+	vectorIndexName := fmt.Sprintf("q_%d_vec_idx", vecSize)
 	_, err = table.CreateIndex(
-		"q_vec_idx",
+		vectorIndexName,
 		infinity.NewIndexInfo(vectorColName, infinity.IndexTypeHnsw, map[string]string{
 			"M":               "16",
 			"ef_construction": "50",
@@ -119,8 +169,9 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID
 		"",
 	)
 	if err != nil {
-		return fmt.Errorf("Failed to create HNSW index: %w", err)
+		return fmt.Errorf("Failed to create HNSW index %s: %w", vectorIndexName, err)
 	}
+	logger.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))

 	// Create full-text indexes for varchar fields with analyzers
 	for _, fieldName := range schema.Keys {