mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-29 04:08:12 +08:00
RAGFlow go API server (#13240)
# RAGFlow Go Implementation Plan 🚀 This repository tracks the progress of porting RAGFlow to Go. We'll implement core features and provide performance comparisons between Python and Go versions. ## Implementation Checklist - [x] User Management APIs - [x] Dataset Management Operations - [x] Retrieval Test - [x] Chat Management Operations - [x] Infinity Go SDK --------- Signed-off-by: Jin Hai <haijin.chn@gmail.com> Co-authored-by: Yingfeng Zhang <yingfeng.zhang@gmail.com>
This commit is contained in:
200
internal/engine/README.md
Normal file
200
internal/engine/README.md
Normal file
@@ -0,0 +1,200 @@
|
||||
# Doc Engine Implementation
|
||||
|
||||
RAGFlow Go document engine implementation, supporting Elasticsearch and Infinity storage engines.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
internal/engine/
|
||||
├── engine.go # DocEngine interface definition
|
||||
├── engine_factory.go # Factory function
|
||||
├── global.go # Global engine instance management
|
||||
├── elasticsearch/ # Elasticsearch implementation
|
||||
│ ├── client.go # ES client initialization
|
||||
│ ├── search.go # Search implementation
|
||||
│ ├── index.go # Index operations
|
||||
│ └── document.go # Document operations
|
||||
└── infinity/ # Infinity implementation
|
||||
├── client.go # Infinity client initialization (placeholder)
|
||||
├── search.go # Search implementation (placeholder)
|
||||
├── index.go # Table operations (placeholder)
|
||||
└── document.go # Document operations (placeholder)
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Using Elasticsearch
|
||||
|
||||
Add to `conf/service_conf.yaml`:
|
||||
|
||||
```yaml
|
||||
doc_engine:
|
||||
type: elasticsearch
|
||||
es:
|
||||
hosts: "http://localhost:9200"
|
||||
username: "elastic"
|
||||
password: "infini_rag_flow"
|
||||
```
|
||||
|
||||
### Using Infinity
|
||||
|
||||
```yaml
|
||||
doc_engine:
|
||||
type: infinity
|
||||
infinity:
|
||||
uri: "localhost:23817"
|
||||
postgres_port: 5432
|
||||
db_name: "default_db"
|
||||
```
|
||||
|
||||
**Note**: Infinity implementation is a placeholder waiting for the official Infinity Go SDK. Only Elasticsearch is fully functional at this time.
|
||||
|
||||
## Usage
|
||||
|
||||
### 1. Initialize Engine
|
||||
|
||||
The engine is automatically initialized on service startup (see `cmd/server_main.go`):
|
||||
|
||||
```go
|
||||
// Initialize doc engine
|
||||
if err := engine.Init(&cfg.DocEngine); err != nil {
|
||||
log.Fatalf("Failed to initialize doc engine: %v", err)
|
||||
}
|
||||
defer engine.Close()
|
||||
```
|
||||
|
||||
### 2. Use in Service
|
||||
|
||||
In `ChunkService`:
|
||||
|
||||
```go
|
||||
type ChunkService struct {
|
||||
docEngine engine.DocEngine
|
||||
engineType config.EngineType
|
||||
}
|
||||
|
||||
func NewChunkService() *ChunkService {
|
||||
cfg := config.Get()
|
||||
return &ChunkService{
|
||||
docEngine: engine.Get(),
|
||||
engineType: cfg.DocEngine.Type,
|
||||
}
|
||||
}
|
||||
|
||||
// Search
|
||||
func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest) (*RetrievalTestResponse, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
switch s.engineType {
|
||||
case config.EngineElasticsearch:
|
||||
// Use Elasticsearch retrieval
|
||||
searchReq := &elasticsearch.SearchRequest{
|
||||
IndexNames: []string{"chunks"},
|
||||
Query: elasticsearch.BuildMatchTextQuery([]string{"content"}, req.Question, "AUTO"),
|
||||
Size: 10,
|
||||
}
|
||||
result, _ := s.docEngine.Search(ctx, searchReq)
|
||||
esResp := result.(*elasticsearch.SearchResponse)
|
||||
// Process result...
|
||||
|
||||
case config.EngineInfinity:
|
||||
// Infinity not implemented yet
|
||||
return nil, fmt.Errorf("infinity not yet implemented")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Direct Use of Global Engine
|
||||
|
||||
```go
|
||||
import "ragflow/internal/engine"
|
||||
|
||||
// Get engine instance
|
||||
docEngine := engine.Get()
|
||||
|
||||
// Search
|
||||
searchReq := &elasticsearch.SearchRequest{
|
||||
IndexNames: []string{"my_index"},
|
||||
Query: elasticsearch.BuildTermQuery("status", "active"),
|
||||
}
|
||||
result, err := docEngine.Search(ctx, searchReq)
|
||||
|
||||
// Index operations
|
||||
err = docEngine.CreateIndex(ctx, "my_index", mapping)
|
||||
err = docEngine.DeleteIndex(ctx, "my_index")
|
||||
exists, _ := docEngine.IndexExists(ctx, "my_index")
|
||||
|
||||
// Document operations
|
||||
err = docEngine.IndexDocument(ctx, "my_index", "doc_id", docData)
|
||||
bulkResp, _ := docEngine.BulkIndex(ctx, "my_index", docs)
|
||||
doc, _ := docEngine.GetDocument(ctx, "my_index", "doc_id")
|
||||
err = docEngine.DeleteDocument(ctx, "my_index", "doc_id")
|
||||
```
|
||||
|
||||
## API Documentation
|
||||
|
||||
### DocEngine Interface
|
||||
|
||||
```go
|
||||
type DocEngine interface {
|
||||
// Search
|
||||
Search(ctx context.Context, req interface{}) (interface{}, error)
|
||||
|
||||
// Index operations
|
||||
CreateIndex(ctx context.Context, indexName string, mapping interface{}) error
|
||||
DeleteIndex(ctx context.Context, indexName string) error
|
||||
IndexExists(ctx context.Context, indexName string) (bool, error)
|
||||
|
||||
// Document operations
|
||||
IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error
|
||||
BulkIndex(ctx context.Context, indexName string, docs []interface{}) (interface{}, error)
|
||||
GetDocument(ctx context.Context, indexName, docID string) (interface{}, error)
|
||||
DeleteDocument(ctx context.Context, indexName, docID string) error
|
||||
|
||||
// Health check
|
||||
Ping(ctx context.Context) error
|
||||
Close() error
|
||||
}
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Elasticsearch
|
||||
- `github.com/elastic/go-elasticsearch/v8`
|
||||
|
||||
### Infinity
|
||||
- **Not available yet** - Waiting for official Infinity Go SDK
|
||||
|
||||
## Notes
|
||||
|
||||
1. **Type Conversion**: The `Search` method returns `interface{}`, requiring type assertion based on engine type
|
||||
2. **Model Definitions**: Each engine has its own request/response models defined in their respective packages
|
||||
3. **Error Handling**: It's recommended to handle errors uniformly in the service layer and return user-friendly error messages
|
||||
4. **Performance Optimization**: For large volumes of documents, prefer using `BulkIndex` for batch operations
|
||||
5. **Connection Management**: The engine is automatically closed when the program exits, no manual management needed
|
||||
6. **Infinity Status**: Infinity implementation is currently a placeholder. Only Elasticsearch is fully functional.
|
||||
|
||||
## Extending with New Engines
|
||||
|
||||
To add a new document engine (e.g., Milvus, Qdrant):
|
||||
|
||||
1. Create a new directory under `internal/engine/`, e.g., `milvus/`
|
||||
2. Implement four files: `client.go`, `search.go`, `index.go`, `document.go`
|
||||
3. Add corresponding creation logic in `engine_factory.go`
|
||||
4. Add configuration structure in `config.go`
|
||||
5. Update service layer code to support the new engine
|
||||
|
||||
## Correspondence with Python Project
|
||||
|
||||
| Python Module | Go Module |
|
||||
|--------------|-----------|
|
||||
| `common/doc_store/doc_store_base.py` | `internal/engine/engine.go` |
|
||||
| `rag/utils/es_conn.py` | `internal/engine/elasticsearch/` |
|
||||
| `rag/utils/infinity_conn.py` | `internal/engine/infinity/` (placeholder) |
|
||||
| `common/settings.py` | `internal/config/config.go` |
|
||||
|
||||
## Current Status
|
||||
|
||||
- ✅ Elasticsearch: Fully implemented and functional
|
||||
- ⏳ Infinity: Placeholder implementation, waiting for official Go SDK
|
||||
- 📋 OceanBase: Not implemented (removed from requirements)
|
||||
103
internal/engine/elasticsearch/client.go
Normal file
103
internal/engine/elasticsearch/client.go
Normal file
@@ -0,0 +1,103 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package elasticsearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"ragflow/internal/server"
|
||||
"time"
|
||||
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/elastic/go-elasticsearch/v8/esapi"
|
||||
)
|
||||
|
||||
// Engine Elasticsearch engine implementation
|
||||
type elasticsearchEngine struct {
|
||||
client *elasticsearch.Client
|
||||
config *server.ElasticsearchConfig
|
||||
}
|
||||
|
||||
// NewEngine creates an Elasticsearch engine
|
||||
func NewEngine(cfg interface{}) (*elasticsearchEngine, error) {
|
||||
esConfig, ok := cfg.(*server.ElasticsearchConfig)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid Elasticsearch config type, expected *config.ElasticsearchConfig")
|
||||
}
|
||||
|
||||
// Create ES client
|
||||
client, err := elasticsearch.NewClient(elasticsearch.Config{
|
||||
Addresses: []string{esConfig.Hosts},
|
||||
Username: esConfig.Username,
|
||||
Password: esConfig.Password,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConnsPerHost: 10,
|
||||
ResponseHeaderTimeout: 30 * time.Second,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Elasticsearch client: %w", err)
|
||||
}
|
||||
|
||||
// Check connection
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req := esapi.InfoRequest{}
|
||||
res, err := req.Do(ctx, client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to ping Elasticsearch: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
engine := &elasticsearchEngine{
|
||||
client: client,
|
||||
config: esConfig,
|
||||
}
|
||||
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
// Type returns the engine type
|
||||
func (e *elasticsearchEngine) Type() string {
|
||||
return "elasticsearch"
|
||||
}
|
||||
|
||||
// Ping health check
|
||||
func (e *elasticsearchEngine) Ping(ctx context.Context) error {
|
||||
req := esapi.InfoRequest{}
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("elasticsearch ping failed: %s", res.Status())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the connection
|
||||
func (e *elasticsearchEngine) Close() error {
|
||||
// Go-elasticsearch client doesn't have a Close method, connection is managed by the transport
|
||||
return nil
|
||||
}
|
||||
238
internal/engine/elasticsearch/document.go
Normal file
238
internal/engine/elasticsearch/document.go
Normal file
@@ -0,0 +1,238 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package elasticsearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/elastic/go-elasticsearch/v8/esapi"
|
||||
)
|
||||
|
||||
// IndexDocument indexes a single document
|
||||
func (e *elasticsearchEngine) IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error {
|
||||
if indexName == "" {
|
||||
return fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
if docID == "" {
|
||||
return fmt.Errorf("document id cannot be empty")
|
||||
}
|
||||
if doc == nil {
|
||||
return fmt.Errorf("document cannot be nil")
|
||||
}
|
||||
|
||||
// Serialize document
|
||||
data, err := json.Marshal(doc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal document: %w", err)
|
||||
}
|
||||
|
||||
// Index document
|
||||
req := esapi.IndexRequest{
|
||||
Index: indexName,
|
||||
DocumentID: docID,
|
||||
Body: bytes.NewReader(data),
|
||||
Refresh: "true",
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to index document: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BulkIndex indexes documents in bulk
|
||||
func (e *elasticsearchEngine) BulkIndex(ctx context.Context, indexName string, docs []interface{}) (interface{}, error) {
|
||||
if indexName == "" {
|
||||
return nil, fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
return nil, fmt.Errorf("documents cannot be empty")
|
||||
}
|
||||
|
||||
// Build bulk request
|
||||
var buf bytes.Buffer
|
||||
for _, doc := range docs {
|
||||
docMap, ok := doc.(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("document must be map[string]interface{}")
|
||||
}
|
||||
|
||||
docID, hasID := docMap["_id"]
|
||||
if !hasID {
|
||||
return nil, fmt.Errorf("document missing _id field")
|
||||
}
|
||||
|
||||
// Delete _id field to avoid duplication
|
||||
delete(docMap, "_id")
|
||||
|
||||
// Add index operation
|
||||
meta := map[string]interface{}{
|
||||
"_index": indexName,
|
||||
"_id": docID,
|
||||
}
|
||||
metaData, _ := json.Marshal(meta)
|
||||
docData, _ := json.Marshal(docMap)
|
||||
|
||||
buf.Write(metaData)
|
||||
buf.WriteByte('\n')
|
||||
buf.Write(docData)
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
|
||||
// Execute bulk request
|
||||
req := esapi.BulkRequest{
|
||||
Body: &buf,
|
||||
Refresh: "true",
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bulk index failed: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
return nil, fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
// Check for errors
|
||||
if errors, ok := result["errors"].(bool); ok && errors {
|
||||
// Get error details
|
||||
if items, ok := result["items"].([]interface{}); ok && len(items) > 0 {
|
||||
for _, item := range items {
|
||||
if itemMap, ok := item.(map[string]interface{}); ok {
|
||||
for _, op := range itemMap {
|
||||
if opMap, ok := op.(map[string]interface{}); ok {
|
||||
if errInfo, ok := opMap["error"].(map[string]interface{}); ok {
|
||||
if reason, ok := errInfo["reason"].(string); ok {
|
||||
return nil, fmt.Errorf("bulk index error: %s", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("bulk index has errors")
|
||||
}
|
||||
|
||||
response := &BulkResponse{
|
||||
Took: int64(result["took"].(float64)),
|
||||
Errors: result["errors"].(bool),
|
||||
Indexed: len(docs),
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// BulkResponse bulk operation response
|
||||
type BulkResponse struct {
|
||||
Took int64
|
||||
Errors bool
|
||||
Indexed int
|
||||
}
|
||||
|
||||
// GetDocument gets a document
|
||||
func (e *elasticsearchEngine) GetDocument(ctx context.Context, indexName, docID string) (interface{}, error) {
|
||||
if indexName == "" {
|
||||
return nil, fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
if docID == "" {
|
||||
return nil, fmt.Errorf("document id cannot be empty")
|
||||
}
|
||||
|
||||
// Get document
|
||||
req := esapi.GetRequest{
|
||||
Index: indexName,
|
||||
DocumentID: docID,
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get document: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode == 404 {
|
||||
return nil, fmt.Errorf("document not found")
|
||||
}
|
||||
|
||||
if res.IsError() {
|
||||
return nil, fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
if found, ok := result["found"].(bool); !ok || !found {
|
||||
return nil, fmt.Errorf("document not found")
|
||||
}
|
||||
|
||||
return result["_source"], nil
|
||||
}
|
||||
|
||||
// DeleteDocument deletes a document
|
||||
func (e *elasticsearchEngine) DeleteDocument(ctx context.Context, indexName, docID string) error {
|
||||
if indexName == "" {
|
||||
return fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
if docID == "" {
|
||||
return fmt.Errorf("document id cannot be empty")
|
||||
}
|
||||
|
||||
// Delete document
|
||||
req := esapi.DeleteRequest{
|
||||
Index: indexName,
|
||||
DocumentID: docID,
|
||||
Refresh: "true",
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete document: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode == 404 {
|
||||
return fmt.Errorf("document not found")
|
||||
}
|
||||
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
144
internal/engine/elasticsearch/index.go
Normal file
144
internal/engine/elasticsearch/index.go
Normal file
@@ -0,0 +1,144 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package elasticsearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/elastic/go-elasticsearch/v8/esapi"
|
||||
)
|
||||
|
||||
// CreateIndex creates an index
|
||||
func (e *elasticsearchEngine) CreateIndex(ctx context.Context, indexName string, mapping interface{}) error {
|
||||
if indexName == "" {
|
||||
return fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
|
||||
// Check if index already exists
|
||||
exists, err := e.IndexExists(ctx, indexName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check index existence: %w", err)
|
||||
}
|
||||
if exists {
|
||||
return fmt.Errorf("index '%s' already exists", indexName)
|
||||
}
|
||||
|
||||
// Prepare request body
|
||||
var body io.Reader
|
||||
if mapping != nil {
|
||||
if str, ok := mapping.(string); ok {
|
||||
body = bytes.NewBufferString(str)
|
||||
} else {
|
||||
data, err := json.Marshal(mapping)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal mapping: %w", err)
|
||||
}
|
||||
body = bytes.NewReader(data)
|
||||
}
|
||||
}
|
||||
|
||||
// Create index
|
||||
req := esapi.IndicesCreateRequest{
|
||||
Index: indexName,
|
||||
Body: body,
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create index: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var result map[string]interface{}
|
||||
if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
|
||||
return fmt.Errorf("failed to parse response: %w", err)
|
||||
}
|
||||
|
||||
acknowledged, ok := result["acknowledged"].(bool)
|
||||
if !ok || !acknowledged {
|
||||
return fmt.Errorf("index creation not acknowledged")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteIndex deletes an index
|
||||
func (e *elasticsearchEngine) DeleteIndex(ctx context.Context, indexName string) error {
|
||||
if indexName == "" {
|
||||
return fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
|
||||
// Check if index exists
|
||||
exists, err := e.IndexExists(ctx, indexName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check index existence: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("index '%s' does not exist", indexName)
|
||||
}
|
||||
|
||||
// Delete index
|
||||
req := esapi.IndicesDeleteRequest{
|
||||
Index: []string{indexName},
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete index: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
return fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IndexExists checks if index exists
|
||||
func (e *elasticsearchEngine) IndexExists(ctx context.Context, indexName string) (bool, error) {
|
||||
if indexName == "" {
|
||||
return false, fmt.Errorf("index name cannot be empty")
|
||||
}
|
||||
|
||||
req := esapi.IndicesExistsRequest{
|
||||
Index: []string{indexName},
|
||||
}
|
||||
|
||||
res, err := req.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check index existence: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode == 200 {
|
||||
return true, nil
|
||||
} else if res.StatusCode == 404 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
528
internal/engine/elasticsearch/search.go
Normal file
528
internal/engine/elasticsearch/search.go
Normal file
@@ -0,0 +1,528 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package elasticsearch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/elastic/go-elasticsearch/v8/esapi"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"ragflow/internal/engine/types"
|
||||
"ragflow/internal/logger"
|
||||
)
|
||||
|
||||
// SearchRequest Elasticsearch search request (legacy, kept for backward compatibility)
|
||||
type SearchRequest struct {
|
||||
IndexNames []string
|
||||
Query map[string]interface{}
|
||||
Filters map[string]interface{} // Filter conditions (e.g., kb_id, doc_id, available_int)
|
||||
Size int
|
||||
From int
|
||||
Highlight map[string]interface{}
|
||||
Source []string
|
||||
Sort []interface{}
|
||||
}
|
||||
|
||||
// SearchResponse Elasticsearch search response
|
||||
type SearchResponse struct {
|
||||
Hits struct {
|
||||
Total struct {
|
||||
Value int64 `json:"value"`
|
||||
} `json:"total"`
|
||||
Hits []struct {
|
||||
ID string `json:"_id"`
|
||||
Score float64 `json:"_score"`
|
||||
Source map[string]interface{} `json:"_source"`
|
||||
} `json:"hits"`
|
||||
} `json:"hits"`
|
||||
Aggregations map[string]interface{} `json:"aggregations"`
|
||||
}
|
||||
|
||||
// Search executes search (supports both unified engine.SearchRequest and legacy SearchRequest)
|
||||
func (e *elasticsearchEngine) Search(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
|
||||
switch searchReq := req.(type) {
|
||||
case *types.SearchRequest:
|
||||
return e.searchUnified(ctx, searchReq)
|
||||
case *SearchRequest:
|
||||
return e.searchLegacy(ctx, searchReq)
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid search request type: %T", req)
|
||||
}
|
||||
}
|
||||
|
||||
// searchUnified handles the unified engine.SearchRequest
|
||||
func (e *elasticsearchEngine) searchUnified(ctx context.Context, req *types.SearchRequest) (*types.SearchResponse, error) {
|
||||
if len(req.IndexNames) == 0 {
|
||||
return nil, fmt.Errorf("index names cannot be empty")
|
||||
}
|
||||
|
||||
// Build pagination parameters
|
||||
offset, limit := calculatePagination(req.Page, req.Size, req.TopK)
|
||||
|
||||
// Build filter clauses (default: available=1, meaning available_int >= 1)
|
||||
// Reference: rag/utils/es_conn.py L60-L78
|
||||
filterClauses := buildFilterClauses(req.KbIDs, req.DocIDs, 1)
|
||||
|
||||
// Build search query body
|
||||
queryBody := make(map[string]interface{})
|
||||
|
||||
// Use MatchText if available (from QueryBuilder), otherwise use original Question
|
||||
matchText := req.MatchText
|
||||
if matchText == "" {
|
||||
matchText = req.Question
|
||||
}
|
||||
|
||||
var vectorFieldName string
|
||||
if req.KeywordOnly || len(req.Vector) == 0 {
|
||||
// Keyword-only search
|
||||
queryBody["query"] = buildESKeywordQuery(matchText, filterClauses, 1.0)
|
||||
} else {
|
||||
// Hybrid search: keyword + vector
|
||||
// Calculate text weight
|
||||
textWeight := 1.0 - req.VectorSimilarityWeight
|
||||
// Build boolean query for text match and filters
|
||||
boolQuery := buildESKeywordQuery(matchText, filterClauses, 1.0)
|
||||
// Add boost to the bool query (as in Python code)
|
||||
if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
|
||||
boolMap["boost"] = textWeight
|
||||
}
|
||||
// Build kNN query
|
||||
dimension := len(req.Vector)
|
||||
var fieldBuilder strings.Builder
|
||||
fieldBuilder.WriteString("q_")
|
||||
fieldBuilder.WriteString(strconv.Itoa(dimension))
|
||||
fieldBuilder.WriteString("_vec")
|
||||
vectorFieldName = fieldBuilder.String()
|
||||
|
||||
k := req.TopK
|
||||
if k <= 0 {
|
||||
k = 1024
|
||||
}
|
||||
numCandidates := k * 2
|
||||
|
||||
knnQuery := map[string]interface{}{
|
||||
"field": vectorFieldName,
|
||||
"query_vector": req.Vector,
|
||||
"k": k,
|
||||
"num_candidates": numCandidates,
|
||||
"filter": boolQuery,
|
||||
"similarity": req.SimilarityThreshold,
|
||||
}
|
||||
|
||||
queryBody["knn"] = knnQuery
|
||||
queryBody["query"] = boolQuery
|
||||
}
|
||||
|
||||
queryBody["size"] = limit
|
||||
queryBody["from"] = offset
|
||||
|
||||
// Serialize query
|
||||
var buf bytes.Buffer
|
||||
if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
|
||||
return nil, fmt.Errorf("error encoding query: %w", err)
|
||||
}
|
||||
|
||||
// Log search details
|
||||
logger.Debug("Elasticsearch searching indices", zap.Strings("indices", req.IndexNames))
|
||||
logger.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody))
|
||||
|
||||
// Build search request
|
||||
reqES := esapi.SearchRequest{
|
||||
Index: req.IndexNames,
|
||||
Body: &buf,
|
||||
}
|
||||
|
||||
// Execute search
|
||||
res, err := reqES.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search failed: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
logger.Error("Elasticsearch failed to read error response body", err)
|
||||
} else {
|
||||
logger.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes)))
|
||||
}
|
||||
return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
// Parse response
|
||||
var esResp SearchResponse
|
||||
if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
|
||||
return nil, fmt.Errorf("error parsing response: %w", err)
|
||||
}
|
||||
|
||||
// Convert to unified response
|
||||
chunks := convertESResponse(&esResp, vectorFieldName)
|
||||
return &types.SearchResponse{
|
||||
Chunks: chunks,
|
||||
Total: esResp.Hits.Total.Value,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// searchLegacy handles the legacy elasticsearch.SearchRequest (backward compatibility)
|
||||
func (e *elasticsearchEngine) searchLegacy(ctx context.Context, searchReq *SearchRequest) (*SearchResponse, error) {
|
||||
if len(searchReq.IndexNames) == 0 {
|
||||
return nil, fmt.Errorf("index names cannot be empty")
|
||||
}
|
||||
|
||||
// Build search query
|
||||
queryBody := make(map[string]interface{})
|
||||
|
||||
// Process Filters first - convert to Elasticsearch filter clauses
|
||||
var filterClauses []map[string]interface{}
|
||||
if searchReq.Filters != nil && len(searchReq.Filters) > 0 {
|
||||
for field, value := range searchReq.Filters {
|
||||
switch v := value.(type) {
|
||||
case map[string]interface{}:
|
||||
filterClauses = append(filterClauses, map[string]interface{}{
|
||||
field: v,
|
||||
})
|
||||
default:
|
||||
filterClauses = append(filterClauses, map[string]interface{}{
|
||||
"term": map[string]interface{}{
|
||||
field: v,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if searchReq.Query != nil {
|
||||
queryCopy := make(map[string]interface{})
|
||||
for k, v := range searchReq.Query {
|
||||
queryCopy[k] = v
|
||||
}
|
||||
|
||||
if knnValue, ok := queryCopy["knn"]; ok {
|
||||
queryBody["knn"] = knnValue
|
||||
delete(queryCopy, "knn")
|
||||
}
|
||||
|
||||
if len(queryCopy) > 0 {
|
||||
if len(filterClauses) > 0 {
|
||||
queryBody["query"] = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must": queryCopy,
|
||||
"filter": filterClauses,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
queryBody["query"] = queryCopy
|
||||
}
|
||||
} else if len(filterClauses) > 0 {
|
||||
queryBody["query"] = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"filter": filterClauses,
|
||||
},
|
||||
}
|
||||
}
|
||||
} else if len(filterClauses) > 0 {
|
||||
queryBody["query"] = map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"filter": filterClauses,
|
||||
},
|
||||
}
|
||||
}
|
||||
if searchReq.Size > 0 {
|
||||
queryBody["size"] = searchReq.Size
|
||||
}
|
||||
if searchReq.From > 0 {
|
||||
queryBody["from"] = searchReq.From
|
||||
}
|
||||
if searchReq.Highlight != nil {
|
||||
queryBody["highlight"] = searchReq.Highlight
|
||||
}
|
||||
if len(searchReq.Source) > 0 {
|
||||
queryBody["_source"] = searchReq.Source
|
||||
}
|
||||
if len(searchReq.Sort) > 0 {
|
||||
queryBody["sort"] = searchReq.Sort
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := json.NewEncoder(&buf).Encode(queryBody); err != nil {
|
||||
return nil, fmt.Errorf("error encoding query: %w", err)
|
||||
}
|
||||
|
||||
logger.Debug("Elasticsearch searching indices", zap.Strings("indices", searchReq.IndexNames))
|
||||
logger.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody))
|
||||
|
||||
reqES := esapi.SearchRequest{
|
||||
Index: searchReq.IndexNames,
|
||||
Body: &buf,
|
||||
}
|
||||
|
||||
res, err := reqES.Do(ctx, e.client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search failed: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.IsError() {
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
logger.Error("Elasticsearch failed to read error response body", err)
|
||||
} else {
|
||||
logger.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes)))
|
||||
}
|
||||
return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status())
|
||||
}
|
||||
|
||||
var response SearchResponse
|
||||
if err := json.NewDecoder(res.Body).Decode(&response); err != nil {
|
||||
return nil, fmt.Errorf("error parsing response: %w", err)
|
||||
}
|
||||
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
// calculatePagination calculates offset and limit based on page, size and topK
|
||||
func calculatePagination(page, size, topK int) (int, int) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if size <= 0 {
|
||||
size = 30
|
||||
}
|
||||
if topK <= 0 {
|
||||
topK = 1024
|
||||
}
|
||||
|
||||
RERANK_LIMIT := max(30, (64/size)*size)
|
||||
if RERANK_LIMIT < size {
|
||||
RERANK_LIMIT = size
|
||||
}
|
||||
if RERANK_LIMIT > topK {
|
||||
RERANK_LIMIT = topK
|
||||
}
|
||||
|
||||
offset := (page - 1) * RERANK_LIMIT
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
return offset, RERANK_LIMIT
|
||||
}
|
||||
|
||||
// buildFilterClauses builds ES filter clauses from kb_ids, doc_ids and available_int
|
||||
// Reference: rag/utils/es_conn.py L60-L78
|
||||
// When available=0: available_int < 1
|
||||
// When available!=0: NOT (available_int < 1)
|
||||
func buildFilterClauses(kbIDs, docIDs []string, available int) []map[string]interface{} {
|
||||
var filters []map[string]interface{}
|
||||
|
||||
if len(kbIDs) > 0 {
|
||||
filters = append(filters, map[string]interface{}{
|
||||
"terms": map[string]interface{}{"kb_id": kbIDs},
|
||||
})
|
||||
}
|
||||
|
||||
if len(docIDs) > 0 {
|
||||
filters = append(filters, map[string]interface{}{
|
||||
"terms": map[string]interface{}{"doc_id": docIDs},
|
||||
})
|
||||
}
|
||||
|
||||
// Add available_int filter
|
||||
// Reference: rag/utils/es_conn.py L63-L68
|
||||
if available == 0 {
|
||||
// available_int < 1
|
||||
filters = append(filters, map[string]interface{}{
|
||||
"range": map[string]interface{}{
|
||||
"available_int": map[string]interface{}{
|
||||
"lt": 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
// must_not: available_int < 1 (i.e., available_int >= 1)
|
||||
filters = append(filters, map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must_not": []map[string]interface{}{
|
||||
{
|
||||
"range": map[string]interface{}{
|
||||
"available_int": map[string]interface{}{
|
||||
"lt": 1,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return filters
|
||||
}
|
||||
|
||||
// buildESKeywordQuery builds keyword-only search query for ES
|
||||
// Uses query_string if matchText is in query_string format, otherwise uses multi_match
|
||||
// boost is applied to the text match clause (query_string or multi_match)
|
||||
func buildESKeywordQuery(matchText string, filterClauses []map[string]interface{}, boost float64) map[string]interface{} {
|
||||
var mustClause map[string]interface{}
|
||||
|
||||
// Use query_string for complex queries
|
||||
queryString := map[string]interface{}{
|
||||
"query": matchText,
|
||||
"fields": []string{"title_tks^10", "title_sm_tks^5", "important_kwd^30", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"},
|
||||
"type": "best_fields",
|
||||
"minimum_should_match": "30%",
|
||||
"boost": boost,
|
||||
}
|
||||
mustClause = map[string]interface{}{
|
||||
"query_string": queryString,
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"bool": map[string]interface{}{
|
||||
"must": mustClause,
|
||||
"filter": filterClauses,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// convertESResponse converts ES SearchResponse to unified chunks format
|
||||
func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[string]interface{} {
|
||||
if esResp == nil || esResp.Hits.Hits == nil {
|
||||
return []map[string]interface{}{}
|
||||
}
|
||||
|
||||
chunks := make([]map[string]interface{}, len(esResp.Hits.Hits))
|
||||
for i, hit := range esResp.Hits.Hits {
|
||||
|
||||
//// vectorField is list of float64, which need to be converted to float32
|
||||
|
||||
chunks[i] = hit.Source
|
||||
chunks[i]["_score"] = hit.Score
|
||||
chunks[i]["_id"] = hit.ID
|
||||
//vectorField := hit.Source[vectorFieldName]
|
||||
//chunks[i][vectorFieldName] = utility.Float64ToFloat32(vectorField)
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
// Helper query builder functions (legacy)
|
||||
|
||||
// BuildMatchTextQuery builds a text match query
|
||||
func BuildMatchTextQuery(fields []string, text string, fuzziness string) map[string]interface{} {
|
||||
query := map[string]interface{}{
|
||||
"multi_match": map[string]interface{}{
|
||||
"query": text,
|
||||
"fields": fields,
|
||||
},
|
||||
}
|
||||
|
||||
if fuzziness != "" {
|
||||
if multiMatch, ok := query["multi_match"].(map[string]interface{}); ok {
|
||||
multiMatch["fuzziness"] = fuzziness
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
// BuildTermQuery builds a term query
|
||||
func BuildTermQuery(field string, value interface{}) map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"term": map[string]interface{}{
|
||||
field: value,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildRangeQuery builds a range query
|
||||
func BuildRangeQuery(field string, from, to interface{}) map[string]interface{} {
|
||||
rangeQuery := make(map[string]interface{})
|
||||
if from != nil {
|
||||
rangeQuery["gte"] = from
|
||||
}
|
||||
if to != nil {
|
||||
rangeQuery["lte"] = to
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"range": map[string]interface{}{
|
||||
field: rangeQuery,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildBoolQuery builds a bool query
|
||||
func BuildBoolQuery() map[string]interface{} {
|
||||
return map[string]interface{}{
|
||||
"bool": make(map[string]interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
// AddMust adds must clause to bool query
|
||||
func AddMust(query map[string]interface{}, clauses ...map[string]interface{}) {
|
||||
if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
|
||||
if _, exists := boolQuery["must"]; !exists {
|
||||
boolQuery["must"] = []map[string]interface{}{}
|
||||
}
|
||||
if must, ok := boolQuery["must"].([]map[string]interface{}); ok {
|
||||
boolQuery["must"] = append(must, clauses...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddShould adds should clause to bool query
|
||||
func AddShould(query map[string]interface{}, clauses ...map[string]interface{}) {
|
||||
if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
|
||||
if _, exists := boolQuery["should"]; !exists {
|
||||
boolQuery["should"] = []map[string]interface{}{}
|
||||
}
|
||||
if should, ok := boolQuery["should"].([]map[string]interface{}); ok {
|
||||
boolQuery["should"] = append(should, clauses...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddFilter adds filter clause to bool query
|
||||
func AddFilter(query map[string]interface{}, clauses ...map[string]interface{}) {
|
||||
if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
|
||||
if _, exists := boolQuery["filter"]; !exists {
|
||||
boolQuery["filter"] = []map[string]interface{}{}
|
||||
}
|
||||
if filter, ok := boolQuery["filter"].([]map[string]interface{}); ok {
|
||||
boolQuery["filter"] = append(filter, clauses...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddMustNot adds must_not clause to bool query
|
||||
func AddMustNot(query map[string]interface{}, clauses ...map[string]interface{}) {
|
||||
if boolQuery, ok := query["bool"].(map[string]interface{}); ok {
|
||||
if _, exists := boolQuery["must_not"]; !exists {
|
||||
boolQuery["must_not"] = []map[string]interface{}{}
|
||||
}
|
||||
if mustNot, ok := boolQuery["must_not"].([]map[string]interface{}); ok {
|
||||
boolQuery["must_not"] = append(mustNot, clauses...)
|
||||
}
|
||||
}
|
||||
}
|
||||
67
internal/engine/engine.go
Normal file
67
internal/engine/engine.go
Normal file
@@ -0,0 +1,67 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"ragflow/internal/engine/types"
|
||||
)
|
||||
|
||||
// EngineType document engine type
|
||||
type EngineType string
|
||||
|
||||
const (
|
||||
EngineElasticsearch EngineType = "elasticsearch"
|
||||
EngineInfinity EngineType = "infinity"
|
||||
)
|
||||
|
||||
// SearchRequest is an alias for types.SearchRequest
|
||||
type SearchRequest = types.SearchRequest
|
||||
|
||||
// SearchResponse is an alias for types.SearchResponse
|
||||
type SearchResponse = types.SearchResponse
|
||||
|
||||
// DocEngine document storage engine interface
|
||||
type DocEngine interface {
|
||||
// Search
|
||||
Search(ctx context.Context, req interface{}) (interface{}, error)
|
||||
|
||||
// Index operations
|
||||
CreateIndex(ctx context.Context, indexName string, mapping interface{}) error
|
||||
DeleteIndex(ctx context.Context, indexName string) error
|
||||
IndexExists(ctx context.Context, indexName string) (bool, error)
|
||||
|
||||
// Document operations
|
||||
IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error
|
||||
BulkIndex(ctx context.Context, indexName string, docs []interface{}) (interface{}, error)
|
||||
GetDocument(ctx context.Context, indexName, docID string) (interface{}, error)
|
||||
DeleteDocument(ctx context.Context, indexName, docID string) error
|
||||
|
||||
// Health check
|
||||
Ping(ctx context.Context) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
// Type returns the engine type (helper method for runtime type checking)
|
||||
// This is a workaround since we can't import elasticsearch or infinity packages directly
|
||||
func Type(docEngine DocEngine) EngineType {
|
||||
// Type checking through interface methods is not straightforward
|
||||
// This is a placeholder that should be implemented differently
|
||||
// or rely on configuration to know the type
|
||||
return EngineType("unknown")
|
||||
}
|
||||
70
internal/engine/global.go
Normal file
70
internal/engine/global.go
Normal file
@@ -0,0 +1,70 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"ragflow/internal/server"
|
||||
"sync"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"ragflow/internal/engine/elasticsearch"
|
||||
"ragflow/internal/engine/infinity"
|
||||
"ragflow/internal/logger"
|
||||
)
|
||||
|
||||
var (
|
||||
globalEngine DocEngine
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// Init initializes document engine
|
||||
func Init(cfg *server.DocEngineConfig) error {
|
||||
var initErr error
|
||||
once.Do(func() {
|
||||
var err error
|
||||
switch EngineType(cfg.Type) {
|
||||
case EngineElasticsearch:
|
||||
globalEngine, err = elasticsearch.NewEngine(cfg.ES)
|
||||
case EngineInfinity:
|
||||
globalEngine, err = infinity.NewEngine(cfg.Infinity)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported doc engine type: %s", cfg.Type)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
initErr = fmt.Errorf("failed to create doc engine: %w", err)
|
||||
return
|
||||
}
|
||||
logger.Info("Doc engine initialized", zap.String("type", string(cfg.Type)))
|
||||
})
|
||||
return initErr
|
||||
}
|
||||
|
||||
// Get gets global document engine instance
|
||||
func Get() DocEngine {
|
||||
return globalEngine
|
||||
}
|
||||
|
||||
// Close closes document engine
|
||||
func Close() error {
|
||||
if globalEngine != nil {
|
||||
return globalEngine.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
59
internal/engine/infinity/client.go
Normal file
59
internal/engine/infinity/client.go
Normal file
@@ -0,0 +1,59 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package infinity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/server"
|
||||
)
|
||||
|
||||
// Engine Infinity engine implementation
|
||||
// Note: Infinity Go SDK is not yet available. This is a placeholder implementation.
|
||||
type infinityEngine struct {
|
||||
config *server.InfinityConfig
|
||||
}
|
||||
|
||||
// NewEngine creates an Infinity engine
|
||||
// Note: This is a placeholder implementation waiting for official Infinity Go SDK
|
||||
func NewEngine(cfg interface{}) (*infinityEngine, error) {
|
||||
infConfig, ok := cfg.(*server.InfinityConfig)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("invalid infinity config type, expected *config.InfinityConfig")
|
||||
}
|
||||
|
||||
engine := &infinityEngine{
|
||||
config: infConfig,
|
||||
}
|
||||
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
// Type returns the engine type
|
||||
func (e *infinityEngine) Type() string {
|
||||
return "infinity"
|
||||
}
|
||||
|
||||
// Ping health check
|
||||
func (e *infinityEngine) Ping(ctx context.Context) error {
|
||||
return fmt.Errorf("infinity engine not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// Close closes the connection
|
||||
func (e *infinityEngine) Close() error {
|
||||
return nil
|
||||
}
|
||||
47
internal/engine/infinity/document.go
Normal file
47
internal/engine/infinity/document.go
Normal file
@@ -0,0 +1,47 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package infinity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// IndexDocument indexes a single document
|
||||
func (e *infinityEngine) IndexDocument(ctx context.Context, tableName, docID string, doc interface{}) error {
|
||||
return fmt.Errorf("infinity insert not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// BulkIndex indexes documents in bulk
|
||||
func (e *infinityEngine) BulkIndex(ctx context.Context, tableName string, docs []interface{}) (interface{}, error) {
|
||||
return nil, fmt.Errorf("infinity bulk insert not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// BulkResponse bulk operation response
|
||||
type BulkResponse struct {
|
||||
Inserted int
|
||||
}
|
||||
|
||||
// GetDocument gets a document
|
||||
func (e *infinityEngine) GetDocument(ctx context.Context, tableName, docID string) (interface{}, error) {
|
||||
return nil, fmt.Errorf("infinity get document not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// DeleteDocument deletes a document
|
||||
func (e *infinityEngine) DeleteDocument(ctx context.Context, tableName, docID string) error {
|
||||
return fmt.Errorf("infinity delete not implemented: waiting for official Go SDK")
|
||||
}
|
||||
37
internal/engine/infinity/index.go
Normal file
37
internal/engine/infinity/index.go
Normal file
@@ -0,0 +1,37 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package infinity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// CreateIndex creates a table/index
|
||||
func (e *infinityEngine) CreateIndex(ctx context.Context, indexName string, mapping interface{}) error {
|
||||
return fmt.Errorf("infinity create table not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// DeleteIndex deletes a table/index
|
||||
func (e *infinityEngine) DeleteIndex(ctx context.Context, indexName string) error {
|
||||
return fmt.Errorf("infinity drop table not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// IndexExists checks if table/index exists
|
||||
func (e *infinityEngine) IndexExists(ctx context.Context, indexName string) (bool, error) {
|
||||
return false, fmt.Errorf("infinity check table existence not implemented: waiting for official Go SDK")
|
||||
}
|
||||
205
internal/engine/infinity/search.go
Normal file
205
internal/engine/infinity/search.go
Normal file
@@ -0,0 +1,205 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package infinity
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/engine/types"
|
||||
)
|
||||
|
||||
// SearchRequest Infinity search request (legacy, kept for backward compatibility)
|
||||
type SearchRequest struct {
|
||||
TableName string
|
||||
ColumnNames []string
|
||||
MatchText *MatchTextExpr
|
||||
MatchDense *MatchDenseExpr
|
||||
Fusion *FusionExpr
|
||||
Offset int
|
||||
Limit int
|
||||
Filter map[string]interface{}
|
||||
}
|
||||
|
||||
// SearchResponse Infinity search response
|
||||
type SearchResponse struct {
|
||||
Rows []map[string]interface{}
|
||||
Total int64
|
||||
}
|
||||
|
||||
// MatchTextExpr text match expression
|
||||
type MatchTextExpr struct {
|
||||
Fields []string
|
||||
MatchingText string
|
||||
TopN int
|
||||
ExtraOptions map[string]interface{}
|
||||
}
|
||||
|
||||
// MatchDenseExpr vector match expression
|
||||
type MatchDenseExpr struct {
|
||||
VectorColumnName string
|
||||
EmbeddingData []float64
|
||||
EmbeddingDataType string
|
||||
DistanceType string
|
||||
TopN int
|
||||
ExtraOptions map[string]interface{}
|
||||
}
|
||||
|
||||
// FusionExpr fusion expression
|
||||
type FusionExpr struct {
|
||||
Method string
|
||||
TopN int
|
||||
Weights []float64
|
||||
FusionParams map[string]interface{}
|
||||
}
|
||||
|
||||
// Search executes search (supports both unified engine.SearchRequest and legacy SearchRequest)
|
||||
func (e *infinityEngine) Search(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
switch searchReq := req.(type) {
|
||||
case *types.SearchRequest:
|
||||
return e.searchUnified(ctx, searchReq)
|
||||
case *SearchRequest:
|
||||
return e.searchLegacy(ctx, searchReq)
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid search request type: %T", req)
|
||||
}
|
||||
}
|
||||
|
||||
// searchUnified handles the unified engine.SearchRequest
|
||||
func (e *infinityEngine) searchUnified(ctx context.Context, req *types.SearchRequest) (*types.SearchResponse, error) {
|
||||
if len(req.IndexNames) == 0 {
|
||||
return nil, fmt.Errorf("index names cannot be empty")
|
||||
}
|
||||
|
||||
// For Infinity, we use the first index name as table name
|
||||
tableName := req.IndexNames[0]
|
||||
|
||||
// Get retrieval parameters with defaults
|
||||
similarityThreshold := req.SimilarityThreshold
|
||||
if similarityThreshold <= 0 {
|
||||
similarityThreshold = 0.1
|
||||
}
|
||||
|
||||
topK := req.TopK
|
||||
if topK <= 0 {
|
||||
topK = 1024
|
||||
}
|
||||
|
||||
vectorSimilarityWeight := req.VectorSimilarityWeight
|
||||
if vectorSimilarityWeight < 0 || vectorSimilarityWeight > 1 {
|
||||
vectorSimilarityWeight = 0.3
|
||||
}
|
||||
|
||||
pageSize := req.Size
|
||||
if pageSize <= 0 {
|
||||
pageSize = 30
|
||||
}
|
||||
|
||||
offset := (req.Page - 1) * pageSize
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
// Build search request
|
||||
searchReq := &SearchRequest{
|
||||
TableName: tableName,
|
||||
Limit: pageSize,
|
||||
Offset: offset,
|
||||
Filter: buildInfinityFilters(req.KbIDs, req.DocIDs),
|
||||
}
|
||||
|
||||
// Add text match (question is always required)
|
||||
searchReq.MatchText = &MatchTextExpr{
|
||||
Fields: []string{"title_tks", "content_ltks"},
|
||||
MatchingText: req.Question,
|
||||
TopN: topK,
|
||||
}
|
||||
|
||||
// Add vector match if vector is provided and not keyword-only mode
|
||||
if !req.KeywordOnly && len(req.Vector) > 0 {
|
||||
fieldName := buildInfinityVectorFieldName(req.Vector)
|
||||
searchReq.MatchDense = &MatchDenseExpr{
|
||||
VectorColumnName: fieldName,
|
||||
EmbeddingData: req.Vector,
|
||||
EmbeddingDataType: "float",
|
||||
DistanceType: "cosine",
|
||||
TopN: topK,
|
||||
ExtraOptions: map[string]interface{}{
|
||||
"similarity": similarityThreshold,
|
||||
},
|
||||
}
|
||||
// Infinity uses weighted_sum fusion with weights
|
||||
searchReq.Fusion = &FusionExpr{
|
||||
Method: "weighted_sum",
|
||||
TopN: topK,
|
||||
Weights: []float64{
|
||||
1.0 - vectorSimilarityWeight, // text weight
|
||||
vectorSimilarityWeight, // vector weight
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Execute the actual search (would call Infinity SDK here)
|
||||
// For now, return not implemented
|
||||
return nil, fmt.Errorf("infinity search unified not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// searchLegacy handles the legacy infinity.SearchRequest (backward compatibility)
|
||||
func (e *infinityEngine) searchLegacy(ctx context.Context, req *SearchRequest) (*SearchResponse, error) {
|
||||
// This would contain the actual Infinity search implementation
|
||||
return nil, fmt.Errorf("infinity search legacy not implemented: waiting for official Go SDK")
|
||||
}
|
||||
|
||||
// buildInfinityFilters builds filter conditions for Infinity
|
||||
func buildInfinityFilters(kbIDs []string, docIDs []string) map[string]interface{} {
|
||||
filters := make(map[string]interface{})
|
||||
|
||||
// kb_id filter
|
||||
if len(kbIDs) > 0 {
|
||||
if len(kbIDs) == 1 {
|
||||
filters["kb_id"] = kbIDs[0]
|
||||
} else {
|
||||
filters["kb_id"] = kbIDs
|
||||
}
|
||||
}
|
||||
|
||||
// doc_id filter
|
||||
if len(docIDs) > 0 {
|
||||
if len(docIDs) == 1 {
|
||||
filters["doc_id"] = docIDs[0]
|
||||
} else {
|
||||
filters["doc_id"] = docIDs
|
||||
}
|
||||
}
|
||||
|
||||
// available_int filter (default to 1 for available chunks)
|
||||
filters["available_int"] = 1
|
||||
|
||||
return filters
|
||||
}
|
||||
|
||||
// buildInfinityVectorFieldName builds vector field name based on dimension
|
||||
func buildInfinityVectorFieldName(vector []float64) string {
|
||||
dimension := len(vector)
|
||||
var fieldBuilder strings.Builder
|
||||
fieldBuilder.WriteString("q_")
|
||||
fieldBuilder.WriteString(strconv.Itoa(dimension))
|
||||
fieldBuilder.WriteString("_vec")
|
||||
return fieldBuilder.String()
|
||||
}
|
||||
54
internal/engine/types/types.go
Normal file
54
internal/engine/types/types.go
Normal file
@@ -0,0 +1,54 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package types
|
||||
|
||||
// SearchRequest unified search request for all engines
|
||||
type SearchRequest struct {
|
||||
// Common fields
|
||||
IndexNames []string // For ES: index names; For Infinity: treated as table names
|
||||
Question string // Search query text
|
||||
Vector []float64 // Embedding vector (optional, for hybrid search)
|
||||
|
||||
// Query analysis results (from QueryBuilder.Question)
|
||||
MatchText string // Processed match text for ES query_string
|
||||
Keywords []string // Extracted keywords from question
|
||||
|
||||
// Filters
|
||||
KbIDs []string // Knowledge base IDs filter
|
||||
DocIDs []string // Document IDs filter
|
||||
|
||||
// Pagination
|
||||
Page int // Page number (1-based)
|
||||
Size int // Page size
|
||||
TopK int // Number of candidates for retrieval
|
||||
|
||||
// Search mode
|
||||
KeywordOnly bool // If true, only do keyword search (no vector search)
|
||||
|
||||
// Scoring parameters
|
||||
SimilarityThreshold float64 // Minimum similarity score (default: 0.1)
|
||||
VectorSimilarityWeight float64 // Weight for vector vs keyword (default: 0.3)
|
||||
|
||||
// Engine-specific options (optional, for advanced use)
|
||||
Options map[string]interface{}
|
||||
}
|
||||
|
||||
// SearchResponse unified search response for all engines
|
||||
type SearchResponse struct {
|
||||
Chunks []map[string]interface{} // Search results
|
||||
Total int64 // Total number of matches
|
||||
}
|
||||
Reference in New Issue
Block a user