zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-04 23:00:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: optimize dataflow indexing and logs (#17737)

Browse Source
This commit is contained in:
buua436
2026-08-03 19:15:44 +08:00
committed by GitHub
parent e997fd655a
commit 1d141aff18
9 changed files with 103 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
rag/utils/es_conn.py
View File

@@ -334,7 +334,7 @@ class ESConnection(ESConnectionBase):
self.logger.error(f"ESConnection.search timeout for {ATTEMPT_TIME} times!")
raise Exception("ESConnection.search timeout.")
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None) -> list[str]:
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None, refresh: str | bool = "wait_for") -> list[str]:
# Refers to https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html
operations = []
for d in documents:
@@ -351,7 +351,7 @@ class ESConnection(ESConnectionBase):
for _ in range(ATTEMPT_TIME):
try:
res = []
r = self.es.bulk(index=index_name, operations=operations, refresh="wait_for", timeout="60s")
r = self.es.bulk(index=index_name, operations=operations, refresh=refresh, timeout="60s")
if re.search(r"False", str(r["errors"]), re.IGNORECASE):
return res

2
rag/utils/infinity_conn.py
View File

@@ -379,7 +379,7 @@ class InfinityConnection(InfinityConnectionBase):
chunk["id"] = chunk_id
return chunk
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None) -> list[str]:
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None, refresh: str | bool = "wait_for") -> list[str]:
"""
# Save input to file to test inserting from file in GO
import datetime

2
rag/utils/ob_conn.py
View File

@@ -1014,7 +1014,7 @@ class OBConnection(OBConnectionBase):
logger.exception(f"OBConnection.get({chunk_id}) got exception")
raise e
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None) -> list[str]:
def insert(self, documents: list[dict], index_name: str, knowledgebase_id: str = None, refresh: str | bool = "wait_for") -> list[str]:
if not documents:
return []

4
rag/utils/opensearch_conn.py
View File

@@ -507,7 +507,7 @@ class OSConnection(DocStoreConnection):
logger.error(f"OSConnection.get timeout for {ATTEMPT_TIME} times!")
raise Exception("OSConnection.get timeout.")
def insert(self, documents: list[dict], indexName: str, knowledgebaseId: str = None) -> list[str]:
def insert(self, documents: list[dict], indexName: str, knowledgebaseId: str = None, refresh: str | bool = "wait_for") -> list[str]:
# Refers to https://opensearch.org/docs/latest/api-reference/document-apis/bulk/
operations = []
for d in documents:
@@ -525,7 +525,7 @@ class OSConnection(DocStoreConnection):
for _ in range(ATTEMPT_TIME):
try:
res = []
r = self.os.bulk(index=(indexName), body=operations, refresh="wait_for", timeout=60)
r = self.os.bulk(index=(indexName), body=operations, refresh=refresh, timeout=60)
if re.search(r"False", str(r["errors"]), re.IGNORECASE):
return res