From 13922209e69f1176e87e39d0a993d2d745576ea0 Mon Sep 17 00:00:00 2001 From: Ricardo-M-L <69202550+Ricardo-M-L@users.noreply.github.com> Date: Mon, 11 May 2026 11:19:07 +0800 Subject: [PATCH] fix(llm): add timeout to HTTP requests in LLM integration layer (#14313) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Multiple `requests.post()` calls across the LLM integration layer lack a `timeout` parameter. Without a timeout, a single unresponsive upstream service can block the calling thread **indefinitely**, eventually exhausting the thread pool and degrading the entire system. This is a well-known issue — Python's `requests` library defaults to `timeout=None` (infinite wait), and [the library docs explicitly recommend](https://requests.readthedocs.io/en/latest/user/advanced/#timeouts) always setting a timeout. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ### Change Added `timeout` to all `requests.post()` calls missing it: | File | Calls fixed | Timeout | |------|-------------|---------| | `rag/llm/rerank_model.py` | 9 | 30s | | `rag/llm/embedding_model.py` | 8 | 30s | | `rag/llm/cv_model.py` | 3 | 60s | | `rag/llm/tts_model.py` | 2 | 60s | | `rag/llm/sequence2txt_model.py` | 2 | 60s | Embedding/rerank calls use 30s (lightweight API calls). Vision, TTS, and audio transcription use 60s (heavier workloads with file uploads). Note: other files in the codebase (e.g. `check_minio_alive`, `check_ragflow_server_alive`) already use `timeout=10`, so this PR brings the LLM layer in line with existing practice. Signed-off-by: Ricardo-M-L Co-authored-by: Kevin Hu --- rag/llm/cv_model.py | 3 +++ rag/llm/embedding_model.py | 16 ++++++++-------- rag/llm/rerank_model.py | 17 +++++++++-------- rag/llm/sequence2txt_model.py | 3 ++- rag/llm/tts_model.py | 6 ++++-- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/rag/llm/cv_model.py b/rag/llm/cv_model.py index d4c9701c2..728f1677d 100644 --- a/rag/llm/cv_model.py +++ b/rag/llm/cv_model.py @@ -446,6 +446,7 @@ class Zhipu4V(GptV4): "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", }, + timeout=60, ) return response.json() @@ -1029,6 +1030,7 @@ class NvidiaCV(Base): "Authorization": f"Bearer {self.key}", }, json={"messages": self.prompt(b64)}, + timeout=60, ) response = response.json() return ( @@ -1046,6 +1048,7 @@ class NvidiaCV(Base): "Authorization": f"Bearer {self.key}", }, json={"messages": msg, **gen_conf}, + timeout=60, ) return response.json() diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py index 9fe109552..e1d0409d0 100644 --- a/rag/llm/embedding_model.py +++ b/rag/llm/embedding_model.py @@ -409,7 +409,7 @@ class JinaMultiVecEmbed(Base): data["task"] = task data["truncate"] = True - response = requests.post(self.base_url, headers=self.headers, json=data) + response = requests.post(self.base_url, headers=self.headers, json=data, timeout=30) try: res = response.json() for d in res["data"]: @@ -687,7 +687,7 @@ class NvidiaEmbed(Base): "encoding_format": "float", "truncate": "END", } - response = requests.post(self.base_url, headers=self.headers, json=payload) + response = requests.post(self.base_url, headers=self.headers, json=payload, timeout=30) try: res = response.json() ress.extend([d["embedding"] for d in res["data"]]) @@ -827,7 +827,7 @@ class SILICONFLOWEmbed(Base): "input": texts_batch, "encoding_format": "float", } - response = requests.post(self.base_url, json=payload, headers=self.headers) + response = requests.post(self.base_url, json=payload, headers=self.headers, timeout=30) try: res = response.json() ress.extend([d["embedding"] for d in res["data"]]) @@ -844,7 +844,7 @@ class SILICONFLOWEmbed(Base): "input": text, "encoding_format": "float", } - response = requests.post(self.base_url, json=payload, headers=self.headers) + response = requests.post(self.base_url, json=payload, headers=self.headers, timeout=30) try: res = response.json() return np.array(res["data"][0]["embedding"]), total_token_count_from_response(res) @@ -954,7 +954,7 @@ class HuggingFaceEmbed(Base): self.base_url = base_url or "http://127.0.0.1:8080" def encode(self, texts: list): - response = requests.post(f"{self.base_url}/embed", json={"inputs": texts}, headers={"Content-Type": "application/json"}) + response = requests.post(f"{self.base_url}/embed", json={"inputs": texts}, headers={"Content-Type": "application/json"}, timeout=30) if response.status_code == 200: embeddings = response.json() else: @@ -962,7 +962,7 @@ class HuggingFaceEmbed(Base): return np.array(embeddings), sum([num_tokens_from_string(text) for text in texts]) def encode_queries(self, text: str): - response = requests.post(f"{self.base_url}/embed", json={"inputs": text}, headers={"Content-Type": "application/json"}) + response = requests.post(f"{self.base_url}/embed", json={"inputs": text}, headers={"Content-Type": "application/json"}, timeout=30) if response.status_code == 200: embedding = response.json()[0] return np.array(embedding), num_tokens_from_string(text) @@ -1163,7 +1163,7 @@ class PerplexityEmbed(Base): "input": [[chunk] for chunk in batch], "encoding_format": "base64_int8", } - response = requests.post(url, headers=self.headers, json=payload) + response = requests.post(url, headers=self.headers, json=payload, timeout=30) try: res = response.json() for doc in res["data"]: @@ -1182,7 +1182,7 @@ class PerplexityEmbed(Base): "input": batch, "encoding_format": "base64_int8", } - response = requests.post(url, headers=self.headers, json=payload) + response = requests.post(url, headers=self.headers, json=payload, timeout=30) try: res = response.json() for d in res["data"]: diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py index 5f1ef3ef2..a150b40e7 100644 --- a/rag/llm/rerank_model.py +++ b/rag/llm/rerank_model.py @@ -65,7 +65,7 @@ class JinaRerank(Base): def similarity(self, query: str, texts: list): texts = [truncate(t, 8196) for t in texts] data = {"model": self.model_name, "query": query, "documents": texts, "top_n": len(texts)} - res = requests.post(self.base_url, headers=self.headers, json=data).json() + res = requests.post(self.base_url, headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["results"]: @@ -97,7 +97,7 @@ class XInferenceRerank(Base): for _, t in pairs: token_count += num_tokens_from_string(t) data = {"model": self.model_name, "query": query, "return_documents": "true", "return_len": "true", "documents": texts} - res = requests.post(self.base_url, headers=self.headers, json=data).json() + res = requests.post(self.base_url, headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["results"]: @@ -130,7 +130,7 @@ class LocalAIRerank(Base): token_count = 0 for t in texts: token_count += num_tokens_from_string(t) - res = requests.post(self.base_url, headers=self.headers, json=data).json() + res = requests.post(self.base_url, headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["results"]: @@ -173,7 +173,7 @@ class NvidiaRerank(Base): "truncate": "END", "top_n": len(texts), } - res = requests.post(self.base_url, headers=self.headers, json=data).json() + res = requests.post(self.base_url, headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["rankings"]: @@ -217,7 +217,7 @@ class OpenAI_APIRerank(Base): token_count = 0 for t in texts: token_count += num_tokens_from_string(t) - res = requests.post(self.base_url, headers=self.headers, json=data).json() + res = requests.post(self.base_url, headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["results"]: @@ -298,7 +298,7 @@ class SILICONFLOWRerank(Base): "max_chunks_per_doc": 1024, "overlap_tokens": 80, } - response_raw = requests.post(self.base_url, json=payload, headers=self.headers) + response_raw = requests.post(self.base_url, json=payload, headers=self.headers, timeout=30) response = response_raw.json() rank = np.zeros(len(texts), dtype=float) try: @@ -421,6 +421,7 @@ class HuggingfaceRerank(Base): endpoint, headers = {"Content-Type": "application/json"}, json = {"query": query, "texts": texts[i: i + batch_size], "raw_scores": False, "truncate": True}, + timeout=30 ) for o in res.json(): scores[o["index"] + i] = o["score"] @@ -468,7 +469,7 @@ class GPUStackRerank(Base): } try: - response = requests.post(self.base_url, json=payload, headers=self.headers) + response = requests.post(self.base_url, json=payload, headers=self.headers, timeout=30) response.raise_for_status() response_json = response.json() @@ -570,7 +571,7 @@ class RAGconRerank(Base): token_count = 0 for t in texts: token_count += num_tokens_from_string(t) - res = requests.post(self._base_url + "/rerank", headers=self.headers, json=data).json() + res = requests.post(self._base_url + "/rerank", headers=self.headers, json=data, timeout=30).json() rank = np.zeros(len(texts), dtype=float) try: for d in res["results"]: diff --git a/rag/llm/sequence2txt_model.py b/rag/llm/sequence2txt_model.py index 563dd47fc..4624a2911 100644 --- a/rag/llm/sequence2txt_model.py +++ b/rag/llm/sequence2txt_model.py @@ -195,7 +195,7 @@ class XinferenceSeq2txt(Base): files = {"file": (audio_file_name, audio_data, "audio/wav")} try: - response = requests.post(f"{self.base_url}/v1/audio/transcriptions", files=files, data=payload) + response = requests.post(f"{self.base_url}/v1/audio/transcriptions", files=files, data=payload, timeout=60) response.raise_for_status() result = response.json() @@ -377,6 +377,7 @@ class ZhipuSeq2txt(Base): data=payload, files=files, headers=headers, + timeout=60, ) body = response.json() if response.status_code == 200: diff --git a/rag/llm/tts_model.py b/rag/llm/tts_model.py index 94a81ceba..f37cd89c2 100644 --- a/rag/llm/tts_model.py +++ b/rag/llm/tts_model.py @@ -116,7 +116,8 @@ class HTTPBasedTTS(Base): url, headers=self.headers, json=payload, - stream=stream + stream=stream, + timeout=60, ) if response.status_code != 200: @@ -532,7 +533,8 @@ class RAGconTTS(Base): f"{self.base_url}/audio/speech", headers=self.headers, json=payload, - stream=stream + stream=stream, + timeout=60, ) if response.status_code != 200: