diff --git a/build.sh b/build.sh index 6ee95086fa..b506871ff7 100755 --- a/build.sh +++ b/build.sh @@ -118,7 +118,7 @@ check_cpp_deps() { print_section "Checking c++ dependencies" command -v cmake >/dev/null 2>&1 || { echo -e "${RED}Error: cmake is required but not installed.${NC}"; exit 1; } - command -v g++ >/dev/null 2>&1 || { echo -e "${RED}Error: g++ is required but not installed.${NC}"; exit 1; } + command -v clang++ >/dev/null 2>&1 || { echo -e "${RED}Error: clang++ is required but not installed.${NC}"; exit 1; } if check_pcre2; then echo "✓ pcre2 library found" diff --git a/internal/development.md b/internal/development.md index 3aa01ef9c2..2a3d24edaa 100644 --- a/internal/development.md +++ b/internal/development.md @@ -1,40 +1,45 @@ -# RAGFlow Go Version - Startup Guide +# RAGFlow Go implementation - Development Guide -## 1. Start Dependencies +## 1. Prepare dependencies -```bash -docker compose -f docker/docker-compose-base.yml --profile ragflow-go --profile infinity up -d +### 1.1 Install CMake and build RAGFlow on Ubuntu 24.04 + +```shell +sudo apt update +sudo apt install ca-certificates gpg wget +test -f /usr/share/doc/kitware-archive-keyring/copyright || wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null +echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ noble main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null +sudo apt update +test -f /usr/share/doc/kitware-archive-keyring/copyright || sudo rm /usr/share/keyrings/kitware-archive-keyring.gpg +sudo apt install -y kitware-archive-keyring +sudo apt update +sudo apt install -y cmake ``` -## 2. Build Go Version RAGFlow -- Build RAGFlow binary with C++ dependencies: +### 1.2 Install clang-20 -```bash -./build.sh +```shell +sudo apt install clang-20 lld-20 +sudo ln -s /usr/bin/clang++-20 /usr/bin/clang++ +sudo ln -s /usr/bin/clang-20 /usr/bin/clang +sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld ``` -- Production builds (strip debug symbols for smaller binaries): +### 1.3 Install golang -```bash -./build.sh --strip --all -# or -./build.sh -s --go +```shell +wget https://go.dev/dl/go1.25.4.linux-amd64.tar.gz +sudo rm -rf /usr/local/go +sudo tar -C /usr/local -xzf go1.23.4.linux-amd64.tar.gz +echo 'export PATH=$PATH:/usr/local/go/bin' >> ~/.bashrc +source ~/.bashrc +go version ``` -### 2.1. Native Static Libraries - -Before building, you need three native static libraries (pdfium, pdf_oxide, office_oxide). -They are **not** downloaded by `build.sh` — use the included download script: - -```bash -# From the project root, download and extract to ~/ragflow-native-libs/ -uv run python3 ragflow_deps/download_deps.py -``` - -This also requires `lld-20` on Linux (for Chromium-built pdfium): - -```bash -sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld +### 1.4 Install dependent library +```shell +sudo apt install libpcre2-dev +python3 ragflow_deps/download_go_deps.py ``` > **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables: @@ -56,7 +61,31 @@ sudo apt install lld-20 && sudo ln -s /usr/bin/ld.lld-20 /usr/bin/ld.lld > > All three native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed. -## 3. Run Go Version RAGFlow + +### 1.5 Build RAGFlow + +- Build binary +```bash +./build.sh +``` + +- Production builds (strip debug symbols for smaller binaries): + +```bash +./build.sh --strip --all +# or +./build.sh -s --go +``` + +## 2. Start RAGFlow + +- Start dependencies +```bash +docker compose -f docker/docker-compose-base.yml --profile ragflow-go --profile infinity up -d +``` + + +- Start RAGFlow Note: admin server must be started first; otherwise, api server will encounter errors when sending heartbeats. ```bash @@ -84,18 +113,18 @@ Note: admin server must be started first; otherwise, api server will encounter e ./bin/ragflow-cli --admin ``` -## 4. Start Frontend +## 3. Start Frontend ```bash cd web && export API_PROXY_SCHEME=hybrid && npm run dev ``` -## 5. Service Ports & API Routing +## 4. Service Ports & API Routing - api server listens on port 9384 by default - admin server listens on port 9383 by default After updating or implementing an API, update the frontend development environment routes in web/vite.config.ts under proxySchemes. -### Proxy Schemes +### 4.1 Proxy Schemes | Scheme | Description | |--------|-------------| @@ -104,11 +133,11 @@ After updating or implementing an API, update the frontend development environme | `go` | All API requests from the frontend are routed to the Go server | -## 6. RAGFlow commands +## 5. RAGFlow commands You can use the following CLI commands to test the corresponding API implementations. -### 6.1. Run ragflow-cli, register user, login, and logout: +### 5.1. Run ragflow-cli, register user, login, and logout: ``` $ ./ragflow-cli @@ -124,19 +153,19 @@ RAGFlow(api/default)> logout; SUCCESS ``` -### 6.2. List currently supported providers +### 5.2. List currently supported providers ``` RAGFlow(api/default)> list available providers; ``` -### 6.3. Add or delete a provider for the current tenant +### 5.3. Add or delete a provider for the current tenant ``` RAGFlow(api/default)> add provider 'openai'; ``` ``` RAGFlow(api/default)> delete provider 'openai'; ``` -### 6.4. Create a model instance for a specific provider +### 5.4. Create a model instance for a specific provider ``` RAGFlow(api/default)> create provider 'openai' instance 'instance_name' key 'api-key'; ``` @@ -148,18 +177,18 @@ For locally deployed models (e.g., ollama, vLLM), use the following command to a ``` RAGFlow(api/default)> create provider 'vllm' instance 'instance_name' key '' url 'http://192.168.1.96:8123/v1'; ``` -### 6.5. List and delete an instance +### 5.5. List and delete an instance ``` RAGFlow(api/default)> list instances from 'openai'; ``` ``` RAGFlow(api/default)> drop instance 'instance_name' from 'openai'; ``` -### 6.6. List models supported by a model instance +### 5.5. List models supported by a model instance ``` RAGFlow(api/default)> list models from 'openai' 'instance_name'; ``` -### 6.7. Chat with LLM +### 5.7. Chat with LLM - Chat ``` RAGFlow(api/default)> chat with 'glm-4.5-flash@test@zhipu-ai' message '20 words introduce LLM'; @@ -196,11 +225,11 @@ Time: 31.600545 ``` RAGFlow(api/default)> chat with 'glm-4.6v-flash@test@zhipu-ai' message 'What are the video talk about?' video 'https://cdn.bigmodel.cn/agent-demos/lark/113123.mov' Answer: Based on the sequence of frames provided, the video is a demonstration of a web search and navigation process... -Time: 76.582520 +Time: 75.582520 ``` Note: Both image and video understanding support streaming and thinking modes as well. -### 6.8. Chat completions +### 5.8. Chat completions ``` RAGFlow(api/default)> chat completion 'hello' @@ -225,7 +254,7 @@ RAGFlow(api/default)> CHAT COMPLETIONS 'Continue' \ history_delimiter ';'; ``` -### 6.9. Chat with OpenAI compatible API +### 5.9. Chat with OpenAI compatible API ``` RAGFlow(api/default)> openai_chat '' 'Hello, how are you?'; @@ -263,17 +292,17 @@ RAGFlow(api/default)> openai_chat '' 'Hello, how are you?' extra_body ' CLI error: OPENAI_CHAT extra_body: unknown field "ref" (valid: reference, reference_metadata, metadata_condition) ``` -### 6.10. Generate Embeddings +### 5.10. Generate Embeddings ``` RAGFlow(api/default)> embed text 'what is rag' 'who are you' with 'embedding-3@test@zhipu-ai' dimension 16; ``` -### 6.11. Document Reranking +### 5.11. Document Reranking ``` RAGFlow(api/default)> rerank query 'what is rag' document 'rag is retrieval augment generation' 'rag need llm' 'famous rag project includes ragflow' with 'rerank@test@zhipu-ai' top 2; ``` -### 6.12. Get supported models from provider API +### 5.12. Get supported models from provider API ``` RAGFlow(api/default)> list supported models from 'gitee' 'test'; @@ -295,7 +324,7 @@ RAGFlow(api/default)> list supported models from 'gitee' 'test'; +-----------+---------------------------+---------------+------------+-----------------------------------------------------------------+----------------------------------------------------------+---------------------------------------------+ ``` -### 6.13. Get preset models of a provider +### 5.13. Get preset models of a provider ``` RAGFlow(api/default)> list models from 'minimax'; @@ -313,7 +342,7 @@ RAGFlow(api/default)> list models from 'minimax'; +------------+-------------+------------------------+ ``` -### 6.14. List instances of a provider +### 5.14. List instances of a provider ``` RAGFlow(api/default)> list instances from 'zhipu-ai'; @@ -324,7 +353,7 @@ RAGFlow(api/default)> list instances from 'zhipu-ai'; +---------+----------------------+----------------------------------+--------------+----------------------------------+--------+ ``` -### 6.15. Show instance of a provider +### 5.15. Show instance of a provider ``` RAGFlow(api/default)> show instance 'test' from 'zhipu-ai'; +----------------------------------+--------------+----------------------------------+---------+--------+ @@ -334,7 +363,7 @@ RAGFlow(api/default)> show instance 'test' from 'zhipu-ai'; +----------------------------------+--------------+----------------------------------+---------+--------+ ``` -### 6.16. List models of a specific instance +### 5.15. List models of a specific instance ``` RAGFlow(api/default)> list models from 'minimax' 'test'; @@ -352,7 +381,7 @@ RAGFlow(api/default)> list models from 'minimax' 'test'; +------------+-------------+------------------------+--------+ ``` -### 6.17. List added providers +### 5.17. List added providers ``` RAGFlow(api/default)> list providers; +--------------------------------------------------------------------------+-------------+--------------+ @@ -364,7 +393,7 @@ RAGFlow(api/default)> list providers; +--------------------------------------------------------------------------+-------------+--------------+ ``` -### 6.18. Deactivate / activate a model +### 5.18. Deactivate / activate a model ``` RAGFlow(api/default)> disable model 'deepseek-v4-pro' from 'deepseek' 'test'; @@ -380,7 +409,7 @@ RAGFlow(api/default)> enable model 'deepseek-v4-pro' from 'deepseek' 'test'; SUCCESS ``` -### 6.19. Set current model +### 5.19. Set current model ``` RAGFlow(api/default)> use model 'glm-4.5-flash@test@zhipu-ai'; SUCCESS @@ -389,7 +418,7 @@ Answer: Large language models are advanced AI systems. They process text to unde Time: 1.680416 ``` -### 6.20. Set, reset, and list default models +### 5.20. Set, reset, and list default models ``` RAGFlow(api/default)> set default chat model 'glm-4.5-flash@test@zhipu-ai'; SUCCESS @@ -433,7 +462,7 @@ RAGFlow(api/default)> list default models; +--------+----------------+--------------+----------------+------------+ ``` -### 6.21. Show current balance of a provider instance +### 5.21. Show current balance of a provider instance ``` RAGFlow(api/default)> show balance from 'gitee' 'test'; +-------------+----------+ @@ -443,13 +472,13 @@ RAGFlow(api/default)> show balance from 'gitee' 'test'; +-------------+----------+ ``` -### 6.22. Check provider instance availability +### 5.22. Check provider instance availability ``` RAGFlow(api/default)> check instance 'test' from 'zhipu-ai'; SUCCESS ``` -### 6.23. Add local model to RAGFlow, only for local deployed inference server, such as ollama +### 5.23. Add local model to RAGFlow, only for local deployed inference server, such as ollama ``` RAGFlow(api/default)> add model 'Qwen/Qwen2.5-0.5B' to provider 'vllm' instance 'test' with tokens 131072 chat; SUCCESS @@ -463,7 +492,7 @@ RAGFlow(api/default)> drop model 'Qwen/Qwen2.5-0.5B' from 'vllm' 'test'; SUCCESS ``` -### 6.24. List datasets +### 5.24. List datasets ``` RAGFlow(api/default)> list datasets; +-------------+--------------+----------------+----------------------+----------------------------------+----------+------+----------+------------+----------------------------------+-----------+---------------+ @@ -474,14 +503,14 @@ RAGFlow(api/default)> list datasets; +-------------+--------------+----------------+----------------------+----------------------------------+----------+------+----------+------------+----------------------------------+-----------+---------------+ ``` -### 6.25. Text to Speech +### 5.25. Text to Speech ``` RAGFlow(api/default)> tts with 'speech-2.8-hd@test@minimax' text 'He who desires but acts not, breeds pestilence.' play format 'wav' save './internal' param '{"voice_setting": {"voice_id": "English_radiant_girl", "speed": 1, "vol": 1, "pitch": 0}, "audio_setting": {"sample_rate": 32000, "bitrate": 128000, "format": "wav", "channel": 1}, "output_format": "hex"}' Saved to directory: /home/infiniflow/Documents/development/ragflow/internal/speech-2.8-hd_output.wav SUCCESS ``` -### 6.26. Audio to Speech +### 5.25. Audio to Speech ``` RAGFlow(api/default)> asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' audio './internal/test.wav' param '' +----------------------------------------------------------------------------------------------------------------------+ @@ -491,7 +520,7 @@ RAGFlow(api/default)> asr with 'FunAudioLLM/SenseVoiceSmall@test@siliconflow' au +----------------------------------------------------------------------------------------------------------------------+ ``` -### 6.27. Optical Character Recognition +### 5.27. Optical Character Recognition ``` RAGFlow(api/default)> ocr with 'paddleocr-vl-0.9b@test@baidu' file './internal/text.jpg' +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -501,7 +530,7 @@ RAGFlow(api/default)> ocr with 'paddleocr-vl-0.9b@test@baidu' file './internal/t +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ``` -### 6.28. Chunk Management Commands +### 5.28. Chunk Management Commands - Create a chunk store with vector size ``` @@ -548,7 +577,7 @@ RAGFlow(api/default)> RETRIEVE 'AI' ON DATASETS 'test' RAGFlow(api/default)> GET CHUNK '29cc4f6d7a5c6e7c' OF DATASET 'test' DOCUMENT 'bbe55942535e11f1bc5184ba59049aa3' IN DATASET 'test' ``` -### 6.29. Metadata Management Commands +### 5.29. Metadata Management Commands - Create metadata store ``` @@ -584,7 +613,7 @@ RAGFlow(api/default)> DROP METADATA STORE RAGFlow(api/default)> GET METADATA OF DATASET 'test' 'test2' ``` -### 6.30. Search datasets +### 5.30. Search datasets - Search datasets using SQL-like dataset search syntax: ``` diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py new file mode 100644 index 0000000000..d3f1e5ed24 --- /dev/null +++ b/ragflow_deps/download_go_deps.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 + +# PEP 723 metadata +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "nltk", +# "huggingface-hub" +# ] +# /// + +# This script downloads every artifact that the `infiniflow/ragflow_deps` +# Docker image bakes in. Run it from anywhere — the `__main__` block +# chdir's into this file's own directory, so all outputs land under +# `ragflow_deps/` regardless of the caller's CWD. +# +# Build-context relationship: `ragflow_deps/Dockerfile` is built with +# `ragflow_deps/` as its build context, so the files written here MUST +# sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume +# top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`, +# `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-`). +# +# Typical workflow: +# +# uv run python3 ragflow_deps/download_deps.py # download +# cd ragflow_deps +# docker build -f Dockerfile -t infiniflow/ragflow_deps . +# +# The main `Dockerfile` (built from the project root) pulls this image +# via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and +# is unaffected by where these files live locally. + +import argparse +import os +import sys +import requests +from typing import Union + +def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]: + if use_china_mirrors: + return [ + # stagehand-server-v3 Node.js SEA binaries (used by Browser + # component in local mode). + # + # The stagehand-go Go module (pinned in go.mod) and the + # stagehand-server binary (this release) are LOOSELY + # MATCHED — both stay on the v3.x line and remain + # protocol-compatible. The two version numbers do NOT + # track each other: the Go SDK is at v3.21.0 while the + # current latest server release is v3.7.2. + # + # On every go.mod bump, refresh this URL to the current + # latest server release. There is no version + # correspondence to maintain; "both on v3.x" is the + # compatibility contract. + "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", + "https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", + # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) + # Used by build.sh's check_*_deps functions — pre-downloaded to avoid + # network access during CI. + ["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], + ["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], + ["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.3/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + ] + else: + return [ + # stagehand-server-v3 Node.js SEA binaries (used by Browser + # component in local mode). + # + # The stagehand-go Go module (pinned in go.mod) and the + # stagehand-server binary (this release) are LOOSELY + # MATCHED — both stay on the v3.x line and remain + # protocol-compatible. The two version numbers do NOT + # track each other: the Go SDK is at v3.21.0 while the + # current latest server release is v3.7.2. + # + # On every go.mod bump, refresh this URL to the current + # latest server release. There is no version + # correspondence to maintain; "both on v3.x" is the + # compatibility contract. + "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", + "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", + # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide) + # Used by build.sh's check_*_deps functions — pre-downloaded to avoid + # network access during CI. + ["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"], + ["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"], + ["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.3/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"], + ] + + +def download_with_progress(url, filename): + response = requests.get(url, stream=True) + total_size = int(response.headers.get('content-length', 0)) + block_size = 1024 + + with open(filename, 'wb') as file: + downloaded = 0 + for data in response.iter_content(block_size): + file.write(data) + downloaded += len(data) + + if total_size > 0: + progress = (downloaded / total_size) * 100 + sys.stdout.write(f'\rProgress: {progress:.1f}% ({downloaded}/{total_size} bytes)') + sys.stdout.flush() + + print() + + +if __name__ == "__main__": + # Anchor CWD to this file's directory so all relative outputs + # (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land + # at the top of ragflow_deps/ regardless of where the user invokes + # the script from. This is the build context for `ragflow_deps/Dockerfile`. + os.chdir(os.path.dirname(os.path.abspath(__file__))) + + parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support") + parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads") + args = parser.parse_args() + + urls = get_urls(args.china_mirrors) + + # Some mirrors (e.g. archive.ubuntu.com) reject the default urllib + # User-Agent with HTTP 403, so install an opener with a browser-like UA. +# opener = urllib.request.build_opener() +# opener.addheaders = [("User-Agent", "Mozilla/5.0")] +# urllib.request.install_opener(opener) + + for url in urls: + download_url = url[0] if isinstance(url, list) else url + filename = url[1] if isinstance(url, list) else url.split("/")[-1] + print(f"Downloading {filename} from {download_url}...") + if not os.path.exists(filename): + download_with_progress(download_url, filename) + + # Extract native static libraries to ~/ragflow-native-libs for Go build. + # Ensures build.sh can find them without network access. + native_deps_dir = os.path.expanduser("~/ragflow-native-libs") + extractions = [ + ("pdfium-linux-x64-static.tgz", "pdfium-static"), + ("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"), + ("office_oxide-linux-x86_64.tar.gz", "office_oxide"), + ] + import tarfile + + for archive, subdir in extractions: + archive_path = os.path.join(os.getcwd(), archive) + if not os.path.isfile(archive_path): + print(f" Skipping extraction: {archive} not found") + continue + target = os.path.join(native_deps_dir, subdir) + if os.path.isdir(target): + print(f" ✓ {subdir} already extracted to {target}") + continue + os.makedirs(target, exist_ok=True) + print(f" Extracting {archive} → {target}") + with tarfile.open(archive_path) as tf: + tf.extractall(target)