#!/bin/bash set -e # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Get script directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$SCRIPT_DIR" # Build directories CPP_DIR="$PROJECT_ROOT/internal/cpp" BUILD_DIR="$CPP_DIR/cmake-build-release" RAGFLOW_SERVER_BINARY="$PROJECT_ROOT/bin/ragflow_server" ADMIN_SERVER_BINARY="$PROJECT_ROOT/bin/admin_server" INGESTOR_BINARY="$PROJECT_ROOT/bin/ingestor" RAGFLOW_CLI_BINARY="$PROJECT_ROOT/bin/ragflow-cli" # Strip symbols from Go binaries (set via --strip / -s) STRIP_SYMBOLS="" # Native static library settings. These are the user-cache paths (~/ragflow-native-libs/). # If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority # and skips the network (download_deps.py) fallback. SYSTEM_DEPS="/opt/ragflow-native-libs" # office_oxide native library settings — static linking OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide" OFFICE_OXIDE_VERSION="0.1.2" # pdfium native library settings — static linking (kognitos/pdfium-static) PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static" PDFIUM_STATIC_VERSION="7809" # pdf_oxide native library settings — static linking (go-ffi tarball) PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide" PDF_OXIDE_VERSION="0.3.67" # Copy a dependency from the system pre-seed directory to the user cache. # Returns 0 if the dep was copied or already exists in cache, 1 otherwise. _seed_from_system() { local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide" local dep_dir="${HOME}/ragflow-native-libs/${dep_name}" local sys_dir="${SYSTEM_DEPS}/${dep_name}" if [ -d "$dep_dir" ]; then return 0 # already cached fi if [ -d "$sys_dir" ]; then echo " ${dep_name} → ${sys_dir} (system)" mkdir -p "$(dirname "$dep_dir")" cp -r "$sys_dir" "$dep_dir" return 0 fi return 1 } echo -e "${GREEN}=== RAGFlow Go Server Build Script ===${NC}" # Function to print section headers print_section() { echo -e "\n${YELLOW}>>> $1${NC}" } # Detect the package-install command for pcre2 development files. # Outputs the command on stdout; empty string if no supported manager is found. detect_pcre2_install_cmd() { if [ "$(uname)" = "Darwin" ]; then echo "brew install pcre2" elif command -v apt-get >/dev/null 2>&1; then echo "sudo apt-get install -y libpcre2-dev" elif command -v zypper >/dev/null 2>&1; then echo "sudo zypper install -y pcre2-devel" elif command -v dnf >/dev/null 2>&1; then echo "sudo dnf install -y pcre2-devel" elif command -v pacman >/dev/null 2>&1; then echo "sudo pacman -S --noconfirm pcre2" else echo "" fi } # Check whether libpcre2-8 is available (static or shared). check_pcre2() { # Prefer pkg-config when available — works across distros. if command -v pkg-config >/dev/null 2>&1 && pkg-config --exists libpcre2-8; then return 0 fi # Fall back to known library paths: # Debian/Ubuntu -> /usr/lib/x86_64-linux-gnu # openSUSE/RHEL -> /usr/lib64 # generic Linux -> /usr/lib, /usr/local/lib # macOS Homebrew -> /opt/homebrew/lib (Apple Silicon), /usr/local/lib (Intel) for p in \ /usr/lib/x86_64-linux-gnu/libpcre2-8.a \ /usr/lib/x86_64-linux-gnu/libpcre2-8.so \ /usr/lib64/libpcre2-8.a \ /usr/lib64/libpcre2-8.so \ /usr/lib/libpcre2-8.a \ /usr/lib/libpcre2-8.so \ /usr/local/lib/libpcre2-8.a \ /usr/local/lib/libpcre2-8.so \ /usr/local/lib/libpcre2-8.dylib \ /opt/homebrew/lib/libpcre2-8.a \ /opt/homebrew/lib/libpcre2-8.dylib; do [ -f "$p" ] && return 0 done return 1 } # Check dependencies check_cpp_deps() { print_section "Checking c++ dependencies" command -v cmake >/dev/null 2>&1 || { echo -e "${RED}Error: cmake is required but not installed.${NC}"; exit 1; } command -v g++ >/dev/null 2>&1 || { echo -e "${RED}Error: g++ is required but not installed.${NC}"; exit 1; } if check_pcre2; then echo "✓ pcre2 library found" else install_cmd="$(detect_pcre2_install_cmd)" echo -e "${YELLOW}Warning: libpcre2-8 not found. You may need to install it:${NC}" if [ -n "$install_cmd" ]; then echo " $install_cmd" else echo " (No supported package manager detected — install pcre2 development files manually)" fi fi echo "✓ Required tools are available" } check_go_deps() { print_section "Checking go dependencies" command -v go >/dev/null 2>&1 || { echo -e "${RED}Error: go is required but not installed.${NC}"; exit 1; } echo "✓ Required tools are available" } # Download and extract a tar.gz from a URL to a target directory _download_and_extract() { local url="$1" target_dir="$2" echo "Downloading ${url} ..." local tmpfile tmpfile="$(mktemp)" if command -v curl >/dev/null 2>&1; then curl -fsSL "$url" -o "$tmpfile" elif command -v wget >/dev/null 2>&1; then wget -q "$url" -O "$tmpfile" else echo -e "${RED}Error: need curl or wget to download office_oxide${NC}" exit 1 fi tar xzf "$tmpfile" -C "$target_dir" rm -f "$tmpfile" } # Check / install office_oxide native library (Rust → C FFI library) check_office_oxide_deps() { print_section "Checking office_oxide native library" _seed_from_system "office_oxide" local lib_file="liboffice_oxide.a" local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}" local header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" if [ -f "$lib_path" ] && [ -f "$header_path" ]; then echo "✓ office_oxide native library found at ${OFFICE_OXIDE_PREFIX}" return 0 fi echo -e "${RED}Error: office_oxide native library not found${NC}" echo " Expected: ${lib_path}" echo " Run: uv run download_deps.py" echo " Or manually download: https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/native-linux-x86_64.tar.gz" exit 1 } # Check pdfium static library (must be pre-installed via download_deps.py or CI image). check_pdfium_deps() { _seed_from_system "pdfium-static" local lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" if [ -f "$lib_path" ]; then echo " pdfium (static) → ${PDFIUM_STATIC_PREFIX}" return 0 fi echo " pdfium (static) not found" echo " Expected: ${lib_path}" echo " Run: uv run download_deps.py" echo " Or: curl -fsSL https://github.com/kognitos/pdfium-static/releases/download/chromium%2F${PDFIUM_STATIC_VERSION}/pdfium-linux-x64-static.tgz | tar xz -C ${PDFIUM_STATIC_PREFIX}" return 1 } # Check / install pdf_oxide static library (go-ffi tarball from GitHub Release). check_pdf_oxide_deps() { _seed_from_system "pdf_oxide" # Map platform to tarball-internal subdirectory. local platform_subdir case "$(uname -s)" in Linux) case "$(uname -m)" in x86_64) platform_subdir="linux_amd64" ;; aarch64|arm64) platform_subdir="linux_arm64" ;; *) echo " pdf_oxide (static) → unsupported arch"; return 1 ;; esac ;; Darwin) case "$(uname -m)" in x86_64) platform_subdir="darwin_amd64" ;; arm64) platform_subdir="darwin_arm64" ;; *) echo " pdf_oxide (static) → unsupported arch"; return 1 ;; esac ;; *) echo " pdf_oxide (static) → unsupported OS"; return 1 ;; esac local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a" if [ -f "$lib_path" ]; then echo " pdf_oxide (static) → ${PDF_OXIDE_PREFIX}" return 0 fi echo " pdf_oxide (static) not found" echo " Expected: ${lib_path}" echo " Run: uv run download_deps.py" echo " Or: curl -fsSL https://github.com/yfedoseev/pdf_oxide/releases/download/v${PDF_OXIDE_VERSION}/pdf_oxide-go-ffi-linux-amd64.tar.gz | tar xz -C ${PDF_OXIDE_PREFIX}" return 1 } # Build C++ static library build_cpp() { print_section "Building C++ static library" mkdir -p "$BUILD_DIR" cd "$BUILD_DIR" echo "Running cmake..." cmake .. -DCMAKE_BUILD_TYPE=Release echo "Building librag_tokenizer_c_api.a..." local jobs jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)" make rag_tokenizer_c_api -j"$jobs" if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then echo -e "${RED}Error: Failed to build C++ static library${NC}" exit 1 fi echo -e "${GREEN}✓ C++ static library built successfully${NC}" } # Build C++ test executable build_cpp_test() { print_section "Building C++ test executable" if [ ! -d "$BUILD_DIR" ]; then echo "Build directory not found, running cmake first..." mkdir -p "$BUILD_DIR" cd "$BUILD_DIR" cmake .. -DCMAKE_BUILD_TYPE=Release else cd "$BUILD_DIR" fi echo "Building rag_analyzer_c_test..." local jobs jobs="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)" make rag_analyzer_c_test -j"$jobs" if [ ! -f "$BUILD_DIR/rag_analyzer_c_test" ]; then echo -e "${RED}Error: Failed to build rag_analyzer_c_test${NC}" exit 1 fi echo -e "${GREEN}✓ C++ test executable built successfully: $BUILD_DIR/rag_analyzer_c_test${NC}" } # Build Go server build_go() { print_section "Building RAGFlow go" cd "$PROJECT_ROOT" # Check if C++ library exists if [ ! -f "$BUILD_DIR/librag_tokenizer_c_api.a" ]; then echo -e "${RED}Error: C++ static library not found. Run with --cpp first.${NC}" exit 1 fi if check_pcre2; then echo "✓ pcre2 library found" else install_cmd="$(detect_pcre2_install_cmd)" if [ -z "$install_cmd" ]; then echo -e "${RED}Error: libpcre2-8 not found and no supported package manager detected.${NC}" echo "Please install pcre2 development files manually." exit 1 fi if [ "$(uname)" = "Darwin" ]; then echo -e "${RED}Error: libpcre2-8 not found. Install with: $install_cmd${NC}" exit 1 fi echo -e "${YELLOW}Warning: libpcre2-8 not found. Installing with: $install_cmd${NC}" eval "$install_cmd" fi setup_cgo_env local strip_flags=() [ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w") echo "Building RAGFlow binary: $RAGFLOW_SERVER_BINARY, $ADMIN_SERVER_BINARY, $INGESTOR_BINARY, and $RAGFLOW_CLI_BINARY" GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go build "${strip_flags[@]}" -o "$RAGFLOW_SERVER_BINARY" cmd/server_main.go GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go build "${strip_flags[@]}" -o "$ADMIN_SERVER_BINARY" cmd/admin_server.go GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go build "${strip_flags[@]}" -o "$INGESTOR_BINARY" cmd/ingestor.go GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go build "${strip_flags[@]}" -o "$RAGFLOW_CLI_BINARY" cmd/ragflow-cli.go if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then echo -e "${RED}Error: Failed to build RAGFlow server binary${NC}" exit 1 fi if [ ! -f "$ADMIN_SERVER_BINARY" ]; then echo -e "${RED}Error: Failed to build Admin server binary${NC}" exit 1 fi if [ ! -f "$INGESTOR_BINARY" ]; then echo -e "${RED}Error: Failed to build Ingestor binary${NC}" exit 1 fi echo -e "${GREEN}✓ Go ragflow_server built successfully: $RAGFLOW_SERVER_BINARY${NC}" echo -e "${GREEN}✓ Go admin_server built successfully: $ADMIN_SERVER_BINARY${NC}" echo -e "${GREEN}✓ Go ragflow-cli built successfully: $RAGFLOW_CLI_BINARY${NC}" echo -e "${GREEN}✓ Go ingestor built successfully: $INGESTOR_BINARY${NC}" } # Configure CGO flags for native libraries (office_oxide, pdfium, pdf_oxide). # All three are statically linked — no LD_LIBRARY_PATH or -Wl,-rpath needed. setup_cgo_env() { # ── office_oxide ────────────────────────────────────────────────── check_office_oxide_deps export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}" export CGO_LDFLAGS="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a" # ── pdfium ──────────────────────────────────────────────────────── check_pdfium_deps || return 1 export CGO_LDFLAGS="$CGO_LDFLAGS ${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" # Linux: Chromium-built objects use Clang's .eh_frame format which GNU ld # cannot merge. Use lld (LLVM linker) which handles them correctly. # --allow-multiple-definition: pdf_oxide and office_oxide are both Rust # staticlibs that embed the Rust runtime; linking them together produces # duplicate rust_eh_personality symbols. if [ "$(uname -s)" = "Linux" ]; then if ! command -v ld.lld >/dev/null 2>&1; then echo -e "${RED}Error: ld.lld not found. Install with: sudo apt install lld-20${NC}" echo " lld is required to static-link Chromium-built pdfium (.eh_frame format)" return 1 fi export CGO_LDFLAGS="$CGO_LDFLAGS \ ${PDFIUM_STATIC_PREFIX}/lib/libc++.a \ ${PDFIUM_STATIC_PREFIX}/lib/libc++abi.a \ -fuse-ld=lld -Wl,--allow-multiple-definition" fi # ── pdf_oxide ───────────────────────────────────────────────────── check_pdf_oxide_deps || return 1 # The go-ffi tarball places the .a under lib//. local pdf_oxide_subdir case "$(uname -s)" in Linux) case "$(uname -m)" in x86_64) pdf_oxide_subdir="linux_amd64" ;; aarch64|arm64) pdf_oxide_subdir="linux_arm64" ;; *) echo "pdf_oxide: unsupported arch"; return 1 ;; esac ;; Darwin) case "$(uname -m)" in x86_64) pdf_oxide_subdir="darwin_amd64" ;; arm64) pdf_oxide_subdir="darwin_arm64" ;; *) echo "pdf_oxide: unsupported arch"; return 1 ;; esac ;; esac export CGO_LDFLAGS="$CGO_LDFLAGS ${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}/libpdf_oxide.a" # ── platform-specific system libraries ──────────────────────────── case "$(uname -s)" in Linux) export CGO_LDFLAGS="$CGO_LDFLAGS -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc" ;; Darwin) export CGO_LDFLAGS="$CGO_LDFLAGS \ -framework CoreFoundation -framework Security \ -framework SystemConfiguration -liconv -lresolv" ;; esac echo "CGO_CFLAGS: $CGO_CFLAGS" echo "CGO_LDFLAGS: $CGO_LDFLAGS" } # Run Go unit tests with the same CGO env as `build_go`. Pass any extra args # to `go test`, e.g. `./build.sh --test -run TestFoo ./internal/admin/...`. run_go_tests() { print_section "Running Go tests" cd "$PROJECT_ROOT" setup_cgo_env if [ "$#" -eq 0 ]; then set -- ./... fi GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} CGO_ENABLED=1 \ CGO_CFLAGS="$CGO_CFLAGS" CGO_LDFLAGS="$CGO_LDFLAGS" \ go test -count=1 "$@" } # Clean build artifacts clean() { print_section "Cleaning build artifacts" rm -rf "$BUILD_DIR" rm -f "$RAGFLOW_SERVER_BINARY" rm -f "$ADMIN_SERVER_BINARY" rm -f "$INGESTOR_BINARY" rm -f "$RAGFLOW_CLI_BINARY" echo -e "${GREEN}✓ Build artifacts cleaned${NC}" } # Run the server run() { if [ ! -f "$ADMIN_SERVER_BINARY" ]; then echo -e "${RED}Error: $ADMIN_SERVER_BINARY not found. Build first with --all or --go${NC}" exit 1 fi if [ ! -f "$RAGFLOW_SERVER_BINARY" ]; then echo -e "${RED}Error: $RAGFLOW_SERVER_BINARY not found. Build first with --all or --go${NC}" exit 1 fi if [ ! -f "$INGESTOR_BINARY" ]; then echo -e "${RED}Error: $INGESTOR_BINARY not found. Build first with --all or --go${NC}" exit 1 fi cd "$PROJECT_ROOT" # admin_server must be running before ragflow_server, otherwise ragflow_server's # heartbeats to admin will error out (see internal/development.md). print_section "Starting admin server (background)" "$ADMIN_SERVER_BINARY" & ADMIN_PID=$! trap 'kill "$ADMIN_PID" 2>/dev/null || true' EXIT INT TERM # Give admin_server a moment to bind its listening port (9383) before # ragflow_server starts sending heartbeats to it. sleep 1 print_section "Starting ingestor (background)" "$INGESTOR_BINARY" & INGESTOR_PID=$! trap 'kill "$INGESTOR_PID" 2>/dev/null || true' EXIT INT TERM sleep 1 print_section "Starting RAGFlow server (foreground)" "$RAGFLOW_SERVER_BINARY" } # Show help show_help() { # Quoted delimiter so backticks, `$var`, and `\$` in the help text are # printed literally instead of being interpreted as command substitution. cat << 'EOF' Usage: $0 [OPTIONS] Build script for RAGFlow Go server with C++ bindings. OPTIONS: --all, -a Build everything (C++ library + Go server) [default] --cpp, -c Build only C++ static library --cpp-test Build C++ test executable (requires --cpp first) --go, -g Build only Go server (requires C++ library to be built) --test, -t Run Go unit tests (sets up CGO env for office_oxide). Pass extra args after `--` to forward to `go test`, e.g. `$0 --test -- -run TestFoo ./internal/admin/...` --clean, -C Clean all build artifacts --run, -r Build and run the server --strip, -s Strip debug symbols from Go binaries (-ldflags="-s -w") (disabled by default, useful for smaller production binaries) --help, -h Show this help message EXAMPLES: $0 # Build everything $0 --cpp # Build only C++ library $0 --go # Build only Go server $0 --cpp-test # Build C++ test executable $0 --test # Run all Go tests $0 --test -- -run TestFoo ./internal/admin/... # Targeted Go tests $0 --run # Build and run $0 --clean # Clean build artifacts DEPENDENCIES: - cmake >= 4.0 - go >= 1.24 - g++ with C++17/23 support - office_oxide native library (auto-downloaded on first build) - pcre2 development files - Debian/Ubuntu: libpcre2-dev - openSUSE/RHEL/Fedora: pcre2-devel - macOS (Homebrew): pcre2 EOF } # Main function main() { # Parse --strip / -s before other arguments local args=() for arg in "$@"; do case "$arg" in --strip|-s) STRIP_SYMBOLS="1" ;; *) args+=("$arg") ;; esac done case "${args[0]:-}" in --cpp|-c) check_cpp_deps build_cpp ;; --cpp-test) check_cpp_deps build_cpp_test ;; --go|-g) check_go_deps build_go ;; --test|-t) check_go_deps # Forward any args after `--` to `go test`. if [ "${2:-}" = "--" ]; then shift 2 run_go_tests "$@" else run_go_tests fi ;; --clean|-C) clean ;; --run|-r) check_cpp_deps check_go_deps build_cpp build_go run ;; --help|-h) show_help ;; --all|-a|"") check_cpp_deps check_go_deps build_cpp build_go echo -e "\n${GREEN}=== Build completed successfully! ===${NC}" echo "Binary: $RAGFLOW_SERVER_BINARY, $ADMIN_SERVER_BINARY, $INGESTOR_BINARY, $RAGFLOW_CLI_BINARY" ;; *) echo -e "${RED}Unknown option: $1${NC}" show_help exit 1 ;; esac } main "$@"