Files
ragflow/.github/workflows/release.yml
Haruko386 dfe841a3e3 fix: Enhance Windows build process for office_oxide and rag tokenizer (#16223)
### What problem does this PR solve?

Updated MSYS2 package list for Windows builds and added Rust target
specifications. Modified build steps for office_oxide and rag tokenizer
libraries to improve compatibility and streamline the build process.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
2026-06-22 10:17:40 +08:00

533 lines
20 KiB
YAML

name: release
on:
schedule:
- cron: '0 13 * * *' # This schedule runs every 13:00:00Z(21:00:00+08:00)
# https://github.com/orgs/community/discussions/26286?utm_source=chatgpt.com#discussioncomment-3251208
# "The create event does not support branch filter and tag filter."
# The "create tags" trigger is specifically focused on the creation of new tags, while the "push tags" trigger is activated when tags are pushed, including both new tag creations and updates to existing tags.
push:
tags:
- "v*.*.*" # normal release
- 'nightly' # mutable tag
permissions:
contents: write
actions: read
checks: read
statuses: read
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
prepare:
runs-on: [ "self-hosted", "ragflow-release" ]
outputs:
release_tag: ${{ steps.release.outputs.release_tag }}
prerelease: ${{ steps.release.outputs.prerelease }}
steps:
- name: Ensure workspace ownership
run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
# https://github.com/actions/checkout/blob/v6/README.md
- name: Check out code
uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }} # Use the secret as an environment variable
fetch-depth: 0
fetch-tags: true
# https://github.com/actions/setup-go
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Prepare release metadata
id: release
run: |
if [[ ${GITHUB_EVENT_NAME} != "schedule" ]]; then
RELEASE_TAG=${GITHUB_REF#refs/tags/}
if [[ ${RELEASE_TAG} == v* ]]; then
PRERELEASE=false
else
PRERELEASE=true
fi
echo "Workflow triggered by create tag: ${RELEASE_TAG}"
else
RELEASE_TAG=nightly
PRERELEASE=true
echo "Workflow triggered by schedule"
fi
echo "RELEASE_TAG=${RELEASE_TAG}" >> ${GITHUB_ENV}
echo "PRERELEASE=${PRERELEASE}" >> ${GITHUB_ENV}
echo "release_tag=${RELEASE_TAG}" >> ${GITHUB_OUTPUT}
echo "prerelease=${PRERELEASE}" >> ${GITHUB_OUTPUT}
- name: Move the existing mutable tag
# https://github.com/softprops/action-gh-release/issues/171
run: |
git fetch --tags
if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
# Determine if a given tag exists and matches a specific Git commit.
# actions/checkout@v6 fetch-tags doesn't work when triggered by schedule
if [ "$(git rev-parse -q --verify "refs/tags/${RELEASE_TAG}")" = "${GITHUB_SHA}" ]; then
echo "mutable tag ${RELEASE_TAG} exists and matches ${GITHUB_SHA}"
else
git tag -f ${RELEASE_TAG} ${GITHUB_SHA}
git push -f origin ${RELEASE_TAG}:refs/tags/${RELEASE_TAG}
echo "created/moved mutable tag ${RELEASE_TAG} to ${GITHUB_SHA}"
fi
fi
build_cli:
needs: prepare
strategy:
fail-fast: false
matrix:
include:
- goos: linux
goarch: amd64
runner: ubuntu-24.04
native_asset: native-linux-x86_64.tar.gz
- goos: linux
goarch: arm64
runner: ubuntu-24.04-arm
native_asset: native-linux-aarch64.tar.gz
- goos: darwin
goarch: amd64
runner: macos-15-intel
native_asset: native-macos-x86_64.tar.gz
- goos: darwin
goarch: arm64
runner: macos-14
native_asset: native-macos-aarch64.tar.gz
- goos: windows
goarch: amd64
runner: windows-latest
native_asset: native-windows-x86_64.zip
msystem: CLANG64
rust_target: x86_64-pc-windows-gnu
msys2_packages: >-
mingw-w64-clang-x86_64-clang
mingw-w64-clang-x86_64-lld
mingw-w64-clang-x86_64-cmake
mingw-w64-clang-x86_64-ninja
mingw-w64-clang-x86_64-pcre2
mingw-w64-clang-x86_64-pkgconf
output_ext: .exe
- goos: windows
goarch: arm64
runner: windows-11-arm
native_asset: native-windows-aarch64.zip
msystem: CLANGARM64
rust_target: aarch64-pc-windows-gnullvm
msys2_packages: >-
mingw-w64-clang-aarch64-clang
mingw-w64-clang-aarch64-lld
mingw-w64-clang-aarch64-cmake
mingw-w64-clang-aarch64-ninja
mingw-w64-clang-aarch64-pcre2
mingw-w64-clang-aarch64-pkgconf
output_ext: .exe
runs-on: ${{ matrix.runner }}
env:
CLI_NAME: ragflow_cli
CLI_MAIN: ./cmd/ragflow_cli.go
DIST_DIR: dist/cli
OFFICE_OXIDE_VERSION: "0.1.2"
RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}
steps:
# https://github.com/actions/checkout/blob/v6/README.md
- name: Check out code
uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0
fetch-tags: true
# https://github.com/actions/setup-go
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Install Unix native build dependencies
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
if [[ "${{ matrix.goos }}" == "linux" ]]; then
sudo apt-get update
sudo apt-get install -y cmake build-essential libpcre2-dev pkg-config
elif [[ "${{ matrix.goos }}" == "darwin" ]]; then
brew list pcre2 >/dev/null 2>&1 || brew install pcre2
fi
- name: Download office_oxide native library
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
OFFICE_OXIDE_PREFIX="${RUNNER_TEMP}/office_oxide"
OFFICE_OXIDE_URL="https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/${{ matrix.native_asset }}"
mkdir -p "${OFFICE_OXIDE_PREFIX}"
curl -fsSL "${OFFICE_OXIDE_URL}" -o "${RUNNER_TEMP}/${{ matrix.native_asset }}"
tar xzf "${RUNNER_TEMP}/${{ matrix.native_asset }}" -C "${OFFICE_OXIDE_PREFIX}"
test -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
test -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
ln -sf "office_oxide_c/office_oxide.h" "${OFFICE_OXIDE_PREFIX}/include/office_oxide.h"
test -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide.h"
echo "OFFICE_OXIDE_PREFIX=${OFFICE_OXIDE_PREFIX}" >> "${GITHUB_ENV}"
- name: Set up MSYS2
if: runner.os == 'Windows'
uses: msys2/setup-msys2@v2
with:
msystem: ${{ matrix.msystem }}
update: true
install: ${{ matrix.msys2_packages }}
path-type: inherit
- name: Configure Windows C compiler
if: runner.os == 'Windows'
shell: msys2 {0}
run: |
set -euo pipefail
cc_path="$(command -v clang.exe || command -v clang)"
cxx_path="$(command -v clang++.exe || command -v clang++)"
cc="$(cygpath -m "${cc_path}")"
cxx="$(cygpath -m "${cxx_path}")"
github_env="$(cygpath -u "${GITHUB_ENV}")"
pcre2_libdir="$(pkg-config --variable=libdir libpcre2-8)"
pcre2_includedir="$(pkg-config --variable=includedir libpcre2-8)"
echo "CC=${cc}" >> "${github_env}"
echo "CXX=${cxx}" >> "${github_env}"
echo "PCRE2_LIBDIR=$(cygpath -m "${pcre2_libdir}")" >> "${github_env}"
echo "PCRE2_INCLUDEDIR=$(cygpath -m "${pcre2_includedir}")" >> "${github_env}"
echo "Resolved MSYS2 clang: ${cc}"
"${cc_path}" --version
echo "Resolved MSYS2 clang++: ${cxx}"
"${cxx_path}" --version
- name: Set up Rust for Windows office_oxide staticlib
if: runner.os == 'Windows'
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.rust_target }}
- name: Build office_oxide native library for Windows GNU ABI
if: runner.os == 'Windows'
shell: msys2 {0}
run: |
set -euo pipefail
office_oxide_prefix="$(cygpath -u "${RUNNER_TEMP}")/office_oxide"
office_oxide_src="$(cygpath -u "${RUNNER_TEMP}")/office_oxide-src"
archive_path="$(cygpath -u "${RUNNER_TEMP}")/office_oxide-v${OFFICE_OXIDE_VERSION}.tar.gz"
github_env="$(cygpath -u "${GITHUB_ENV}")"
rm -rf "${office_oxide_prefix}" "${office_oxide_src}"
mkdir -p "${office_oxide_prefix}/include/office_oxide_c" "${office_oxide_prefix}/lib" "${office_oxide_src}"
curl -fsSL "https://github.com/yfedoseev/office_oxide/archive/refs/tags/v${OFFICE_OXIDE_VERSION}.tar.gz" -o "${archive_path}"
tar xzf "${archive_path}" -C "${office_oxide_src}" --strip-components=1
cd "${office_oxide_src}"
export CC="$(command -v clang)"
export CXX="$(command -v clang++)"
export AR="$(command -v llvm-ar || command -v ar)"
export CARGO_BUILD_TARGET="${{ matrix.rust_target }}"
export RUSTFLAGS="-C target-feature=+crt-static"
case "${{ matrix.rust_target }}" in
x86_64-pc-windows-gnu)
export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER="${CC}"
export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_AR="${AR}"
;;
aarch64-pc-windows-gnullvm)
export CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_LINKER="${CC}"
export CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_AR="${AR}"
;;
*)
echo "Unsupported Rust target: ${{ matrix.rust_target }}"
exit 1
;;
esac
cargo build --release --lib --target "${CARGO_BUILD_TARGET}" --no-default-features
cp "include/office_oxide_c/office_oxide.h" "${office_oxide_prefix}/include/office_oxide_c/office_oxide.h"
cp "include/office_oxide_c/office_oxide.h" "${office_oxide_prefix}/include/office_oxide.h"
cp "target/${CARGO_BUILD_TARGET}/release/liboffice_oxide.a" "${office_oxide_prefix}/lib/liboffice_oxide.a"
test -f "${office_oxide_prefix}/include/office_oxide_c/office_oxide.h"
test -f "${office_oxide_prefix}/include/office_oxide.h"
test -f "${office_oxide_prefix}/lib/liboffice_oxide.a"
echo "OFFICE_OXIDE_PREFIX=$(cygpath -m "${office_oxide_prefix}")" >> "${github_env}"
- name: Build rag tokenizer native library
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
cmake -S internal/cpp -B internal/cpp/cmake-build-release -DCMAKE_BUILD_TYPE=Release
cmake --build internal/cpp/cmake-build-release --target rag_tokenizer_c_api --parallel
test -f internal/cpp/cmake-build-release/librag_tokenizer_c_api.a
- name: Build rag tokenizer native library
if: runner.os == 'Windows'
shell: msys2 {0}
run: |
set -euo pipefail
cmake -S internal/cpp -B internal/cpp/cmake-build-release -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER="$(command -v clang)" \
-DCMAKE_CXX_COMPILER="$(command -v clang++)"
cmake --build internal/cpp/cmake-build-release --target rag_tokenizer_c_api --parallel
test -f internal/cpp/cmake-build-release/librag_tokenizer_c_api.a
- name: Build Go CLI release binaries
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
mkdir -p "${DIST_DIR}"
if [[ ! -e "${CLI_MAIN}" ]]; then
echo "::error::Go CLI entry does not exist: ${CLI_MAIN}"
echo "::error::Please update CLI_MAIN in .github/workflows/release.yml"
exit 1
fi
echo "Building Go CLI release binaries"
echo "CLI name: ${CLI_NAME}"
echo "CLI main: ${CLI_MAIN}"
echo "Release tag: ${RELEASE_TAG}"
echo "Commit: ${GITHUB_SHA}"
output="${DIST_DIR}/${CLI_NAME}-${RELEASE_TAG}-${{ matrix.goos }}-${{ matrix.goarch }}"
echo "Building ${{ matrix.goos }}/${{ matrix.goarch }} -> ${output}"
case "${{ matrix.goos }}" in
linux)
cgo_ldflags="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a -lm -ldl -lpthread"
;;
darwin)
cgo_ldflags="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
;;
*)
echo "::error::Unsupported Unix target: ${{ matrix.goos }}"
exit 1
;;
esac
cgo_cflags="-I${OFFICE_OXIDE_PREFIX}/include -I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c"
echo "office_oxide prefix: ${OFFICE_OXIDE_PREFIX}"
echo "CGO_CFLAGS: ${cgo_cflags}"
echo "CGO_LDFLAGS: ${cgo_ldflags}"
CGO_ENABLED=1 \
CGO_CFLAGS="${cgo_cflags}" \
CGO_LDFLAGS="${cgo_ldflags}" \
GOOS="${{ matrix.goos }}" \
GOARCH="${{ matrix.goarch }}" \
go build \
-trimpath \
-ldflags="-s -w -X main.version=${RELEASE_TAG} -X main.commit=${GITHUB_SHA}" \
-o "${output}" \
"${CLI_MAIN}"
chmod +x "${output}"
if [[ "${{ matrix.goos }}" == "linux" ]] && command -v ldd >/dev/null 2>&1; then
if ldd "${output}" 2>&1 | grep -q "liboffice_oxide"; then
echo "::error::linux CLI unexpectedly links liboffice_oxide dynamically"
ldd "${output}" || true
exit 1
fi
echo "Verified linux CLI does not require liboffice_oxide.so at runtime"
fi
- name: Build Go CLI release binaries
if: runner.os == 'Windows'
shell: pwsh
run: |
New-Item -ItemType Directory -Force -Path $env:DIST_DIR | Out-Null
if (-not (Test-Path $env:CLI_MAIN)) {
Write-Error "Go CLI entry does not exist: $env:CLI_MAIN"
exit 1
}
$output = Join-Path $env:DIST_DIR "${env:CLI_NAME}-${env:RELEASE_TAG}-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.output_ext }}"
Write-Host "Building ${{ matrix.goos }}/${{ matrix.goarch }} -> $output"
$officeOxidePrefix = $env:OFFICE_OXIDE_PREFIX -replace "\\", "/"
$cc = $env:CC
if ([string]::IsNullOrWhiteSpace($cc)) {
Write-Error "CC is not set"
exit 1
}
if (-not (Test-Path $cc)) {
Write-Error "C compiler does not exist: $cc"
exit 1
}
if (-not (Test-Path "${officeOxidePrefix}/lib/liboffice_oxide.a")) {
Write-Error "liboffice_oxide.a does not exist: ${officeOxidePrefix}/lib/liboffice_oxide.a"
exit 1
}
if (-not (Test-Path "internal/cpp/cmake-build-release/librag_tokenizer_c_api.a")) {
Write-Error "librag_tokenizer_c_api.a does not exist"
exit 1
}
if ([string]::IsNullOrWhiteSpace($env:PCRE2_LIBDIR) -or -not (Test-Path $env:PCRE2_LIBDIR)) {
Write-Error "PCRE2_LIBDIR is not set or does not exist: $env:PCRE2_LIBDIR"
exit 1
}
$ragTokenizerLib = (Resolve-Path "internal/cpp/cmake-build-release/librag_tokenizer_c_api.a").Path -replace '\\', '/'
$pcre2LibDir = $env:PCRE2_LIBDIR -replace '\\', '/'
$pcre2IncludeDir = $env:PCRE2_INCLUDEDIR -replace '\\', '/'
$env:CGO_ENABLED = "1"
$env:CC = $cc
$env:CGO_CFLAGS = "-I${officeOxidePrefix}/include -I${officeOxidePrefix}/include/office_oxide_c -I${pcre2IncludeDir}"
$env:CGO_LDFLAGS = "${officeOxidePrefix}/lib/liboffice_oxide.a ${ragTokenizerLib} -L${pcre2LibDir} -lpcre2-8 -lc++ -static -static-libgcc -static-libstdc++ -lws2_32 -lbcrypt -lntdll -luserenv -ladvapi32"
$env:GOOS = "${{ matrix.goos }}"
$env:GOARCH = "${{ matrix.goarch }}"
Write-Host "CC: $env:CC"
& $env:CC --version
Write-Host "office_oxide prefix: $officeOxidePrefix"
Write-Host "CGO_CFLAGS: $env:CGO_CFLAGS"
Write-Host "CGO_LDFLAGS: $env:CGO_LDFLAGS"
go build `
-trimpath `
-ldflags="-s -w -X main.version=$env:RELEASE_TAG -X main.commit=$env:GITHUB_SHA" `
-o "$output" `
"$env:CLI_MAIN"
- name: Upload CLI artifact
uses: actions/upload-artifact@v4
with:
name: cli-${{ matrix.goos }}-${{ matrix.goarch }}
path: dist/cli/*
if-no-files-found: error
publish_cli_assets:
needs:
- prepare
- build_cli
runs-on: [ "self-hosted", "ragflow-release" ]
steps:
- name: Ensure workspace ownership
run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
# https://github.com/actions/checkout/blob/v6/README.md
- name: Check out code
uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0
fetch-tags: true
- name: Download CLI artifacts
uses: actions/download-artifact@v5
with:
pattern: cli-*
path: dist/cli
merge-multiple: true
- name: Prepare CLI release assets
env:
RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}
run: |
set -euo pipefail
RELEASE_DATETIME=$(date --rfc-3339=seconds)
echo Release ${RELEASE_TAG} created from ${GITHUB_SHA} at ${RELEASE_DATETIME} > release_body.md
cd dist/cli
sha256sum * > SHA256SUMS
cd -
echo "Generated CLI release assets:"
ls -lh dist/cli
- name: Upload Go CLI release assets
uses: softprops/action-gh-release@v2
with:
token: ${{ secrets.GITHUB_TOKEN }}
prerelease: ${{ needs.prepare.outputs.prerelease }}
tag_name: ${{ needs.prepare.outputs.release_tag }}
body_path: release_body.md
files: |
dist/cli/*
tools/scripts/install.sh
tools/scripts/install.ps1
release:
needs:
- prepare
- publish_cli_assets
runs-on: [ "self-hosted", "ragflow-release" ]
env:
RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}
steps:
- name: Ensure workspace ownership
run: echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
# https://github.com/actions/checkout/blob/v6/README.md
- name: Check out code
uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0
fetch-tags: true
- name: Build and push image
run: |
sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }}
sudo docker build -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
sudo docker push infiniflow/ragflow:${RELEASE_TAG}
sudo docker push infiniflow/ragflow:latest
- name: Build and push ragflow-sdk
if: startsWith(github.ref, 'refs/tags/v')
run: |
cd sdk/python && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}
- name: Build and push ragflow-cli
if: startsWith(github.ref, 'refs/tags/v')
run: |
cd admin/client && uv build && uv publish --token ${{ secrets.PYPI_API_TOKEN }}