mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-17 05:07:23 +08:00
Compare commits
208 Commits
v0.26.3
...
dev-202607
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
999eb533a9 | ||
|
|
d279aee1ff | ||
|
|
e54e7ec7ef | ||
|
|
d48fd37ff1 | ||
|
|
844e3ea64d | ||
|
|
45862bf95d | ||
|
|
8bc18154d2 | ||
|
|
09abe5f513 | ||
|
|
466f33e6b4 | ||
|
|
d03b09b6a3 | ||
|
|
3bfad1f00e | ||
|
|
d35e957252 | ||
|
|
cfacaccad7 | ||
|
|
80a7a87427 | ||
|
|
2a482f3ae7 | ||
|
|
547bc86141 | ||
|
|
9f403ac3ca | ||
|
|
f21368057b | ||
|
|
2b9569ff51 | ||
|
|
d549194562 | ||
|
|
ed27b5f9f8 | ||
|
|
b0cac0ac9d | ||
|
|
3c0f8fc192 | ||
|
|
347a45967e | ||
|
|
a7ef7be056 | ||
|
|
891261e108 | ||
|
|
bdef878821 | ||
|
|
156a11c56b | ||
|
|
8a3699fa87 | ||
|
|
0ee02fb6d8 | ||
|
|
e6e99b86a6 | ||
|
|
4060cd1440 | ||
|
|
5e60fcec9f | ||
|
|
9b60870fd6 | ||
|
|
d291e4641d | ||
|
|
30739b7f8e | ||
|
|
ea9b3789ce | ||
|
|
0aa2993d55 | ||
|
|
106c2d0a41 | ||
|
|
8ced3d1f47 | ||
|
|
e53235ed3d | ||
|
|
2a83ad6cb2 | ||
|
|
07a3523b09 | ||
|
|
fb42e5531d | ||
|
|
d317742975 | ||
|
|
7db39822db | ||
|
|
0b01171a86 | ||
|
|
c33a5a9d49 | ||
|
|
5797f81fea | ||
|
|
28340f6218 | ||
|
|
868e524f29 | ||
|
|
eca8e87f44 | ||
|
|
a1a2ba8f6c | ||
|
|
15d1ee7654 | ||
|
|
add7b9486f | ||
|
|
289ca28ce2 | ||
|
|
12787996d1 | ||
|
|
74bbbba3e0 | ||
|
|
d48a5622df | ||
|
|
8236f2cabb | ||
|
|
0095fa048f | ||
|
|
5c8b51cbbf | ||
|
|
0083ad0deb | ||
|
|
5de823eab9 | ||
|
|
8bd62573eb | ||
|
|
b29a4a61eb | ||
|
|
8ac80284c4 | ||
|
|
0ba1d37a10 | ||
|
|
a9420a7832 | ||
|
|
cc94639555 | ||
|
|
7fa2a0b607 | ||
|
|
6a77523bf0 | ||
|
|
794fcc2517 | ||
|
|
b9ed0773ec | ||
|
|
ae96e636e9 | ||
|
|
1430d0e431 | ||
|
|
575984877f | ||
|
|
3ec9187cd2 | ||
|
|
080dd84fed | ||
|
|
36f053a248 | ||
|
|
70019810a1 | ||
|
|
74d3508a37 | ||
|
|
a41fef49d0 | ||
|
|
2c59d07bdb | ||
|
|
8e3bbad4da | ||
|
|
0f08dc070d | ||
|
|
0dd0ac06f8 | ||
|
|
c8d1b21ae3 | ||
|
|
e548108fff | ||
|
|
330033d7c2 | ||
|
|
21266286cb | ||
|
|
dc95b1d291 | ||
|
|
3d167204e7 | ||
|
|
b9432bb43f | ||
|
|
5baf4089bf | ||
|
|
18ea093344 | ||
|
|
40cb581d16 | ||
|
|
699a25c19c | ||
|
|
0ae5961e1c | ||
|
|
cb93883f3f | ||
|
|
2b95fa2ba4 | ||
|
|
fe835fd19c | ||
|
|
9e87fc6036 | ||
|
|
49d9f6f98e | ||
|
|
568bac673b | ||
|
|
4da9429451 | ||
|
|
41801ad2b8 | ||
|
|
28a41ed070 | ||
|
|
6cd03d7a70 | ||
|
|
705754ea8b | ||
|
|
7db3eea37b | ||
|
|
ee7edddea3 | ||
|
|
f8f049e663 | ||
|
|
7df7384b21 | ||
|
|
5236c8f659 | ||
|
|
2de5940325 | ||
|
|
b82169fba1 | ||
|
|
ac5860c3e2 | ||
|
|
332d34c495 | ||
|
|
b7945f3a64 | ||
|
|
dd2f27d6a3 | ||
|
|
9eba45249c | ||
|
|
f082675e6f | ||
|
|
5aa3e81a93 | ||
|
|
3660b98ae9 | ||
|
|
a0bda639e0 | ||
|
|
318045dda5 | ||
|
|
b4540672e4 | ||
|
|
d03a360fb1 | ||
|
|
7a4e1b6034 | ||
|
|
863b35db7f | ||
|
|
ab5958f518 | ||
|
|
f477d3329d | ||
|
|
d8cefcf052 | ||
|
|
dd20561fca | ||
|
|
82f3735770 | ||
|
|
5a8660df23 | ||
|
|
8a19c6aa5a | ||
|
|
85b565244d | ||
|
|
0c2fb622e9 | ||
|
|
779bf52549 | ||
|
|
8fb8e4197c | ||
|
|
55af1d70f3 | ||
|
|
3044283442 | ||
|
|
8ee3f097b9 | ||
|
|
7e0ccee7e2 | ||
|
|
b2e82a42d6 | ||
|
|
52f985f43e | ||
|
|
bfb9641128 | ||
|
|
3a247dbb3c | ||
|
|
b3d536c48e | ||
|
|
ed52255868 | ||
|
|
57625c919a | ||
|
|
290ab557a5 | ||
|
|
98189cd20a | ||
|
|
b633ff0435 | ||
|
|
4effc242fd | ||
|
|
c4166a91e0 | ||
|
|
c9f064d5fd | ||
|
|
d607b55c24 | ||
|
|
48ef1f4965 | ||
|
|
aaade3530e | ||
|
|
534a3a7faa | ||
|
|
3d2f60c34f | ||
|
|
09eb9dbd21 | ||
|
|
fe2f3b60a1 | ||
|
|
358152f758 | ||
|
|
e5d217993b | ||
|
|
0265ffbc53 | ||
|
|
8b065d3ddd | ||
|
|
1d3c100acb | ||
|
|
014c3f634f | ||
|
|
0fcfb38365 | ||
|
|
a0e65637eb | ||
|
|
cf634b92b4 | ||
|
|
06aa169df7 | ||
|
|
63a4ed55d8 | ||
|
|
7da4f200e5 | ||
|
|
0f4f2135f3 | ||
|
|
6b571694df | ||
|
|
1861087787 | ||
|
|
fd7fb6669a | ||
|
|
83d09b16ce | ||
|
|
dde8b6d54c | ||
|
|
226d0ff77c | ||
|
|
488574fd80 | ||
|
|
706fa4e87a | ||
|
|
ffc4d29a06 | ||
|
|
8db68e3eec | ||
|
|
3cba34d67f | ||
|
|
1880e65e99 | ||
|
|
79518973e5 | ||
|
|
4effd057f0 | ||
|
|
a4c370c5ba | ||
|
|
994b603374 | ||
|
|
7b341539e7 | ||
|
|
ac5d0c4615 | ||
|
|
ee942711c4 | ||
|
|
b2e4740acd | ||
|
|
383d059969 | ||
|
|
e65bac238e | ||
|
|
6a4b9be426 | ||
|
|
19fcb4a981 | ||
|
|
918229613a | ||
|
|
83540185e1 | ||
|
|
1aa8abe373 | ||
|
|
62f94cd59b | ||
|
|
7d64a78f83 |
@@ -27,9 +27,9 @@ docker/oceanbase/
|
||||
docker/seekdb/
|
||||
|
||||
# ── Go and C++ build outputs ────────────────────────────────────────────────
|
||||
internal/cpp/build/
|
||||
internal/cpp/cmake-build-release/
|
||||
internal/cpp/cmake-build-debug/
|
||||
internal/binding/cpp/build/
|
||||
internal/binding/cpp/cmake-build-release/
|
||||
internal/binding/cpp/cmake-build-debug/
|
||||
target/
|
||||
|
||||
# ── ragflow_deps build context (built as a separate image, mounted ──
|
||||
|
||||
340
.github/workflows/release.yml
vendored
340
.github/workflows/release.yml
vendored
@@ -94,53 +94,28 @@ jobs:
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
runner: ubuntu-24.04
|
||||
native_asset: native-linux-x86_64.tar.gz
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
native_asset: native-linux-aarch64.tar.gz
|
||||
- goos: darwin
|
||||
goarch: amd64
|
||||
runner: macos-15-intel
|
||||
native_asset: native-macos-x86_64.tar.gz
|
||||
- goos: darwin
|
||||
goarch: arm64
|
||||
runner: macos-14
|
||||
native_asset: native-macos-aarch64.tar.gz
|
||||
- goos: windows
|
||||
goarch: amd64
|
||||
runner: windows-latest
|
||||
native_asset: native-windows-x86_64.zip
|
||||
msystem: CLANG64
|
||||
rust_target: x86_64-pc-windows-gnullvm
|
||||
msys2_packages: >-
|
||||
mingw-w64-clang-x86_64-clang
|
||||
mingw-w64-clang-x86_64-lld
|
||||
mingw-w64-clang-x86_64-cmake
|
||||
mingw-w64-clang-x86_64-ninja
|
||||
mingw-w64-clang-x86_64-pcre2
|
||||
mingw-w64-clang-x86_64-pkgconf
|
||||
output_ext: .exe
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
runner: windows-11-arm
|
||||
native_asset: native-windows-aarch64.zip
|
||||
msystem: CLANGARM64
|
||||
rust_target: aarch64-pc-windows-gnullvm
|
||||
msys2_packages: >-
|
||||
mingw-w64-clang-aarch64-clang
|
||||
mingw-w64-clang-aarch64-lld
|
||||
mingw-w64-clang-aarch64-cmake
|
||||
mingw-w64-clang-aarch64-ninja
|
||||
mingw-w64-clang-aarch64-pcre2
|
||||
mingw-w64-clang-aarch64-pkgconf
|
||||
output_ext: .exe
|
||||
runs-on: ${{ matrix.runner }}
|
||||
env:
|
||||
CLI_NAME: ragflow-cli
|
||||
CLI_MAIN: ./cmd/ragflow-cli.go
|
||||
DIST_DIR: dist/cli
|
||||
OFFICE_OXIDE_VERSION: "0.1.2"
|
||||
RELEASE_TAG: ${{ needs.prepare.outputs.release_tag }}
|
||||
|
||||
steps:
|
||||
@@ -159,249 +134,7 @@ jobs:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Install Unix native build dependencies
|
||||
if: runner.os != 'Windows'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${{ matrix.goos }}" == "linux" ]]; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libpcre2-dev libsimde-dev pkg-config python3-pip
|
||||
elif [[ "${{ matrix.goos }}" == "darwin" ]]; then
|
||||
brew list pcre2 >/dev/null 2>&1 || brew install pcre2
|
||||
brew list simde >/dev/null 2>&1 || brew install simde
|
||||
fi
|
||||
|
||||
python3 -m pip install --user --upgrade 'cmake>=4.0,<5' || \
|
||||
python3 -m pip install --user --break-system-packages --upgrade 'cmake>=4.0,<5'
|
||||
python_user_base="$(python3 -m site --user-base)"
|
||||
echo "${python_user_base}/bin" >> "${GITHUB_PATH}"
|
||||
export PATH="${python_user_base}/bin:${PATH}"
|
||||
cmake --version
|
||||
|
||||
- name: Download office_oxide native library
|
||||
if: runner.os != 'Windows'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
OFFICE_OXIDE_PREFIX="${RUNNER_TEMP}/office_oxide"
|
||||
OFFICE_OXIDE_URL="https://github.com/yfedoseev/office_oxide/releases/download/v${OFFICE_OXIDE_VERSION}/${{ matrix.native_asset }}"
|
||||
|
||||
mkdir -p "${OFFICE_OXIDE_PREFIX}"
|
||||
curl -fsSL "${OFFICE_OXIDE_URL}" -o "${RUNNER_TEMP}/${{ matrix.native_asset }}"
|
||||
tar xzf "${RUNNER_TEMP}/${{ matrix.native_asset }}" -C "${OFFICE_OXIDE_PREFIX}"
|
||||
|
||||
test -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"
|
||||
test -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
|
||||
ln -sf "office_oxide_c/office_oxide.h" "${OFFICE_OXIDE_PREFIX}/include/office_oxide.h"
|
||||
test -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide.h"
|
||||
|
||||
echo "OFFICE_OXIDE_PREFIX=${OFFICE_OXIDE_PREFIX}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Set up MSYS2
|
||||
if: runner.os == 'Windows'
|
||||
uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: ${{ matrix.msystem }}
|
||||
update: true
|
||||
install: ${{ matrix.msys2_packages }}
|
||||
path-type: inherit
|
||||
|
||||
- name: Install Windows SIMDe headers
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
simde_dir="$(cygpath -u "${RUNNER_TEMP}")/simde"
|
||||
simde_archive="$(cygpath -u "${RUNNER_TEMP}")/simde-v0.8.2.tar.gz"
|
||||
github_env="$(cygpath -u "${GITHUB_ENV}")"
|
||||
|
||||
rm -rf "${simde_dir}"
|
||||
mkdir -p "${simde_dir}"
|
||||
curl -fsSL "https://github.com/simd-everywhere/simde/archive/refs/tags/v0.8.2.tar.gz" -o "${simde_archive}"
|
||||
tar xzf "${simde_archive}" -C "${simde_dir}" --strip-components=1
|
||||
|
||||
test -f "${simde_dir}/simde/x86/sse4.1.h"
|
||||
|
||||
# Install SIMDe headers into the MSYS2 toolchain include directory.
|
||||
# CMake/Ninja invokes clang-scan-deps as a Windows-native executable;
|
||||
# a POSIX-style include path like /c/a/_temp/simde may not be resolved
|
||||
# there, so keep the headers under ${MINGW_PREFIX}/include instead.
|
||||
rm -rf "${MINGW_PREFIX}/include/simde"
|
||||
cp -R "${simde_dir}/simde" "${MINGW_PREFIX}/include/simde"
|
||||
test -f "${MINGW_PREFIX}/include/simde/x86/sse4.1.h"
|
||||
|
||||
- name: Configure Windows C compiler
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
cc_path="$(command -v clang.exe 2>/dev/null || command -v clang)"
|
||||
cxx_path="$(command -v clang++.exe 2>/dev/null || command -v clang++)"
|
||||
|
||||
if [[ ! -f "${cc_path}" && -f "${cc_path}.exe" ]]; then
|
||||
cc_path="${cc_path}.exe"
|
||||
fi
|
||||
if [[ ! -f "${cxx_path}" && -f "${cxx_path}.exe" ]]; then
|
||||
cxx_path="${cxx_path}.exe"
|
||||
fi
|
||||
test -f "${cc_path}"
|
||||
test -f "${cxx_path}"
|
||||
|
||||
cc="$(cygpath -m "${cc_path}")"
|
||||
cxx="$(cygpath -m "${cxx_path}")"
|
||||
github_env="$(cygpath -u "${GITHUB_ENV}")"
|
||||
|
||||
pcre2_libdir="$(pkg-config --variable=libdir libpcre2-8)"
|
||||
pcre2_includedir="$(pkg-config --variable=includedir libpcre2-8)"
|
||||
|
||||
echo "CC=${cc}" >> "${github_env}"
|
||||
echo "CXX=${cxx}" >> "${github_env}"
|
||||
echo "PCRE2_LIBDIR=$(cygpath -m "${pcre2_libdir}")" >> "${github_env}"
|
||||
echo "PCRE2_INCLUDEDIR=$(cygpath -m "${pcre2_includedir}")" >> "${github_env}"
|
||||
|
||||
echo "Resolved MSYS2 clang: ${cc}"
|
||||
"${cc_path}" --version
|
||||
echo "Resolved MSYS2 clang++: ${cxx}"
|
||||
"${cxx_path}" --version
|
||||
|
||||
- name: Set up Rust for Windows office_oxide staticlib
|
||||
if: runner.os == 'Windows'
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.rust_target }}
|
||||
|
||||
- name: Build office_oxide native library for Windows GNU ABI
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
office_oxide_prefix="$(cygpath -u "${RUNNER_TEMP}")/office_oxide"
|
||||
office_oxide_src="$(cygpath -u "${RUNNER_TEMP}")/office_oxide-src"
|
||||
archive_path="$(cygpath -u "${RUNNER_TEMP}")/office_oxide-v${OFFICE_OXIDE_VERSION}.tar.gz"
|
||||
github_env="$(cygpath -u "${GITHUB_ENV}")"
|
||||
|
||||
rm -rf "${office_oxide_prefix}" "${office_oxide_src}"
|
||||
mkdir -p "${office_oxide_prefix}/include/office_oxide_c" "${office_oxide_prefix}/lib" "${office_oxide_src}"
|
||||
|
||||
curl -fsSL "https://github.com/yfedoseev/office_oxide/archive/refs/tags/v${OFFICE_OXIDE_VERSION}.tar.gz" -o "${archive_path}"
|
||||
tar xzf "${archive_path}" -C "${office_oxide_src}" --strip-components=1
|
||||
|
||||
cd "${office_oxide_src}"
|
||||
|
||||
cc_path="$(command -v clang.exe 2>/dev/null || command -v clang)"
|
||||
cxx_path="$(command -v clang++.exe 2>/dev/null || command -v clang++)"
|
||||
if [[ ! -f "${cc_path}" && -f "${cc_path}.exe" ]]; then
|
||||
cc_path="${cc_path}.exe"
|
||||
fi
|
||||
if [[ ! -f "${cxx_path}" && -f "${cxx_path}.exe" ]]; then
|
||||
cxx_path="${cxx_path}.exe"
|
||||
fi
|
||||
test -f "${cc_path}"
|
||||
test -f "${cxx_path}"
|
||||
|
||||
export CC="${cc_path}"
|
||||
export CXX="${cxx_path}"
|
||||
export AR="$(command -v llvm-ar || command -v ar)"
|
||||
export CARGO_BUILD_TARGET="${{ matrix.rust_target }}"
|
||||
export RUSTFLAGS="-C target-feature=+crt-static -C link-arg=-fuse-ld=lld"
|
||||
|
||||
case "${{ matrix.rust_target }}" in
|
||||
x86_64-pc-windows-gnullvm)
|
||||
export CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_LINKER="${CC}"
|
||||
export CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_AR="${AR}"
|
||||
;;
|
||||
aarch64-pc-windows-gnullvm)
|
||||
export CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_LINKER="${CC}"
|
||||
export CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_AR="${AR}"
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported Rust target: ${{ matrix.rust_target }}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# The release workflow only needs the static archive for cgo.
|
||||
# Building cdylib on Windows pulls in extra MinGW runtime libraries,
|
||||
# which can fail under the CLANG64/CLANGARM64 environments.
|
||||
perl -0pi -e 's/crate-type\s*=\s*\[[^\]]+\]/crate-type = ["staticlib"]/s' Cargo.toml
|
||||
|
||||
cargo build --release --lib --target "${CARGO_BUILD_TARGET}" --no-default-features
|
||||
|
||||
cp "include/office_oxide_c/office_oxide.h" "${office_oxide_prefix}/include/office_oxide_c/office_oxide.h"
|
||||
cp "include/office_oxide_c/office_oxide.h" "${office_oxide_prefix}/include/office_oxide.h"
|
||||
cp "target/${CARGO_BUILD_TARGET}/release/liboffice_oxide.a" "${office_oxide_prefix}/lib/liboffice_oxide.a"
|
||||
|
||||
test -f "${office_oxide_prefix}/include/office_oxide_c/office_oxide.h"
|
||||
test -f "${office_oxide_prefix}/include/office_oxide.h"
|
||||
test -f "${office_oxide_prefix}/lib/liboffice_oxide.a"
|
||||
|
||||
echo "OFFICE_OXIDE_PREFIX=$(cygpath -m "${office_oxide_prefix}")" >> "${github_env}"
|
||||
|
||||
- name: Build rag tokenizer native library
|
||||
if: runner.os != 'Windows'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
cmake_args=(
|
||||
-S internal/cpp
|
||||
-B internal/cpp/cmake-build-release
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
)
|
||||
|
||||
if [[ "${{ matrix.goos }}" == "darwin" ]]; then
|
||||
simde_prefix="$(brew --prefix simde)"
|
||||
pcre2_prefix="$(brew --prefix pcre2)"
|
||||
cmake_args+=(
|
||||
-DCMAKE_PREFIX_PATH="${pcre2_prefix};${simde_prefix}"
|
||||
-DCMAKE_C_FLAGS="-I${simde_prefix}/include"
|
||||
-DCMAKE_CXX_FLAGS="-I${simde_prefix}/include"
|
||||
)
|
||||
fi
|
||||
|
||||
cmake "${cmake_args[@]}"
|
||||
cmake --build internal/cpp/cmake-build-release --target rag_tokenizer_c_api --parallel
|
||||
|
||||
test -f internal/cpp/cmake-build-release/librag_tokenizer_c_api.a
|
||||
|
||||
- name: Build rag tokenizer native library
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
cc_path="$(command -v clang.exe 2>/dev/null || command -v clang)"
|
||||
cxx_path="$(command -v clang++.exe 2>/dev/null || command -v clang++)"
|
||||
|
||||
if [[ ! -f "${cc_path}" && -f "${cc_path}.exe" ]]; then
|
||||
cc_path="${cc_path}.exe"
|
||||
fi
|
||||
if [[ ! -f "${cxx_path}" && -f "${cxx_path}.exe" ]]; then
|
||||
cxx_path="${cxx_path}.exe"
|
||||
fi
|
||||
test -f "${cc_path}"
|
||||
test -f "${cxx_path}"
|
||||
|
||||
test -f "${MINGW_PREFIX}/include/simde/x86/sse4.1.h"
|
||||
|
||||
cmake -S internal/cpp -B internal/cpp/cmake-build-release -G Ninja \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_COMPILER="$(cygpath -m "${cc_path}")" \
|
||||
-DCMAKE_CXX_COMPILER="$(cygpath -m "${cxx_path}")" \
|
||||
-DCMAKE_C_FLAGS="-I${MINGW_PREFIX}/include" \
|
||||
-DCMAKE_CXX_FLAGS="-I${MINGW_PREFIX}/include"
|
||||
|
||||
cmake --build internal/cpp/cmake-build-release --target rag_tokenizer_c_api --parallel
|
||||
|
||||
test -f internal/cpp/cmake-build-release/librag_tokenizer_c_api.a
|
||||
|
||||
- name: Build Go CLI release binaries
|
||||
- name: Build Go CLI release binaries on non-Windows
|
||||
if: runner.os != 'Windows'
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -424,27 +157,7 @@ jobs:
|
||||
output="${DIST_DIR}/${CLI_NAME}-${RELEASE_TAG}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
echo "Building ${{ matrix.goos }}/${{ matrix.goarch }} -> ${output}"
|
||||
|
||||
case "${{ matrix.goos }}" in
|
||||
linux)
|
||||
cgo_ldflags="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a -lm -ldl -lpthread"
|
||||
;;
|
||||
darwin)
|
||||
cgo_ldflags="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unsupported Unix target: ${{ matrix.goos }}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
cgo_cflags="-I${OFFICE_OXIDE_PREFIX}/include -I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c"
|
||||
|
||||
echo "office_oxide prefix: ${OFFICE_OXIDE_PREFIX}"
|
||||
echo "CGO_CFLAGS: ${cgo_cflags}"
|
||||
echo "CGO_LDFLAGS: ${cgo_ldflags}"
|
||||
|
||||
CGO_ENABLED=1 \
|
||||
CGO_CFLAGS="${cgo_cflags}" \
|
||||
CGO_LDFLAGS="${cgo_ldflags}" \
|
||||
CGO_ENABLED=0 \
|
||||
GOOS="${{ matrix.goos }}" \
|
||||
GOARCH="${{ matrix.goarch }}" \
|
||||
go build \
|
||||
@@ -455,16 +168,7 @@ jobs:
|
||||
|
||||
chmod +x "${output}"
|
||||
|
||||
if [[ "${{ matrix.goos }}" == "linux" ]] && command -v ldd >/dev/null 2>&1; then
|
||||
if ldd "${output}" 2>&1 | grep -q "liboffice_oxide"; then
|
||||
echo "::error::linux CLI unexpectedly links liboffice_oxide dynamically"
|
||||
ldd "${output}" || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Verified linux CLI does not require liboffice_oxide.so at runtime"
|
||||
fi
|
||||
|
||||
- name: Build Go CLI release binaries
|
||||
- name: Build Go CLI release binaries on Windows
|
||||
if: runner.os == 'Windows'
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -478,46 +182,10 @@ jobs:
|
||||
$output = Join-Path $env:DIST_DIR "${env:CLI_NAME}-${env:RELEASE_TAG}-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.output_ext }}"
|
||||
Write-Host "Building ${{ matrix.goos }}/${{ matrix.goarch }} -> $output"
|
||||
|
||||
$officeOxidePrefix = $env:OFFICE_OXIDE_PREFIX -replace "\\", "/"
|
||||
$cc = $env:CC
|
||||
if ([string]::IsNullOrWhiteSpace($cc)) {
|
||||
Write-Error "CC is not set"
|
||||
exit 1
|
||||
}
|
||||
if (-not (Test-Path $cc)) {
|
||||
Write-Error "C compiler does not exist: $cc"
|
||||
exit 1
|
||||
}
|
||||
if (-not (Test-Path "${officeOxidePrefix}/lib/liboffice_oxide.a")) {
|
||||
Write-Error "liboffice_oxide.a does not exist: ${officeOxidePrefix}/lib/liboffice_oxide.a"
|
||||
exit 1
|
||||
}
|
||||
if (-not (Test-Path "internal/cpp/cmake-build-release/librag_tokenizer_c_api.a")) {
|
||||
Write-Error "librag_tokenizer_c_api.a does not exist"
|
||||
exit 1
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($env:PCRE2_LIBDIR) -or -not (Test-Path $env:PCRE2_LIBDIR)) {
|
||||
Write-Error "PCRE2_LIBDIR is not set or does not exist: $env:PCRE2_LIBDIR"
|
||||
exit 1
|
||||
}
|
||||
|
||||
$ragTokenizerLib = (Resolve-Path "internal/cpp/cmake-build-release/librag_tokenizer_c_api.a").Path -replace '\\', '/'
|
||||
$pcre2LibDir = $env:PCRE2_LIBDIR -replace '\\', '/'
|
||||
$pcre2IncludeDir = $env:PCRE2_INCLUDEDIR -replace '\\', '/'
|
||||
|
||||
$env:CGO_ENABLED = "1"
|
||||
$env:CC = $cc
|
||||
$env:CGO_CFLAGS = "-I${officeOxidePrefix}/include -I${officeOxidePrefix}/include/office_oxide_c -I${pcre2IncludeDir}"
|
||||
$env:CGO_LDFLAGS = "${officeOxidePrefix}/lib/liboffice_oxide.a ${ragTokenizerLib} -L${pcre2LibDir} -lpcre2-8 -lc++ -static -static-libgcc -static-libstdc++ -lws2_32 -lbcrypt -lntdll -luserenv -ladvapi32"
|
||||
$env:CGO_ENABLED = "0"
|
||||
$env:GOOS = "${{ matrix.goos }}"
|
||||
$env:GOARCH = "${{ matrix.goarch }}"
|
||||
|
||||
Write-Host "CC: $env:CC"
|
||||
& $env:CC --version
|
||||
Write-Host "office_oxide prefix: $officeOxidePrefix"
|
||||
Write-Host "CGO_CFLAGS: $env:CGO_CFLAGS"
|
||||
Write-Host "CGO_LDFLAGS: $env:CGO_LDFLAGS"
|
||||
|
||||
go build `
|
||||
-trimpath `
|
||||
-ldflags="-s -w -X main.version=$env:RELEASE_TAG -X main.commit=$env:GITHUB_SHA" `
|
||||
|
||||
200
.github/workflows/sep-tests.yml
vendored
200
.github/workflows/sep-tests.yml
vendored
@@ -40,6 +40,7 @@ jobs:
|
||||
has_go_changes: ${{ steps.detect_changes.outputs.has_go_changes }}
|
||||
has_python_changes: ${{ steps.detect_changes.outputs.has_python_changes }}
|
||||
has_web_changes: ${{ steps.detect_changes.outputs.has_web_changes }}
|
||||
api_proxy_schemes: ${{ steps.detect_changes.outputs.api_proxy_schemes }}
|
||||
steps:
|
||||
- name: Ensure workspace ownership
|
||||
run: |
|
||||
@@ -135,7 +136,9 @@ jobs:
|
||||
trap 'rm -f "$changed_files"' EXIT
|
||||
git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
|
||||
| while read -r file; do
|
||||
[[ -f "$file" ]] && printf '%s\0' "$file"
|
||||
if [[ -f "$file" ]]; then
|
||||
printf '%s\0' "$file"
|
||||
fi
|
||||
done > "$changed_files"
|
||||
echo "Changed files to run lefthook on:"
|
||||
if [[ -s "$changed_files" ]]; then
|
||||
@@ -174,16 +177,24 @@ jobs:
|
||||
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
|
||||
CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
|
||||
else
|
||||
CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD)
|
||||
BASE_SHA="${{ github.event.before }}"
|
||||
if [[ -n "${BASE_SHA}" && ! "${BASE_SHA}" =~ ^0+$ ]]; then
|
||||
CHANGED_FILES=$(git diff --name-only "${BASE_SHA}" "${GITHUB_SHA}")
|
||||
else
|
||||
CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD)
|
||||
fi
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
case "$file" in
|
||||
*.go|go.mod|go.sum) has_go=true ;;
|
||||
# Docker, test, SDK, and workflow changes can alter either server path.
|
||||
Dockerfile|docker/**|.github/workflows/**|test/**|sdk/**)
|
||||
has_go=true
|
||||
has_python=true
|
||||
;;
|
||||
*.go|go.mod|go.sum|build.sh) has_go=true ;;
|
||||
*.py|pyproject.toml|requirements*.txt) has_python=true ;;
|
||||
web/*) has_web=true ;;
|
||||
Dockerfile|docker-compose*.yml) has_go=true; has_python=true ;;
|
||||
build.sh) has_go=true ;;
|
||||
esac
|
||||
done <<< "$CHANGED_FILES"
|
||||
|
||||
@@ -194,10 +205,19 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$has_go" == "true" && "$has_python" == "true" ]]; then
|
||||
api_proxy_schemes='["go","python"]'
|
||||
elif [[ "$has_go" == "true" ]]; then
|
||||
api_proxy_schemes='["go"]'
|
||||
else
|
||||
api_proxy_schemes='["python"]'
|
||||
fi
|
||||
|
||||
echo "has_go_changes=${has_go}" >> $GITHUB_OUTPUT
|
||||
echo "has_python_changes=${has_python}" >> $GITHUB_OUTPUT
|
||||
echo "has_web_changes=${has_web}" >> $GITHUB_OUTPUT
|
||||
echo "Go: ${has_go}, Python: ${has_python}, Web: ${has_web}"
|
||||
echo "api_proxy_schemes=${api_proxy_schemes}" >> $GITHUB_OUTPUT
|
||||
echo "Go: ${has_go}, Python: ${has_python}, Web: ${has_web}, proxy schemes: ${api_proxy_schemes}"
|
||||
|
||||
- name: Prepare Python test environment
|
||||
if: steps.detect_changes.outputs.has_python_changes == 'true'
|
||||
@@ -216,13 +236,18 @@ jobs:
|
||||
|
||||
|
||||
ragflow_tests_infinity:
|
||||
name: ragflow_tests_infinity
|
||||
name: ragflow_tests_infinity (${{ matrix.api_proxy_scheme }})
|
||||
needs: ragflow_preflight
|
||||
if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
|
||||
runs-on: [ "self-hosted", "ragflow-test" ]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }}
|
||||
env:
|
||||
DOC_ENGINE: infinity
|
||||
RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity
|
||||
API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }}
|
||||
RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity-${{ matrix.api_proxy_scheme }}
|
||||
HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
|
||||
HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }}
|
||||
HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }}
|
||||
@@ -243,7 +268,7 @@ jobs:
|
||||
fetch-tags: true
|
||||
|
||||
- name: Build ragflow go server
|
||||
if: env.HAS_GO == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
@@ -259,13 +284,13 @@ jobs:
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
|
||||
- name: Run Go unit tests
|
||||
if: env.HAS_GO == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
# Runs after `./build.sh --go`, which guarantees the C++ static
|
||||
# library (librag_tokenizer_c_api.a) is present on disk. The Go
|
||||
# test binaries link against it transitively through
|
||||
@@ -295,30 +320,35 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build ragflow:nightly
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker pull ubuntu:24.04
|
||||
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
|
||||
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
|
||||
BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock
|
||||
mkdir -p "$(dirname "${BUILD_LOCK_FILE}")"
|
||||
(
|
||||
flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; }
|
||||
echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}"
|
||||
sudo docker pull ubuntu:24.04
|
||||
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
|
||||
) 9>"${BUILD_LOCK_FILE}"
|
||||
|
||||
- name: Prepare Python test environment
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
run: |
|
||||
uv sync --python 3.13 --group test --frozen
|
||||
uv pip install -e sdk/python
|
||||
|
||||
- name: Prepare function test environment
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
working-directory: docker
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# install ss
|
||||
sudo chmod 1777 /tmp
|
||||
sudo apt update && sudo apt install -y iproute2
|
||||
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
|
||||
COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
|
||||
COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}"
|
||||
echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
|
||||
echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
|
||||
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
|
||||
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}
|
||||
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
|
||||
rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
|
||||
|
||||
@@ -329,7 +359,7 @@ jobs:
|
||||
# Engine-specific offset partitions keep concurrent engine jobs from
|
||||
# choosing the same host ports when they land on the same self-hosted runner.
|
||||
# A lock plus reservation file closes the check/start race between parallel jobs.
|
||||
PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
|
||||
PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222)
|
||||
PARTITION_SIZE=6000
|
||||
case "${DOC_ENGINE}" in
|
||||
elasticsearch) PARTITION_BASE=1000 ;;
|
||||
@@ -397,7 +427,8 @@ jobs:
|
||||
MINIO_PORT=$((9000 + PORT_OFFSET))
|
||||
MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
|
||||
REDIS_PORT=$((6379 + PORT_OFFSET))
|
||||
NATS_PORT=$((4222 + PORT_OFFSET))
|
||||
EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET))
|
||||
NATS_MONITORING_PORT=$((8222 + PORT_OFFSET))
|
||||
TEI_PORT=$((6380 + PORT_OFFSET))
|
||||
KIBANA_PORT=$((6601 + PORT_OFFSET))
|
||||
SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
|
||||
@@ -411,7 +442,7 @@ jobs:
|
||||
|
||||
# Persist computed ports into .env so docker-compose uses the correct host bindings.
|
||||
# Remove previous CI overrides first; docker compose uses the last duplicate key.
|
||||
sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
|
||||
sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env
|
||||
{
|
||||
echo ""
|
||||
echo "ES_PORT=${ES_PORT}"
|
||||
@@ -423,7 +454,8 @@ jobs:
|
||||
echo "MINIO_PORT=${MINIO_PORT}"
|
||||
echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
|
||||
echo "REDIS_PORT=${REDIS_PORT}"
|
||||
echo "NATS_PORT=${NATS_PORT}"
|
||||
echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}"
|
||||
echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}"
|
||||
echo "TEI_PORT=${TEI_PORT}"
|
||||
echo "KIBANA_PORT=${KIBANA_PORT}"
|
||||
echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
|
||||
@@ -434,26 +466,37 @@ jobs:
|
||||
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
|
||||
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
|
||||
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
|
||||
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"
|
||||
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"
|
||||
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
|
||||
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
|
||||
echo "DOC_ENGINE=${DOC_ENGINE}"
|
||||
echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}"
|
||||
} >> .env
|
||||
echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
|
||||
RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
|
||||
else
|
||||
RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT}
|
||||
fi
|
||||
echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
|
||||
sed -i \
|
||||
-e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \
|
||||
-e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \
|
||||
docker-compose-base.yml
|
||||
|
||||
# Patch entrypoint.sh for coverage
|
||||
sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
|
||||
|
||||
|
||||
- name: Start ragflow:nightly for Infinity
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
run: |
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
|
||||
sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
|
||||
|
||||
- name: Run sdk tests against Infinity
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'python'
|
||||
run: |
|
||||
export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
|
||||
svc_ready=0
|
||||
@@ -474,7 +517,6 @@ jobs:
|
||||
source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log
|
||||
|
||||
- name: Run New RESTFUL api tests against Infinity
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
run: |
|
||||
export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
|
||||
svc_ready=0
|
||||
@@ -494,7 +536,7 @@ jobs:
|
||||
source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee infinity_restful_api_test.log
|
||||
|
||||
- name: RAGFlow CLI retrieval test Infinity
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'python'
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
run: |
|
||||
@@ -548,9 +590,8 @@ jobs:
|
||||
source docker/.env
|
||||
set +a
|
||||
|
||||
HOST_ADDRESS="http://host.docker.internal:${SVR_HTTP_PORT}"
|
||||
USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
|
||||
USER_PORT="${SVR_HTTP_PORT}"
|
||||
USER_PORT="${RAGFLOW_HTTP_PORT}"
|
||||
ADMIN_HOST="${USER_HOST}"
|
||||
ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
|
||||
|
||||
@@ -606,7 +647,7 @@ jobs:
|
||||
run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"
|
||||
|
||||
- name: Stop ragflow to save coverage Infinity
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
|
||||
run: |
|
||||
# Send SIGINT to ragflow_server.py to trigger coverage save
|
||||
PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
|
||||
@@ -621,7 +662,7 @@ jobs:
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
|
||||
|
||||
- name: Generate server coverage report Infinity
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
|
||||
run: |
|
||||
# .coverage file should be in docker/ragflow-logs/.coverage
|
||||
if [ -f docker/ragflow-logs/.coverage ]; then
|
||||
@@ -647,18 +688,20 @@ jobs:
|
||||
fail_ci_if_error: false
|
||||
|
||||
- name: Collect ragflow log Infinity
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
if [ -d docker/ragflow-logs ]; then
|
||||
cp -r docker/ragflow-logs ${ARTIFACTS_DIR}/ragflow-logs-infinity
|
||||
echo "ragflow log" && tail -n 200 docker/ragflow-logs/ragflow_server.log || true
|
||||
destination="${ARTIFACTS_DIR}/ragflow-logs-infinity"
|
||||
sudo cp -r docker/ragflow-logs "${destination}"
|
||||
sudo chown -R "$(id -u):$(id -g)" "${destination}"
|
||||
echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
|
||||
else
|
||||
echo "No docker/ragflow-logs directory found; skipping log collection"
|
||||
fi
|
||||
sudo rm -rf docker/ragflow-logs || true
|
||||
|
||||
- name: Stop ragflow:nightly for Infinity
|
||||
if: ${{ always() && (env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true') }}
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
# Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
|
||||
@@ -673,13 +716,18 @@ jobs:
|
||||
|
||||
|
||||
ragflow_tests_elasticsearch:
|
||||
name: ragflow_tests_elasticsearch
|
||||
name: ragflow_tests_elasticsearch (${{ matrix.api_proxy_scheme }})
|
||||
needs: ragflow_preflight
|
||||
if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
|
||||
runs-on: [ "self-hosted", "ragflow-test" ]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
api_proxy_scheme: ${{ fromJSON(needs.ragflow_preflight.outputs.api_proxy_schemes) }}
|
||||
env:
|
||||
DOC_ENGINE: elasticsearch
|
||||
RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch
|
||||
API_PROXY_SCHEME: ${{ matrix.api_proxy_scheme }}
|
||||
RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch-${{ matrix.api_proxy_scheme }}
|
||||
HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
|
||||
HAS_GO: ${{ needs.ragflow_preflight.outputs.has_go_changes }}
|
||||
HAS_PYTHON: ${{ needs.ragflow_preflight.outputs.has_python_changes }}
|
||||
@@ -700,7 +748,7 @@ jobs:
|
||||
fetch-tags: true
|
||||
|
||||
- name: Build ragflow go server
|
||||
if: env.HAS_GO == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
|
||||
@@ -716,13 +764,13 @@ jobs:
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
|
||||
- name: Run Go unit tests
|
||||
if: env.HAS_GO == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'go'
|
||||
# Runs after `./build.sh --go`, which guarantees the C++ static
|
||||
# library (librag_tokenizer_c_api.a) is present on disk. The Go
|
||||
# test binaries link against it transitively through
|
||||
@@ -752,30 +800,35 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build ragflow:nightly
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker pull ubuntu:24.04
|
||||
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
|
||||
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
|
||||
BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock
|
||||
mkdir -p "$(dirname "${BUILD_LOCK_FILE}")"
|
||||
(
|
||||
flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; }
|
||||
echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}"
|
||||
sudo docker pull ubuntu:24.04
|
||||
sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
|
||||
) 9>"${BUILD_LOCK_FILE}"
|
||||
|
||||
- name: Prepare Python test environment
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
run: |
|
||||
uv sync --python 3.13 --group test --frozen
|
||||
uv pip install -e sdk/python
|
||||
|
||||
- name: Prepare function test environment
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
working-directory: docker
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# install ss
|
||||
sudo chmod 1777 /tmp
|
||||
sudo apt update && sudo apt install -y iproute2
|
||||
RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
|
||||
COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
|
||||
COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}-${API_PROXY_SCHEME}"
|
||||
echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
|
||||
echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
|
||||
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
|
||||
ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}/${API_PROXY_SCHEME}
|
||||
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
|
||||
rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
|
||||
|
||||
@@ -786,7 +839,7 @@ jobs:
|
||||
# Engine-specific offset partitions keep concurrent engine jobs from
|
||||
# choosing the same host ports when they land on the same self-hosted runner.
|
||||
# A lock plus reservation file closes the check/start race between parallel jobs.
|
||||
PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
|
||||
PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222 8222)
|
||||
PARTITION_SIZE=6000
|
||||
case "${DOC_ENGINE}" in
|
||||
elasticsearch) PARTITION_BASE=1000 ;;
|
||||
@@ -854,7 +907,8 @@ jobs:
|
||||
MINIO_PORT=$((9000 + PORT_OFFSET))
|
||||
MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
|
||||
REDIS_PORT=$((6379 + PORT_OFFSET))
|
||||
NATS_PORT=$((4222 + PORT_OFFSET))
|
||||
EXPOSE_NATS_PORT=$((4222 + PORT_OFFSET))
|
||||
NATS_MONITORING_PORT=$((8222 + PORT_OFFSET))
|
||||
TEI_PORT=$((6380 + PORT_OFFSET))
|
||||
KIBANA_PORT=$((6601 + PORT_OFFSET))
|
||||
SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
|
||||
@@ -868,7 +922,7 @@ jobs:
|
||||
|
||||
# Persist computed ports into .env so docker-compose uses the correct host bindings.
|
||||
# Remove previous CI overrides first; docker compose uses the last duplicate key.
|
||||
sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
|
||||
sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^EXPOSE_NATS_PORT=/d;/^NATS_MONITORING_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d;/^API_PROXY_SCHEME=/d' .env
|
||||
{
|
||||
echo ""
|
||||
echo "ES_PORT=${ES_PORT}"
|
||||
@@ -880,7 +934,8 @@ jobs:
|
||||
echo "MINIO_PORT=${MINIO_PORT}"
|
||||
echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
|
||||
echo "REDIS_PORT=${REDIS_PORT}"
|
||||
echo "NATS_PORT=${NATS_PORT}"
|
||||
echo "EXPOSE_NATS_PORT=${EXPOSE_NATS_PORT}"
|
||||
echo "NATS_MONITORING_PORT=${NATS_MONITORING_PORT}"
|
||||
echo "TEI_PORT=${TEI_PORT}"
|
||||
echo "KIBANA_PORT=${KIBANA_PORT}"
|
||||
echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
|
||||
@@ -891,26 +946,37 @@ jobs:
|
||||
echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
|
||||
echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
|
||||
echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
|
||||
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc"
|
||||
echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,ragflow-go,tei-cpu,deepdoc"
|
||||
echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
|
||||
echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
|
||||
echo "DOC_ENGINE=${DOC_ENGINE}"
|
||||
echo "API_PROXY_SCHEME=${API_PROXY_SCHEME}"
|
||||
} >> .env
|
||||
echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
if [[ "${API_PROXY_SCHEME}" == "go" ]]; then
|
||||
RAGFLOW_HTTP_PORT=${SVR_WEB_HTTP_PORT}
|
||||
else
|
||||
RAGFLOW_HTTP_PORT=${SVR_HTTP_PORT}
|
||||
fi
|
||||
echo "RAGFLOW_HTTP_PORT=${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
echo "HOST_ADDRESS=http://host.docker.internal:${RAGFLOW_HTTP_PORT}" >> ${GITHUB_ENV}
|
||||
|
||||
sed -i \
|
||||
-e 's#${NATS_PORT:-4222}:4222#${EXPOSE_NATS_PORT}:4222#' \
|
||||
-e 's#"8222:8222"#"${NATS_MONITORING_PORT}:8222"#' \
|
||||
docker-compose-base.yml
|
||||
|
||||
# Patch entrypoint.sh for coverage
|
||||
sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
|
||||
|
||||
|
||||
- name: Start ragflow:nightly for Elasticsearch
|
||||
if: env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true'
|
||||
run: |
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
|
||||
sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
|
||||
|
||||
- name: Run sdk tests against Elasticsearch
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'python'
|
||||
run: |
|
||||
export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
|
||||
svc_ready=0
|
||||
@@ -931,7 +997,6 @@ jobs:
|
||||
source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-es-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log
|
||||
|
||||
- name: Run New RESTFUL api tests against Elasticsearch
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
run: |
|
||||
export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
|
||||
svc_ready=0
|
||||
@@ -951,7 +1016,7 @@ jobs:
|
||||
source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee es_restful_api_test.log
|
||||
|
||||
- name: RAGFlow CLI retrieval test Elasticsearch
|
||||
if: env.HAS_PYTHON == 'true'
|
||||
if: env.API_PROXY_SCHEME == 'python'
|
||||
env:
|
||||
PYTHONPATH: ${{ github.workspace }}
|
||||
run: |
|
||||
@@ -1005,9 +1070,8 @@ jobs:
|
||||
source docker/.env
|
||||
set +a
|
||||
|
||||
HOST_ADDRESS="http://host.docker.internal:${SVR_HTTP_PORT}"
|
||||
USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
|
||||
USER_PORT="${SVR_HTTP_PORT}"
|
||||
USER_PORT="${RAGFLOW_HTTP_PORT}"
|
||||
ADMIN_HOST="${USER_HOST}"
|
||||
ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
|
||||
|
||||
@@ -1063,7 +1127,7 @@ jobs:
|
||||
run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"
|
||||
|
||||
- name: Stop ragflow to save coverage Elasticsearch
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
|
||||
run: |
|
||||
# Send SIGINT to ragflow_server.py to trigger coverage save
|
||||
PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
|
||||
@@ -1078,7 +1142,7 @@ jobs:
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
|
||||
|
||||
- name: Generate server coverage report Elasticsearch
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() && env.API_PROXY_SCHEME == 'python' }}
|
||||
run: |
|
||||
# .coverage file should be in docker/ragflow-logs/.coverage
|
||||
if [ -f docker/ragflow-logs/.coverage ]; then
|
||||
@@ -1099,18 +1163,20 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Collect ragflow log Elasticsearch
|
||||
if: ${{ !cancelled() && env.HAS_PYTHON == 'true' }}
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
if [ -d docker/ragflow-logs ]; then
|
||||
cp -r docker/ragflow-logs ${ARTIFACTS_DIR}/ragflow-logs-es
|
||||
echo "ragflow log" && tail -n 200 docker/ragflow-logs/ragflow_server.log || true
|
||||
destination="${ARTIFACTS_DIR}/ragflow-logs-es"
|
||||
sudo cp -r docker/ragflow-logs "${destination}"
|
||||
sudo chown -R "$(id -u):$(id -g)" "${destination}"
|
||||
echo "ragflow log" && sudo tail -n 200 docker/ragflow-logs/ragflow_server.log || true
|
||||
else
|
||||
echo "No docker/ragflow-logs directory found; skipping log collection"
|
||||
fi
|
||||
sudo rm -rf docker/ragflow-logs || true
|
||||
|
||||
- name: Stop ragflow:nightly for Elasticsearch
|
||||
if: ${{ always() && (env.HAS_GO == 'true' || env.HAS_PYTHON == 'true' || env.HAS_WEB == 'true') }}
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
# Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
|
||||
sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
|
||||
@@ -1120,4 +1186,4 @@ jobs:
|
||||
fi
|
||||
if [[ -n ${PORT_RESERVATION:-} ]]; then
|
||||
rm -f "${PORT_RESERVATION}"
|
||||
fi
|
||||
fi
|
||||
|
||||
8
.github/workflows/tests.yml
vendored
8
.github/workflows/tests.yml
vendored
@@ -100,7 +100,9 @@ jobs:
|
||||
trap 'rm -f "$changed_files"' EXIT
|
||||
git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
|
||||
| while read -r file; do
|
||||
[[ -f "$file" ]] && printf '%s\0' "$file"
|
||||
if [[ -f "$file" ]]; then
|
||||
printf '%s\0' "$file"
|
||||
fi
|
||||
done > "$changed_files"
|
||||
echo "Changed files to run lefthook on:"
|
||||
if [[ -s "$changed_files" ]]; then
|
||||
@@ -178,7 +180,7 @@ jobs:
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
@@ -622,7 +624,7 @@ jobs:
|
||||
-e TZ="${TZ}" \
|
||||
-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
|
||||
-v "${PWD}:/ragflow" \
|
||||
-v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
|
||||
-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
|
||||
infiniflow/infinity_builder:ubuntu22_clang20
|
||||
sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
|
||||
./build.sh --go
|
||||
|
||||
18
.gitignore
vendored
18
.gitignore
vendored
@@ -22,6 +22,7 @@ Cargo.lock
|
||||
.idea/
|
||||
.vscode/
|
||||
.cursor/settings.json
|
||||
.opencode/
|
||||
|
||||
# Exclude Mac generated files
|
||||
.DS_Store
|
||||
@@ -225,9 +226,9 @@ uv-aarch64-unknown-linux-gnu.tar.gz
|
||||
docker/launch_backend_service_windows.sh
|
||||
|
||||
# C++ build directories
|
||||
internal/cpp/build/
|
||||
internal/cpp/cmake-build-release/
|
||||
internal/cpp/cmake-build-debug/
|
||||
internal/binding/cpp/build/
|
||||
internal/binding/cpp/cmake-build-release/
|
||||
internal/binding/cpp/cmake-build-debug/
|
||||
|
||||
# Trae IDE config
|
||||
.trae/
|
||||
@@ -245,5 +246,16 @@ bin/*
|
||||
# Parser test fixtures and python tools
|
||||
internal/deepdoc/parser/pdf/testdata/
|
||||
internal/deepdoc/parser/pdf/tools-py/
|
||||
|
||||
# IDE tooling artifacts
|
||||
.codebuddy/
|
||||
|
||||
# Local build output
|
||||
build/
|
||||
internal/deepdoc/parser/docx/testdata/
|
||||
internal/deepdoc/parser/docx/tool/
|
||||
|
||||
# test data compare tool
|
||||
internal/ingestion/task/tool/generate_dataflow_golden.py
|
||||
internal/ingestion/task/tool/README.md
|
||||
internal/cpp/cmake-build-release
|
||||
@@ -72,9 +72,9 @@ docker/seekdb
|
||||
# Native / compiled build dirs
|
||||
target/
|
||||
bin/
|
||||
internal/cpp/build/
|
||||
internal/cpp/cmake-build-release/
|
||||
internal/cpp/cmake-build-debug/
|
||||
internal/binding/cpp/build/
|
||||
internal/binding/cpp/cmake-build-release/
|
||||
internal/binding/cpp/cmake-build-debug/
|
||||
|
||||
# Optional: skip tests and docs from indexing
|
||||
# test/
|
||||
|
||||
192
AGENTS.md
192
AGENTS.md
@@ -1,109 +1,109 @@
|
||||
# RAGFlow Project Instructions for GitHub Copilot
|
||||
# RAGFlow Instructions
|
||||
|
||||
This file provides context, build instructions, and coding standards for the RAGFlow project.
|
||||
It is structured to follow GitHub Copilot's [customization guidelines](https://docs.github.com/en/copilot/concepts/prompting/response-customization).
|
||||
Use this file as the local operating guide for the current codebase. Prefer the code and the current CLAUDE.md over any older convention or remembered project shape.
|
||||
|
||||
## 1. Project Overview
|
||||
RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It is a full-stack application with a Python backend and a React/TypeScript frontend.
|
||||
## Core stance
|
||||
- Treat legacy code as liability, not as a compatibility target.
|
||||
- Prefer deletion over shims, deprecated branches, wrapper APIs, and dual-track migration notes.
|
||||
- If old and new implementations coexist, converge to one path unless an external contract forces compatibility.
|
||||
- Remove dead tests, commented-out code, stale docs, and "move later" notes instead of preserving them.
|
||||
- Reduce public surface area when a helper can be made private or internal.
|
||||
- Keep refactors centered on the owning abstraction, not on adjacent compatibility layers.
|
||||
|
||||
- **Backend**: Python 3.10+ (Flask/Quart)
|
||||
- **Frontend**: TypeScript, React, UmiJS
|
||||
- **Architecture**: Microservices based on Docker.
|
||||
- `api/`: Backend API server.
|
||||
- `rag/`: Core RAG logic (indexing, retrieval).
|
||||
- `deepdoc/`: Document parsing and OCR.
|
||||
- `web/`: Frontend application.
|
||||
## Current stack
|
||||
- Backend: Python 3.13+, Quart-based API server, Peewee ORM, async workers.
|
||||
- Frontend: React + TypeScript + Vite in `web/`.
|
||||
- Go: the repository also has a substantial Go module for servers, ingestion, parser/runtime, CLI, and supporting services.
|
||||
- Runtime services commonly include MySQL/PostgreSQL, Redis, MinIO, and Elasticsearch/Infinity/OpenSearch depending on configuration.
|
||||
|
||||
## 2. Directory Structure
|
||||
- `api/`: Backend API server (Flask/Quart).
|
||||
- `apps/`: API Blueprints (Knowledge Base, Chat, etc.).
|
||||
- `db/`: Database models and services.
|
||||
- `rag/`: Core RAG logic.
|
||||
- `llm/`: LLM, Embedding, and Rerank model abstractions.
|
||||
- `deepdoc/`: Document parsing and OCR modules.
|
||||
- `agent/`: Agentic reasoning components.
|
||||
- `web/`: Frontend application (React + UmiJS).
|
||||
- `docker/`: Docker deployment configurations.
|
||||
- `sdk/`: Python SDK.
|
||||
- `test/`: Backend tests.
|
||||
## Code layout to expect
|
||||
- `api/`: Python API server entrypoints, blueprints, services, and database code.
|
||||
- `rag/`: ingestion, retrieval, LLM integration, and graph RAG logic.
|
||||
- `deepdoc/`: parsing and OCR.
|
||||
- `agent/`: workflow canvas, components, tools, and templates.
|
||||
- `cmd/`: Go entrypoints. `ragflow_main` is the main server/admin/ingestor binary surface; `ragflow-cli` is the CLI entrypoint.
|
||||
- `internal/`: main Go application code. Important subtrees:
|
||||
- `internal/agent/`: Go agent runtime, canvas execution, components, tool bindings, workflow helpers.
|
||||
- `internal/cli/`: CLI parsing, HTTP transport, command execution, response formatting.
|
||||
- `internal/dao/`: Go data-access layer and persistence-facing helpers.
|
||||
- `internal/deepdoc/`: Go DeepDOC integrations, especially native-backed PDF/DOCX parsing.
|
||||
- `internal/engine/`: search/index backends such as Elasticsearch and Infinity.
|
||||
- `internal/entity/`: shared Go entities and model definitions.
|
||||
- `internal/handler/`: HTTP handlers and route-facing request logic.
|
||||
- `internal/ingestion/`: Go ingestion pipeline, canvas adapter, components, wiring, service orchestration.
|
||||
- `internal/ingestion/component/`: stage implementations such as file/parser/chunker/tokenizer/extractor.
|
||||
- `internal/ingestion/pipeline/`: DSL translation, canvas-driven execution, checkpoints, resume/run logic.
|
||||
- `internal/parser/`: parser and chunk libraries used by ingestion and other Go paths.
|
||||
- `internal/parser/parser/`: typed parse-result parsers for markdown/html/pdf/docx/xlsx/text and related families.
|
||||
- `internal/parser/chunk/`: chunk operator library and DSL/typed execution helpers.
|
||||
- `internal/service/`: higher-level business services used by handlers and server flows.
|
||||
- `internal/storage/`: storage backends and in-memory test doubles.
|
||||
- `internal/router/`: HTTP route registration.
|
||||
- `internal/server/`: server bootstrap/config wiring.
|
||||
- `internal/cpp/`: C++ sources used by native-backed Go features.
|
||||
- `web/`: frontend application.
|
||||
- `docker/`: local and production compose files.
|
||||
- `sdk/` and `test/`: SDK and automated tests.
|
||||
|
||||
## 3. Build Instructions
|
||||
## Go-specific rules
|
||||
- Treat `internal/ingestion`, `internal/parser`, and `internal/deepdoc` as actively refactored code. Prefer collapsing duplicate paths over preserving transitional wrappers.
|
||||
- Do not add or preserve deprecated Go APIs just to ease migration inside the repo.
|
||||
- Remove commented-out Go code instead of leaving recovery notes in place.
|
||||
- Keep package comments and doc comments aligned with the current runtime path, not with migration history.
|
||||
|
||||
### Backend (Python)
|
||||
The project uses **uv** for dependency management.
|
||||
## Working rules
|
||||
- Before editing, inspect the nearest code path that actually owns the behavior.
|
||||
- Keep changes small and local unless the task is explicitly a broader refactor.
|
||||
- Prefer one implementation path instead of preserving old and new versions side by side.
|
||||
- Preserve behavior with focused tests when the behavior is still valid; do not keep tests that protect obsolete behavior.
|
||||
- If a surface is only there for compatibility, remove it unless the user asks to keep it.
|
||||
- Do not add new compatibility wording in comments or docs.
|
||||
- When a maintainer takes over a community PR, a new commit generated by rewriting history (e.g. `merge`, `rebase -i`) must preserve the original author and add the maintainer as co-author (via a `Co-authored-by:` trailer) instead of overwriting the author with the maintainer alone.
|
||||
|
||||
1. **Setup Environment**:
|
||||
```bash
|
||||
uv sync --python 3.13 --all-extras
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
```
|
||||
|
||||
2. **Run Server**:
|
||||
- **Pre-requisite**: Start dependent services (MySQL, ES/Infinity, Redis, MinIO).
|
||||
```bash
|
||||
docker compose -f docker/docker-compose-base.yml up -d
|
||||
```
|
||||
- **Launch**:
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=$(pwd)
|
||||
bash docker/launch_backend_service.sh
|
||||
```
|
||||
|
||||
### Frontend (TypeScript/React)
|
||||
Located in `web/`.
|
||||
|
||||
1. **Install Dependencies**:
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
```
|
||||
|
||||
2. **Run Dev Server**:
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
Runs on port 8000 by default.
|
||||
|
||||
### Docker Deployment
|
||||
To run the full stack using Docker:
|
||||
## Commands
|
||||
### Backend
|
||||
```bash
|
||||
cd docker
|
||||
docker compose -f docker-compose.yml up -d
|
||||
uv sync --python 3.13 --all-extras
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
docker compose -f docker/docker-compose-base.yml up -d
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=$(pwd)
|
||||
bash docker/launch_backend_service.sh
|
||||
uv run pytest
|
||||
ruff check
|
||||
ruff format
|
||||
```
|
||||
|
||||
## 4. Testing Instructions
|
||||
### Frontend
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev
|
||||
npm run build
|
||||
npm run lint
|
||||
npm run test
|
||||
npm run type-check
|
||||
```
|
||||
|
||||
### Backend Tests
|
||||
- **Run All Tests**:
|
||||
```bash
|
||||
uv run pytest
|
||||
```
|
||||
- **Run Specific Test**:
|
||||
```bash
|
||||
uv run pytest test/test_api.py
|
||||
```
|
||||
### Go
|
||||
```bash
|
||||
uv run ragflow_deps/download_deps.py
|
||||
bash build.sh --test ./path/to/package/...
|
||||
bash build.sh --go
|
||||
# or build specific binaries:
|
||||
bash build.sh --all
|
||||
```
|
||||
|
||||
### Frontend Tests
|
||||
- **Run Tests**:
|
||||
```bash
|
||||
cd web
|
||||
npm run test
|
||||
```
|
||||
## Validation preference
|
||||
- Run the narrowest relevant test, lint, or build command after a change.
|
||||
- For backend changes, prefer targeted pytest or ruff checks over full-suite runs.
|
||||
- For frontend changes, prefer the touched-package lint, type-check, or test command.
|
||||
- For Go changes, prefer package-scoped `bash build.sh --test ...` first.
|
||||
- Do not default to raw `go test`, `go build`, or IDE Run/Debug for Go in this repo. They often miss the required CGO flags and native static libraries (`office_oxide`, `pdfium-static`, `pdf_oxide`) that `build.sh` wires correctly.
|
||||
- If Go native builds fail, inspect `build.sh` and `internal/development.md` before changing code. Common environment issues are missing downloaded native deps and missing `lld` on Linux.
|
||||
|
||||
## 5. Coding Standards & Guidelines
|
||||
- **Python Formatting**: Use `ruff` for linting and formatting.
|
||||
```bash
|
||||
ruff check
|
||||
ruff format
|
||||
```
|
||||
- **Frontend Linting**:
|
||||
```bash
|
||||
cd web
|
||||
npm run lint
|
||||
```
|
||||
- **Git Hooks**: Run this once after the first clone to enable local Git hooks.
|
||||
```bash
|
||||
lefthook install
|
||||
lefthook run pre-commit --all-files
|
||||
```
|
||||
## Default review checklist
|
||||
- Remove instead of retaining `deprecated`, `legacy`, or compatibility-only code.
|
||||
- Collapse duplicate implementations to one path.
|
||||
- Drop stale comments and documentation that describe a superseded design.
|
||||
- Keep exported APIs only when the current code actually needs them.
|
||||
|
||||
308
CLAUDE.md
308
CLAUDE.md
@@ -1,308 +0,0 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It's a full-stack application with:
|
||||
|
||||
- Python backend (Quart-based async API server — Quart is the async reimplementation of Flask)
|
||||
- React/TypeScript frontend (built with vitejs)
|
||||
- Background task executor workers (separate Python processes, Redis-queue-driven)
|
||||
- Peewee ORM for database models (not SQLAlchemy)
|
||||
- Multiple data stores (MySQL/PostgreSQL, Elasticsearch/Infinity/OpenSearch/OceanBase, Redis, MinIO)
|
||||
|
||||
## Architecture
|
||||
|
||||
### Runtime Architecture
|
||||
|
||||
RAGFlow runs as **two separate Python process types**, orchestrated by `docker/launch_backend_service.sh`:
|
||||
|
||||
- **API Server** (`api/ragflow_server.py`): Quart-based async HTTP server
|
||||
- **Task Executors** (`rag/svr/task_executor.py`): Background workers processing documents from Redis streams. Multiple instances run in parallel (controlled by `WS` env var). Each consumes from priority-ordered Redis streams (`te.1.common`, `te.0.common`), using consumer groups for load distribution.
|
||||
|
||||
Key consequence: task executors import a different code surface than the API server, so always check which process a module is meant for.
|
||||
|
||||
### Backend API (`/api/`)
|
||||
|
||||
- **App factory**: `api/apps/__init__.py` — creates the Quart app, configures auth (`login_required` decorator, JWT + API token + session fallback), and dynamically discovers/registers blueprints
|
||||
- **Two API coexisting patterns**:
|
||||
- **RESTful APIs** in `api/apps/restful_apis/` — newer pattern with Pydantic request validation, service layer in `api/apps/services/`, routes registered under `/api/v1`
|
||||
- **Legacy APIs** in `api/apps/*_app.py` — older pattern using `@validate_request()`, routes registered under `/v1/<page_name>`
|
||||
- **SDK APIs** in `api/apps/sdk/` — registered under `/v1/`
|
||||
- **Services**: `api/db/services/` — business logic wrapping Peewee model operations. `api/apps/services/` — service layer for the RESTful APIs
|
||||
- **Models**: `api/db/db_models.py` — Peewee ORM models with pooled MySQL/PostgreSQL connections, custom `JSONField`/`ListField` types, retry logic on connection loss
|
||||
|
||||
### Core Processing (`/rag/`)
|
||||
|
||||
- **Document ingestion pipeline**: `rag/flow/pipeline.py` — `Pipeline` (extends `agent.canvas.Graph`) orchestrates the ingestion DAG. Components: File (fetches binary from storage), Parser (dispatches to `deepdoc.parser` based on file type), TokenChunker/TitleChunker (splits into chunks), Tokenizer (computes full-text tokens + embedding vectors), Extractor (LLM-based extraction). Data flows via Pydantic `*FromUpstream` schemas.
|
||||
- **Document parsing**: `deepdoc/` — PDF parsing (vision-based OCR, layout analysis, table structure recognition) and format-specific parsers (DOCX, XLSX, PPT, Markdown, HTML, images). All parsers normalize to a common structure (list of bbox dicts for PDFs, `{text, doc_type_kwd}` for others).
|
||||
- **DeepDoc HTTP API service** (`deepdoc/server/`): OSS ONNX models (DLA, OCR, TSR) wrapped with LitServe as a standalone HTTP API on port 8124. The Go parser (`internal/parser/`) calls this service via `DeepDocClient`. Endpoints: `GET /health`, `GET /model`, `POST /predict/dla`, `POST /predict/tsr`, `POST /predict/ocr` (with `operator=det` or `operator=rec` form field). Docker image: `deepdoc_oss:latest`. See `deepdoc/server/README.md` for the full API reference.
|
||||
- **LLM Integration**: `rag/llm/` — factory pattern with runtime class discovery. `chat_model.py` (30+ providers via OpenAI SDK and LiteLLM wrappers), `embedding_model.py`, `rerank_model.py`, `cv_model.py` (image-to-text), `sequence2txt_model.py` (ASR), `tts_model.py`. Use `LLMBundle` (from `api.db.services.llm_service`) as the unified interface.
|
||||
- **Graph RAG**: `rag/graphrag/` — multi-phase pipeline: per-document subgraph extraction (LLM or spaCy NER), Leiden community detection, entity resolution, community summarization. Entities/relations/reports are indexed as chunks alongside regular text chunks, differentiated by `knowledge_graph_kwd`.
|
||||
- **Search**: `rag/nlp/search.py` — `Dealer` class combines vector similarity + BM25 + re-ranking. `KGSearch` extends it for graph-aware retrieval (entity resolution, n-hop enrichment).
|
||||
|
||||
### Agent System (`/agent/`)
|
||||
|
||||
- **Execution engine**: `agent/canvas.py` — `Canvas` (extends `Graph`) executes the DAG. Components are run in topological order via `_run_batch`, each receiving upstream outputs as kwargs. Control-flow components (`Categorize`, `Switch`, `Iteration`, `Loop`) dynamically modify the execution path.
|
||||
- **Component base**: `agent/component/base.py` — `ComponentBase` with `invoke(**kwargs)` / `invoke_async(**kwargs)` lifecycle. Variable references (`{component_id@output_var}` or `{sys.query}`) are resolved from the canvas graph at runtime.
|
||||
- **Components**: Modular workflow components in `agent/component/` — Begin, LLM, Agent (tool-calling LLM), Categorize, Switch, Iteration, Loop, Message, Invoke (HTTP), and data manipulation nodes. Auto-discovered by `__init__.py`.
|
||||
- **Templates**: Pre-built agent workflows as JSON DSL files in `agent/templates/`. Each contains a complete `components` DAG, `path`, and `globals`.
|
||||
- **Tools**: `agent/tools/` — Retrieval, web search (DuckDuckGo, Google, Tavily, SearXNG), academic search (ArXiv, PubMed, Google Scholar, Wikipedia), code execution, SQL execution, email, GitHub, finance data, translation, weather. Tools implement `ToolBase` (extends `ComponentBase`) and produce OpenAI-compatible function descriptors.
|
||||
- **Plugins**: `agent/plugin/` — plugin system using `pluginlib` for loading external LLM tool plugins from `embedded_plugins/`.
|
||||
|
||||
### Frontend (`/web/`)
|
||||
|
||||
- React/TypeScript with vitejs framework
|
||||
- shadcn/ui components (Radix UI primitives + Tailwind CSS)
|
||||
- `@tanstack/react-query` for server state (cache keys, mutations, invalidation)
|
||||
- Zustand for local state (primarily agent canvas graph store)
|
||||
- `react-router` v7 with lazy-loaded pages
|
||||
- `react-i18next` for i18n (17 languages)
|
||||
- Axios for HTTP with a layered pattern: endpoint definitions (`utils/api.ts`) → HTTP client (`utils/next-request.ts`) → service layer (`services/`) → query hooks (`hooks/use-*-request.ts`) → components
|
||||
- `@xyflow/react` for the agent workflow canvas
|
||||
- `react-hook-form` + `zod` for form validation
|
||||
- Two API proxy prefixes: `webAPI = '/v1'` (legacy) and `restAPIv1 = '/api/v1'` (RESTful)
|
||||
|
||||
## Common Development Commands
|
||||
|
||||
### Backend Development
|
||||
|
||||
```bash
|
||||
# Install Python dependencies
|
||||
uv sync --python 3.13 --all-extras
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
|
||||
# Run once after the first clone to enable local Git hooks
|
||||
lefthook install
|
||||
|
||||
# Start dependent services
|
||||
docker compose -f docker/docker-compose-base.yml up -d
|
||||
|
||||
# Run backend (requires services to be running)
|
||||
source .venv/bin/activate
|
||||
export PYTHONPATH=$(pwd)
|
||||
bash docker/launch_backend_service.sh
|
||||
|
||||
# Run tests
|
||||
uv run pytest
|
||||
|
||||
# Linting
|
||||
ruff check
|
||||
ruff format
|
||||
```
|
||||
|
||||
### Frontend Development
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev # Development server
|
||||
npm run build # Production build
|
||||
npm run lint # ESLint
|
||||
npm run test # Jest tests
|
||||
```
|
||||
|
||||
### Docker Development
|
||||
|
||||
```bash
|
||||
# Full stack with Docker (includes deepdoc vision service)
|
||||
cd docker
|
||||
docker compose -f docker-compose.yml up -d
|
||||
|
||||
# Check server status
|
||||
docker logs -f ragflow-server
|
||||
|
||||
# Build the OSS deepdoc vision service standalone
|
||||
docker build -f docker/Dockerfile_deepdoc_oss -t deepdoc_oss:latest .
|
||||
docker run -p 8124:8124 deepdoc_oss:latest
|
||||
|
||||
# Rebuild images
|
||||
docker build --platform linux/amd64 -f Dockerfile -t infiniflow/ragflow:nightly .
|
||||
```
|
||||
|
||||
## Key Configuration Files
|
||||
|
||||
- `docker/.env` - Environment variables for Docker deployment
|
||||
- `docker/service_conf.yaml.template` - Backend service configuration
|
||||
- `pyproject.toml` - Python dependencies and project configuration
|
||||
- `web/package.json` - Frontend dependencies and scripts
|
||||
|
||||
## Testing
|
||||
|
||||
- **Python**: pytest with markers (p1/p2/p3 priority levels)
|
||||
- **Frontend**: Jest with React Testing Library
|
||||
- **API Tests**: HTTP API and SDK tests in `test/` and `sdk/python/test/`
|
||||
|
||||
## Database Engines
|
||||
|
||||
RAGFlow supports switching between Elasticsearch (default) and Infinity:
|
||||
|
||||
- Set `DOC_ENGINE=infinity` in `docker/.env` to use Infinity
|
||||
- Requires container restart: `docker compose down -v && docker compose up -d`
|
||||
|
||||
## Account Password Handling (Critical for Login Flow)
|
||||
|
||||
### Password Encryption Pipeline (Browser → Backend → DB Hash)
|
||||
|
||||
The login password verification chain is counterintuitive. Understanding this is essential when generating or verifying password hashes.
|
||||
|
||||
**Complete flow:**
|
||||
|
||||
```
|
||||
Browser input: "demo"
|
||||
→ Base64("demo") = "ZGVtbw=="
|
||||
→ RSA encrypt with conf/public.pem
|
||||
→ POST to /api/v1/auth/login
|
||||
|
||||
Backend DecryptPassword():
|
||||
→ RSA decrypt with conf/private.pem (passphrase: "Welcome")
|
||||
→ Returns "ZGVtbw==" (NOT "demo"!)
|
||||
|
||||
VerifyPassword("ZGVtbw==", storedHash) ← hash is of Base64(password), not raw password
|
||||
```
|
||||
|
||||
**Consequences:**
|
||||
- The string verified against the hash is **Base64(original password)**, never the raw password
|
||||
- `DecryptPassword()` handles both RSA-encrypted (browser) and plaintext (curl/API key) inputs: if base64 decode fails, the input is returned as-is for backward compatibility
|
||||
- Python backend has the same design: `api/utils/crypt.py:decrypt()` RSA-decrypts and returns the Base64-encoded string directly, no further decode
|
||||
|
||||
### How to Generate a Valid Password Hash
|
||||
|
||||
```bash
|
||||
# For password "demo" (user input in browser):
|
||||
# The actual verified string = Base64("demo") = "ZGVtbw=="
|
||||
# Generate hash with: common.GenerateWerkzeugPasswordHash("ZGVtbw==")
|
||||
# or use the scrypt template:
|
||||
# scrypt:32768:8:1$<random-b64-salt>$<hex-hash-of-ZGVtbw==>
|
||||
```
|
||||
|
||||
**To update a user's password in the running database:**
|
||||
```bash
|
||||
docker exec docker-mysql-1 mysql -u root -pinfini_rag_flow rag_flow \
|
||||
-e "UPDATE user SET password='<hash>' WHERE email='<email>';"
|
||||
```
|
||||
|
||||
### RSA Keys
|
||||
- `conf/public.pem` — frontend uses this to encrypt Base64(password) before sending
|
||||
- `conf/private.pem` — backend uses this to decrypt, passphrase `"Welcome"`
|
||||
- Both referenced in `internal/common/password.go:DecryptPassword()`
|
||||
|
||||
### Obtaining an API Token for a Tenant
|
||||
|
||||
When testing APIs manually (curl, Go scripts, etc.), you need a valid auth token. The login endpoint returns **two different tokens**:
|
||||
|
||||
| Field | Format | Purpose |
|
||||
|-------|--------|---------|
|
||||
| `response.body.data.access_token` | Raw UUID | Stored in DB, NOT used for API auth |
|
||||
| `response.Header["Authorization"]` | itsdangerous-signed token | Used as `Bearer <token>` for all subsequent API requests |
|
||||
|
||||
**How to obtain the correct token:**
|
||||
|
||||
```bash
|
||||
# Step 1: Construct the encrypted password
|
||||
# Raw password → Base64 → RSA encrypt with conf/public.pem
|
||||
PASSWORD="demo"
|
||||
PASSWORD_B64=$(echo -n "$PASSWORD" | base64)
|
||||
|
||||
# Step 2: POST to login (use RSA encryption — easiest via a Go/Python script)
|
||||
# Response header contains: Authorization: <itsdangerous-signed-token>
|
||||
|
||||
# Step 3: Use the Authorization header value for all API requests
|
||||
curl -H "Authorization: <itsdangerous-signed-token>" \
|
||||
http://127.0.0.1:9222/api/v1/agents
|
||||
```
|
||||
|
||||
**Go snippet (complete login + token extraction):**
|
||||
|
||||
```go
|
||||
// Login
|
||||
passwordB64 := base64.StdEncoding.EncodeToString([]byte(password))
|
||||
pubData, _ := os.ReadFile("conf/public.pem")
|
||||
block, _ := pem.Decode(pubData)
|
||||
pubKey, _ := x509.ParsePKIXPublicKey(block.Bytes)
|
||||
ciphertext, _ := rsa.EncryptPKCS1v15(rand.Reader, pubKey.(*rsa.PublicKey), []byte(passwordB64))
|
||||
encryptedB64 := base64.StdEncoding.EncodeToString(ciphertext)
|
||||
|
||||
body, _ := json.Marshal(map[string]string{"email": email, "password": encryptedB64})
|
||||
resp, _ := http.Post(baseURL+"/api/v1/auth/login", "application/json", bytes.NewReader(body))
|
||||
|
||||
// KEY: use the Authorization header, NOT body.access_token
|
||||
authToken := resp.Header.Get("Authorization")
|
||||
|
||||
// Use for API calls
|
||||
req, _ := http.NewRequest("GET", baseURL+"/api/v1/agents", nil)
|
||||
req.Header.Set("Authorization", authToken)
|
||||
```
|
||||
|
||||
**The raw `access_token` (UUID) in the response body** is the internal DB token used only by the `itsdangerous` middleware to verify the signed token — it is never passed directly in API Authorization headers.
|
||||
|
||||
---
|
||||
|
||||
## Agent Run E2E Tests
|
||||
|
||||
### Running the Tests
|
||||
|
||||
```bash
|
||||
# Run all agent run e2e tests (in-memory SQLite + miniredis, no Docker needed)
|
||||
cd /home/zhichyu/github.com/infiniflow/ragflow
|
||||
go test -count=1 -v -run 'TestRunAgent_RealCanvas|TestRunAgent_RunTracker' ./internal/service/
|
||||
```
|
||||
|
||||
### Test Architecture
|
||||
|
||||
All e2e tests live in `internal/service/agent_run_e2e_test.go`. They exercise the full production chain:
|
||||
|
||||
```
|
||||
loadCanvasForUser → versionDAO.GetLatest → decodeCanvasFromDSL →
|
||||
canvas.Compile → cc.Workflow.Invoke → answer extraction
|
||||
```
|
||||
|
||||
**Test isolation**: Each test stands up its own in-memory SQLite DB (pushed as `dao.DB`), seeds User/Tenant/UserCanvas/UserCanvasVersion rows, and tears down in `t.Cleanup`. Tests use **miniredis** for Redis-backed CheckPointStore + RunTracker — no external services needed.
|
||||
|
||||
**Key test helpers:**
|
||||
- `makeCanvasWithDSL(t, canvasID, userID, tenantID, versionID, dsl)` — seeds all required DB rows
|
||||
- `drainAgentEvents(t, events)` — drains the `<-chan canvas.RunEvent` channel, buckets results into `messages`, `waiting`, `errors_`, `done`
|
||||
- `newRunTrackerForTest(t, ttl)` — wires a `canvas.RunTracker` against in-memory miniredis
|
||||
|
||||
**Existing e2e tests:**
|
||||
|
||||
| Test | What it covers |
|
||||
|------|---------------|
|
||||
| `TestRunAgent_RealCanvas_BeginMessage` | Happy path: Begin→Message, verifies `"{{sys.query}}"` resolution |
|
||||
| `TestRunAgent_RealCanvas_WaitForUserResume` | Resume path: Begin→Message→UserFillUp, two-run cycle |
|
||||
| `TestRunAgent_RealCanvas_CompileFails` | Error path: unknown component name → sanitized error |
|
||||
| `TestRunAgent_RealCanvas_InvokeFails` | Error path: unresolvable template ref |
|
||||
| `TestRunAgent_RunTracker_AttachCheckpoint_CallSequence` | Production boot: Start→AttachCheckpoint→MarkSucceeded with Redis/miniredis |
|
||||
|
||||
**Test DSL data files** are in `internal/agent/dsl/testdata/`:
|
||||
- `agent_msg.json` — Agent+Message with Begin, LLM-powered agent component
|
||||
- `all.json` — Complex: Begin→UserFillUp→Switch→Loop→Message
|
||||
- `switch.json`, `resume.json`, `browser.json`, `subagent.json`, etc.
|
||||
|
||||
**Handler-level SSE streaming tests** in `internal/handler/agent_test.go` use a `stubChatRunner` that emits pre-configured `canvas.RunEvent` values without a real DB or eino runner, verifying:
|
||||
- SSE `Content-Type: text/event-stream`
|
||||
- `data: {...}\n\n` framing
|
||||
- Trailing `data: [DONE]\n\n` terminator
|
||||
- OpenAI-compatible non-stream `choices` response shape
|
||||
|
||||
**Important**: `_ "ragflow/internal/agent/component"` (blank import in test) is required — it triggers `init()` to register all component factories. Without it, `canvas.Compile` fails to resolve any component type.
|
||||
|
||||
---
|
||||
|
||||
## Development Environment Requirements
|
||||
|
||||
- Python 3.10-3.13
|
||||
- Node.js >=18.20.4
|
||||
- Docker & Docker Compose
|
||||
- uv package manager
|
||||
- 16GB+ RAM, 50GB+ disk space
|
||||
|
||||
1. Think before acting. Read existing files before writing code.
|
||||
2. Be concise in output but thorough in reasoning.
|
||||
3. Prefer editing over rewriting whole files.
|
||||
4. Do not re-read files you have already read.
|
||||
5. Test your code before declaring done.
|
||||
6. No sycophantic openers or closing fluff.
|
||||
7. Keep solutions simple and direct.
|
||||
8. User instructions always override this file.
|
||||
@@ -59,7 +59,7 @@ RUN mkdir -p /usr/share/infinity/resource && \
|
||||
cp -r /tmp/resource/* /usr/share/infinity/resource && \
|
||||
rm -rf /tmp/resource
|
||||
|
||||
ARG NGINX_VERSION=1.31.0-1~noble
|
||||
ARG NGINX_VERSION=1.31.2-1~noble
|
||||
RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
|
||||
mkdir -p /etc/apt/keyrings && \
|
||||
curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://nginx.org/keys/nginx_signing.key | gpg --dearmor -o /etc/apt/keyrings/nginx-archive-keyring.gpg && \
|
||||
|
||||
12
README.md
12
README.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -193,12 +193,12 @@ releases! 🌟
|
||||
> All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
|
||||
> If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
|
||||
|
||||
> The command below downloads the `v0.26.3` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.26.3`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
|
||||
> The command below downloads the `v0.26.4` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.26.4`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
|
||||
# This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
|
||||
|
||||
@@ -320,7 +320,7 @@ docker build --platform linux/amd64 \
|
||||
## 🔨 Launch service from source for development
|
||||
|
||||
> [!IMPORTANT]
|
||||
> After cloning the repository for the first time, run `lefthook install` once from the repo root to enable local Git hooks.
|
||||
> After cloning the repository for the first time, run `git config --local --unset core.hooksPath`, `uv tool install lefthook` and `lefthook install` once from the repo root to enable local Git hooks.
|
||||
|
||||
1. Install `uv`, or skip this step if it is already installed:
|
||||
|
||||
@@ -334,6 +334,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Launch the dependent services (MinIO, Elasticsearch, Redis, and MySQL) using Docker Compose:
|
||||
|
||||
10
README_ar.md
10
README_ar.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -193,12 +193,12 @@
|
||||
> جميع الصور Docker مصممة لمنصات x86. لا نعرض حاليًا صور Docker لـ ARM64.
|
||||
> إذا كنت تستخدم نظامًا أساسيًا ARM64، فاتبع [هذا الدليل](https://ragflow.io/docs/dev/build_docker_image) لإنشاء صورة Docker متوافقة مع نظامك.
|
||||
|
||||
> يقوم الأمر أدناه بتنزيل إصدار `v0.26.3` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.26.3`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
|
||||
> يقوم الأمر أدناه بتنزيل إصدار `v0.26.4` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.26.4`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
|
||||
# This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
|
||||
|
||||
@@ -331,6 +331,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. قم بتشغيل الخدمات التابعة (MinIO وElasticsearch وRedis وMySQL) باستخدام Docker Compose:
|
||||
|
||||
12
README_fr.md
12
README_fr.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,10 +25,10 @@
|
||||
<img alt="Badge statique" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Dernière%20version" alt="Dernière version">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Derniere%20version" alt="Dernière version">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
|
||||
<img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="licence">
|
||||
@@ -190,12 +190,12 @@ Essayez notre service cloud sur [https://cloud.ragflow.io](https://cloud.ragflow
|
||||
> Toutes les images Docker sont construites pour les plateformes x86. Nous ne proposons pas actuellement d'images Docker pour ARM64.
|
||||
> Si vous êtes sur une plateforme ARM64, suivez [ce guide](https://ragflow.io/docs/dev/build_docker_image) pour construire une image Docker compatible avec votre système.
|
||||
|
||||
> La commande ci-dessous télécharge l'édition `v0.26.3` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.26.3`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
|
||||
> La commande ci-dessous télécharge l'édition `v0.26.4` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.26.4`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Optionnel : utiliser un tag stable (voir les versions : https://github.com/infiniflow/ragflow/releases)
|
||||
# Cette étape garantit que le fichier **entrypoint.sh** dans le code correspond à la version de l'image Docker.
|
||||
|
||||
@@ -322,6 +322,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Lancez les services dépendants (MinIO, Elasticsearch, Redis et MySQL) avec Docker Compose :
|
||||
|
||||
10
README_id.md
10
README_id.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="Logo ragflow">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="Logo ragflow">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Lencana Daring" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
|
||||
@@ -191,12 +191,12 @@ Coba layanan cloud kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
|
||||
> Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
|
||||
> Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
|
||||
|
||||
> Perintah di bawah ini mengunduh edisi v0.26.3 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.26.3, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
|
||||
> Perintah di bawah ini mengunduh edisi v0.26.4 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.26.4, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases)
|
||||
# This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
|
||||
|
||||
@@ -303,6 +303,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Jalankan aplikasi yang diperlukan (MinIO, Elasticsearch, Redis, dan MySQL) menggunakan Docker Compose:
|
||||
|
||||
10
README_ja.md
10
README_ja.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -172,12 +172,12 @@
|
||||
> 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
|
||||
> ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
|
||||
|
||||
> 以下のコマンドは、RAGFlow Docker イメージの v0.26.3 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.26.3 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
|
||||
> 以下のコマンドは、RAGFlow Docker イメージの v0.26.4 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.26.4 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases)
|
||||
# この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。
|
||||
|
||||
@@ -304,6 +304,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Docker Compose を使用して依存サービス(MinIO、Elasticsearch、Redis、MySQL)を起動する:
|
||||
|
||||
10
README_ko.md
10
README_ko.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -174,12 +174,12 @@
|
||||
> 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
|
||||
> ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
|
||||
|
||||
> 아래 명령어는 RAGFlow Docker 이미지의 v0.26.3 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.26.3와 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
|
||||
> 아래 명령어는 RAGFlow Docker 이미지의 v0.26.4 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.26.4와 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
|
||||
# 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다.
|
||||
|
||||
@@ -299,6 +299,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,10 +25,10 @@
|
||||
<img alt="Badge Estático" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=%C3%9Altima%20Release" alt="Última Release">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
|
||||
<img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="licença">
|
||||
@@ -191,12 +191,12 @@ Experimente o nosso serviço na nuvem em [https://cloud.ragflow.io](https://clou
|
||||
> Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
|
||||
> Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
|
||||
|
||||
> O comando abaixo baixa a edição`v0.26.3` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.26.3`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
|
||||
> O comando abaixo baixa a edição`v0.26.4` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.26.4`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases)
|
||||
# Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker.
|
||||
|
||||
@@ -320,6 +320,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # instala os módulos Python dependentes do RAGFlow
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Inicie os serviços dependentes (MinIO, Elasticsearch, Redis e MySQL) usando Docker Compose:
|
||||
|
||||
12
README_tr.md
12
README_tr.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,10 +25,10 @@
|
||||
<img alt="Çevrimiçi Demo" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Son%20Sürüm" alt="Son Sürüm">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Son%20S%C3%BCr%C3%BCm" alt="Son Sürüm">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/blob/main/LICENSE">
|
||||
<img height="21" src="https://img.shields.io/badge/Lisans-Apache--2.0-ffffff?labelColor=d4eaf7&color=2e6cc4" alt="lisans">
|
||||
@@ -191,12 +191,12 @@ Bulut hizmetimizi [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinde
|
||||
> Tüm Docker imajları x86 platformları için oluşturulmuştur. Şu anda ARM64 için Docker imajı sunmuyoruz.
|
||||
> ARM64 platformundaysanız, sisteminizle uyumlu bir Docker imajı oluşturmak için [bu kılavuzu](https://ragflow.io/docs/dev/build_docker_image) takip edin.
|
||||
|
||||
> Aşağıdaki komut RAGFlow Docker imajının `v0.26.3` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.26.3` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
|
||||
> Aşağıdaki komut RAGFlow Docker imajının `v0.26.4` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.26.4` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# İsteğe bağlı: Kararlı bir etiket kullanın (sürümler: https://github.com/infiniflow/ragflow/releases)
|
||||
# Bu adım, koddaki **entrypoint.sh** dosyasının Docker imaj sürümüyle eşleşmesini sağlar.
|
||||
|
||||
@@ -326,6 +326,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # RAGFlow'un bağımlı Python modüllerini yükler
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. Bağımlı hizmetleri (MinIO, Elasticsearch, Redis ve MySQL) Docker Compose kullanarak başlatın:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -191,12 +191,12 @@
|
||||
> 所有 Docker 映像檔都是為 x86 平台建置的。目前,我們不提供 ARM64 平台的 Docker 映像檔。
|
||||
> 如果您使用的是 ARM64 平台,請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
|
||||
|
||||
> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.26.3`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.26.3` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
|
||||
> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.26.4`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.26.4` 的 Docker 映像,請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# 可選:使用穩定版標籤(查看發佈:https://github.com/infiniflow/ragflow/releases)
|
||||
# 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。
|
||||
|
||||
@@ -331,6 +331,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
3. 透過 Docker Compose 啟動依賴的服務(MinIO, Elasticsearch, Redis, and MySQL):
|
||||
|
||||
10
README_zh.md
10
README_zh.md
@@ -1,6 +1,6 @@
|
||||
<div align="center">
|
||||
<a href="https://cloud.ragflow.io/">
|
||||
<img src="web/src/assets/logo-with-text.svg" width="350" alt="ragflow logo">
|
||||
<img src="https://raw.githubusercontent.com/infiniflow/ragflow/main/web/src/assets/logo-with-text.svg" width="520" alt="ragflow logo">
|
||||
</a>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
<img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
|
||||
</a>
|
||||
<a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
|
||||
<img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.3">
|
||||
<img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/infiniflow/ragflow-stats/main/badges/docker-pulls.json&style=flat-square&logo=docker&logoColor=white" alt="docker pull infiniflow/ragflow:v0.26.4">
|
||||
</a>
|
||||
<a href="https://github.com/infiniflow/ragflow/releases/latest">
|
||||
<img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
|
||||
@@ -192,12 +192,12 @@
|
||||
> 请注意,目前官方提供的所有 Docker 镜像均基于 x86 架构构建,并不提供基于 ARM64 的 Docker 镜像。
|
||||
> 如果你的操作系统是 ARM64 架构,请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
|
||||
|
||||
> 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.26.3`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.26.3` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
|
||||
> 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.26.4`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.26.4` 的 Docker 镜像,请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
|
||||
|
||||
```bash
|
||||
$ cd ragflow/docker
|
||||
|
||||
# git checkout v0.26.3
|
||||
git checkout v0.26.4
|
||||
# 可选:使用稳定版本标签(查看发布:https://github.com/infiniflow/ragflow/releases)
|
||||
# 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。
|
||||
|
||||
@@ -331,6 +331,8 @@ docker build --platform linux/amd64 \
|
||||
cd ragflow/
|
||||
uv sync --python 3.13 # install RAGFlow dependent python modules
|
||||
uv run python3 ragflow_deps/download_deps.py
|
||||
git config --local --unset core.hooksPath
|
||||
uv tool install lefthook
|
||||
lefthook install
|
||||
```
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
|
||||
```bash
|
||||
python admin/server/admin_server.py
|
||||
```
|
||||
The service will start and listen for incoming connections from the CLI on the configured port.
|
||||
The service will start and listen for incoming connections from the CLI on the configured port.
|
||||
|
||||
#### Using docker image
|
||||
|
||||
@@ -48,7 +48,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
|
||||
1. Ensure the Admin Service is running.
|
||||
2. Install ragflow-cli.
|
||||
```bash
|
||||
pip install ragflow-cli==0.26.3
|
||||
pip install ragflow-cli==0.26.4
|
||||
```
|
||||
3. Launch the CLI client:
|
||||
```bash
|
||||
@@ -58,9 +58,9 @@ It consists of a server-side Service and a command-line client (CLI), both imple
|
||||
The default password is admin.
|
||||
|
||||
**Parameters:**
|
||||
|
||||
|
||||
- -h: RAGFlow admin server host address
|
||||
|
||||
|
||||
- -p: RAGFlow admin server port
|
||||
|
||||
|
||||
|
||||
@@ -25,14 +25,14 @@ import requests
|
||||
|
||||
class HttpClient:
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 9381,
|
||||
api_version: str = "v1",
|
||||
api_key: Optional[str] = None,
|
||||
connect_timeout: float = 5.0,
|
||||
read_timeout: float = 60.0,
|
||||
verify_ssl: bool = False,
|
||||
self,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 9381,
|
||||
api_version: str = "v1",
|
||||
api_key: Optional[str] = None,
|
||||
connect_timeout: float = 5.0,
|
||||
read_timeout: float = 60.0,
|
||||
verify_ssl: bool = False,
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
@@ -71,19 +71,19 @@ class HttpClient:
|
||||
return headers
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
use_api_base: bool = True,
|
||||
auth_kind: Optional[str] = "api",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
json_body: Optional[Dict[str, Any]] = None,
|
||||
data: Any = None,
|
||||
files: Any = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
stream: bool = False,
|
||||
iterations: int = 1,
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
use_api_base: bool = True,
|
||||
auth_kind: Optional[str] = "api",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
json_body: Optional[Dict[str, Any]] = None,
|
||||
data: Any = None,
|
||||
files: Any = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
stream: bool = False,
|
||||
iterations: int = 1,
|
||||
) -> requests.Response | dict:
|
||||
url = self.build_url(path, use_api_base=use_api_base)
|
||||
merged_headers = self._headers(auth_kind, headers)
|
||||
@@ -144,18 +144,18 @@ class HttpClient:
|
||||
# )
|
||||
|
||||
def request_json(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
use_api_base: bool = True,
|
||||
auth_kind: Optional[str] = "api",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
json_body: Optional[Dict[str, Any]] = None,
|
||||
data: Any = None,
|
||||
files: Any = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
stream: bool = False,
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
use_api_base: bool = True,
|
||||
auth_kind: Optional[str] = "api",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
json_body: Optional[Dict[str, Any]] = None,
|
||||
data: Any = None,
|
||||
files: Any = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
stream: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
response = self.request(
|
||||
method,
|
||||
|
||||
@@ -336,8 +336,8 @@ reset_default_asr: RESET DEFAULT ASR ";"
|
||||
reset_default_tts: RESET DEFAULT TTS ";"
|
||||
|
||||
list_user_datasets: LIST DATASETS ";"
|
||||
create_user_dataset_with_parser: CREATE DATASET quoted_string WITH EMBEDDING quoted_string PARSER quoted_string ";"
|
||||
create_user_dataset_with_pipeline: CREATE DATASET quoted_string WITH EMBEDDING quoted_string PIPELINE quoted_string ";"
|
||||
create_user_dataset_with_parser: CREATE DATASET quoted_string WITH EMBEDDING quoted_string PARSER quoted_string ";"
|
||||
create_user_dataset_with_pipeline: CREATE DATASET quoted_string WITH EMBEDDING quoted_string PIPELINE quoted_string ";"
|
||||
drop_user_dataset: DROP DATASET quoted_string ";"
|
||||
list_user_dataset_files: LIST FILES OF DATASET quoted_string ";"
|
||||
list_user_dataset_documents: LIST DOCUMENTS OF DATASET quoted_string ";"
|
||||
@@ -640,15 +640,13 @@ class RAGFlowCLITransformer(Transformer):
|
||||
dataset_name = items[2].children[0].strip("'\"")
|
||||
embedding = items[5].children[0].strip("'\"")
|
||||
parser_type = items[7].children[0].strip("'\"")
|
||||
return {"type": "create_user_dataset", "dataset_name": dataset_name, "embedding": embedding,
|
||||
"parser_type": parser_type}
|
||||
return {"type": "create_user_dataset", "dataset_name": dataset_name, "embedding": embedding, "parser_type": parser_type}
|
||||
|
||||
def create_user_dataset_with_pipeline(self, items):
|
||||
dataset_name = items[2].children[0].strip("'\"")
|
||||
embedding = items[5].children[0].strip("'\"")
|
||||
pipeline = items[7].children[0].strip("'\"")
|
||||
return {"type": "create_user_dataset", "dataset_name": dataset_name, "embedding": embedding,
|
||||
"pipeline": pipeline}
|
||||
return {"type": "create_user_dataset", "dataset_name": dataset_name, "embedding": embedding, "pipeline": pipeline}
|
||||
|
||||
def drop_user_dataset(self, items):
|
||||
dataset_name = items[2].children[0].strip("'\"")
|
||||
@@ -666,7 +664,7 @@ class RAGFlowCLITransformer(Transformer):
|
||||
dataset_names = []
|
||||
dataset_names.append(items[4].children[0].strip("'\""))
|
||||
for i in range(5, len(items)):
|
||||
if items[i] and hasattr(items[i], 'children') and items[i].children:
|
||||
if items[i] and hasattr(items[i], "children") and items[i].children:
|
||||
dataset_names.append(items[i].children[0].strip("'\""))
|
||||
return {"type": "list_user_datasets_metadata", "dataset_names": dataset_names}
|
||||
|
||||
@@ -675,7 +673,7 @@ class RAGFlowCLITransformer(Transformer):
|
||||
doc_ids = []
|
||||
if len(items) > 6 and items[6] == "DOCUMENTS":
|
||||
for i in range(7, len(items)):
|
||||
if items[i] and hasattr(items[i], 'children') and items[i].children:
|
||||
if items[i] and hasattr(items[i], "children") and items[i].children:
|
||||
doc_id = items[i].children[0].strip("'\"")
|
||||
doc_ids.append(doc_id)
|
||||
return {"type": "list_user_documents_metadata_summary", "dataset_name": dataset_name, "document_ids": doc_ids}
|
||||
@@ -698,17 +696,17 @@ class RAGFlowCLITransformer(Transformer):
|
||||
dataset_name = None
|
||||
vector_size = None
|
||||
for i, item in enumerate(items):
|
||||
if hasattr(item, 'data') and item.data == 'quoted_string':
|
||||
if hasattr(item, "data") and item.data == "quoted_string":
|
||||
dataset_name = item.children[0].strip("'\"")
|
||||
if hasattr(item, 'type') and item.type == 'NUMBER':
|
||||
if i > 0 and items[i-1].type == 'SIZE' and items[i-2].type == 'VECTOR':
|
||||
if hasattr(item, "type") and item.type == "NUMBER":
|
||||
if i > 0 and items[i - 1].type == "SIZE" and items[i - 2].type == "VECTOR":
|
||||
vector_size = int(item)
|
||||
return {"type": "create_dataset_table", "dataset_name": dataset_name, "vector_size": vector_size}
|
||||
|
||||
def drop_dataset_table(self, items):
|
||||
dataset_name = None
|
||||
for item in items:
|
||||
if hasattr(item, 'data') and item.data == 'quoted_string':
|
||||
if hasattr(item, "data") and item.data == "quoted_string":
|
||||
dataset_name = item.children[0].strip("'\"")
|
||||
return {"type": "drop_dataset_table", "dataset_name": dataset_name}
|
||||
|
||||
@@ -792,7 +790,7 @@ class RAGFlowCLITransformer(Transformer):
|
||||
|
||||
def update_chunk(self, items):
|
||||
def get_quoted_value(item):
|
||||
if hasattr(item, 'children') and item.children:
|
||||
if hasattr(item, "children") and item.children:
|
||||
return item.children[0].strip("'\"")
|
||||
return str(item).strip("'\"")
|
||||
|
||||
@@ -813,16 +811,16 @@ class RAGFlowCLITransformer(Transformer):
|
||||
for i in range(2, len(items)):
|
||||
item = items[i]
|
||||
# Check for FROM token to stop
|
||||
if hasattr(item, 'type') and item.type == 'FROM':
|
||||
if hasattr(item, "type") and item.type == "FROM":
|
||||
break
|
||||
if hasattr(item, 'children') and item.children:
|
||||
if hasattr(item, "children") and item.children:
|
||||
tag = item.children[0].strip("'\"")
|
||||
tags.append(tag)
|
||||
# Find dataset_name: quoted_string after DATASET
|
||||
dataset_name = None
|
||||
for i, item in enumerate(items):
|
||||
# Check if item is a DATASET token
|
||||
if hasattr(item, 'type') and item.type == 'DATASET':
|
||||
if hasattr(item, "type") and item.type == "DATASET":
|
||||
# Next item should be quoted_string
|
||||
dataset_name = items[i + 1].children[0].strip("'\"")
|
||||
break
|
||||
@@ -835,10 +833,10 @@ class RAGFlowCLITransformer(Transformer):
|
||||
|
||||
# Check if it's "REMOVE ALL CHUNKS"
|
||||
for item in items:
|
||||
if hasattr(item, 'type') and item.type == 'ALL':
|
||||
if hasattr(item, "type") and item.type == "ALL":
|
||||
# Find doc_id
|
||||
for j, inner_item in enumerate(items):
|
||||
if hasattr(inner_item, 'type') and inner_item.type == 'DOCUMENT':
|
||||
if hasattr(inner_item, "type") and inner_item.type == "DOCUMENT":
|
||||
doc_id = items[j + 1].children[0].strip("'\"")
|
||||
return {"type": "remove_chunks", "doc_id": doc_id, "delete_all": True}
|
||||
|
||||
@@ -846,12 +844,12 @@ class RAGFlowCLITransformer(Transformer):
|
||||
chunk_ids = []
|
||||
doc_id = None
|
||||
for i, item in enumerate(items):
|
||||
if hasattr(item, 'type') and item.type == 'DOCUMENT':
|
||||
if hasattr(item, "type") and item.type == "DOCUMENT":
|
||||
doc_id = items[i + 1].children[0].strip("'\"")
|
||||
elif hasattr(item, 'children') and item.children:
|
||||
elif hasattr(item, "children") and item.children:
|
||||
val = item.children[0].strip("'\"")
|
||||
# Skip if it's "FROM" or "DOCUMENT"
|
||||
if val.upper() in ['FROM', 'DOCUMENT']:
|
||||
if val.upper() in ["FROM", "DOCUMENT"]:
|
||||
continue
|
||||
chunk_ids.append(val)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "ragflow-cli"
|
||||
version = "0.26.3"
|
||||
version = "0.26.4"
|
||||
description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
|
||||
authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
|
||||
license = { text = "Apache License, Version 2.0" }
|
||||
@@ -8,7 +8,7 @@ readme = "README.md"
|
||||
requires-python = ">=3.13,<3.14"
|
||||
dependencies = [
|
||||
"requests>=2.30.0,<3.0.0",
|
||||
"beartype>=0.20.0,<1.0.0",
|
||||
"beartype>=0.22.9,<1.0.0",
|
||||
"pycryptodomex>=3.10.0",
|
||||
"lark>=1.1.0",
|
||||
"requests-toolbelt>=1.0.0",
|
||||
|
||||
@@ -36,6 +36,7 @@ from user import login_user
|
||||
|
||||
warnings.filterwarnings("ignore", category=getpass.GetPassWarning)
|
||||
|
||||
|
||||
def encrypt(input_string):
|
||||
pub = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArq9XTUSeYr2+N1h3Afl/z8Dse/2yD0ZGrKwx+EEEcdsBLca9Ynmx3nIB5obmLlSfmskLpBo0UACBmB5rEjBp2Q2f3AG3Hjd4B+gNCG6BDaawuDlgANIhGnaTLrIqWrrcm4EMzJOnAOI1fgzJRsOOUEfaS318Eq9OVO3apEyCCt0lOQK6PuksduOjVxtltDav+guVAA068NrPYmRNabVKRNLJpL8w4D44sfth5RvZ3q9t+6RTArpEtc5sh5ChzvqPOzKGMXW83C95TxmXqpbK6olN4RevSfVjEAgCydH6HN6OhtOQEcnrU97r9H0iZOWwbw3pVrZiUkuRD1R56Wzs2wIDAQAB\n-----END PUBLIC KEY-----"
|
||||
pub_key = RSA.importKey(pub)
|
||||
@@ -49,9 +50,6 @@ def encode_to_base64(input_string):
|
||||
return base64_encoded.decode("utf-8")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class RAGFlowCLI(Cmd):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -240,9 +238,9 @@ class RAGFlowCLI(Cmd):
|
||||
print(r"""
|
||||
____ ___ ______________ ________ ____
|
||||
/ __ \/ | / ____/ ____/ /___ _ __ / ____/ / / _/
|
||||
/ /_/ / /| |/ / __/ /_ / / __ \ | /| / / / / / / / /
|
||||
/ _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ / / /___/ /____/ /
|
||||
/_/ |_/_/ |_\____/_/ /_/\____/|__/|__/ \____/_____/___/
|
||||
/ /_/ / /| |/ / __/ /_ / / __ \ | /| / / / / / / / /
|
||||
/ _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ / / /___/ /____/ /
|
||||
/_/ |_/_/ |_\____/_/ /_/\____/|__/|__/ \____/_____/___/
|
||||
""")
|
||||
self.cmdloop()
|
||||
|
||||
@@ -254,15 +252,13 @@ class RAGFlowCLI(Cmd):
|
||||
result = self.parse_command(command)
|
||||
self.execute_command(result)
|
||||
|
||||
|
||||
def parse_connection_args(self, args: List[str]) -> Dict[str, Any]:
|
||||
parser = argparse.ArgumentParser(description="RAGFlow CLI Client", add_help=False)
|
||||
parser.add_argument("-h", "--host", default="127.0.0.1", help="Admin or RAGFlow service host")
|
||||
parser.add_argument("-p", "--port", type=int, default=9381, help="Admin or RAGFlow service port")
|
||||
parser.add_argument("-w", "--password", default="admin", type=str, help="Superuser password")
|
||||
parser.add_argument("-t", "--type", default="admin", type=str, help="CLI mode, admin or user")
|
||||
parser.add_argument("-u", "--username", default=None,
|
||||
help="Username (email). In admin mode defaults to admin@ragflow.io, in user mode required.")
|
||||
parser.add_argument("-u", "--username", default=None, help="Username (email). In admin mode defaults to admin@ragflow.io, in user mode required.")
|
||||
parser.add_argument("command", nargs="?", help="Single command")
|
||||
try:
|
||||
parsed_args, remaining_args = parser.parse_known_args(args)
|
||||
@@ -274,7 +270,7 @@ class RAGFlowCLI(Cmd):
|
||||
|
||||
if remaining_args:
|
||||
if remaining_args[0] == "command":
|
||||
command_str = ' '.join(remaining_args[1:]) + ';'
|
||||
command_str = " ".join(remaining_args[1:]) + ";"
|
||||
auth = True
|
||||
if remaining_args[1] == "register":
|
||||
auth = False
|
||||
@@ -282,28 +278,14 @@ class RAGFlowCLI(Cmd):
|
||||
if username is None:
|
||||
print("Error: username (-u) is required in user mode")
|
||||
return {"error": "Username required"}
|
||||
return {
|
||||
"host": parsed_args.host,
|
||||
"port": parsed_args.port,
|
||||
"password": parsed_args.password,
|
||||
"type": parsed_args.type,
|
||||
"username": username,
|
||||
"command": command_str,
|
||||
"auth": auth
|
||||
}
|
||||
return {"host": parsed_args.host, "port": parsed_args.port, "password": parsed_args.password, "type": parsed_args.type, "username": username, "command": command_str, "auth": auth}
|
||||
else:
|
||||
return {"error": "Invalid command"}
|
||||
else:
|
||||
auth = True
|
||||
if username is None:
|
||||
auth = False
|
||||
return {
|
||||
"host": parsed_args.host,
|
||||
"port": parsed_args.port,
|
||||
"type": parsed_args.type,
|
||||
"username": username,
|
||||
"auth": auth
|
||||
}
|
||||
return {"host": parsed_args.host, "port": parsed_args.port, "type": parsed_args.type, "username": username, "auth": auth}
|
||||
except SystemExit:
|
||||
return {"error": "Invalid connection arguments"}
|
||||
|
||||
@@ -321,6 +303,7 @@ class RAGFlowCLI(Cmd):
|
||||
# print(f"Parsed command: {command_dict}")
|
||||
run_command(self.ragflow_client, command_dict)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
cli = RAGFlowCLI()
|
||||
|
||||
@@ -71,6 +71,7 @@ class RAGFlowClient:
|
||||
user_password: str = command.get("password")
|
||||
if not user_password:
|
||||
import getpass
|
||||
|
||||
user_password = getpass.getpass("Password: ")
|
||||
try:
|
||||
token = login_user(self.http_client, self.server_type, email, user_password)
|
||||
@@ -86,8 +87,7 @@ class RAGFlowClient:
|
||||
def ping_server(self, command):
|
||||
iterations = command.get("iterations", 1)
|
||||
if iterations > 1:
|
||||
response = self.http_client.request("GET", "/system/ping", use_api_base=True, auth_kind="web",
|
||||
iterations=iterations)
|
||||
response = self.http_client.request("GET", "/system/ping", use_api_base=True, auth_kind="web", iterations=iterations)
|
||||
return response
|
||||
else:
|
||||
response = self.http_client.request("GET", "/system/ping", use_api_base=True, auth_kind="web")
|
||||
@@ -106,8 +106,7 @@ class RAGFlowClient:
|
||||
enc_password = encrypt_password(password)
|
||||
print(f"Register user: {nickname}, email: {username}, password: ******")
|
||||
payload = {"email": username, "nickname": nickname, "password": enc_password}
|
||||
response = self.http_client.request(method="POST", path="/users",
|
||||
json_body=payload, use_api_base=True, auth_kind="web")
|
||||
response = self.http_client.request(method="POST", path="/users", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
@@ -135,8 +134,7 @@ class RAGFlowClient:
|
||||
|
||||
service_id: int = command["number"]
|
||||
|
||||
response = self.http_client.request("GET", f"/admin/services/{service_id}", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/services/{service_id}", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
res_data = res_json["data"]
|
||||
@@ -226,9 +224,7 @@ class RAGFlowClient:
|
||||
password_tree: Tree = command["password"]
|
||||
password: str = password_tree.children[0].strip("'\"")
|
||||
print(f"Alter user: {user_name}, password: ******")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/password",
|
||||
json_body={"new_password": encrypt_password(password)}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/password", json_body={"new_password": encrypt_password(password)}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
@@ -247,9 +243,7 @@ class RAGFlowClient:
|
||||
print(f"Create user: {user_name}, password: ******, role: {role}")
|
||||
# enpass1 = encrypt(password)
|
||||
enc_password = encrypt_password(password)
|
||||
response = self.http_client.request(method="POST", path="/admin/users",
|
||||
json_body={"username": user_name, "password": enc_password, "role": role},
|
||||
use_api_base=True, auth_kind="admin")
|
||||
response = self.http_client.request(method="POST", path="/admin/users", json_body={"username": user_name, "password": enc_password, "role": role}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -266,9 +260,7 @@ class RAGFlowClient:
|
||||
activate_status: str = activate_tree.children[0].strip("'\"")
|
||||
if activate_status.lower() in ["on", "off"]:
|
||||
print(f"Alter user {user_name} activate status, turn {activate_status.lower()}.")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/activate",
|
||||
json_body={"activate_status": activate_status}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/activate", json_body={"activate_status": activate_status}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
@@ -283,14 +275,12 @@ class RAGFlowClient:
|
||||
|
||||
user_name_tree: Tree = command["user_name"]
|
||||
user_name: str = user_name_tree.children[0].strip("'\"")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/admin", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name}/admin", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to grant {user_name} admin authorization, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to grant {user_name} admin authorization, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def revoke_admin(self, command):
|
||||
if self.server_type != "admin":
|
||||
@@ -298,14 +288,12 @@ class RAGFlowClient:
|
||||
|
||||
user_name_tree: Tree = command["user_name"]
|
||||
user_name: str = user_name_tree.children[0].strip("'\"")
|
||||
response = self.http_client.request("DELETE", f"/admin/users/{user_name}/admin", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("DELETE", f"/admin/users/{user_name}/admin", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to revoke {user_name} admin authorization, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to revoke {user_name} admin authorization, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def create_role(self, command):
|
||||
if self.server_type != "admin":
|
||||
@@ -319,10 +307,7 @@ class RAGFlowClient:
|
||||
desc_str = desc_tree.children[0].strip("'\"")
|
||||
|
||||
print(f"create role name: {role_name}, description: {desc_str}")
|
||||
response = self.http_client.request("POST", "/admin/roles",
|
||||
json_body={"role_name": role_name, "description": desc_str},
|
||||
use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("POST", "/admin/roles", json_body={"role_name": role_name, "description": desc_str}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -336,9 +321,7 @@ class RAGFlowClient:
|
||||
role_name_tree: Tree = command["role_name"]
|
||||
role_name: str = role_name_tree.children[0].strip("'\"")
|
||||
print(f"drop role name: {role_name}")
|
||||
response = self.http_client.request("DELETE", f"/admin/roles/{role_name}",
|
||||
use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("DELETE", f"/admin/roles/{role_name}", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -355,24 +338,18 @@ class RAGFlowClient:
|
||||
desc_str: str = desc_tree.children[0].strip("'\"")
|
||||
|
||||
print(f"alter role name: {role_name}, description: {desc_str}")
|
||||
response = self.http_client.request("PUT", f"/admin/roles/{role_name}",
|
||||
json_body={"description": desc_str},
|
||||
use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", f"/admin/roles/{role_name}", json_body={"description": desc_str}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to update role {role_name} with description: {desc_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to update role {role_name} with description: {desc_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def list_roles(self, command):
|
||||
if self.server_type != "admin":
|
||||
print("This command is only allowed in ADMIN mode")
|
||||
|
||||
response = self.http_client.request("GET", "/admin/roles",
|
||||
use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", "/admin/roles", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -386,9 +363,7 @@ class RAGFlowClient:
|
||||
role_name_tree: Tree = command["role_name"]
|
||||
role_name: str = role_name_tree.children[0].strip("'\"")
|
||||
print(f"show role: {role_name}")
|
||||
response = self.http_client.request("GET", f"/admin/roles/{role_name}/permission",
|
||||
use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/roles/{role_name}/permission", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -409,15 +384,12 @@ class RAGFlowClient:
|
||||
action_str: str = action_tree.children[0].strip("'\"")
|
||||
actions.append(action_str)
|
||||
print(f"grant role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
|
||||
response = self.http_client.request("POST", f"/admin/roles/{role_name_str}/permission",
|
||||
json_body={"actions": actions, "resource": resource_str}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("POST", f"/admin/roles/{role_name_str}/permission", json_body={"actions": actions, "resource": resource_str}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to grant role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to grant role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def revoke_permission(self, command):
|
||||
if self.server_type != "admin":
|
||||
@@ -433,15 +405,12 @@ class RAGFlowClient:
|
||||
action_str: str = action_tree.children[0].strip("'\"")
|
||||
actions.append(action_str)
|
||||
print(f"revoke role_name: {role_name_str}, resource: {resource_str}, actions: {actions}")
|
||||
response = self.http_client.request("DELETE", f"/admin/roles/{role_name_str}/permission",
|
||||
json_body={"actions": actions, "resource": resource_str}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("DELETE", f"/admin/roles/{role_name_str}/permission", json_body={"actions": actions, "resource": resource_str}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to revoke role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to revoke role {role_name_str} with {actions} on {resource_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def alter_user_role(self, command):
|
||||
if self.server_type != "admin":
|
||||
@@ -452,15 +421,12 @@ class RAGFlowClient:
|
||||
user_name_tree: Tree = command["user_name"]
|
||||
user_name_str: str = user_name_tree.children[0].strip("'\"")
|
||||
print(f"alter_user_role user_name: {user_name_str}, role_name: {role_name_str}")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name_str}/role",
|
||||
json_body={"role_name": role_name_str}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", f"/admin/users/{user_name_str}/role", json_body={"role_name": role_name_str}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to alter user: {user_name_str} to role {role_name_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to alter user: {user_name_str} to role {role_name_str}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def show_user_permission(self, command):
|
||||
if self.server_type != "admin":
|
||||
@@ -469,14 +435,12 @@ class RAGFlowClient:
|
||||
user_name_tree: Tree = command["user_name"]
|
||||
user_name_str: str = user_name_tree.children[0].strip("'\"")
|
||||
print(f"show_user_permission user_name: {user_name_str}")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name_str}/permission", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name_str}/permission", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to show user: {user_name_str} permission, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to show user: {user_name_str} permission, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def generate_key(self, command: dict[str, Any]) -> None:
|
||||
if self.server_type != "admin":
|
||||
@@ -485,14 +449,12 @@ class RAGFlowClient:
|
||||
username_tree: Tree = command["user_name"]
|
||||
user_name: str = username_tree.children[0].strip("'\"")
|
||||
print(f"Generating API key for user: {user_name}")
|
||||
response = self.http_client.request("POST", f"/admin/users/{user_name}/keys", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("POST", f"/admin/users/{user_name}/keys", use_api_base=True, auth_kind="admin")
|
||||
res_json: dict[str, Any] = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
else:
|
||||
print(
|
||||
f"Failed to generate key for user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Failed to generate key for user {user_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def list_keys(self, command: dict[str, Any]) -> None:
|
||||
if self.server_type != "admin":
|
||||
@@ -501,8 +463,7 @@ class RAGFlowClient:
|
||||
username_tree: Tree = command["user_name"]
|
||||
user_name: str = username_tree.children[0].strip("'\"")
|
||||
print(f"Listing API keys for user: {user_name}")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/keys", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/keys", use_api_base=True, auth_kind="admin")
|
||||
res_json: dict[str, Any] = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -520,8 +481,7 @@ class RAGFlowClient:
|
||||
print(f"Dropping API key for user: {user_name}")
|
||||
# URL encode the key to handle special characters
|
||||
encoded_key: str = urllib.parse.quote(key, safe="")
|
||||
response = self.http_client.request("DELETE", f"/admin/users/{user_name}/keys/{encoded_key}", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("DELETE", f"/admin/users/{user_name}/keys/{encoded_key}", use_api_base=True, auth_kind="admin")
|
||||
res_json: dict[str, Any] = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
@@ -534,23 +494,19 @@ class RAGFlowClient:
|
||||
|
||||
var_name = _strip_tree_value(command["var_name"])
|
||||
var_value = _strip_tree_value(command["var_value"])
|
||||
response = self.http_client.request("PUT", "/admin/variables",
|
||||
json_body={"var_name": var_name, "var_value": var_value}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("PUT", "/admin/variables", json_body={"var_name": var_name, "var_value": var_value}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print(res_json["message"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to set variable {var_name} to {var_value}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to set variable {var_name} to {var_value}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def show_variable(self, command):
|
||||
if self.server_type != "admin":
|
||||
print("This command is only allowed in ADMIN mode")
|
||||
|
||||
var_name = _strip_tree_value(command["var_name"])
|
||||
response = self.http_client.request(method="GET", path="/admin/variables", json_body={"var_name": var_name},
|
||||
use_api_base=True, auth_kind="admin")
|
||||
response = self.http_client.request(method="GET", path="/admin/variables", json_body={"var_name": var_name}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -604,8 +560,7 @@ class RAGFlowClient:
|
||||
if self.server_type != "admin":
|
||||
print("This command is only allowed in ADMIN mode")
|
||||
license = command["license"]
|
||||
response = self.http_client.request("POST", "/admin/license", json_body={"license": license}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("POST", "/admin/license", json_body={"license": license}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print("Set license successfully")
|
||||
@@ -617,9 +572,7 @@ class RAGFlowClient:
|
||||
print("This command is only allowed in ADMIN mode")
|
||||
value1 = command["value1"]
|
||||
value2 = command["value2"]
|
||||
response = self.http_client.request("POST", "/admin/license/config",
|
||||
json_body={"value1": value1, "value2": value2}, use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("POST", "/admin/license/config", json_body={"value1": value1, "value2": value2}, use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
print("Set license successfully")
|
||||
@@ -690,8 +643,7 @@ class RAGFlowClient:
|
||||
user_name: str = username_tree.children[0].strip("'\"")
|
||||
print(f"Listing all datasets of user: {user_name}")
|
||||
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/datasets", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/datasets", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
table_data = res_json["data"]
|
||||
@@ -708,8 +660,7 @@ class RAGFlowClient:
|
||||
username_tree: Tree = command["user_name"]
|
||||
user_name: str = username_tree.children[0].strip("'\"")
|
||||
print(f"Listing all agents of user: {user_name}")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/agents", use_api_base=True,
|
||||
auth_kind="admin")
|
||||
response = self.http_client.request("GET", f"/admin/users/{user_name}/agents", use_api_base=True, auth_kind="admin")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
table_data = res_json["data"]
|
||||
@@ -733,8 +684,7 @@ class RAGFlowClient:
|
||||
|
||||
# Step 1: Add provider
|
||||
provider_payload = {"provider_name": provider_name}
|
||||
provider_response = self.http_client.request("PUT", "/providers", json_body=provider_payload,
|
||||
use_api_base=True, auth_kind="web")
|
||||
provider_response = self.http_client.request("PUT", "/providers", json_body=provider_payload, use_api_base=True, auth_kind="web")
|
||||
provider_res = provider_response.json()
|
||||
if provider_response.status_code == 200 and provider_res.get("code") == 0:
|
||||
print(f"Success to add provider {provider_name}")
|
||||
@@ -747,15 +697,8 @@ class RAGFlowClient:
|
||||
return
|
||||
|
||||
# Step 2: Add instance
|
||||
instance_payload = {
|
||||
"instance_name": "default",
|
||||
"api_key": api_key,
|
||||
"region": "default",
|
||||
"base_url": ""
|
||||
}
|
||||
instance_response = self.http_client.request("POST", f"/providers/{provider_name}/instances",
|
||||
json_body=instance_payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
instance_payload = {"instance_name": "default", "api_key": api_key, "region": "default", "base_url": ""}
|
||||
instance_response = self.http_client.request("POST", f"/providers/{provider_name}/instances", json_body=instance_payload, use_api_base=True, auth_kind="web")
|
||||
instance_res = instance_response.json()
|
||||
if instance_response.status_code == 200 and instance_res.get("code") == 0:
|
||||
print(f"Success to add instance for provider {provider_name}")
|
||||
@@ -771,8 +714,7 @@ class RAGFlowClient:
|
||||
print("This command is only allowed in USER mode")
|
||||
return
|
||||
provider_name: str = command["provider_name"]
|
||||
response = self.http_client.request("DELETE", f"/providers/{provider_name}", use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("DELETE", f"/providers/{provider_name}", use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print(f"Success to drop model provider {provider_name}")
|
||||
@@ -810,8 +752,7 @@ class RAGFlowClient:
|
||||
"model_type": model_type,
|
||||
"model_name": model_name,
|
||||
}
|
||||
response = self.http_client.request("PATCH", "/models/default", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("PATCH", "/models/default", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print(f"Success to set default {model_type} to {model_id}")
|
||||
@@ -830,8 +771,7 @@ class RAGFlowClient:
|
||||
return
|
||||
|
||||
payload = {"model_type": model_type}
|
||||
response = self.http_client.request("PATCH", "/models/default", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("PATCH", "/models/default", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print(f"Success to reset default {model_type}")
|
||||
@@ -861,8 +801,7 @@ class RAGFlowClient:
|
||||
|
||||
iterations = command.get("iterations", 1)
|
||||
if iterations > 1:
|
||||
response = self.http_client.request("GET", "/datasets", use_api_base=True, auth_kind="web",
|
||||
iterations=iterations)
|
||||
response = self.http_client.request("GET", "/datasets", use_api_base=True, auth_kind="web", iterations=iterations)
|
||||
return response
|
||||
else:
|
||||
response = self.http_client.request("GET", "/datasets", use_api_base=True, auth_kind="web")
|
||||
@@ -876,16 +815,12 @@ class RAGFlowClient:
|
||||
def create_user_dataset(self, command):
|
||||
if self.server_type != "user":
|
||||
print("This command is only allowed in USER mode")
|
||||
payload = {
|
||||
"name": command["dataset_name"],
|
||||
"embedding_model": command["embedding"]
|
||||
}
|
||||
payload = {"name": command["dataset_name"], "embedding_model": command["embedding"]}
|
||||
if "parser_id" in command:
|
||||
payload["chunk_method"] = command["parser"]
|
||||
if "pipeline" in command:
|
||||
payload["pipeline_id"] = command["pipeline"]
|
||||
response = self.http_client.request("POST", "/datasets", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("POST", "/datasets", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
self._print_table_simple(res_json["data"])
|
||||
@@ -981,8 +916,7 @@ class RAGFlowClient:
|
||||
|
||||
dataset_ids = [dataset_id for _, dataset_id in valid_datasets]
|
||||
kb_ids_param = ",".join(dataset_ids)
|
||||
response = self.http_client.request("GET", f"/kb/get_meta?kb_ids={kb_ids_param}",
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("GET", f"/kb/get_meta?kb_ids={kb_ids_param}", use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code != 200:
|
||||
print(f"Fail to get metadata, code: {res_json.get('code')}, message: {res_json.get('message')}")
|
||||
@@ -996,11 +930,7 @@ class RAGFlowClient:
|
||||
table_data = []
|
||||
for field_name, values_dict in meta.items():
|
||||
for value, docs in values_dict.items():
|
||||
table_data.append({
|
||||
"field": field_name,
|
||||
"value": value,
|
||||
"doc_ids": ", ".join(docs)
|
||||
})
|
||||
table_data.append({"field": field_name, "value": value, "doc_ids": ", ".join(docs)})
|
||||
self._print_table_simple(table_data)
|
||||
|
||||
def list_user_documents_metadata_summary(self, command_dict):
|
||||
@@ -1018,8 +948,7 @@ class RAGFlowClient:
|
||||
payload = {"kb_id": kb_id}
|
||||
if doc_ids:
|
||||
payload["doc_ids"] = doc_ids
|
||||
response = self.http_client.request("POST", "/document/metadata/summary", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/document/metadata/summary", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
summary = res_json.get("data", {}).get("summary", {})
|
||||
@@ -1086,16 +1015,11 @@ class RAGFlowClient:
|
||||
"quote": True,
|
||||
"keyword": False,
|
||||
"tts": False,
|
||||
"system": "You are an intelligent assistant. Your primary function is to answer questions based strictly on the provided knowledge base.\n\n **Essential Rules:**\n - Your answer must be derived **solely** from this knowledge base: `{knowledge}`.\n - **When information is available**: Summarize the content to give a detailed answer.\n - **When information is unavailable**: Your response must contain this exact sentence: \"The answer you are looking for is not found in the knowledge base!\"\n - **Always consider** the entire conversation history.",
|
||||
"system": 'You are an intelligent assistant. Your primary function is to answer questions based strictly on the provided knowledge base.\n\n **Essential Rules:**\n - Your answer must be derived **solely** from this knowledge base: `{knowledge}`.\n - **When information is available**: Summarize the content to give a detailed answer.\n - **When information is unavailable**: Your response must contain this exact sentence: "The answer you are looking for is not found in the knowledge base!"\n - **Always consider** the entire conversation history.',
|
||||
"refine_multiturn": False,
|
||||
"use_kg": False,
|
||||
"reasoning": False,
|
||||
"parameters": [
|
||||
{
|
||||
"key": "knowledge",
|
||||
"optional": False
|
||||
}
|
||||
],
|
||||
"parameters": [{"key": "knowledge", "optional": False}],
|
||||
"toc_enhance": False,
|
||||
},
|
||||
"similarity_threshold": 0.2,
|
||||
@@ -1136,8 +1060,7 @@ class RAGFlowClient:
|
||||
# Build payload
|
||||
payload = {"kb_id": dataset_id, "vector_size": vector_size}
|
||||
# Call API
|
||||
response = self.http_client.request("POST", "/kb/doc_engine_table", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/kb/doc_engine_table", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print(f"Success to create table for dataset: {dataset_name}")
|
||||
@@ -1155,8 +1078,7 @@ class RAGFlowClient:
|
||||
return
|
||||
# Call API to delete table
|
||||
payload = {"kb_id": dataset_id}
|
||||
response = self.http_client.request("DELETE", "/kb/doc_engine_table", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("DELETE", "/kb/doc_engine_table", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print(f"Success to drop table for dataset: {dataset_name}")
|
||||
@@ -1168,8 +1090,7 @@ class RAGFlowClient:
|
||||
print("This command is only allowed in USER mode")
|
||||
return
|
||||
# Call API to create metadata table
|
||||
response = self.http_client.request("POST", "/tenant/doc_engine_metadata_table",
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/tenant/doc_engine_metadata_table", use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print("Success to create metadata table")
|
||||
@@ -1181,8 +1102,7 @@ class RAGFlowClient:
|
||||
print("This command is only allowed in USER mode")
|
||||
return
|
||||
# Call API to delete metadata table
|
||||
response = self.http_client.request("DELETE", "/tenant/doc_engine_metadata_table",
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("DELETE", "/tenant/doc_engine_metadata_table", use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json.get("code") == 0:
|
||||
print("Success to drop metadata table")
|
||||
@@ -1225,8 +1145,7 @@ class RAGFlowClient:
|
||||
|
||||
def _list_chat_sessions(self, dialog_id):
|
||||
"""List all sessions (conversations) for a given dialog."""
|
||||
response = self.http_client.request("GET", f"/chats/{dialog_id}/conversations", use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("GET", f"/chats/{dialog_id}/conversations", use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json["code"] == 0:
|
||||
return res_json["data"]
|
||||
@@ -1242,14 +1161,12 @@ class RAGFlowClient:
|
||||
if dialog_id is None:
|
||||
return
|
||||
payload = {"name": "New conversation"}
|
||||
response = self.http_client.request("POST", f"/chats/{dialog_id}/conversations", json_body=payload,
|
||||
use_api_base=True, auth_kind="web")
|
||||
response = self.http_client.request("POST", f"/chats/{dialog_id}/conversations", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json["code"] == 0:
|
||||
print(f"Success to create chat session for chat: {chat_name}")
|
||||
else:
|
||||
print(
|
||||
f"Fail to create chat session for chat {chat_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to create chat session for chat {chat_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def drop_chat_session(self, command):
|
||||
if self.server_type != "user":
|
||||
@@ -1270,14 +1187,12 @@ class RAGFlowClient:
|
||||
print(f"Chat session '{session_id}' not found in chat '{chat_name}'")
|
||||
return
|
||||
payload = {"ids": to_drop_session_ids}
|
||||
response = self.http_client.request("DELETE", f"/chats/{dialog_id}/conversations", json_body=payload,
|
||||
use_api_base=True, auth_kind="web")
|
||||
response = self.http_client.request("DELETE", f"/chats/{dialog_id}/conversations", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json["code"] == 0:
|
||||
print(f"Success to drop chat session '{session_id}' from chat: {chat_name}")
|
||||
else:
|
||||
print(
|
||||
f"Fail to drop chat session '{session_id}' from chat {chat_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to drop chat session '{session_id}' from chat {chat_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def list_chat_sessions(self, command):
|
||||
if self.server_type != "user":
|
||||
@@ -1305,13 +1220,9 @@ class RAGFlowClient:
|
||||
|
||||
# Prepare payload for completion API
|
||||
# Note: stream parameter is not sent, server defaults to stream=True
|
||||
payload = {
|
||||
"session_id": session_id,
|
||||
"messages": [{"role": "user", "content": message}]
|
||||
}
|
||||
payload = {"session_id": session_id, "messages": [{"role": "user", "content": message}]}
|
||||
|
||||
response = self.http_client.request("POST", "/chat/completions", json_body=payload,
|
||||
use_api_base=True, auth_kind="web", stream=True)
|
||||
response = self.http_client.request("POST", "/chat/completions", json_body=payload, use_api_base=True, auth_kind="web", stream=True)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Fail to chat on session, status code: {response.status_code}")
|
||||
@@ -1322,17 +1233,16 @@ class RAGFlowClient:
|
||||
for line in response.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
line_str = line.decode('utf-8')
|
||||
if not line_str.startswith('data:'):
|
||||
line_str = line.decode("utf-8")
|
||||
if not line_str.startswith("data:"):
|
||||
continue
|
||||
data_str = line_str[5:].strip()
|
||||
if data_str == '[DONE]':
|
||||
if data_str == "[DONE]":
|
||||
break
|
||||
try:
|
||||
data_json = json.loads(data_str)
|
||||
if data_json.get("code") != 0:
|
||||
print(
|
||||
f"\nFail to chat on session, code: {data_json.get('code')}, message: {data_json.get('message', '')}")
|
||||
print(f"\nFail to chat on session, code: {data_json.get('code')}, message: {data_json.get('message', '')}")
|
||||
return
|
||||
# Check if it's the final message
|
||||
if data_json.get("data") is True:
|
||||
@@ -1416,14 +1326,12 @@ class RAGFlowClient:
|
||||
print(f"Documents {document_names} not found in {dataset_name}")
|
||||
|
||||
payload = {"doc_ids": document_ids, "run": 1}
|
||||
response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json["code"] == 0:
|
||||
print(f"Success to parse {to_parse_doc_names} of {dataset_name}")
|
||||
else:
|
||||
print(
|
||||
f"Fail to parse documents {res_json["data"]["docs"]}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to parse documents {res_json['data']['docs']}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def parse_dataset(self, command_dict):
|
||||
if self.server_type != "user":
|
||||
@@ -1442,8 +1350,7 @@ class RAGFlowClient:
|
||||
document_ids.append(doc["id"])
|
||||
|
||||
payload = {"doc_ids": document_ids, "run": 1}
|
||||
response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200 and res_json["code"] == 0:
|
||||
pass
|
||||
@@ -1483,15 +1390,7 @@ class RAGFlowClient:
|
||||
encoder = MultipartEncoder(fields=fields)
|
||||
headers = {"Content-Type": encoder.content_type}
|
||||
response = self.http_client.request(
|
||||
"POST",
|
||||
f"/datasets/{dataset_id}/documents?return_raw_files=true",
|
||||
headers=headers,
|
||||
data=encoder,
|
||||
json_body=None,
|
||||
params=None,
|
||||
stream=False,
|
||||
auth_kind="web",
|
||||
use_api_base=True
|
||||
"POST", f"/datasets/{dataset_id}/documents?return_raw_files=true", headers=headers, data=encoder, json_body=None, params=None, stream=False, auth_kind="web", use_api_base=True
|
||||
)
|
||||
res = response.json()
|
||||
if res.get("code") == 0:
|
||||
@@ -1526,22 +1425,18 @@ class RAGFlowClient:
|
||||
}
|
||||
iterations = command_dict.get("iterations", 1)
|
||||
if iterations > 1:
|
||||
response = self.http_client.request("POST", "/retrieval", json_body=payload, use_api_base=True,
|
||||
auth_kind="web", iterations=iterations)
|
||||
response = self.http_client.request("POST", "/retrieval", json_body=payload, use_api_base=True, auth_kind="web", iterations=iterations)
|
||||
return response
|
||||
else:
|
||||
response = self.http_client.request("POST", "/retrieval", json_body=payload, use_api_base=True,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("POST", "/retrieval", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
self._print_table_simple(res_json["data"]["chunks"])
|
||||
else:
|
||||
print(
|
||||
f"Fail to search datasets: {dataset_names}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to search datasets: {dataset_names}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
else:
|
||||
print(
|
||||
f"Fail to search datasets: {dataset_names}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to search datasets: {dataset_names}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
|
||||
def get_chunk(self, command_dict):
|
||||
if self.server_type != "user":
|
||||
@@ -1549,8 +1444,7 @@ class RAGFlowClient:
|
||||
return
|
||||
|
||||
chunk_id = command_dict["chunk_id"]
|
||||
response = self.http_client.request("GET", f"/chunk/get?chunk_id={chunk_id}", use_api_base=False,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("GET", f"/chunk/get?chunk_id={chunk_id}", use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
@@ -1568,8 +1462,7 @@ class RAGFlowClient:
|
||||
|
||||
file_path = command_dict["file_path"]
|
||||
payload = {"file_path": file_path}
|
||||
response = self.http_client.request("POST", "/kb/insert_from_file", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/kb/insert_from_file", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
@@ -1589,8 +1482,7 @@ class RAGFlowClient:
|
||||
|
||||
file_path = command_dict["file_path"]
|
||||
payload = {"file_path": file_path}
|
||||
response = self.http_client.request("POST", "/tenant/insert_metadata_from_file", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/tenant/insert_metadata_from_file", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
@@ -1617,8 +1509,7 @@ class RAGFlowClient:
|
||||
return
|
||||
|
||||
# Get doc_id from chunk_id via GET /chunk/get
|
||||
response = self.http_client.request("GET", f"/chunk/get?chunk_id={chunk_id}", use_api_base=False,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("GET", f"/chunk/get?chunk_id={chunk_id}", use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code != 200:
|
||||
print(f"Fail to get chunk info, code: {res_json.get('code')}, message: {res_json.get('message')}")
|
||||
@@ -1655,14 +1546,8 @@ class RAGFlowClient:
|
||||
else:
|
||||
print(f"Fail to update chunk, HTTP {response.status_code}")
|
||||
|
||||
def _get_documents_by_ids(self, ids:list[str]):
|
||||
response = self.http_client.request(
|
||||
"POST",
|
||||
"/document/infos",
|
||||
json_body={"doc_ids": ids},
|
||||
use_api_base=False,
|
||||
auth_kind="web"
|
||||
)
|
||||
def _get_documents_by_ids(self, ids: list[str]):
|
||||
response = self.http_client.request("POST", "/document/infos", json_body={"doc_ids": ids}, use_api_base=False, auth_kind="web")
|
||||
|
||||
if response.status_code != 200:
|
||||
return f"Fail to get document info, HTTP {response.status_code}", None
|
||||
@@ -1687,6 +1572,7 @@ class RAGFlowClient:
|
||||
|
||||
# Parse JSON string to dict
|
||||
import json
|
||||
|
||||
try:
|
||||
meta_fields = json.loads(meta_json_str)
|
||||
except json.JSONDecodeError as e:
|
||||
@@ -1713,13 +1599,7 @@ class RAGFlowClient:
|
||||
"meta_fields": meta_fields,
|
||||
}
|
||||
|
||||
response = self.http_client.request(
|
||||
"PATCH",
|
||||
f"/datasets/{dataset_id}/documents/{doc_id}",
|
||||
json_body=payload,
|
||||
use_api_base=True,
|
||||
auth_kind="web"
|
||||
)
|
||||
response = self.http_client.request("PATCH", f"/datasets/{dataset_id}/documents/{doc_id}", json_body=payload, use_api_base=True, auth_kind="web")
|
||||
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
@@ -1747,8 +1627,7 @@ class RAGFlowClient:
|
||||
"tags": tags,
|
||||
}
|
||||
|
||||
response = self.http_client.request("POST", f"/kb/{dataset_id}/rm_tags", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", f"/kb/{dataset_id}/rm_tags", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json.get("code") == 0:
|
||||
@@ -1771,8 +1650,7 @@ class RAGFlowClient:
|
||||
elif command_dict.get("chunk_ids"):
|
||||
payload["chunk_ids"] = command_dict["chunk_ids"]
|
||||
|
||||
response = self.http_client.request("POST", "/chunk/rm", json_body=payload,
|
||||
use_api_base=False, auth_kind="web")
|
||||
response = self.http_client.request("POST", "/chunk/rm", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json.get("code") == 0:
|
||||
@@ -1803,15 +1681,14 @@ class RAGFlowClient:
|
||||
if "available_int" in command_dict:
|
||||
payload["available_int"] = command_dict["available_int"]
|
||||
|
||||
response = self.http_client.request("POST", "/chunk/list", json_body=payload, use_api_base=False,
|
||||
auth_kind="web")
|
||||
response = self.http_client.request("POST", "/chunk/list", json_body=payload, use_api_base=False, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code == 200:
|
||||
if res_json["code"] == 0:
|
||||
chunks = res_json["data"]["chunks"]
|
||||
if chunks:
|
||||
for i, chunk in enumerate(chunks):
|
||||
print(f"\n--- Chunk {i+1} ---")
|
||||
print(f"\n--- Chunk {i + 1} ---")
|
||||
for key, value in chunk.items():
|
||||
print(f" {key}: {value}")
|
||||
else:
|
||||
@@ -1845,7 +1722,7 @@ class RAGFlowClient:
|
||||
all_done = True
|
||||
for doc in docs:
|
||||
if doc.get("run") != "DONE":
|
||||
print(f"Document {doc["name"]} is not done, status: {doc.get("run")}")
|
||||
print(f"Document {doc['name']} is not done, status: {doc.get('run')}")
|
||||
all_done = False
|
||||
break
|
||||
if all_done:
|
||||
@@ -1856,16 +1733,10 @@ class RAGFlowClient:
|
||||
|
||||
def _list_documents(self, dataset_name: str, dataset_id: str):
|
||||
# Use the new RESTful API: GET /api/v1/datasets/<dataset_id>/documents
|
||||
response = self.http_client.request(
|
||||
"GET",
|
||||
f"/datasets/{dataset_id}/documents",
|
||||
use_api_base=True,
|
||||
auth_kind="web"
|
||||
)
|
||||
response = self.http_client.request("GET", f"/datasets/{dataset_id}/documents", use_api_base=True, auth_kind="web")
|
||||
res_json = response.json()
|
||||
if response.status_code != 200:
|
||||
print(
|
||||
f"Fail to list files from dataset {dataset_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
print(f"Fail to list files from dataset {dataset_name}, code: {res_json['code']}, message: {res_json['message']}")
|
||||
return None
|
||||
return res_json["data"]["docs"]
|
||||
|
||||
@@ -2254,22 +2125,14 @@ def run_benchmark(client: RAGFlowClient, command_dict: dict):
|
||||
total_duration = result["duration"]
|
||||
qps = iterations / total_duration if total_duration > 0 else None
|
||||
print(f"command: {command}, Concurrency: {concurrency}, iterations: {iterations}")
|
||||
print(
|
||||
f"total duration: {total_duration:.4f}s, QPS: {qps}, COMMAND_COUNT: {iterations}, SUCCESS: {success_count}, FAILURE: {iterations - success_count}")
|
||||
print(f"total duration: {total_duration:.4f}s, QPS: {qps}, COMMAND_COUNT: {iterations}, SUCCESS: {success_count}, FAILURE: {iterations - success_count}")
|
||||
pass
|
||||
else:
|
||||
results: List[Optional[dict]] = [None] * concurrency
|
||||
mp_context = mp.get_context("spawn")
|
||||
start_time = time.perf_counter()
|
||||
with ProcessPoolExecutor(max_workers=concurrency, mp_context=mp_context) as executor:
|
||||
future_map = {
|
||||
executor.submit(
|
||||
run_command,
|
||||
client,
|
||||
command
|
||||
): idx
|
||||
for idx in range(concurrency)
|
||||
}
|
||||
future_map = {executor.submit(run_command, client, command): idx for idx in range(concurrency)}
|
||||
for future in as_completed(future_map):
|
||||
idx = future_map[future]
|
||||
results[idx] = future.result()
|
||||
@@ -2291,7 +2154,6 @@ def run_benchmark(client: RAGFlowClient, command_dict: dict):
|
||||
total_command_count = iterations * concurrency
|
||||
qps = total_command_count / total_duration if total_duration > 0 else None
|
||||
print(f"command: {command}, Concurrency: {concurrency} , iterations: {iterations}")
|
||||
print(
|
||||
f"total duration: {total_duration:.4f}s, QPS: {qps}, COMMAND_COUNT: {total_command_count}, SUCCESS: {success_count}, FAILURE: {total_command_count - success_count}")
|
||||
print(f"total duration: {total_duration:.4f}s, QPS: {qps}, COMMAND_COUNT: {total_command_count}, SUCCESS: {success_count}, FAILURE: {total_command_count - success_count}")
|
||||
|
||||
pass
|
||||
|
||||
@@ -29,6 +29,7 @@ def encrypt_password(password_plain: str) -> str:
|
||||
import base64
|
||||
from Cryptodome.PublicKey import RSA
|
||||
from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5
|
||||
|
||||
def crypt(line):
|
||||
"""
|
||||
decrypt(crypt(input_string)) == base64(input_string), which frontend and ragflow_cli use.
|
||||
@@ -36,13 +37,11 @@ def encrypt_password(password_plain: str) -> str:
|
||||
pub = "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEArq9XTUSeYr2+N1h3Afl/z8Dse/2yD0ZGrKwx+EEEcdsBLca9Ynmx3nIB5obmLlSfmskLpBo0UACBmB5rEjBp2Q2f3AG3Hjd4B+gNCG6BDaawuDlgANIhGnaTLrIqWrrcm4EMzJOnAOI1fgzJRsOOUEfaS318Eq9OVO3apEyCCt0lOQK6PuksduOjVxtltDav+guVAA068NrPYmRNabVKRNLJpL8w4D44sfth5RvZ3q9t+6RTArpEtc5sh5ChzvqPOzKGMXW83C95TxmXqpbK6olN4RevSfVjEAgCydH6HN6OhtOQEcnrU97r9H0iZOWwbw3pVrZiUkuRD1R56Wzs2wIDAQAB\n-----END PUBLIC KEY-----"
|
||||
rsa_key = RSA.importKey(pub)
|
||||
cipher = Cipher_pkcs1_v1_5.new(rsa_key)
|
||||
password_base64 = base64.b64encode(line.encode('utf-8')).decode("utf-8")
|
||||
password_base64 = base64.b64encode(line.encode("utf-8")).decode("utf-8")
|
||||
encrypted_password = cipher.encrypt(password_base64.encode())
|
||||
return base64.b64encode(encrypted_password).decode('utf-8')
|
||||
return base64.b64encode(encrypted_password).decode("utf-8")
|
||||
except Exception as exc:
|
||||
raise AuthException(
|
||||
"Password encryption unavailable; install pycryptodomex (uv sync --python 3.13 --group test)."
|
||||
) from exc
|
||||
raise AuthException("Password encryption unavailable; install pycryptodomex (uv sync --python 3.13 --group test).") from exc
|
||||
return crypt(password_plain)
|
||||
|
||||
|
||||
|
||||
2
admin/client/uv.lock
generated
2
admin/client/uv.lock
generated
@@ -156,7 +156,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ragflow-cli"
|
||||
version = "0.26.3"
|
||||
version = "0.26.4"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "beartype" },
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#
|
||||
|
||||
import time
|
||||
|
||||
start_ts = time.time()
|
||||
|
||||
import os
|
||||
@@ -38,26 +39,24 @@ from common.versions import get_ragflow_version
|
||||
|
||||
stop_event = threading.Event()
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
faulthandler.enable()
|
||||
init_root_logger("admin_service")
|
||||
logging.info(r"""
|
||||
____ ___ ______________ ___ __ _
|
||||
/ __ \/ | / ____/ ____/ /___ _ __ / | ____/ /___ ___ (_)___
|
||||
____ ___ ______________ ___ __ _
|
||||
/ __ \/ | / ____/ ____/ /___ _ __ / | ____/ /___ ___ (_)___
|
||||
/ /_/ / /| |/ / __/ /_ / / __ \ | /| / / / /| |/ __ / __ `__ \/ / __ \
|
||||
/ _, _/ ___ / /_/ / __/ / / /_/ / |/ |/ / / ___ / /_/ / / / / / / / / / /
|
||||
/_/ |_/_/ |_\____/_/ /_/\____/|__/|__/ /_/ |_\__,_/_/ /_/ /_/_/_/ /_/
|
||||
/_/ |_/_/ |_\____/_/ /_/\____/|__/|__/ /_/ |_\__,_/_/ /_/ /_/_/_/ /_/
|
||||
""")
|
||||
|
||||
app = Flask(__name__)
|
||||
app.register_blueprint(admin_bp)
|
||||
app.config["SESSION_PERMANENT"] = False
|
||||
app.config["SESSION_TYPE"] = "filesystem"
|
||||
app.config["MAX_CONTENT_LENGTH"] = int(
|
||||
os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
|
||||
)
|
||||
app.config["MAX_CONTENT_LENGTH"] = int(os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024))
|
||||
Session(app)
|
||||
logging.info(f'RAGFlow admin version: {get_ragflow_version()}')
|
||||
logging.info(f"RAGFlow admin version: {get_ragflow_version()}")
|
||||
show_configs()
|
||||
login_manager = LoginManager()
|
||||
login_manager.init_app(app)
|
||||
|
||||
@@ -70,9 +70,7 @@ def setup_auth(login_manager):
|
||||
logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
|
||||
return None
|
||||
|
||||
user = UserService.query(
|
||||
access_token=access_token, status=StatusEnum.VALID.value
|
||||
)
|
||||
user = UserService.query(access_token=access_token, status=StatusEnum.VALID.value)
|
||||
if user:
|
||||
if not user[0].access_token or not user[0].access_token.strip():
|
||||
logging.warning(f"User {user[0].email} has empty access_token in database")
|
||||
@@ -123,22 +121,16 @@ def add_tenant_for_admin(user_info: dict, role: str):
|
||||
"embd_id": settings.EMBEDDING_MDL,
|
||||
"asr_id": settings.ASR_MDL,
|
||||
"parser_ids": settings.PARSERS,
|
||||
"img2txt_id": settings.IMAGE2TEXT_MDL,
|
||||
"img2txt_id": settings.VISION_MDL,
|
||||
"rerank_id": settings.RERANK_MDL,
|
||||
}
|
||||
usr_tenant = {
|
||||
"tenant_id": user_info["id"],
|
||||
"user_id": user_info["id"],
|
||||
"invited_by": user_info["id"],
|
||||
"role": role
|
||||
}
|
||||
usr_tenant = {"tenant_id": user_info["id"], "user_id": user_info["id"], "invited_by": user_info["id"], "role": role}
|
||||
|
||||
# tenant_llm = get_init_tenant_llm(user_info["id"])
|
||||
TenantService.insert(**tenant)
|
||||
UserTenantService.insert(**usr_tenant)
|
||||
# TenantLLMService.insert_many(tenant_llm)
|
||||
logging.info(
|
||||
f"Added tenant for email: {user_info['email']}, A default tenant has been set; changing the default models after login is strongly recommended.")
|
||||
logging.info(f"Added tenant for email: {user_info['email']}, A default tenant has been set; changing the default models after login is strongly recommended.")
|
||||
|
||||
|
||||
def check_admin_auth(func):
|
||||
@@ -160,13 +152,13 @@ def check_admin_auth(func):
|
||||
def login_admin(email: str, password: str):
|
||||
"""
|
||||
:param email: admin email
|
||||
:param password: string before decrypt
|
||||
:param password: string before decrypt (RSA encrypted + base64 encoded)
|
||||
"""
|
||||
users = UserService.query(email=email)
|
||||
if not users:
|
||||
raise UserNotFoundError(email)
|
||||
psw = decrypt(password)
|
||||
user = UserService.query_user(email, psw)
|
||||
decrypted = decrypt(password)
|
||||
user = UserService.query_user(email, decrypted)
|
||||
if not user:
|
||||
raise AdminException("Email and password do not match!")
|
||||
if not user.is_superuser:
|
||||
@@ -212,28 +204,17 @@ def login_verify(f):
|
||||
@wraps(f)
|
||||
def decorated(*args, **kwargs):
|
||||
auth = request.authorization
|
||||
if not auth or 'username' not in auth.parameters or 'password' not in auth.parameters:
|
||||
return jsonify({
|
||||
"code": 401,
|
||||
"message": "Authentication required",
|
||||
"data": None
|
||||
}), 200
|
||||
if not auth or "username" not in auth.parameters or "password" not in auth.parameters:
|
||||
return jsonify({"code": 401, "message": "Authentication required", "data": None}), 200
|
||||
|
||||
username = auth.parameters['username']
|
||||
password = auth.parameters['password']
|
||||
username = auth.parameters["username"]
|
||||
password = auth.parameters["password"]
|
||||
try:
|
||||
if not check_admin(username, password):
|
||||
return jsonify({
|
||||
"code": 500,
|
||||
"message": "Access denied",
|
||||
"data": None
|
||||
}), 200
|
||||
return jsonify({"code": 500, "message": "Access denied", "data": None}), 200
|
||||
except Exception:
|
||||
logging.exception("An error occurred during admin login verification.")
|
||||
return jsonify({
|
||||
"code": 500,
|
||||
"message": "An internal server error occurred."
|
||||
}), 200
|
||||
return jsonify({"code": 500, "message": "An internal server error occurred."}), 200
|
||||
|
||||
return f(*args, **kwargs)
|
||||
|
||||
|
||||
@@ -34,8 +34,7 @@ class BaseConfig(BaseModel):
|
||||
detail_func_name: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port,
|
||||
'service_type': self.service_type}
|
||||
return {"id": self.id, "name": self.name, "host": self.host, "port": self.port, "service_type": self.service_type}
|
||||
|
||||
|
||||
class ServiceConfigs:
|
||||
@@ -63,11 +62,11 @@ class MetaConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['meta_type'] = self.meta_type
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["meta_type"] = self.meta_type
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -77,21 +76,20 @@ class MySQLConfig(MetaConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['username'] = self.username
|
||||
extra_dict['password'] = self.password
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["username"] = self.username
|
||||
extra_dict["password"] = self.password
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
class PostgresConfig(MetaConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
return result
|
||||
|
||||
|
||||
@@ -100,11 +98,11 @@ class RetrievalConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['retrieval_type'] = self.retrieval_type
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["retrieval_type"] = self.retrieval_type
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -113,11 +111,11 @@ class InfinityConfig(RetrievalConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['db_name'] = self.db_name
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["db_name"] = self.db_name
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -127,12 +125,12 @@ class ElasticsearchConfig(RetrievalConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['username'] = self.username
|
||||
extra_dict['password'] = self.password
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["username"] = self.username
|
||||
extra_dict["password"] = self.password
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -141,11 +139,11 @@ class MessageQueueConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['mq_type'] = self.mq_type
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["mq_type"] = self.mq_type
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -155,30 +153,28 @@ class RedisConfig(MessageQueueConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['database'] = self.database
|
||||
extra_dict['password'] = self.password
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["database"] = self.database
|
||||
extra_dict["password"] = self.password
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
class RabbitMQConfig(MessageQueueConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
return result
|
||||
|
||||
|
||||
class RAGFlowServerConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
return result
|
||||
|
||||
|
||||
@@ -187,9 +183,9 @@ class TaskExecutorConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
result['extra']['message_queue_type'] = self.message_queue_type
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
result["extra"]["message_queue_type"] = self.message_queue_type
|
||||
return result
|
||||
|
||||
|
||||
@@ -198,11 +194,11 @@ class FileStoreConfig(BaseConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['store_type'] = self.store_type
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["store_type"] = self.store_type
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -212,12 +208,12 @@ class MinioConfig(FileStoreConfig):
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = super().to_dict()
|
||||
if 'extra' not in result:
|
||||
result['extra'] = dict()
|
||||
extra_dict = result['extra'].copy()
|
||||
extra_dict['user'] = self.user
|
||||
extra_dict['password'] = self.password
|
||||
result['extra'] = extra_dict
|
||||
if "extra" not in result:
|
||||
result["extra"] = dict()
|
||||
extra_dict = result["extra"].copy()
|
||||
extra_dict["user"] = self.user
|
||||
extra_dict["password"] = self.password
|
||||
result["extra"] = extra_dict
|
||||
return result
|
||||
|
||||
|
||||
@@ -229,106 +225,105 @@ def load_configurations(config_path: str) -> list[BaseConfig]:
|
||||
for k, v in raw_configs.items():
|
||||
match k:
|
||||
case "ragflow":
|
||||
name: str = f'ragflow_{ragflow_count}'
|
||||
host: str = v['host']
|
||||
http_port: int = v['http_port']
|
||||
config = RAGFlowServerConfig(id=id_count, name=name, host=host, port=http_port,
|
||||
service_type="ragflow_server",
|
||||
detail_func_name="check_ragflow_server_alive")
|
||||
name: str = f"ragflow_{ragflow_count}"
|
||||
host: str = v["host"]
|
||||
http_port: int = v["http_port"]
|
||||
config = RAGFlowServerConfig(id=id_count, name=name, host=host, port=http_port, service_type="ragflow_server", detail_func_name="check_ragflow_server_alive")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "es":
|
||||
name: str = 'elasticsearch'
|
||||
url = v['hosts']
|
||||
name: str = "elasticsearch"
|
||||
url = v["hosts"]
|
||||
parsed = urlparse(url)
|
||||
host: str = parsed.hostname
|
||||
port: int = parsed.port
|
||||
username: str = v.get('username')
|
||||
password: str = v.get('password')
|
||||
config = ElasticsearchConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval",
|
||||
retrieval_type="elasticsearch",
|
||||
username=username, password=password,
|
||||
detail_func_name="get_es_cluster_stats")
|
||||
username: str = v.get("username")
|
||||
password: str = v.get("password")
|
||||
config = ElasticsearchConfig(
|
||||
id=id_count,
|
||||
name=name,
|
||||
host=host,
|
||||
port=port,
|
||||
service_type="retrieval",
|
||||
retrieval_type="elasticsearch",
|
||||
username=username,
|
||||
password=password,
|
||||
detail_func_name="get_es_cluster_stats",
|
||||
)
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
|
||||
case "infinity":
|
||||
name: str = 'infinity'
|
||||
url = v['uri']
|
||||
parts = url.split(':', 1)
|
||||
name: str = "infinity"
|
||||
url = v["uri"]
|
||||
parts = url.split(":", 1)
|
||||
host = parts[0]
|
||||
port = int(parts[1])
|
||||
database: str = v.get('db_name', 'default_db')
|
||||
config = InfinityConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval",
|
||||
retrieval_type="infinity",
|
||||
db_name=database, detail_func_name="get_infinity_status")
|
||||
database: str = v.get("db_name", "default_db")
|
||||
config = InfinityConfig(id=id_count, name=name, host=host, port=port, service_type="retrieval", retrieval_type="infinity", db_name=database, detail_func_name="get_infinity_status")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "minio_0":
|
||||
name: str = 'minio_0'
|
||||
url = v['host']
|
||||
parts = url.split(':', 1)
|
||||
name: str = "minio_0"
|
||||
url = v["host"]
|
||||
parts = url.split(":", 1)
|
||||
host = parts[0]
|
||||
port = int(parts[1])
|
||||
user = v.get('user')
|
||||
password = v.get('password')
|
||||
config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password,
|
||||
service_type="file_store",
|
||||
store_type="minio", detail_func_name="check_minio_alive")
|
||||
user = v.get("user")
|
||||
password = v.get("password")
|
||||
config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password, service_type="file_store", store_type="minio", detail_func_name="check_minio_alive")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "minio":
|
||||
name: str = 'minio'
|
||||
url = v['host']
|
||||
parts = url.split(':', 1)
|
||||
name: str = "minio"
|
||||
url = v["host"]
|
||||
parts = url.split(":", 1)
|
||||
host = parts[0]
|
||||
port = int(parts[1])
|
||||
user = v.get('user')
|
||||
password = v.get('password')
|
||||
config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password,
|
||||
service_type="file_store",
|
||||
store_type="minio", detail_func_name="check_minio_alive")
|
||||
user = v.get("user")
|
||||
password = v.get("password")
|
||||
config = MinioConfig(id=id_count, name=name, host=host, port=port, user=user, password=password, service_type="file_store", store_type="minio", detail_func_name="check_minio_alive")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "redis":
|
||||
name: str = 'redis'
|
||||
url = v['host']
|
||||
parts = url.split(':', 1)
|
||||
name: str = "redis"
|
||||
url = v["host"]
|
||||
parts = url.split(":", 1)
|
||||
host = parts[0]
|
||||
port = int(parts[1])
|
||||
password = v.get('password')
|
||||
db: int = v.get('db')
|
||||
config = RedisConfig(id=id_count, name=name, host=host, port=port, password=password, database=db,
|
||||
service_type="message_queue", mq_type="redis", detail_func_name="get_redis_info")
|
||||
password = v.get("password")
|
||||
db: int = v.get("db")
|
||||
config = RedisConfig(id=id_count, name=name, host=host, port=port, password=password, database=db, service_type="message_queue", mq_type="redis", detail_func_name="get_redis_info")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "mysql":
|
||||
name: str = 'mysql'
|
||||
host: str = v.get('host')
|
||||
port: int = v.get('port')
|
||||
username = v.get('user')
|
||||
password = v.get('password')
|
||||
config = MySQLConfig(id=id_count, name=name, host=host, port=port, username=username, password=password,
|
||||
service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status")
|
||||
name: str = "mysql"
|
||||
host: str = v.get("host")
|
||||
port: int = v.get("port")
|
||||
username = v.get("user")
|
||||
password = v.get("password")
|
||||
config = MySQLConfig(
|
||||
id=id_count, name=name, host=host, port=port, username=username, password=password, service_type="meta_data", meta_type="mysql", detail_func_name="get_mysql_status"
|
||||
)
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "admin":
|
||||
pass
|
||||
case "task_executor":
|
||||
name: str = 'task_executor'
|
||||
host: str = v.get('host', '')
|
||||
port: int = v.get('port', 0)
|
||||
message_queue_type: str = v.get('message_queue_type')
|
||||
config = TaskExecutorConfig(id=id_count, name=name, host=host, port=port, message_queue_type=message_queue_type,
|
||||
service_type="task_executor", detail_func_name="check_task_executor_alive")
|
||||
name: str = "task_executor"
|
||||
host: str = v.get("host", "")
|
||||
port: int = v.get("port", 0)
|
||||
message_queue_type: str = v.get("message_queue_type")
|
||||
config = TaskExecutorConfig(
|
||||
id=id_count, name=name, host=host, port=port, message_queue_type=message_queue_type, service_type="task_executor", detail_func_name="check_task_executor_alive"
|
||||
)
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case "rabbitmq":
|
||||
name: str = 'rabbitmq'
|
||||
host: str = v.get('host')
|
||||
port: int = v.get('port')
|
||||
config = RabbitMQConfig(id=id_count, name=name, host=host, port=port,
|
||||
service_type="message_queue", mq_type="rabbitmq", detail_func_name="check_rabbitmq_alive")
|
||||
name: str = "rabbitmq"
|
||||
host: str = v.get("host")
|
||||
port: int = v.get("port")
|
||||
config = RabbitMQConfig(id=id_count, name=name, host=host, port=port, service_type="message_queue", mq_type="rabbitmq", detail_func_name="check_rabbitmq_alive")
|
||||
configurations.append(config)
|
||||
id_count += 1
|
||||
case _:
|
||||
|
||||
@@ -4,14 +4,17 @@ class AdminException(Exception):
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
class UserNotFoundError(AdminException):
|
||||
def __init__(self, username):
|
||||
super().__init__(f"User '{username}' not found", 404)
|
||||
|
||||
|
||||
class UserAlreadyExistsError(AdminException):
|
||||
def __init__(self, username):
|
||||
super().__init__(f"User '{username}' already exists", 409)
|
||||
|
||||
|
||||
class CannotDeleteAdminError(AdminException):
|
||||
def __init__(self):
|
||||
super().__init__("Cannot delete admin account", 403)
|
||||
super().__init__("Cannot delete admin account", 403)
|
||||
|
||||
@@ -17,16 +17,8 @@ from flask import jsonify
|
||||
|
||||
|
||||
def success_response(data=None, message="Success", code=0):
|
||||
return jsonify({
|
||||
"code": code,
|
||||
"message": message,
|
||||
"data": data
|
||||
}), 200
|
||||
return jsonify({"code": code, "message": message, "data": data}), 200
|
||||
|
||||
|
||||
def error_response(message="Error", code=-1, data=None):
|
||||
return jsonify({
|
||||
"code": code,
|
||||
"message": message,
|
||||
"data": data
|
||||
}), 400
|
||||
return jsonify({"code": code, "message": message, "data": data}), 400
|
||||
|
||||
@@ -153,6 +153,8 @@ def change_password(username):
|
||||
def alter_user_activate_status(username):
|
||||
try:
|
||||
data = request.get_json()
|
||||
if current_user.email == username:
|
||||
return error_response(f"can't alter current user status: {username}", 409)
|
||||
if not data or "activate_status" not in data:
|
||||
return error_response("Activation status is required", 400)
|
||||
activate_status = data["activate_status"]
|
||||
|
||||
@@ -489,10 +489,7 @@ class SandboxMgr:
|
||||
"""List all available sandbox providers."""
|
||||
result = []
|
||||
for provider_id, metadata in SandboxMgr.PROVIDER_REGISTRY.items():
|
||||
result.append({
|
||||
"id": provider_id,
|
||||
**metadata
|
||||
})
|
||||
result.append({"id": provider_id, **metadata})
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
@@ -635,6 +632,7 @@ class SandboxMgr:
|
||||
config_json = json.dumps(config)
|
||||
SettingsMgr.update_by_name(f"sandbox.{provider_type}", config_json)
|
||||
from agent.sandbox.client import reload_provider
|
||||
|
||||
reload_provider()
|
||||
|
||||
return {"provider_type": provider_type, "config": config}
|
||||
@@ -727,11 +725,7 @@ def main() -> dict:
|
||||
# Build detailed result message
|
||||
success = execution_result.exit_code == 0 and "TEST_PASSED" in execution_result.stdout
|
||||
|
||||
message_parts = [
|
||||
f"Test {success and 'PASSED' or 'FAILED'}",
|
||||
f"Exit code: {execution_result.exit_code}",
|
||||
f"Execution time: {execution_result.execution_time:.2f}s"
|
||||
]
|
||||
message_parts = [f"Test {success and 'PASSED' or 'FAILED'}", f"Exit code: {execution_result.exit_code}", f"Execution time: {execution_result.execution_time:.2f}s"]
|
||||
|
||||
if execution_result.stdout.strip():
|
||||
stdout_preview = execution_result.stdout.strip()[:200]
|
||||
@@ -751,12 +745,13 @@ def main() -> dict:
|
||||
"execution_time": execution_result.execution_time,
|
||||
"stdout": execution_result.stdout,
|
||||
"stderr": execution_result.stderr,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
except AdminException:
|
||||
raise
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
error_details = traceback.format_exc()
|
||||
raise AdminException(f"Connection test failed: {str(e)}\\n\\nStack trace:\\n{error_details}")
|
||||
|
||||
@@ -25,18 +25,19 @@ import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
from typing import Any, Union, Tuple
|
||||
from typing import Any, Tuple, Union
|
||||
|
||||
from agent.component import component_class
|
||||
from agent.component.base import ComponentBase
|
||||
from agent.dsl_migration import normalize_chunker_dsl
|
||||
from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type
|
||||
from api.db.services.file_service import FileService
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.services.task_service import has_canceled
|
||||
from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type
|
||||
from common.constants import LLMType
|
||||
from common.misc_utils import get_uuid, hash_str2int
|
||||
from common.llm_request_context import set_llm_request_context, reset_llm_request_context
|
||||
from common.exceptions import TaskCanceledException
|
||||
from common.misc_utils import get_uuid, hash_str2int
|
||||
from common.token_utils import token_usage_sink, langfuse_run_attrs
|
||||
from rag.prompts.generator import chunks_format
|
||||
from rag.utils.redis_conn import REDIS_CONN
|
||||
@@ -161,6 +162,7 @@ class Graph:
|
||||
|
||||
def close(self):
|
||||
from common.mcp_tool_call_conn import MCPToolCallSession
|
||||
|
||||
seen = set()
|
||||
for cpn in self.components.values():
|
||||
obj = cpn.get("obj")
|
||||
@@ -198,7 +200,9 @@ class Graph:
|
||||
return self._tenant_id
|
||||
|
||||
def get_value_with_variable(self, value: str) -> Any:
|
||||
pat = re.compile(r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*")
|
||||
# Reference the canonical pre-compiled regex from ComponentBase so
|
||||
# the source-pattern and the runtime-pattern can never drift apart.
|
||||
pat = ComponentBase.variable_ref_patt_re
|
||||
out_parts = []
|
||||
last = 0
|
||||
|
||||
@@ -228,7 +232,13 @@ class Graph:
|
||||
exp = exp.strip("{").strip("}").strip(" ").strip("{").strip("}")
|
||||
if exp.find("@") < 0:
|
||||
return self.globals[exp]
|
||||
cpn_id, var_nm = exp.split("@")
|
||||
# Split from the left with maxsplit=1 so the trailing var_nm can
|
||||
# legitimately contain '@' characters (defensive: although the
|
||||
# upstream regex in `get_value_with_variable` constrains `var_nm`
|
||||
# to `[A-Za-z0-9_.-]+`, direct callers of this method may pass
|
||||
# any string and should not raise `ValueError: too many values
|
||||
# to unpack`). `cpn_id` is system-generated and never contains '@'.
|
||||
cpn_id, var_nm = exp.split("@", 1)
|
||||
cpn = self.get_component(cpn_id)
|
||||
if not cpn:
|
||||
raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
|
||||
@@ -275,7 +285,10 @@ class Graph:
|
||||
if exp.find("@") < 0:
|
||||
self.globals[exp] = value
|
||||
return
|
||||
cpn_id, var_nm = exp.split("@")
|
||||
# See `get_variable_value` above for rationale on `maxsplit=1`.
|
||||
# Without it, a var_nm containing '@' would raise
|
||||
# `ValueError: too many values to unpack` instead of being preserved.
|
||||
cpn_id, var_nm = exp.split("@", 1)
|
||||
cpn = self.get_component(cpn_id)
|
||||
if not cpn:
|
||||
raise Exception(f"Can't find variable: '{cpn_id}@{var_nm}'")
|
||||
@@ -428,6 +441,14 @@ class Canvas(Graph):
|
||||
_lf_attrs["session_id"] = str(_session_id)[:200]
|
||||
sink_token = token_usage_sink.set(self._run_token_usage)
|
||||
attrs_token = langfuse_run_attrs.set(_lf_attrs)
|
||||
# Forward the originating session/user to upstream LLM providers (as the
|
||||
# OpenAI `user` field) for the duration of this run, and reset afterwards so
|
||||
# the value never leaks to later calls in the same task. Reuse the same
|
||||
# session/user already derived above so both integrations stay consistent.
|
||||
_req_ctx_token = set_llm_request_context(
|
||||
session_id=_session_id,
|
||||
user_id=_user_id,
|
||||
)
|
||||
try:
|
||||
async for ev in self._run_impl(**kwargs):
|
||||
yield ev
|
||||
@@ -444,6 +465,7 @@ class Canvas(Graph):
|
||||
except ValueError:
|
||||
logging.debug("Failed to reset Langfuse run attributes ContextVar", exc_info=True)
|
||||
langfuse_run_attrs.set(None)
|
||||
reset_llm_request_context(_req_ctx_token)
|
||||
|
||||
async def _run_impl(self, **kwargs):
|
||||
self.globals["sys.date"] = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
@@ -455,7 +477,15 @@ class Canvas(Graph):
|
||||
path_set = set(self.path)
|
||||
for k, cpn in self.components.items():
|
||||
if k in path_set:
|
||||
self.components[k]["obj"].reset(True)
|
||||
# Begin is intentionally kept as `only_output=True` to preserve existing behavior.
|
||||
# (Begin/UserFillUp may populate `_param.inputs` during invocation; we leave that unchanged here.)
|
||||
# All other path components must clear both
|
||||
# inputs and outputs so the next run resolves refs against
|
||||
# this run's runtime values (e.g. Await-response capture
|
||||
# propagating to a downstream Agent's user_prompt), not
|
||||
# against stale values from the previous canvas run.
|
||||
is_begin = self.components[k]["obj"].component_name.lower() == "begin"
|
||||
self.components[k]["obj"].reset(only_output=is_begin)
|
||||
|
||||
if kwargs.get("webhook_payload"):
|
||||
for k, cpn in self.components.items():
|
||||
@@ -529,11 +559,13 @@ class Canvas(Graph):
|
||||
if use_async:
|
||||
await cpn_obj.invoke_async(**(call_kwargs or {}))
|
||||
return
|
||||
# run_in_executor does not propagate context variables; copy the
|
||||
# current context so the token usage sink / Langfuse attributes set
|
||||
# by run() remain visible to LLMBundle calls inside sync components.
|
||||
ctx = contextvars.copy_context()
|
||||
await loop.run_in_executor(self._thread_pool, lambda: ctx.run(partial(sync_fn, **(call_kwargs or {}))))
|
||||
# run_in_executor does not carry context variables into the worker
|
||||
# thread; copy the current context so the LLM request context (the
|
||||
# `user` forwarding), token usage sink, and Langfuse attributes set
|
||||
# by run() remain visible to sync components.
|
||||
bound_call = partial(sync_fn, **(call_kwargs or {}))
|
||||
call_ctx = contextvars.copy_context()
|
||||
await loop.run_in_executor(self._thread_pool, partial(call_ctx.run, bound_call))
|
||||
|
||||
i = f
|
||||
while i < t:
|
||||
|
||||
@@ -22,8 +22,9 @@ from typing import Dict, Type
|
||||
_package_path = os.path.dirname(__file__)
|
||||
__all_classes: Dict[str, Type] = {}
|
||||
|
||||
|
||||
def _import_submodules() -> None:
|
||||
for filename in os.listdir(_package_path): # noqa: F821
|
||||
for filename in os.listdir(_package_path): # noqa: F821
|
||||
if filename.startswith("__") or not filename.endswith(".py") or filename.startswith("base"):
|
||||
continue
|
||||
module_name = filename[:-3]
|
||||
@@ -34,13 +35,14 @@ def _import_submodules() -> None:
|
||||
except ImportError as e:
|
||||
print(f"Warning: Failed to import module {module_name}: {str(e)}")
|
||||
|
||||
|
||||
def _extract_classes_from_module(module: ModuleType) -> None:
|
||||
for name, obj in inspect.getmembers(module):
|
||||
if (inspect.isclass(obj) and
|
||||
obj.__module__ == module.__name__ and not name.startswith("_")):
|
||||
if inspect.isclass(obj) and obj.__module__ == module.__name__ and not name.startswith("_"):
|
||||
__all_classes[name] = obj
|
||||
globals()[name] = obj
|
||||
|
||||
|
||||
_import_submodules()
|
||||
|
||||
__all__ = list(__all_classes.keys()) + ["__all_classes"]
|
||||
|
||||
@@ -27,7 +27,7 @@ import json_repair
|
||||
|
||||
from agent.component.llm import LLM, LLMParam
|
||||
from agent.tools.base import LLMToolPluginCallSession, ToolBase, ToolMeta, ToolParamBase
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
|
||||
from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.services.mcp_server_service import MCPServerService
|
||||
from common.connection_utils import timeout
|
||||
@@ -82,9 +82,9 @@ class Agent(LLM, ToolBase):
|
||||
original_name = cpn.get_meta()["function"]["name"]
|
||||
indexed_name = f"{original_name}_{idx}"
|
||||
self.tools[indexed_name] = cpn
|
||||
model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
model_type = "chat" if "chat" in model_types else model_types[0]
|
||||
chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
chat_model_config = resolve_model_config(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
self.chat_mdl = LLMBundle(
|
||||
self._canvas.get_tenant_id(),
|
||||
chat_model_config,
|
||||
|
||||
@@ -15,20 +15,21 @@
|
||||
#
|
||||
|
||||
import asyncio
|
||||
import builtins
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from abc import ABC
|
||||
import builtins
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
from typing import Any, List, Union
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from agent import settings
|
||||
from common.connection_utils import timeout
|
||||
|
||||
|
||||
|
||||
from common.misc_utils import thread_pool_exec
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
@@ -101,6 +102,7 @@ class ComponentParamBase(ABC):
|
||||
return None
|
||||
logging.warning("ComponentParamBase.__str__: JSON fallback via str() for type=%s", type(obj).__name__)
|
||||
return str(obj)
|
||||
|
||||
return json.dumps(self.as_dict(), ensure_ascii=False, default=_serialize_default)
|
||||
|
||||
def as_dict(self):
|
||||
@@ -135,15 +137,11 @@ class ComponentParamBase(ABC):
|
||||
update_from_raw_conf = conf.get(_IS_RAW_CONF, True)
|
||||
if update_from_raw_conf:
|
||||
deprecated_params_set = self._get_or_init_deprecated_params_set()
|
||||
feeded_deprecated_params_set = (
|
||||
self._get_or_init_feeded_deprecated_params_set()
|
||||
)
|
||||
feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set()
|
||||
user_feeded_params_set = self._get_or_init_user_feeded_params_set()
|
||||
setattr(self, _IS_RAW_CONF, False)
|
||||
else:
|
||||
feeded_deprecated_params_set = (
|
||||
self._get_or_init_feeded_deprecated_params_set(conf)
|
||||
)
|
||||
feeded_deprecated_params_set = self._get_or_init_feeded_deprecated_params_set(conf)
|
||||
user_feeded_params_set = self._get_or_init_user_feeded_params_set(conf)
|
||||
|
||||
def _recursive_update_param(param, config, depth, prefix):
|
||||
@@ -179,15 +177,11 @@ class ComponentParamBase(ABC):
|
||||
|
||||
else:
|
||||
# recursive set obj attr
|
||||
sub_params = _recursive_update_param(
|
||||
attr, config_value, depth + 1, prefix=f"{prefix}{config_key}."
|
||||
)
|
||||
sub_params = _recursive_update_param(attr, config_value, depth + 1, prefix=f"{prefix}{config_key}.")
|
||||
setattr(param, config_key, sub_params)
|
||||
|
||||
if not allow_redundant and redundant_attrs:
|
||||
raise ValueError(
|
||||
f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`"
|
||||
)
|
||||
raise ValueError(f"cpn `{getattr(self, '_name', type(self))}` has redundant parameters: `{[redundant_attrs]}`")
|
||||
|
||||
return param
|
||||
|
||||
@@ -218,9 +212,7 @@ class ComponentParamBase(ABC):
|
||||
param_validation_path_prefix = home_dir + "/param_validation/"
|
||||
|
||||
param_name = type(self).__name__
|
||||
param_validation_path = "/".join(
|
||||
[param_validation_path_prefix, param_name + ".json"]
|
||||
)
|
||||
param_validation_path = "/".join([param_validation_path_prefix, param_name + ".json"])
|
||||
|
||||
validation_json = None
|
||||
|
||||
@@ -253,11 +245,7 @@ class ComponentParamBase(ABC):
|
||||
break
|
||||
|
||||
if not value_legal:
|
||||
raise ValueError(
|
||||
"Please check runtime conf, {} = {} does not match user-parameter restriction".format(
|
||||
variable, value
|
||||
)
|
||||
)
|
||||
raise ValueError("Please check runtime conf, {} = {} does not match user-parameter restriction".format(variable, value))
|
||||
|
||||
elif variable in validation_json:
|
||||
self._validate_param(attr, validation_json)
|
||||
@@ -335,11 +323,7 @@ class ComponentParamBase(ABC):
|
||||
def _range(value, ranges):
|
||||
in_range = False
|
||||
for left_limit, right_limit in ranges:
|
||||
if (
|
||||
left_limit - settings.FLOAT_ZERO
|
||||
<= value
|
||||
<= right_limit + settings.FLOAT_ZERO
|
||||
):
|
||||
if left_limit - settings.FLOAT_ZERO <= value <= right_limit + settings.FLOAT_ZERO:
|
||||
in_range = True
|
||||
break
|
||||
|
||||
@@ -355,16 +339,11 @@ class ComponentParamBase(ABC):
|
||||
|
||||
def _warn_deprecated_param(self, param_name, description):
|
||||
if self._deprecated_params_set.get(param_name):
|
||||
logging.warning(
|
||||
f"{description} {param_name} is deprecated and ignored in this version."
|
||||
)
|
||||
logging.warning(f"{description} {param_name} is deprecated and ignored in this version.")
|
||||
|
||||
def _warn_to_deprecate_param(self, param_name, description, new_param):
|
||||
if self._deprecated_params_set.get(param_name):
|
||||
logging.warning(
|
||||
f"{description} {param_name} will be deprecated in future release; "
|
||||
f"please use {new_param} instead."
|
||||
)
|
||||
logging.warning(f"{description} {param_name} will be deprecated in future release; please use {new_param} instead.")
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -372,8 +351,14 @@ class ComponentParamBase(ABC):
|
||||
class ComponentBase(ABC):
|
||||
component_name: str
|
||||
thread_limiter = asyncio.Semaphore(int(os.environ.get("MAX_CONCURRENT_CHATS", 10)))
|
||||
variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
|
||||
# Match `cpn_id@var_nm` / `sys.var_nm` / `env.var_nm` style template refs.
|
||||
# `cpn_id` allows underscores (frontend ids like `userfillup_abc`,
|
||||
# `retrieval_xyz`) and colons (legacy DSL ids like `UserFillUp:CateInput`,
|
||||
# `Retrieval:KBSearch`).
|
||||
variable_ref_patt = r"\{* *\{([a-zA-Z0-9_:]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
|
||||
variable_ref_patt_re = re.compile(variable_ref_patt, flags=re.IGNORECASE | re.DOTALL)
|
||||
iteration_alias_patt = r"\{* *\{(item|index|result)\} *\}*"
|
||||
iteration_alias_patt_re = re.compile(iteration_alias_patt, flags=re.IGNORECASE | re.DOTALL)
|
||||
|
||||
def __str__(self):
|
||||
"""
|
||||
@@ -385,9 +370,7 @@ class ComponentBase(ABC):
|
||||
return """{{
|
||||
"component_name": "{}",
|
||||
"params": {}
|
||||
}}""".format(self.component_name,
|
||||
self._param
|
||||
)
|
||||
}}""".format(self.component_name, self._param)
|
||||
|
||||
def __init__(self, canvas, id, param: ComponentParamBase):
|
||||
from agent.canvas import Graph # Local import to avoid cyclic dependency
|
||||
@@ -403,7 +386,7 @@ class ComponentBase(ABC):
|
||||
|
||||
def check_if_canceled(self, message: str = "") -> bool:
|
||||
if self.is_canceled():
|
||||
task_id = getattr(self._canvas, 'task_id', 'unknown')
|
||||
task_id = getattr(self._canvas, "task_id", "unknown")
|
||||
log_message = f"Task {task_id} has been canceled"
|
||||
if message:
|
||||
log_message += f" during {message}"
|
||||
@@ -491,7 +474,9 @@ class ComponentBase(ABC):
|
||||
input_elements = self.get_input_elements()
|
||||
_logger.debug(
|
||||
"[Base] Component '%s' (%s) resolving inputs. Input element keys: %s",
|
||||
self._id, self.component_name, list(input_elements.keys()),
|
||||
self._id,
|
||||
self.component_name,
|
||||
list(input_elements.keys()),
|
||||
)
|
||||
for var, o in input_elements.items():
|
||||
v = self.get_param(var)
|
||||
@@ -502,9 +487,9 @@ class ComponentBase(ABC):
|
||||
resolved = self._canvas.get_variable_value(v)
|
||||
self.set_input_value(var, resolved)
|
||||
_logger.debug("[Base] var '%s': resolved ref '%s' -> %s", var, v, json.dumps(resolved, ensure_ascii=False, default=str)[:200])
|
||||
elif isinstance(v, str) and re.search(self.variable_ref_patt, v):
|
||||
elif isinstance(v, str) and self.variable_ref_patt_re.search(v):
|
||||
elements = self.get_input_elements_from_text(v)
|
||||
kv = {k: e.get('value', '') for k, e in elements.items()}
|
||||
kv = {k: e.get("value", "") for k, e in elements.items()}
|
||||
self.set_input_value(var, self.string_format(v, kv))
|
||||
_logger.debug("[Base] var '%s': resolved text refs '%s' -> %s", var, v, json.dumps(kv, ensure_ascii=False, default=str)[:200])
|
||||
else:
|
||||
@@ -538,16 +523,21 @@ class ComponentBase(ABC):
|
||||
|
||||
def get_input_elements_from_text(self, txt: str) -> dict[str, dict[str, str]]:
|
||||
res = {}
|
||||
for r in re.finditer(self.variable_ref_patt, txt, flags=re.IGNORECASE | re.DOTALL):
|
||||
for r in self.variable_ref_patt_re.finditer(txt):
|
||||
exp = r.group(1)
|
||||
cpn_id, var_nm = exp.split("@") if exp.find("@") > 0 else ("", exp)
|
||||
# Use maxsplit=1 to be defensive: although `exp` here comes
|
||||
# from `variable_ref_patt` (which constrains `var_nm` to
|
||||
# `[A-Za-z0-9_.-]+`), a future regex relaxation or a non-
|
||||
# pattern caller should not raise `ValueError: too many values
|
||||
# to unpack` if the trailing part happens to contain '@'.
|
||||
cpn_id, var_nm = exp.split("@", 1) if exp.find("@") > 0 else ("", exp)
|
||||
res[exp] = {
|
||||
"name": (self._canvas.get_component_name(cpn_id) + f"@{var_nm}") if cpn_id else exp,
|
||||
"value": self._canvas.get_variable_value(exp),
|
||||
"_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references") if cpn_id else None,
|
||||
"_cpn_id": cpn_id
|
||||
"_cpn_id": cpn_id,
|
||||
}
|
||||
for r in re.finditer(self.iteration_alias_patt, txt, flags=re.IGNORECASE | re.DOTALL):
|
||||
for r in self.iteration_alias_patt_re.finditer(txt):
|
||||
exp = r.group(1)
|
||||
if exp in res:
|
||||
continue
|
||||
@@ -559,7 +549,7 @@ class ComponentBase(ABC):
|
||||
"name": (self._canvas.get_component_name(cpn_id) + f"@{var_nm}"),
|
||||
"value": self._canvas.get_variable_value(ref),
|
||||
"_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references"),
|
||||
"_cpn_id": cpn_id
|
||||
"_cpn_id": cpn_id,
|
||||
}
|
||||
return res
|
||||
|
||||
@@ -601,33 +591,27 @@ class ComponentBase(ABC):
|
||||
return self._canvas.get_component(pid)["obj"]
|
||||
|
||||
def get_upstream(self) -> List[str]:
|
||||
cpn_nms = self._canvas.get_component(self._id)['upstream']
|
||||
cpn_nms = self._canvas.get_component(self._id)["upstream"]
|
||||
return cpn_nms
|
||||
|
||||
def get_downstream(self) -> List[str]:
|
||||
cpn_nms = self._canvas.get_component(self._id)['downstream']
|
||||
cpn_nms = self._canvas.get_component(self._id)["downstream"]
|
||||
return cpn_nms
|
||||
|
||||
@staticmethod
|
||||
def string_format(content: str, kv: dict[str, str]) -> str:
|
||||
for n, v in kv.items():
|
||||
|
||||
def repl(_match, val=v):
|
||||
return str(val) if val is not None else ""
|
||||
|
||||
content = re.sub(
|
||||
r"\{%s\}" % re.escape(n),
|
||||
repl,
|
||||
content
|
||||
)
|
||||
content = re.sub(r"\{%s\}" % re.escape(n), repl, content)
|
||||
return content
|
||||
|
||||
def exception_handler(self):
|
||||
if not self._param.exception_method:
|
||||
return None
|
||||
return {
|
||||
"goto": self._param.exception_goto,
|
||||
"default_value": self._param.exception_default_value
|
||||
}
|
||||
return {"goto": self._param.exception_goto, "default_value": self._param.exception_default_value}
|
||||
|
||||
def get_exception_default_value(self):
|
||||
if self._param.exception_method != "comment":
|
||||
|
||||
@@ -17,17 +17,17 @@ from agent.component.fillup import UserFillUpParam, UserFillUp
|
||||
|
||||
|
||||
class BeginParam(UserFillUpParam):
|
||||
|
||||
"""
|
||||
Define the Begin component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.mode = "conversational"
|
||||
self.prologue = "Hi! I'm your smart assistant. What can I do for you?"
|
||||
|
||||
def check(self):
|
||||
self.check_valid_value(self.mode, "The 'mode' should be either `conversational` or `task`", ["conversational", "task","Webhook"])
|
||||
self.check_valid_value(self.mode, "The 'mode' should be either `conversational` or `task`", ["conversational", "task", "Webhook"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return getattr(self, "inputs")
|
||||
|
||||
@@ -33,7 +33,7 @@ from urllib.request import Request, urlopen
|
||||
from agent.component.base import ComponentBase
|
||||
from agent.component.llm import LLMParam
|
||||
from api.db import FileType
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
|
||||
from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type
|
||||
from api.db.services import duplicate_name
|
||||
from api.db.services.file_service import FileService
|
||||
from api.utils.file_utils import filename_type
|
||||
@@ -371,9 +371,7 @@ class Browser(ComponentBase, ABC):
|
||||
sort_keys=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
raw_canvas_id = (
|
||||
f"dsl_{hashlib.sha1(graph_text.encode('utf-8')).hexdigest()[:12]}"
|
||||
)
|
||||
raw_canvas_id = f"dsl_{hashlib.sha1(graph_text.encode('utf-8')).hexdigest()[:12]}"
|
||||
canvas_id = self._safe_path_segment(raw_canvas_id)
|
||||
node_id = self._safe_path_segment(self._id)
|
||||
return os.path.join(root, tenant, canvas_id, node_id)
|
||||
@@ -402,9 +400,9 @@ class Browser(ComponentBase, ABC):
|
||||
def _build_browser_llm(self):
|
||||
from browser_use.llm import ChatBrowserUse, ChatOpenAI
|
||||
|
||||
chat_model_config = get_model_config_from_provider_instance(
|
||||
chat_model_config = resolve_model_config(
|
||||
self._canvas.get_tenant_id(),
|
||||
get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id),
|
||||
resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id),
|
||||
self._param.llm_id,
|
||||
)
|
||||
cfg = self._as_model_config_dict(chat_model_config)
|
||||
@@ -488,10 +486,7 @@ class Browser(ComponentBase, ABC):
|
||||
# Keep browser-use watchdog fallback in sync with our resolved path.
|
||||
os.environ["BROWSER_USE_BROWSER_BINARY_PATH"] = executable_path
|
||||
else:
|
||||
logging.warning(
|
||||
"Browser no local browser executable found. "
|
||||
"Set BROWSER_USE_EXECUTABLE_PATH or preinstall chromium in image to avoid runtime playwright install."
|
||||
)
|
||||
logging.warning("Browser no local browser executable found. Set BROWSER_USE_EXECUTABLE_PATH or preinstall chromium in image to avoid runtime playwright install.")
|
||||
if profile_dir:
|
||||
browser_kwargs["user_data_dir"] = profile_dir
|
||||
# browser-use expects profile_directory to be a profile name
|
||||
@@ -682,21 +677,13 @@ class Browser(ComponentBase, ABC):
|
||||
try:
|
||||
self._prepare_input_values()
|
||||
user_prompt = self._resolve_text(kwargs.get("prompts", self._param.prompts))
|
||||
with tempfile.TemporaryDirectory(prefix="browser_use_upload_") as upload_dir, tempfile.TemporaryDirectory(
|
||||
prefix="browser_use_download_"
|
||||
) as download_dir:
|
||||
with tempfile.TemporaryDirectory(prefix="browser_use_upload_") as upload_dir, tempfile.TemporaryDirectory(prefix="browser_use_download_") as download_dir:
|
||||
uploaded_files = self._prepare_upload_files(upload_dir)
|
||||
|
||||
upload_lines = [
|
||||
f"- file_id={item['file_id']}, name={item['name']}, local_path={item['local_path']}"
|
||||
for item in uploaded_files
|
||||
]
|
||||
upload_lines = [f"- file_id={item['file_id']}, name={item['name']}, local_path={item['local_path']}" for item in uploaded_files]
|
||||
task_text = user_prompt
|
||||
if upload_lines:
|
||||
task_text += (
|
||||
"\n\nYou can upload files from these local paths when operating web pages:\n"
|
||||
+ "\n".join(upload_lines)
|
||||
)
|
||||
task_text += "\n\nYou can upload files from these local paths when operating web pages:\n" + "\n".join(upload_lines)
|
||||
|
||||
upload_local_paths = [item.get("local_path", "") for item in uploaded_files if item.get("local_path")]
|
||||
if persist_session:
|
||||
@@ -707,11 +694,7 @@ class Browser(ComponentBase, ABC):
|
||||
profile_dir = tempfile.mkdtemp(prefix="browser_use_profile_")
|
||||
except OSError:
|
||||
profile_dir = None
|
||||
history = asyncio.run(
|
||||
self._run_browser_use_async(
|
||||
task_text, download_dir, upload_local_paths, profile_dir
|
||||
)
|
||||
)
|
||||
history = asyncio.run(self._run_browser_use_async(task_text, download_dir, upload_local_paths, profile_dir))
|
||||
target_dir_id = FileService.get_root_folder(self._canvas.get_tenant_id())["id"]
|
||||
downloaded_files = self._save_downloads(download_dir, target_dir_id)
|
||||
|
||||
|
||||
@@ -21,17 +21,17 @@ from abc import ABC
|
||||
|
||||
from common.constants import LLMType
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance
|
||||
from api.db.joint_services.tenant_model_service import resolve_model_config
|
||||
from agent.component.llm import LLMParam, LLM
|
||||
from common.connection_utils import timeout
|
||||
from rag.llm.chat_model import ERROR_PREFIX
|
||||
|
||||
|
||||
class CategorizeParam(LLMParam):
|
||||
|
||||
"""
|
||||
Define the categorize component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.category_description = {}
|
||||
@@ -50,12 +50,7 @@ class CategorizeParam(LLMParam):
|
||||
raise ValueError(f"[Categorize] 'To' of category {k} can not be empty!")
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"query": {
|
||||
"type": "line",
|
||||
"name": "Query"
|
||||
}
|
||||
}
|
||||
return {"query": {"type": "line", "name": "Query"}}
|
||||
|
||||
def update_prompt(self):
|
||||
cate_lines = []
|
||||
@@ -63,13 +58,12 @@ class CategorizeParam(LLMParam):
|
||||
for line in desc.get("examples", []):
|
||||
if not line:
|
||||
continue
|
||||
cate_lines.append("USER: \"" + re.sub(r"\n", " ", line, flags=re.DOTALL) + "\" → "+c)
|
||||
cate_lines.append('USER: "' + re.sub(r"\n", " ", line, flags=re.DOTALL) + '" → ' + c)
|
||||
|
||||
descriptions = []
|
||||
for c, desc in self.category_description.items():
|
||||
if desc.get("description"):
|
||||
descriptions.append(
|
||||
"\n------\nCategory: {}\nDescription: {}".format(c, desc["description"]))
|
||||
descriptions.append("\n------\nCategory: {}\nDescription: {}".format(c, desc["description"]))
|
||||
|
||||
self.sys_prompt = """
|
||||
You are an advanced classification system that categorizes user questions into specific types. Analyze the input question and classify it into ONE of the following categories:
|
||||
@@ -84,10 +78,7 @@ Here's description of each category:
|
||||
- Return only the category name without explanations
|
||||
- Use "Other" only when no other category fits
|
||||
|
||||
""".format(
|
||||
"\n - ".join(list(self.category_description.keys())),
|
||||
"\n".join(descriptions)
|
||||
)
|
||||
""".format("\n - ".join(list(self.category_description.keys())), "\n".join(descriptions))
|
||||
|
||||
if cate_lines:
|
||||
self.sys_prompt += """
|
||||
@@ -106,7 +97,7 @@ class Categorize(LLM, ABC):
|
||||
logging.warning(f"[Categorize] input element not detected for query key: {query_key}")
|
||||
return elements
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
async def _invoke_async(self, **kwargs):
|
||||
if self.check_if_canceled("Categorize processing"):
|
||||
return
|
||||
@@ -124,13 +115,13 @@ class Categorize(LLM, ABC):
|
||||
msg[-1]["content"] = query_value
|
||||
self.set_input_value(query_key, msg[-1]["content"])
|
||||
self._param.update_prompt()
|
||||
chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
|
||||
chat_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
|
||||
chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config)
|
||||
|
||||
user_prompt = """
|
||||
---- Real Data ----
|
||||
{} →
|
||||
""".format(" | ".join(["{}: \"{}\"".format(c["role"].upper(), re.sub(r"\n", "", c["content"], flags=re.DOTALL)) for c in msg]))
|
||||
""".format(" | ".join(['{}: "{}"'.format(c["role"].upper(), re.sub(r"\n", "", c["content"], flags=re.DOTALL)) for c in msg]))
|
||||
|
||||
if self.check_if_canceled("Categorize processing"):
|
||||
return
|
||||
@@ -158,7 +149,7 @@ class Categorize(LLM, ABC):
|
||||
self.set_output("category_name", max_category)
|
||||
self.set_output("_next", cpn_ids)
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
return asyncio.run(self._invoke_async(**kwargs))
|
||||
|
||||
|
||||
@@ -19,55 +19,47 @@ import os
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from api.utils.api_utils import timeout
|
||||
|
||||
|
||||
class DataOperationsParam(ComponentParamBase):
|
||||
"""
|
||||
Define the Data Operations component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.query = []
|
||||
self.operations = "literal_eval"
|
||||
self.select_keys = []
|
||||
self.filter_values=[]
|
||||
self.updates=[]
|
||||
self.remove_keys=[]
|
||||
self.rename_keys=[]
|
||||
self.outputs = {
|
||||
"result": {
|
||||
"value": [],
|
||||
"type": "Array of Object"
|
||||
}
|
||||
}
|
||||
|
||||
def check(self):
|
||||
self.check_valid_value(self.operations, "Support operations", ["select_keys", "literal_eval","combine","filter_values","append_or_update","remove_keys","rename_keys"])
|
||||
|
||||
|
||||
self.filter_values = []
|
||||
self.updates = []
|
||||
self.remove_keys = []
|
||||
self.rename_keys = []
|
||||
self.outputs = {"result": {"value": [], "type": "Array of Object"}}
|
||||
|
||||
class DataOperations(ComponentBase,ABC):
|
||||
def check(self):
|
||||
self.check_valid_value(self.operations, "Support operations", ["select_keys", "literal_eval", "combine", "filter_values", "append_or_update", "remove_keys", "rename_keys"])
|
||||
|
||||
|
||||
class DataOperations(ComponentBase, ABC):
|
||||
component_name = "DataOperations"
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
k: {"name": o.get("name", ""), "type": "line"}
|
||||
for input_item in (self._param.query or [])
|
||||
for k, o in self.get_input_elements_from_text(input_item).items()
|
||||
}
|
||||
return {k: {"name": o.get("name", ""), "type": "line"} for input_item in (self._param.query or []) for k, o in self.get_input_elements_from_text(input_item).items()}
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
self.input_objects=[]
|
||||
self.input_objects = []
|
||||
inputs = getattr(self._param, "query", None)
|
||||
if not isinstance(inputs, (list, tuple)):
|
||||
inputs = [inputs]
|
||||
for input_ref in inputs:
|
||||
input_object=self._canvas.get_variable_value(input_ref)
|
||||
input_object = self._canvas.get_variable_value(input_ref)
|
||||
self.set_input_value(input_ref, input_object)
|
||||
if input_object is None:
|
||||
continue
|
||||
if isinstance(input_object,dict):
|
||||
if isinstance(input_object, dict):
|
||||
self.input_objects.append(input_object)
|
||||
elif isinstance(input_object,list):
|
||||
elif isinstance(input_object, list):
|
||||
self.input_objects.extend(x for x in input_object if isinstance(x, dict))
|
||||
else:
|
||||
continue
|
||||
@@ -85,13 +77,12 @@ class DataOperations(ComponentBase,ABC):
|
||||
self._remove_keys()
|
||||
else:
|
||||
self._rename_keys()
|
||||
|
||||
|
||||
def _select_keys(self):
|
||||
filter_criteria: list[str] = self._param.select_keys
|
||||
results = [{key: value for key, value in data_dict.items() if key in filter_criteria} for data_dict in self.input_objects]
|
||||
self.set_output("result", results)
|
||||
|
||||
|
||||
def _recursive_eval(self, data):
|
||||
if isinstance(data, dict):
|
||||
return {k: self._recursive_eval(v) for k, v in data.items()}
|
||||
@@ -99,23 +90,19 @@ class DataOperations(ComponentBase,ABC):
|
||||
return [self._recursive_eval(item) for item in data]
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
if (
|
||||
data.strip().startswith(("{", "[", "(", "'", '"'))
|
||||
or data.strip().lower() in ("true", "false", "none")
|
||||
or data.strip().replace(".", "").isdigit()
|
||||
):
|
||||
if data.strip().startswith(("{", "[", "(", "'", '"')) or data.strip().lower() in ("true", "false", "none") or data.strip().replace(".", "").isdigit():
|
||||
return ast.literal_eval(data)
|
||||
except (ValueError, SyntaxError, TypeError, MemoryError):
|
||||
return data
|
||||
else:
|
||||
return data
|
||||
return data
|
||||
|
||||
|
||||
def _literal_eval(self):
|
||||
self.set_output("result", self._recursive_eval(self.input_objects))
|
||||
|
||||
def _combine(self):
|
||||
result={}
|
||||
result = {}
|
||||
for obj in self.input_objects:
|
||||
for key, value in obj.items():
|
||||
if key not in result:
|
||||
@@ -126,15 +113,13 @@ class DataOperations(ComponentBase,ABC):
|
||||
else:
|
||||
result[key].append(value)
|
||||
else:
|
||||
result[key] = (
|
||||
[result[key], value] if not isinstance(value, list) else [result[key], *value]
|
||||
)
|
||||
result[key] = [result[key], value] if not isinstance(value, list) else [result[key], *value]
|
||||
self.set_output("result", result)
|
||||
|
||||
def norm(self,v):
|
||||
|
||||
def norm(self, v):
|
||||
s = "" if v is None else str(v)
|
||||
return s
|
||||
|
||||
|
||||
def match_rule(self, obj, rule):
|
||||
key = rule.get("key")
|
||||
op = (rule.get("operator") or "equals").lower()
|
||||
@@ -155,10 +140,10 @@ class DataOperations(ComponentBase,ABC):
|
||||
if op == "end with":
|
||||
return v.endswith(target)
|
||||
return False
|
||||
|
||||
|
||||
def _filter_values(self):
|
||||
results=[]
|
||||
rules = (getattr(self._param, "filter_values", None) or [])
|
||||
results = []
|
||||
rules = getattr(self._param, "filter_values", None) or []
|
||||
for obj in self.input_objects:
|
||||
if not rules:
|
||||
results.append(obj)
|
||||
@@ -166,11 +151,10 @@ class DataOperations(ComponentBase,ABC):
|
||||
if all(self.match_rule(obj, r) for r in rules):
|
||||
results.append(obj)
|
||||
self.set_output("result", results)
|
||||
|
||||
|
||||
|
||||
def _append_or_update(self):
|
||||
results=[]
|
||||
updates = getattr(self._param, "updates", []) or []
|
||||
results = []
|
||||
updates = getattr(self._param, "updates", []) or []
|
||||
for obj in self.input_objects:
|
||||
new_obj = dict(obj)
|
||||
for item in updates:
|
||||
@@ -187,7 +171,7 @@ class DataOperations(ComponentBase,ABC):
|
||||
results = []
|
||||
remove_keys = getattr(self._param, "remove_keys", []) or []
|
||||
|
||||
for obj in (self.input_objects or []):
|
||||
for obj in self.input_objects or []:
|
||||
new_obj = dict(obj)
|
||||
for k in remove_keys:
|
||||
if not isinstance(k, str):
|
||||
@@ -200,7 +184,7 @@ class DataOperations(ComponentBase,ABC):
|
||||
results = []
|
||||
rename_pairs = getattr(self._param, "rename_keys", []) or []
|
||||
|
||||
for obj in (self.input_objects or []):
|
||||
for obj in self.input_objects or []:
|
||||
new_obj = dict(obj)
|
||||
for pair in rename_pairs:
|
||||
if not isinstance(pair, dict):
|
||||
|
||||
@@ -13,6 +13,7 @@ from xml.sax.saxutils import escape
|
||||
|
||||
from agent.component.base import ComponentParamBase
|
||||
from api.utils.api_utils import timeout
|
||||
from api.utils.file_response import agent_attachment_preview_path
|
||||
from common import settings
|
||||
from common.misc_utils import get_uuid
|
||||
from .message import Message
|
||||
@@ -136,6 +137,7 @@ class DocGenerator(Message, ABC):
|
||||
"mime_type": mime_type,
|
||||
"size": file_size,
|
||||
"base64": file_base64,
|
||||
"preview_url": agent_attachment_preview_path(doc_id, ext=output_format, mime_type=mime_type),
|
||||
"include_download_info_in_content": self._param.include_download_info_in_content,
|
||||
}
|
||||
self.set_output("doc_id", doc_id)
|
||||
@@ -163,6 +165,7 @@ class DocGenerator(Message, ABC):
|
||||
logging.info("Starting document generation, content length: %s chars", len(content))
|
||||
|
||||
if content:
|
||||
|
||||
def _replace_variable(match_obj: re.Match[str]) -> str:
|
||||
match = match_obj.group(1)
|
||||
try:
|
||||
@@ -186,7 +189,7 @@ class DocGenerator(Message, ABC):
|
||||
flags=re.DOTALL,
|
||||
)
|
||||
|
||||
return content
|
||||
return self._strip_thinking(content)
|
||||
|
||||
def _get_output_directory(self) -> str:
|
||||
os.makedirs(self._default_output_directory, exist_ok=True)
|
||||
|
||||
@@ -39,71 +39,52 @@ class ExcelProcessorParam(ComponentParamBase):
|
||||
"""
|
||||
Define the ExcelProcessor component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Input configuration
|
||||
self.input_files = [] # Variable references to uploaded files
|
||||
self.operation = "read" # read, merge, transform, output
|
||||
|
||||
|
||||
# Processing options
|
||||
self.sheet_selection = "all" # all, first, or comma-separated sheet names
|
||||
self.merge_strategy = "concat" # concat, join
|
||||
self.join_on = "" # Column name for join operations
|
||||
|
||||
|
||||
# Transform options (for LLM-guided transformations)
|
||||
self.transform_instructions = ""
|
||||
self.transform_data = "" # Variable reference to transformation data
|
||||
|
||||
|
||||
# Output options
|
||||
self.output_format = "xlsx" # xlsx, csv
|
||||
self.output_filename = "output"
|
||||
|
||||
|
||||
# Component outputs
|
||||
self.outputs = {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"value": {}
|
||||
},
|
||||
"summary": {
|
||||
"type": "str",
|
||||
"value": ""
|
||||
},
|
||||
"markdown": {
|
||||
"type": "str",
|
||||
"value": ""
|
||||
}
|
||||
}
|
||||
|
||||
self.outputs = {"data": {"type": "object", "value": {}}, "summary": {"type": "str", "value": ""}, "markdown": {"type": "str", "value": ""}}
|
||||
|
||||
def check(self):
|
||||
self.check_valid_value(
|
||||
self.operation,
|
||||
"[ExcelProcessor] Operation",
|
||||
["read", "merge", "transform", "output"]
|
||||
)
|
||||
self.check_valid_value(
|
||||
self.output_format,
|
||||
"[ExcelProcessor] Output format",
|
||||
["xlsx", "csv"]
|
||||
)
|
||||
self.check_valid_value(self.operation, "[ExcelProcessor] Operation", ["read", "merge", "transform", "output"])
|
||||
self.check_valid_value(self.output_format, "[ExcelProcessor] Output format", ["xlsx", "csv"])
|
||||
return True
|
||||
|
||||
|
||||
class ExcelProcessor(ComponentBase, ABC):
|
||||
"""
|
||||
Excel processing component for RAGFlow agents.
|
||||
|
||||
|
||||
Operations:
|
||||
- read: Parse Excel files into structured data
|
||||
- merge: Combine multiple Excel files
|
||||
- transform: Apply data transformations based on instructions
|
||||
- output: Generate Excel file output
|
||||
"""
|
||||
|
||||
component_name = "ExcelProcessor"
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
"""Define input form for the component."""
|
||||
res = {}
|
||||
for ref in (self._param.input_files or []):
|
||||
for ref in self._param.input_files or []:
|
||||
for k, o in self.get_input_elements_from_text(ref).items():
|
||||
res[k] = {"name": o.get("name", ""), "type": "file"}
|
||||
if self._param.transform_data:
|
||||
@@ -111,13 +92,13 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
res[k] = {"name": o.get("name", ""), "type": "object"}
|
||||
return res
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("ExcelProcessor processing"):
|
||||
return
|
||||
|
||||
operation = self._param.operation.lower()
|
||||
|
||||
|
||||
if operation == "read":
|
||||
self._read_excels()
|
||||
elif operation == "merge":
|
||||
@@ -137,7 +118,7 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
value = self._canvas.get_variable_value(file_ref)
|
||||
if value is None:
|
||||
return None, None
|
||||
|
||||
|
||||
# Handle different value formats
|
||||
if isinstance(value, dict):
|
||||
# File reference from Begin/UserFillUp component
|
||||
@@ -154,12 +135,13 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
# Could be base64 encoded or a path
|
||||
if value.startswith("data:"):
|
||||
import base64
|
||||
|
||||
# Extract base64 content
|
||||
_, encoded = value.split(",", 1)
|
||||
return base64.b64decode(encoded), "uploaded.xlsx"
|
||||
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def _get_file_content_from_list(self, item) -> tuple[bytes, str]:
|
||||
"""Extract file content from a list item."""
|
||||
if isinstance(item, dict):
|
||||
@@ -170,15 +152,15 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
"""Parse Excel content into a dictionary of DataFrames (one per sheet)."""
|
||||
try:
|
||||
excel_file = BytesIO(content)
|
||||
|
||||
|
||||
if filename.lower().endswith(".csv"):
|
||||
df = pd.read_csv(excel_file)
|
||||
return {"Sheet1": df}
|
||||
else:
|
||||
# Read all sheets
|
||||
xlsx = pd.ExcelFile(excel_file, engine='openpyxl')
|
||||
xlsx = pd.ExcelFile(excel_file, engine="openpyxl")
|
||||
sheet_selection = self._param.sheet_selection
|
||||
|
||||
|
||||
if sheet_selection == "all":
|
||||
sheets_to_read = xlsx.sheet_names
|
||||
elif sheet_selection == "first":
|
||||
@@ -187,12 +169,12 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
# Comma-separated sheet names
|
||||
requested = [s.strip() for s in sheet_selection.split(",")]
|
||||
sheets_to_read = [s for s in requested if s in xlsx.sheet_names]
|
||||
|
||||
|
||||
dfs = {}
|
||||
for sheet in sheets_to_read:
|
||||
dfs[sheet] = pd.read_excel(xlsx, sheet_name=sheet)
|
||||
return dfs
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error parsing Excel file {filename}: {e}")
|
||||
return {}
|
||||
@@ -202,36 +184,36 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
all_data = {}
|
||||
summaries = []
|
||||
markdown_parts = []
|
||||
|
||||
for file_ref in (self._param.input_files or []):
|
||||
|
||||
for file_ref in self._param.input_files or []:
|
||||
if self.check_if_canceled("ExcelProcessor reading"):
|
||||
return
|
||||
|
||||
|
||||
# Get variable value
|
||||
value = self._canvas.get_variable_value(file_ref)
|
||||
self.set_input_value(file_ref, str(value)[:200] if value else "")
|
||||
|
||||
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
|
||||
# Handle file content
|
||||
content, filename = self._get_file_content(file_ref)
|
||||
if content is None:
|
||||
continue
|
||||
|
||||
|
||||
# Parse Excel
|
||||
dfs = self._parse_excel_to_dataframes(content, filename)
|
||||
|
||||
|
||||
for sheet_name, df in dfs.items():
|
||||
key = f"{filename}_{sheet_name}" if len(dfs) > 1 else filename
|
||||
all_data[key] = df.to_dict(orient="records")
|
||||
|
||||
|
||||
# Build summary
|
||||
summaries.append(f"**{key}**: {len(df)} rows, {len(df.columns)} columns ({', '.join(df.columns.tolist()[:5])}{'...' if len(df.columns) > 5 else ''})")
|
||||
|
||||
|
||||
# Build markdown table
|
||||
markdown_parts.append(f"### {key}\n\n{df.head(10).to_markdown(index=False)}\n")
|
||||
|
||||
|
||||
# Set outputs
|
||||
self.set_output("data", all_data)
|
||||
self.set_output("summary", "\n".join(summaries) if summaries else "No Excel files found")
|
||||
@@ -240,29 +222,29 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
def _merge_excels(self):
|
||||
"""Merge multiple Excel files/sheets into one."""
|
||||
all_dfs = []
|
||||
|
||||
for file_ref in (self._param.input_files or []):
|
||||
|
||||
for file_ref in self._param.input_files or []:
|
||||
if self.check_if_canceled("ExcelProcessor merging"):
|
||||
return
|
||||
|
||||
|
||||
value = self._canvas.get_variable_value(file_ref)
|
||||
self.set_input_value(file_ref, str(value)[:200] if value else "")
|
||||
|
||||
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
|
||||
content, filename = self._get_file_content(file_ref)
|
||||
if content is None:
|
||||
continue
|
||||
|
||||
|
||||
dfs = self._parse_excel_to_dataframes(content, filename)
|
||||
all_dfs.extend(dfs.values())
|
||||
|
||||
|
||||
if not all_dfs:
|
||||
self.set_output("data", {})
|
||||
self.set_output("summary", "No data to merge")
|
||||
return
|
||||
|
||||
|
||||
# Merge strategy
|
||||
if self._param.merge_strategy == "concat":
|
||||
merged_df = pd.concat(all_dfs, ignore_index=True)
|
||||
@@ -273,7 +255,7 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
merged_df = merged_df.merge(df, on=self._param.join_on, how="outer")
|
||||
else:
|
||||
merged_df = pd.concat(all_dfs, ignore_index=True)
|
||||
|
||||
|
||||
self.set_output("data", {"merged": merged_df.to_dict(orient="records")})
|
||||
self.set_output("summary", f"Merged {len(all_dfs)} sources into {len(merged_df)} rows, {len(merged_df.columns)} columns")
|
||||
self.set_output("markdown", merged_df.head(20).to_markdown(index=False))
|
||||
@@ -285,14 +267,14 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
if not transform_ref:
|
||||
self.set_output("summary", "No transform data reference provided")
|
||||
return
|
||||
|
||||
|
||||
data = self._canvas.get_variable_value(transform_ref)
|
||||
self.set_input_value(transform_ref, str(data)[:300] if data else "")
|
||||
|
||||
|
||||
if data is None:
|
||||
self.set_output("summary", "Transform data is empty")
|
||||
return
|
||||
|
||||
|
||||
# Convert to DataFrame
|
||||
if isinstance(data, dict):
|
||||
# Could be {"sheet": [rows]} format
|
||||
@@ -315,7 +297,7 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
else:
|
||||
self.set_output("data", {"raw": str(data)})
|
||||
self.set_output("markdown", str(data))
|
||||
|
||||
|
||||
self.set_output("summary", "Transformed data ready for processing")
|
||||
|
||||
def _output_excel(self):
|
||||
@@ -325,14 +307,14 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
if not transform_ref:
|
||||
self.set_output("summary", "No data reference for output")
|
||||
return
|
||||
|
||||
|
||||
data = self._canvas.get_variable_value(transform_ref)
|
||||
self.set_input_value(transform_ref, str(data)[:300] if data else "")
|
||||
|
||||
|
||||
if data is None:
|
||||
self.set_output("summary", "No data to output")
|
||||
return
|
||||
|
||||
|
||||
try:
|
||||
# Prepare DataFrames
|
||||
if isinstance(data, dict):
|
||||
@@ -346,10 +328,10 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
else:
|
||||
self.set_output("summary", "Invalid data format for Excel output")
|
||||
return
|
||||
|
||||
|
||||
# Generate output
|
||||
doc_id = get_uuid()
|
||||
|
||||
|
||||
if self._param.output_format == "csv":
|
||||
# For CSV, only output first sheet
|
||||
first_df = list(dfs.values())[0]
|
||||
@@ -358,7 +340,7 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
else:
|
||||
# Excel output
|
||||
excel_io = BytesIO()
|
||||
with pd.ExcelWriter(excel_io, engine='openpyxl') as writer:
|
||||
with pd.ExcelWriter(excel_io, engine="openpyxl") as writer:
|
||||
for sheet_name, df in dfs.items():
|
||||
# Sanitize sheet name (max 31 chars, no special chars)
|
||||
safe_name = sheet_name[:31].replace("/", "_").replace("\\", "_")
|
||||
@@ -366,23 +348,19 @@ class ExcelProcessor(ComponentBase, ABC):
|
||||
excel_io.seek(0)
|
||||
binary_content = excel_io.read()
|
||||
filename = f"{self._param.output_filename}.xlsx"
|
||||
|
||||
|
||||
# Store file
|
||||
settings.STORAGE_IMPL.put(self._canvas._tenant_id, doc_id, binary_content)
|
||||
|
||||
|
||||
# Set attachment output
|
||||
self.set_output("attachment", {
|
||||
"doc_id": doc_id,
|
||||
"format": self._param.output_format,
|
||||
"file_name": filename
|
||||
})
|
||||
|
||||
self.set_output("attachment", {"doc_id": doc_id, "format": self._param.output_format, "file_name": filename})
|
||||
|
||||
total_rows = sum(len(df) for df in dfs.values())
|
||||
self.set_output("summary", f"Generated {filename} with {len(dfs)} sheet(s), {total_rows} total rows")
|
||||
self.set_output("data", {k: v.to_dict(orient="records") for k, v in dfs.items()})
|
||||
|
||||
|
||||
logging.info(f"ExcelProcessor: Generated {filename} as {doc_id}")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"ExcelProcessor output error: {e}")
|
||||
self.set_output("summary", f"Error generating output: {str(e)}")
|
||||
|
||||
@@ -29,4 +29,4 @@ class ExitLoop(ComponentBase, ABC):
|
||||
pass
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return ""
|
||||
return ""
|
||||
|
||||
@@ -25,7 +25,6 @@ _INITIAL_USER_INPUT_CONSUMED_KEY = "sys.__initial_user_input_consumed__"
|
||||
|
||||
|
||||
class UserFillUpParam(ComponentParamBase):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.enable_tips = True
|
||||
@@ -55,11 +54,7 @@ class UserFillUp(ComponentBase):
|
||||
return {}
|
||||
|
||||
if isinstance(query, dict):
|
||||
matched = {
|
||||
key: value if isinstance(value, dict) else {"value": value}
|
||||
for key, value in query.items()
|
||||
if key in fields
|
||||
}
|
||||
matched = {key: value if isinstance(value, dict) else {"value": value} for key, value in query.items() if key in fields}
|
||||
if matched:
|
||||
self._canvas.globals[_INITIAL_USER_INPUT_CONSUMED_KEY] = True
|
||||
return matched
|
||||
@@ -81,7 +76,13 @@ class UserFillUp(ComponentBase):
|
||||
return FileService.get_files(files, layout_recognize=layout_recognize)
|
||||
|
||||
if isinstance(value, dict):
|
||||
return value.get("value")
|
||||
raw = value.get("value")
|
||||
if value.get("type") == "object" and isinstance(raw, str) and raw.strip():
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except Exception:
|
||||
return raw
|
||||
return raw
|
||||
|
||||
return value
|
||||
|
||||
@@ -108,11 +109,18 @@ class UserFillUp(ComponentBase):
|
||||
ans = v
|
||||
if not ans:
|
||||
ans = ""
|
||||
content = re.sub(r"\{%s\}"%k, ans, content)
|
||||
content = re.sub(r"\{%s\}" % k, ans, content)
|
||||
|
||||
self.set_output("tips", content)
|
||||
layout_recognize = self._param.layout_recognize or None
|
||||
merged_inputs = self._merge_runtime_inputs(kwargs.get("inputs", {}))
|
||||
if not merged_inputs:
|
||||
# No fresh user answer was supplied on this entry. Clear any values
|
||||
# retained from a previous response so the canvas wait-check treats
|
||||
# the form as unsatisfied and pauses for input again. Without this,
|
||||
# an Await Response node inside a Loop would only pause on the first
|
||||
# iteration and silently reuse the earlier answer afterwards.
|
||||
self._clear_form_values()
|
||||
for k, v in merged_inputs.items():
|
||||
if self.check_if_canceled("UserFillUp processing"):
|
||||
return
|
||||
@@ -120,5 +128,18 @@ class UserFillUp(ComponentBase):
|
||||
self.set_output(k, resolved)
|
||||
self.set_input_value(k, resolved)
|
||||
|
||||
def _clear_form_values(self):
|
||||
for field in self.get_input_elements().values():
|
||||
if not isinstance(field, dict):
|
||||
continue
|
||||
field_type = str(field.get("type", "")).lower()
|
||||
# An optional file input is already treated as satisfied when empty
|
||||
# (see Canvas._is_input_field_satisfied), so clearing it would not
|
||||
# force a re-prompt and would only drop a previously uploaded file.
|
||||
# Leave it untouched to avoid unexpected data loss.
|
||||
if "file" in field_type and field.get("optional"):
|
||||
continue
|
||||
field["value"] = None
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Waiting for your input..."
|
||||
|
||||
@@ -24,6 +24,7 @@ class VariableModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
"""
|
||||
|
||||
|
||||
class IterationParam(ComponentParamBase):
|
||||
"""
|
||||
Define the Iteration component parameters.
|
||||
@@ -32,15 +33,10 @@ class IterationParam(ComponentParamBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.items_ref = ""
|
||||
self.variable={}
|
||||
self.variable = {}
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"items": {
|
||||
"type": "json",
|
||||
"name": "Items"
|
||||
}
|
||||
}
|
||||
return {"items": {"type": "json", "name": "Items"}}
|
||||
|
||||
def check(self):
|
||||
return True
|
||||
@@ -62,10 +58,7 @@ class Iteration(ComponentBase, ABC):
|
||||
|
||||
arr = self._canvas.get_variable_value(self._param.items_ref)
|
||||
if not isinstance(arr, list):
|
||||
self.set_output("_ERROR", self._param.items_ref + " must be an array, but its type is "+str(type(arr)))
|
||||
self.set_output("_ERROR", self._param.items_ref + " must be an array, but its type is " + str(type(arr)))
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Need to process {} items.".format(len(self._canvas.get_variable_value(self._param.items_ref)))
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
from abc import ABC
|
||||
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
|
||||
|
||||
@@ -21,6 +22,7 @@ class IterationItemParam(ComponentParamBase):
|
||||
"""
|
||||
Define the IterationItem component parameters.
|
||||
"""
|
||||
|
||||
def check(self):
|
||||
return True
|
||||
|
||||
@@ -40,7 +42,7 @@ class IterationItem(ComponentBase, ABC):
|
||||
arr = self._canvas.get_variable_value(parent._param.items_ref)
|
||||
if not isinstance(arr, list):
|
||||
self._idx = -1
|
||||
raise Exception(parent._param.items_ref + " must be an array, but its type is "+str(type(arr)))
|
||||
raise Exception(parent._param.items_ref + " must be an array, but its type is " + str(type(arr)))
|
||||
|
||||
if self._idx > 0:
|
||||
if self.check_if_canceled("IterationItem processing"):
|
||||
@@ -79,7 +81,11 @@ class IterationItem(ComponentBase, ABC):
|
||||
for k, o in p._param.outputs.items():
|
||||
if "ref" not in o:
|
||||
continue
|
||||
_cid, var = o["ref"].split("@")
|
||||
# Use maxsplit=1 so an `@` legitimately embedded in `var`
|
||||
# (e.g. a user-defined output key that happens to contain
|
||||
# '@') does not raise `ValueError: too many values to unpack`.
|
||||
# `_cid` is system-generated and never contains '@'.
|
||||
_cid, var = o["ref"].split("@", 1)
|
||||
if _cid != cid:
|
||||
continue
|
||||
res = p.output(k)
|
||||
|
||||
@@ -3,10 +3,12 @@ import os
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from api.utils.api_utils import timeout
|
||||
|
||||
|
||||
class ListOperationsParam(ComponentParamBase):
|
||||
"""
|
||||
Define the List Operations component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.query = ""
|
||||
@@ -20,24 +22,8 @@ class ListOperationsParam(ComponentParamBase):
|
||||
# first field). Mirrors internal/agent/component/list_operations.go
|
||||
# parseSortByFieldList + opSort's SortBy path.
|
||||
self.sort_by = ""
|
||||
self.filter = {
|
||||
"operator": "=",
|
||||
"value": ""
|
||||
}
|
||||
self.outputs = {
|
||||
"result": {
|
||||
"value": [],
|
||||
"type": "Array of ?"
|
||||
},
|
||||
"first": {
|
||||
"value": "",
|
||||
"type": "?"
|
||||
},
|
||||
"last": {
|
||||
"value": "",
|
||||
"type": "?"
|
||||
}
|
||||
}
|
||||
self.filter = {"operator": "=", "value": ""}
|
||||
self.outputs = {"result": {"value": [], "type": "Array of ?"}, "first": {"value": "", "type": "?"}, "last": {"value": "", "type": "?"}}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_operation_name(operation):
|
||||
@@ -45,7 +31,7 @@ class ListOperationsParam(ComponentParamBase):
|
||||
if op.lower() == "topn":
|
||||
return "head"
|
||||
return op or "nth"
|
||||
|
||||
|
||||
def check(self):
|
||||
self.check_empty(self.query, "query")
|
||||
self.operations = self._normalize_operation_name(self.operations)
|
||||
@@ -57,14 +43,14 @@ class ListOperationsParam(ComponentParamBase):
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {}
|
||||
|
||||
|
||||
class ListOperations(ComponentBase,ABC):
|
||||
|
||||
class ListOperations(ComponentBase, ABC):
|
||||
component_name = "ListOperations"
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
self.input_objects=[]
|
||||
self.input_objects = []
|
||||
inputs = getattr(self._param, "query", None)
|
||||
self.inputs = self._canvas.get_variable_value(inputs)
|
||||
if not isinstance(self.inputs, list):
|
||||
@@ -83,7 +69,6 @@ class ListOperations(ComponentBase,ABC):
|
||||
elif self._param.operations == "drop_duplicates":
|
||||
self._drop_duplicates()
|
||||
|
||||
|
||||
def _coerce_n(self):
|
||||
try:
|
||||
return int(getattr(self._param, "n", 0))
|
||||
@@ -99,12 +84,10 @@ class ListOperations(ComponentBase,ABC):
|
||||
def _set_outputs(self, outputs):
|
||||
self._param.outputs["result"]["value"] = outputs
|
||||
self._param.outputs["first"]["value"] = outputs[0] if outputs else None
|
||||
self._param.outputs["last"]["value"] = outputs[-1] if outputs else None
|
||||
self._param.outputs["last"]["value"] = outputs[-1] if outputs else None
|
||||
|
||||
def _raise_strict_range_error(self, operation, n):
|
||||
raise ValueError(
|
||||
f"{operation} requires n to be within the valid range in strict mode, got {n}."
|
||||
)
|
||||
raise ValueError(f"{operation} requires n to be within the valid range in strict mode, got {n}.")
|
||||
|
||||
def _nth(self):
|
||||
n = self._coerce_n()
|
||||
@@ -160,9 +143,9 @@ class ListOperations(ComponentBase,ABC):
|
||||
self._set_outputs(outputs)
|
||||
|
||||
def _filter(self):
|
||||
self._set_outputs([i for i in self.inputs if self._eval(self._norm(i),self._param.filter["operator"],self._param.filter["value"])])
|
||||
self._set_outputs([i for i in self.inputs if self._eval(self._norm(i), self._param.filter["operator"], self._param.filter["value"])])
|
||||
|
||||
def _norm(self,v):
|
||||
def _norm(self, v):
|
||||
s = "" if v is None else str(v)
|
||||
return s
|
||||
|
||||
@@ -222,7 +205,7 @@ class ListOperations(ComponentBase,ABC):
|
||||
outs.append(item)
|
||||
self._set_outputs(outs)
|
||||
|
||||
def _hashable(self,x):
|
||||
def _hashable(self, x):
|
||||
if isinstance(x, dict):
|
||||
return tuple(sorted((k, self._hashable(v)) for k, v in x.items()))
|
||||
if isinstance(x, (list, tuple)):
|
||||
|
||||
@@ -25,7 +25,7 @@ from functools import partial
|
||||
from common.constants import LLMType
|
||||
from api.db.services.dialog_service import _stream_with_think_delta
|
||||
from api.db.services.llm_service import LLMBundle
|
||||
from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
|
||||
from api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from common.connection_utils import timeout
|
||||
from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt
|
||||
@@ -78,6 +78,8 @@ class LLMParam(ComponentParamBase):
|
||||
conf["presence_penalty"] = float(self.presence_penalty)
|
||||
if float(self.frequency_penalty) > 0 and get_attr("frequencyPenaltyEnabled"):
|
||||
conf["frequency_penalty"] = float(self.frequency_penalty)
|
||||
if hasattr(self, "thinking") and self.thinking and self.thinking != "default":
|
||||
conf["thinking"] = self.thinking
|
||||
return conf
|
||||
|
||||
|
||||
@@ -86,9 +88,9 @@ class LLM(ComponentBase):
|
||||
|
||||
def __init__(self, canvas, component_id, param: ComponentParamBase):
|
||||
super().__init__(canvas, component_id, param)
|
||||
model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
model_type = "chat" if "chat" in model_types else model_types[0]
|
||||
chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
chat_model_config = resolve_model_config(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)
|
||||
self.imgs = []
|
||||
|
||||
@@ -316,14 +318,14 @@ class LLM(ComponentBase):
|
||||
len(sys_file_imgs),
|
||||
max(0, prev_img_count + len(sys_file_imgs) - len(self.imgs)),
|
||||
)
|
||||
model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
if self.imgs and LLMType.IMAGE2TEXT.value in model_types:
|
||||
model_type = LLMType.IMAGE2TEXT.value
|
||||
model_types = resolve_model_type(self._canvas.get_tenant_id(), self._param.llm_id)
|
||||
if self.imgs and LLMType.VISION.value in model_types:
|
||||
model_type = LLMType.VISION.value
|
||||
elif LLMType.CHAT.value in model_types:
|
||||
model_type = LLMType.CHAT.value
|
||||
else:
|
||||
model_type = model_types[0]
|
||||
model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
model_config = resolve_model_config(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
|
||||
if self.imgs:
|
||||
self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), model_config, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)
|
||||
|
||||
|
||||
@@ -25,16 +25,11 @@ class LoopParam(ComponentParamBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.loop_variables = []
|
||||
self.loop_termination_condition=[]
|
||||
self.loop_termination_condition = []
|
||||
self.maximum_loop_count = 0
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"items": {
|
||||
"type": "json",
|
||||
"name": "Items"
|
||||
}
|
||||
}
|
||||
return {"items": {"type": "json", "name": "Items"}}
|
||||
|
||||
def check(self):
|
||||
return True
|
||||
@@ -83,10 +78,10 @@ class Loop(ComponentBase, ABC):
|
||||
for item in self._param.loop_variables:
|
||||
if self._is_incomplete_loop_variable(item):
|
||||
raise ValueError("Loop Variable is not complete.")
|
||||
if item["input_mode"]=="variable":
|
||||
self.set_output(item["variable"],self._canvas.get_variable_value(item["value"]))
|
||||
elif item["input_mode"]=="constant":
|
||||
self.set_output(item["variable"],item["value"])
|
||||
if item["input_mode"] == "variable":
|
||||
self.set_output(item["variable"], self._canvas.get_variable_value(item["value"]))
|
||||
elif item["input_mode"] == "constant":
|
||||
self.set_output(item["variable"], item["value"])
|
||||
else:
|
||||
if item["type"] == "number":
|
||||
self.set_output(item["variable"], 0)
|
||||
@@ -101,6 +96,5 @@ class Loop(ComponentBase, ABC):
|
||||
else:
|
||||
self.set_output(item["variable"], "")
|
||||
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Loop from canvas."
|
||||
return "Loop from canvas."
|
||||
|
||||
@@ -21,9 +21,11 @@ class LoopItemParam(ComponentParamBase):
|
||||
"""
|
||||
Define the LoopItem component parameters.
|
||||
"""
|
||||
|
||||
def check(self):
|
||||
return True
|
||||
|
||||
|
||||
class LoopItem(ComponentBase, ABC):
|
||||
component_name = "LoopItem"
|
||||
|
||||
@@ -31,7 +33,6 @@ class LoopItem(ComponentBase, ABC):
|
||||
super().__init__(canvas, id, param)
|
||||
self._idx = 0
|
||||
|
||||
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("LoopItem processing"):
|
||||
return
|
||||
@@ -45,7 +46,7 @@ class LoopItem(ComponentBase, ABC):
|
||||
return
|
||||
self._idx += 1
|
||||
|
||||
def evaluate_condition(self,var, operator, value):
|
||||
def evaluate_condition(self, var, operator, value):
|
||||
if isinstance(var, str):
|
||||
if operator == "contains":
|
||||
return value in var
|
||||
@@ -140,11 +141,7 @@ class LoopItem(ComponentBase, ABC):
|
||||
else:
|
||||
raise ValueError("Invalid input mode.")
|
||||
conditions.append(self.evaluate_condition(var, operator, value))
|
||||
should_end = (
|
||||
all(conditions) if logical_operator == "and"
|
||||
else any(conditions) if logical_operator == "or"
|
||||
else None
|
||||
)
|
||||
should_end = all(conditions) if logical_operator == "and" else any(conditions) if logical_operator == "or" else None
|
||||
if should_end is None:
|
||||
raise ValueError("Invalid logical operator,should be 'and' or 'or'.")
|
||||
|
||||
@@ -164,4 +161,4 @@ class LoopItem(ComponentBase, ABC):
|
||||
return False
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Next turn..."
|
||||
return "Next turn..."
|
||||
|
||||
@@ -14,8 +14,10 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
import nest_asyncio
|
||||
|
||||
nest_asyncio.apply()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -45,20 +47,14 @@ class MessageParam(ComponentParamBase):
|
||||
"""
|
||||
Define the Message component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.content = []
|
||||
self.stream = True
|
||||
self.output_format = None # default output format
|
||||
self.auto_play = False
|
||||
self.outputs = {
|
||||
"content": {
|
||||
"type": "str"
|
||||
},
|
||||
"downloads": {
|
||||
"type": "list"
|
||||
}
|
||||
}
|
||||
self.outputs = {"content": {"type": "str"}, "downloads": {"type": "list"}}
|
||||
|
||||
def check(self):
|
||||
self.check_empty(self.content, "[Message] Content")
|
||||
@@ -71,9 +67,7 @@ class Message(ComponentBase):
|
||||
|
||||
@staticmethod
|
||||
def _is_download_info(value: Any) -> bool:
|
||||
return isinstance(value, dict) and all(
|
||||
key in value for key in ("doc_id", "filename", "mime_type")
|
||||
)
|
||||
return isinstance(value, dict) and all(key in value for key in ("doc_id", "filename", "mime_type"))
|
||||
|
||||
@staticmethod
|
||||
def _download_info_includes_content(value: Any) -> bool:
|
||||
@@ -157,7 +151,7 @@ class Message(ComponentBase):
|
||||
delimiter: str = None,
|
||||
downloads: list[dict[str, Any]] | None = None,
|
||||
) -> tuple[str, dict[str, str | list | Any]]:
|
||||
for k,v in self.get_input_elements_from_text(script).items():
|
||||
for k, v in self.get_input_elements_from_text(script).items():
|
||||
if k in kwargs:
|
||||
continue
|
||||
v = v["value"]
|
||||
@@ -191,7 +185,7 @@ class Message(ComponentBase):
|
||||
buf += t
|
||||
return buf
|
||||
|
||||
async def _stream(self, rand_cnt:str):
|
||||
async def _stream(self, rand_cnt: str):
|
||||
s = 0
|
||||
all_content = ""
|
||||
cache = {}
|
||||
@@ -200,8 +194,8 @@ class Message(ComponentBase):
|
||||
if self.check_if_canceled("Message streaming"):
|
||||
return
|
||||
|
||||
all_content += rand_cnt[s: r.start()]
|
||||
yield rand_cnt[s: r.start()]
|
||||
all_content += rand_cnt[s : r.start()]
|
||||
yield rand_cnt[s : r.start()]
|
||||
s = r.end()
|
||||
exp = r.group(1)
|
||||
if exp in cache:
|
||||
@@ -235,9 +229,7 @@ class Message(ComponentBase):
|
||||
continue
|
||||
elif inspect.isawaitable(v):
|
||||
v = await v
|
||||
v = self._stringify_message_value(
|
||||
v, downloads=downloads, fallback_to_str=True
|
||||
)
|
||||
v = self._stringify_message_value(v, downloads=downloads, fallback_to_str=True)
|
||||
yield v
|
||||
self.set_input_value(exp, v)
|
||||
all_content += v
|
||||
@@ -247,21 +239,19 @@ class Message(ComponentBase):
|
||||
if self.check_if_canceled("Message streaming"):
|
||||
return
|
||||
|
||||
all_content += rand_cnt[s: ]
|
||||
yield rand_cnt[s: ]
|
||||
all_content += rand_cnt[s:]
|
||||
yield rand_cnt[s:]
|
||||
|
||||
self.set_output("downloads", downloads)
|
||||
self.set_output("content", all_content)
|
||||
self._convert_content(all_content)
|
||||
await self._save_to_memory(all_content)
|
||||
|
||||
def _is_jinjia2(self, content:str) -> bool:
|
||||
patt = [
|
||||
r"\{%.*%\}", "{{", "}}"
|
||||
]
|
||||
def _is_jinjia2(self, content: str) -> bool:
|
||||
patt = [r"\{%.*%\}", "{{", "}}"]
|
||||
return any([re.search(p, content) for p in patt])
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("Message processing"):
|
||||
return
|
||||
@@ -303,19 +293,19 @@ class Message(ComponentBase):
|
||||
def _parse_markdown_table_lines(self, table_lines: list):
|
||||
"""
|
||||
Parse a list of Markdown table lines into a pandas DataFrame.
|
||||
|
||||
|
||||
Args:
|
||||
table_lines: List of strings, each representing a row in the Markdown table
|
||||
(excluding separator lines like |---|---|)
|
||||
|
||||
|
||||
Returns:
|
||||
pandas DataFrame with the table data, or None if parsing fails
|
||||
"""
|
||||
import pandas as pd
|
||||
|
||||
|
||||
if not table_lines:
|
||||
return None
|
||||
|
||||
|
||||
rows = []
|
||||
headers = None
|
||||
|
||||
@@ -350,36 +340,58 @@ class Message(ComponentBase):
|
||||
return cell
|
||||
|
||||
return cell
|
||||
|
||||
|
||||
for line in table_lines:
|
||||
# Split by | and clean up
|
||||
cells = [cell.strip() for cell in line.split('|')]
|
||||
cells = [cell.strip() for cell in line.split("|")]
|
||||
# Remove empty first and last elements from split (caused by leading/trailing |)
|
||||
cells = [c for c in cells if c]
|
||||
|
||||
|
||||
if headers is None:
|
||||
headers = cells
|
||||
else:
|
||||
cells = [_coerce_excel_cell_type(c) for c in cells]
|
||||
rows.append(cells)
|
||||
|
||||
|
||||
if headers and rows:
|
||||
# Ensure all rows have same number of columns as headers
|
||||
normalized_rows = []
|
||||
for row in rows:
|
||||
while len(row) < len(headers):
|
||||
row.append('')
|
||||
normalized_rows.append(row[:len(headers)])
|
||||
|
||||
row.append("")
|
||||
normalized_rows.append(row[: len(headers)])
|
||||
|
||||
return pd.DataFrame(normalized_rows, columns=headers)
|
||||
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _strip_thinking(content: str) -> str:
|
||||
"""Remove <think>...</think> reasoning blocks before document export.
|
||||
|
||||
Reasoning models (e.g. DeepSeek-R1, OpenAI o1) embed chain-of-thought
|
||||
inside ``<think>`` tags. These blocks must not leak into exported
|
||||
Word/PDF/Excel documents.
|
||||
"""
|
||||
if not isinstance(content, str) or not content:
|
||||
return content
|
||||
# Remove complete think blocks (DOTALL so newlines are matched)
|
||||
cleaned = re.sub(r"<think>.*?</think>", "", content, flags=re.DOTALL)
|
||||
# Remove any dangling unclosed <think> opening tag + trailing content
|
||||
cleaned = re.sub(r"<think>.*$", "", cleaned, flags=re.DOTALL)
|
||||
# Remove leftover standalone tags
|
||||
cleaned = re.sub(r"</?think>", "", cleaned)
|
||||
# Collapse 3+ consecutive newlines left behind by removed blocks
|
||||
cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
|
||||
return cleaned.strip()
|
||||
|
||||
def _convert_content(self, content):
|
||||
if not self._param.output_format:
|
||||
return
|
||||
|
||||
content = self._strip_thinking(content)
|
||||
import pypandoc
|
||||
|
||||
doc_id = get_uuid()
|
||||
|
||||
if self._param.output_format.lower() not in {"markdown", "html", "pdf", "docx", "xlsx"}:
|
||||
@@ -408,49 +420,47 @@ class Message(ComponentBase):
|
||||
|
||||
# Debug: log the content being parsed
|
||||
logging.info(f"XLSX Parser: Content length={len(content) if content else 0}, first 500 chars: {content[:500] if content else 'None'}")
|
||||
|
||||
|
||||
# Try to parse ALL Markdown tables from the content
|
||||
# Each table will be written to a separate sheet
|
||||
tables = [] # List of (sheet_name, dataframe)
|
||||
|
||||
|
||||
if isinstance(content, str):
|
||||
lines = content.strip().split('\n')
|
||||
lines = content.strip().split("\n")
|
||||
logging.info(f"XLSX Parser: Total lines={len(lines)}, lines starting with '|': {sum(1 for line in lines if line.strip().startswith('|'))}")
|
||||
current_table_lines = []
|
||||
current_table_title = None
|
||||
pending_title = None
|
||||
in_table = False
|
||||
table_count = 0
|
||||
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.strip()
|
||||
|
||||
|
||||
# Check for potential table title (lines before a table)
|
||||
# Look for patterns like "Table 1:", "## Table", or markdown headers
|
||||
if not in_table and stripped and not stripped.startswith('|'):
|
||||
if not in_table and stripped and not stripped.startswith("|"):
|
||||
# Check if this could be a table title
|
||||
lower_stripped = stripped.lower()
|
||||
if (lower_stripped.startswith('table') or
|
||||
stripped.startswith('#') or
|
||||
':' in stripped):
|
||||
pending_title = stripped.lstrip('#').strip()
|
||||
|
||||
if stripped.startswith('|') and '|' in stripped[1:]:
|
||||
if lower_stripped.startswith("table") or stripped.startswith("#") or ":" in stripped:
|
||||
pending_title = stripped.lstrip("#").strip()
|
||||
|
||||
if stripped.startswith("|") and "|" in stripped[1:]:
|
||||
# Check if this is a separator line (|---|---|)
|
||||
cleaned = stripped.replace(' ', '').replace('|', '').replace('-', '').replace(':', '')
|
||||
if cleaned == '':
|
||||
cleaned = stripped.replace(" ", "").replace("|", "").replace("-", "").replace(":", "")
|
||||
if cleaned == "":
|
||||
continue # Skip separator line
|
||||
|
||||
|
||||
if not in_table:
|
||||
# Starting a new table
|
||||
in_table = True
|
||||
current_table_lines = []
|
||||
current_table_title = pending_title
|
||||
pending_title = None
|
||||
|
||||
|
||||
current_table_lines.append(stripped)
|
||||
|
||||
elif in_table and not stripped.startswith('|'):
|
||||
|
||||
elif in_table and not stripped.startswith("|"):
|
||||
# End of current table - save it
|
||||
if current_table_lines:
|
||||
df = self._parse_markdown_table_lines(current_table_lines)
|
||||
@@ -460,24 +470,22 @@ class Message(ComponentBase):
|
||||
if current_table_title:
|
||||
# Clean and truncate title for sheet name
|
||||
sheet_name = current_table_title[:31]
|
||||
sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
|
||||
sheet_name = sheet_name.replace("/", "_").replace("\\", "_").replace("*", "").replace("?", "").replace("[", "").replace("]", "").replace(":", "")
|
||||
else:
|
||||
sheet_name = f"Table_{table_count}"
|
||||
tables.append((sheet_name, df))
|
||||
|
||||
|
||||
# Reset for next table
|
||||
in_table = False
|
||||
current_table_lines = []
|
||||
current_table_title = None
|
||||
|
||||
|
||||
# Check if this line could be a title for the next table
|
||||
if stripped:
|
||||
lower_stripped = stripped.lower()
|
||||
if (lower_stripped.startswith('table') or
|
||||
stripped.startswith('#') or
|
||||
':' in stripped):
|
||||
pending_title = stripped.lstrip('#').strip()
|
||||
|
||||
if lower_stripped.startswith("table") or stripped.startswith("#") or ":" in stripped:
|
||||
pending_title = stripped.lstrip("#").strip()
|
||||
|
||||
# Don't forget the last table if content ends with a table
|
||||
if in_table and current_table_lines:
|
||||
df = self._parse_markdown_table_lines(current_table_lines)
|
||||
@@ -485,11 +493,11 @@ class Message(ComponentBase):
|
||||
table_count += 1
|
||||
if current_table_title:
|
||||
sheet_name = current_table_title[:31]
|
||||
sheet_name = sheet_name.replace('/', '_').replace('\\', '_').replace('*', '').replace('?', '').replace('[', '').replace(']', '').replace(':', '')
|
||||
sheet_name = sheet_name.replace("/", "_").replace("\\", "_").replace("*", "").replace("?", "").replace("[", "").replace("]", "").replace(":", "")
|
||||
else:
|
||||
sheet_name = f"Table_{table_count}"
|
||||
tables.append((sheet_name, df))
|
||||
|
||||
|
||||
# Fallback: if no tables found, create single sheet with content
|
||||
if not tables:
|
||||
df = pd.DataFrame({"Content": [content if content else ""]})
|
||||
@@ -497,7 +505,7 @@ class Message(ComponentBase):
|
||||
|
||||
# Write all tables to Excel, each in a separate sheet
|
||||
excel_io = BytesIO()
|
||||
with pd.ExcelWriter(excel_io, engine='openpyxl') as writer:
|
||||
with pd.ExcelWriter(excel_io, engine="openpyxl") as writer:
|
||||
used_names = set()
|
||||
for sheet_name, df in tables:
|
||||
# Ensure unique sheet names
|
||||
@@ -505,14 +513,14 @@ class Message(ComponentBase):
|
||||
counter = 1
|
||||
while sheet_name in used_names:
|
||||
suffix = f"_{counter}"
|
||||
sheet_name = original_name[:31-len(suffix)] + suffix
|
||||
sheet_name = original_name[: 31 - len(suffix)] + suffix
|
||||
counter += 1
|
||||
used_names.add(sheet_name)
|
||||
df.to_excel(writer, sheet_name=sheet_name, index=False)
|
||||
|
||||
|
||||
excel_io.seek(0)
|
||||
binary_content = excel_io.read()
|
||||
|
||||
|
||||
logging.info(f"Generated Excel with {len(tables)} sheet(s): {[t[0] for t in tables]}")
|
||||
|
||||
else: # pdf, docx
|
||||
@@ -543,10 +551,7 @@ class Message(ComponentBase):
|
||||
os.remove(tmp_name)
|
||||
|
||||
settings.STORAGE_IMPL.put(self._canvas._tenant_id, doc_id, binary_content)
|
||||
self.set_output("attachment", {
|
||||
"doc_id":doc_id,
|
||||
"format":self._param.output_format,
|
||||
"file_name":f"{doc_id[:8]}.{self._param.output_format}"})
|
||||
self.set_output("attachment", {"doc_id": doc_id, "format": self._param.output_format, "file_name": f"{doc_id[:8]}.{self._param.output_format}"})
|
||||
|
||||
logging.info(f"Converted content uploaded as {doc_id} (format={self._param.output_format})")
|
||||
|
||||
@@ -560,15 +565,10 @@ class Message(ComponentBase):
|
||||
user_id = self._param.user_id if hasattr(self._param, "user_id") else ""
|
||||
if user_id:
|
||||
import re
|
||||
|
||||
# is variable
|
||||
if re.match(r"^{.*}$", user_id):
|
||||
user_id = self._canvas.get_variable_value(user_id)
|
||||
|
||||
message_dict = {
|
||||
"user_id": user_id,
|
||||
"agent_id": self._canvas._id,
|
||||
"session_id": self._canvas.task_id,
|
||||
"user_input": self._canvas.get_sys_query(),
|
||||
"agent_response": content
|
||||
}
|
||||
message_dict = {"user_id": user_id, "agent_id": self._canvas._id, "session_id": self._canvas.task_id, "user_input": self._canvas.get_sys_query(), "agent_response": content}
|
||||
return await queue_save_to_memory_task(self._param.memory_ids, message_dict)
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
#
|
||||
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
PipelineChunker Component
|
||||
|
||||
Run RAGFlow Pipeline-style chunkers (rag.app.*) against uploaded files inside an
|
||||
Agent workflow. Emits plain text chunks for downstream Agent nodes — no
|
||||
embedding, no persistence. Wraps existing chunker functions; does not
|
||||
re-implement chunking logic.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
from abc import ABC
|
||||
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from api.db.services.file_service import FileService
|
||||
from common.connection_utils import timeout
|
||||
|
||||
|
||||
# Parser id -> dotted module path under rag.app. Imported lazily so we don't
|
||||
# pull deepdoc/OCR/VLM machinery at component-discovery time.
|
||||
_PARSER_MODULES: dict[str, str] = {
|
||||
"general": "rag.app.naive",
|
||||
"naive": "rag.app.naive",
|
||||
"paper": "rag.app.paper",
|
||||
"book": "rag.app.book",
|
||||
"presentation": "rag.app.presentation",
|
||||
"manual": "rag.app.manual",
|
||||
"laws": "rag.app.laws",
|
||||
"qa": "rag.app.qa",
|
||||
"table": "rag.app.table",
|
||||
"resume": "rag.app.resume",
|
||||
"picture": "rag.app.picture",
|
||||
"one": "rag.app.one",
|
||||
"audio": "rag.app.audio",
|
||||
"email": "rag.app.email",
|
||||
"tag": "rag.app.tag",
|
||||
}
|
||||
|
||||
|
||||
def _load_chunker(parser_id: str):
|
||||
"""Resolve a parser id to the underlying ``rag.app.<module>.chunk`` callable."""
|
||||
module_path = _PARSER_MODULES[parser_id.lower()]
|
||||
return importlib.import_module(module_path).chunk
|
||||
|
||||
|
||||
class PipelineChunkerParam(ComponentParamBase):
|
||||
"""
|
||||
Define the PipelineChunker component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise PipelineChunker defaults and declare component outputs."""
|
||||
super().__init__()
|
||||
self.inputs = [] # variable references to uploaded files
|
||||
self.parser_id = "naive"
|
||||
self.lang = "English"
|
||||
self.from_page = 0
|
||||
self.to_page = 100000000
|
||||
self.parser_config = {}
|
||||
|
||||
self.outputs = {
|
||||
"chunks": {"type": "list", "value": []},
|
||||
"chunks_full": {"type": "list", "value": []},
|
||||
"summary": {"type": "str", "value": ""},
|
||||
}
|
||||
|
||||
def check(self):
|
||||
"""Validate parser id, page range, and parser_config shape."""
|
||||
self.check_valid_value(
|
||||
self.parser_id.lower(),
|
||||
"[PipelineChunker] parser_id",
|
||||
list(_PARSER_MODULES.keys()),
|
||||
)
|
||||
self.check_nonnegative_number(self.from_page, "[PipelineChunker] from_page")
|
||||
self.check_nonnegative_number(self.to_page, "[PipelineChunker] to_page")
|
||||
if isinstance(self.from_page, (int, float)) and isinstance(self.to_page, (int, float)) and self.from_page > self.to_page:
|
||||
raise ValueError("[PipelineChunker] from_page must be <= to_page")
|
||||
if not isinstance(self.parser_config, dict):
|
||||
raise ValueError("[PipelineChunker] parser_config must be a dict.")
|
||||
return True
|
||||
|
||||
|
||||
class PipelineChunker(ComponentBase, ABC):
|
||||
"""
|
||||
Run a Pipeline-style chunker (naive, paper, qa, manual, book, ...) against
|
||||
one or more uploaded files and surface the resulting chunks to downstream
|
||||
Agent nodes.
|
||||
"""
|
||||
|
||||
component_name = "PipelineChunker"
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
"""Expose each referenced file input as a file-typed form element."""
|
||||
res = {}
|
||||
for ref in self._param.inputs or []:
|
||||
for k, o in self.get_input_elements_from_text(ref).items():
|
||||
res[k] = {"name": o.get("name", ""), "type": "file"}
|
||||
return res
|
||||
|
||||
def _get_file_content(self, file_ref: str) -> tuple[bytes | None, str | None]:
|
||||
"""Resolve a canvas variable reference to ``(content_bytes, filename)``."""
|
||||
value = self._canvas.get_variable_value(file_ref)
|
||||
if value is None:
|
||||
return None, None
|
||||
|
||||
if isinstance(value, list) and value:
|
||||
value = value[0]
|
||||
|
||||
if isinstance(value, dict):
|
||||
file_id = value.get("id") or value.get("file_id")
|
||||
created_by = value.get("created_by") or self._canvas.get_tenant_id()
|
||||
filename = value.get("name") or value.get("filename") or "uploaded"
|
||||
if file_id:
|
||||
try:
|
||||
return FileService.get_blob(created_by, file_id), filename
|
||||
except Exception as e:
|
||||
logging.exception(
|
||||
f"[PipelineChunker] FileService.get_blob failed for "
|
||||
f"file_id={file_id} created_by={created_by} filename={filename}: {e}"
|
||||
)
|
||||
return None, None
|
||||
return None, None
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
"""Run the configured chunker over every referenced file and publish outputs."""
|
||||
if self.check_if_canceled("PipelineChunker processing"):
|
||||
return
|
||||
|
||||
chunker = _load_chunker(self._param.parser_id)
|
||||
tenant_id = self._canvas.get_tenant_id()
|
||||
chunk_kwargs = dict(
|
||||
lang=self._param.lang,
|
||||
tenant_id=tenant_id,
|
||||
from_page=self._param.from_page,
|
||||
to_page=self._param.to_page,
|
||||
parser_config=self._param.parser_config or {},
|
||||
callback=lambda prog=0, msg="": logging.info(f"[PipelineChunker] {prog}: {msg}"),
|
||||
)
|
||||
|
||||
all_chunks: list[dict] = []
|
||||
per_file_counts: list[str] = []
|
||||
|
||||
for file_ref in self._param.inputs or []:
|
||||
if self.check_if_canceled("PipelineChunker processing"):
|
||||
return
|
||||
|
||||
content, filename = self._get_file_content(file_ref)
|
||||
self.set_input_value(file_ref, filename or "")
|
||||
if content is None:
|
||||
logging.warning(f"[PipelineChunker] could not resolve file ref: {file_ref}")
|
||||
per_file_counts.append(f"{filename or file_ref}: error (could not resolve file)")
|
||||
continue
|
||||
|
||||
try:
|
||||
file_chunks = chunker(filename, binary=content, **chunk_kwargs) or []
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
per_file_counts.append(f"{filename}: error (chunking failed)")
|
||||
continue
|
||||
|
||||
all_chunks.extend(file_chunks)
|
||||
per_file_counts.append(f"{filename}: {len(file_chunks)} chunks")
|
||||
|
||||
text_only = [(c.get("content_with_weight") or c.get("text") or "") for c in all_chunks if isinstance(c, dict)]
|
||||
text_only = [t for t in text_only if t]
|
||||
|
||||
self.set_output("chunks", text_only)
|
||||
self.set_output("chunks_full", all_chunks)
|
||||
self.set_output(
|
||||
"summary",
|
||||
f"Parser: {self._param.parser_id} | Files: {len(self._param.inputs or [])} | Chunks: {len(text_only)}" + (" | " + "; ".join(per_file_counts) if per_file_counts else ""),
|
||||
)
|
||||
|
||||
def thoughts(self) -> str:
|
||||
"""Return a short status line for UI display."""
|
||||
return f"Chunking with `{self._param.parser_id}` strategy..."
|
||||
@@ -52,18 +52,10 @@ class StringTransform(Message, ABC):
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
if self._param.method == "split":
|
||||
return {
|
||||
"line": {
|
||||
"name": "String",
|
||||
"type": "line"
|
||||
}
|
||||
}
|
||||
return {k: {
|
||||
"name": o["name"],
|
||||
"type": "line"
|
||||
} for k, o in self.get_input_elements_from_text(self._param.script).items()}
|
||||
return {"line": {"name": "String", "type": "line"}}
|
||||
return {k: {"name": o["name"], "type": "line"} for k, o in self.get_input_elements_from_text(self._param.script).items()}
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("StringTransform processing"):
|
||||
return
|
||||
@@ -73,7 +65,7 @@ class StringTransform(Message, ABC):
|
||||
else:
|
||||
self._merge(kwargs)
|
||||
|
||||
def _split(self, line:str|None = None):
|
||||
def _split(self, line: str | None = None):
|
||||
if self.check_if_canceled("StringTransform split processing"):
|
||||
return
|
||||
|
||||
@@ -84,13 +76,13 @@ class StringTransform(Message, ABC):
|
||||
self.set_input_value(self._param.split_ref, var)
|
||||
|
||||
res = []
|
||||
for i,s in enumerate(re.split(r"(%s)"%("|".join([re.escape(d) for d in self._param.delimiters])), var, flags=re.DOTALL)):
|
||||
for i, s in enumerate(re.split(r"(%s)" % ("|".join([re.escape(d) for d in self._param.delimiters])), var, flags=re.DOTALL)):
|
||||
if i % 2 == 1:
|
||||
continue
|
||||
res.append(s)
|
||||
self.set_output("result", res)
|
||||
|
||||
def _merge(self, kwargs:dict[str, str] = {}):
|
||||
def _merge(self, kwargs: dict[str, str] = {}):
|
||||
if self.check_if_canceled("StringTransform merge processing"):
|
||||
return
|
||||
|
||||
@@ -104,7 +96,7 @@ class StringTransform(Message, ABC):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for k,v in kwargs.items():
|
||||
for k, v in kwargs.items():
|
||||
if v is None:
|
||||
v = ""
|
||||
script = re.sub(k, lambda match: v, script)
|
||||
@@ -113,5 +105,3 @@ class StringTransform(Message, ABC):
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return f"It's {self._param.method}ing."
|
||||
|
||||
|
||||
|
||||
@@ -40,8 +40,7 @@ class SwitchParam(ComponentParamBase):
|
||||
"""
|
||||
self.conditions = []
|
||||
self.end_cpn_ids = []
|
||||
self.operators = ['contains', 'not contains', 'start with', 'end with', 'empty', 'not empty', '=', '≠', '>',
|
||||
'<', '≥', '≤']
|
||||
self.operators = ["contains", "not contains", "start with", "end with", "empty", "not empty", "=", "≠", ">", "<", "≥", "≤"]
|
||||
|
||||
def check(self):
|
||||
self.check_empty(self.conditions, "[Switch] conditions")
|
||||
@@ -51,12 +50,8 @@ class SwitchParam(ComponentParamBase):
|
||||
self.check_empty(self.end_cpn_ids, "[Switch] the ELSE/Other destination can not be empty.")
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"urls": {
|
||||
"name": "URLs",
|
||||
"type": "line"
|
||||
}
|
||||
}
|
||||
return {"urls": {"name": "URLs", "type": "line"}}
|
||||
|
||||
|
||||
class Switch(ComponentBase, ABC):
|
||||
component_name = "Switch"
|
||||
@@ -137,7 +132,7 @@ class Switch(ComponentBase, ABC):
|
||||
except Exception:
|
||||
return True if input <= value else False
|
||||
|
||||
raise ValueError(f'Not supported operator: {operator}')
|
||||
raise ValueError(f"Not supported operator: {operator}")
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "I’m weighing a few options and will pick the next step shortly."
|
||||
|
||||
@@ -38,13 +38,9 @@ class VariableAggregatorParam(ComponentParamBase):
|
||||
if not g.get("group_name"):
|
||||
raise ValueError("[VariableAggregator] group_name can not be empty!")
|
||||
if not g.get("variables"):
|
||||
raise ValueError(
|
||||
f"[VariableAggregator] variables of group `{g.get('group_name')}` can not be empty"
|
||||
)
|
||||
raise ValueError(f"[VariableAggregator] variables of group `{g.get('group_name')}` can not be empty")
|
||||
if not isinstance(g.get("variables"), list):
|
||||
raise ValueError(
|
||||
f"[VariableAggregator] variables of group `{g.get('group_name')}` should be a list of strings"
|
||||
)
|
||||
raise ValueError(f"[VariableAggregator] variables of group `{g.get('group_name')}` should be a list of strings")
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
@@ -67,11 +63,11 @@ class VariableAggregator(ComponentBase):
|
||||
# record candidate selectors within this group
|
||||
self.set_input_value(f"{gname}.variables", list(group.get("variables", [])))
|
||||
for selector in group.get("variables", []):
|
||||
val = self._canvas.get_variable_value(selector['value'])
|
||||
val = self._canvas.get_variable_value(selector["value"])
|
||||
if val:
|
||||
self.set_output(gname, val)
|
||||
break
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _to_object(value: Any) -> Any:
|
||||
# Try to convert value to serializable object if it has to_object()
|
||||
|
||||
@@ -19,32 +19,30 @@ import numbers
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from api.utils.api_utils import timeout
|
||||
|
||||
|
||||
class VariableAssignerParam(ComponentParamBase):
|
||||
"""
|
||||
Define the Variable Assigner component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.variables=[]
|
||||
self.variables = []
|
||||
|
||||
def check(self):
|
||||
return True
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"items": {
|
||||
"type": "json",
|
||||
"name": "Items"
|
||||
}
|
||||
}
|
||||
|
||||
class VariableAssigner(ComponentBase,ABC):
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {"items": {"type": "json", "name": "Items"}}
|
||||
|
||||
|
||||
class VariableAssigner(ComponentBase, ABC):
|
||||
component_name = "VariableAssigner"
|
||||
_NO_PARAMETER_OPERATORS = {"clear", "remove_first", "remove_last"}
|
||||
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10*60)))
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
if not isinstance(self._param.variables,list):
|
||||
if not isinstance(self._param.variables, list):
|
||||
return
|
||||
else:
|
||||
for item in self._param.variables:
|
||||
@@ -56,105 +54,105 @@ class VariableAssigner(ComponentBase,ABC):
|
||||
raise ValueError("Variable is not complete.")
|
||||
if operator not in self._NO_PARAMETER_OPERATORS and parameter is None:
|
||||
raise ValueError("Variable is not complete.")
|
||||
variable_value=self._canvas.get_variable_value(variable)
|
||||
new_variable=self._operate(variable_value,operator,parameter)
|
||||
variable_value = self._canvas.get_variable_value(variable)
|
||||
new_variable = self._operate(variable_value, operator, parameter)
|
||||
self._canvas.set_variable_value(variable, new_variable)
|
||||
|
||||
def _operate(self,variable,operator,parameter):
|
||||
def _operate(self, variable, operator, parameter):
|
||||
if operator == "overwrite":
|
||||
return self._overwrite(parameter)
|
||||
elif operator == "clear":
|
||||
return self._clear(variable)
|
||||
elif operator == "set":
|
||||
return self._set(variable,parameter)
|
||||
return self._set(variable, parameter)
|
||||
elif operator == "append":
|
||||
return self._append(variable,parameter)
|
||||
return self._append(variable, parameter)
|
||||
elif operator == "extend":
|
||||
return self._extend(variable,parameter)
|
||||
return self._extend(variable, parameter)
|
||||
elif operator == "remove_first":
|
||||
return self._remove_first(variable)
|
||||
elif operator == "remove_last":
|
||||
return self._remove_last(variable)
|
||||
elif operator == "+=":
|
||||
return self._add(variable,parameter)
|
||||
return self._add(variable, parameter)
|
||||
elif operator == "-=":
|
||||
return self._subtract(variable,parameter)
|
||||
return self._subtract(variable, parameter)
|
||||
elif operator == "*=":
|
||||
return self._multiply(variable,parameter)
|
||||
return self._multiply(variable, parameter)
|
||||
elif operator == "/=":
|
||||
return self._divide(variable,parameter)
|
||||
return self._divide(variable, parameter)
|
||||
else:
|
||||
return
|
||||
|
||||
def _overwrite(self,parameter):
|
||||
|
||||
def _overwrite(self, parameter):
|
||||
return self._canvas.get_variable_value(parameter)
|
||||
|
||||
def _clear(self,variable):
|
||||
if isinstance(variable,list):
|
||||
def _clear(self, variable):
|
||||
if isinstance(variable, list):
|
||||
return []
|
||||
elif isinstance(variable,str):
|
||||
elif isinstance(variable, str):
|
||||
return ""
|
||||
elif isinstance(variable,dict):
|
||||
elif isinstance(variable, dict):
|
||||
return {}
|
||||
elif isinstance(variable,bool):
|
||||
elif isinstance(variable, bool):
|
||||
return False
|
||||
elif isinstance(variable,int):
|
||||
elif isinstance(variable, int):
|
||||
return 0
|
||||
elif isinstance(variable,float):
|
||||
elif isinstance(variable, float):
|
||||
return 0.0
|
||||
else:
|
||||
return None
|
||||
|
||||
def _set(self,variable,parameter):
|
||||
def _set(self, variable, parameter):
|
||||
if variable is None:
|
||||
return self._canvas.get_value_with_variable(parameter)
|
||||
elif isinstance(variable,str):
|
||||
elif isinstance(variable, str):
|
||||
return self._canvas.get_value_with_variable(parameter)
|
||||
elif isinstance(variable,bool):
|
||||
elif isinstance(variable, bool):
|
||||
return parameter
|
||||
elif isinstance(variable,int):
|
||||
elif isinstance(variable, int):
|
||||
return parameter
|
||||
elif isinstance(variable,float):
|
||||
elif isinstance(variable, float):
|
||||
return parameter
|
||||
else:
|
||||
return parameter
|
||||
|
||||
def _append(self,variable,parameter):
|
||||
parameter=self._canvas.get_variable_value(parameter)
|
||||
def _append(self, variable, parameter):
|
||||
parameter = self._canvas.get_variable_value(parameter)
|
||||
if variable is None:
|
||||
variable=[]
|
||||
if not isinstance(variable,list):
|
||||
variable = []
|
||||
if not isinstance(variable, list):
|
||||
return "ERROR:VARIABLE_NOT_LIST"
|
||||
elif len(variable)!=0 and not isinstance(parameter,type(variable[0])):
|
||||
elif len(variable) != 0 and not isinstance(parameter, type(variable[0])):
|
||||
return "ERROR:PARAMETER_NOT_LIST_ELEMENT_TYPE"
|
||||
else:
|
||||
variable.append(parameter)
|
||||
return variable
|
||||
|
||||
def _extend(self,variable,parameter):
|
||||
parameter=self._canvas.get_variable_value(parameter)
|
||||
def _extend(self, variable, parameter):
|
||||
parameter = self._canvas.get_variable_value(parameter)
|
||||
if variable is None:
|
||||
variable=[]
|
||||
if not isinstance(variable,list):
|
||||
variable = []
|
||||
if not isinstance(variable, list):
|
||||
return "ERROR:VARIABLE_NOT_LIST"
|
||||
elif not isinstance(parameter,list):
|
||||
elif not isinstance(parameter, list):
|
||||
return "ERROR:PARAMETER_NOT_LIST"
|
||||
elif len(variable)!=0 and len(parameter)!=0 and not isinstance(parameter[0],type(variable[0])):
|
||||
elif len(variable) != 0 and len(parameter) != 0 and not isinstance(parameter[0], type(variable[0])):
|
||||
return "ERROR:PARAMETER_NOT_LIST_ELEMENT_TYPE"
|
||||
else:
|
||||
return variable + parameter
|
||||
|
||||
def _remove_first(self,variable):
|
||||
if not isinstance(variable,list):
|
||||
def _remove_first(self, variable):
|
||||
if not isinstance(variable, list):
|
||||
return "ERROR:VARIABLE_NOT_LIST"
|
||||
if len(variable)==0:
|
||||
if len(variable) == 0:
|
||||
return variable
|
||||
return variable[1:]
|
||||
|
||||
def _remove_last(self,variable):
|
||||
if not isinstance(variable,list):
|
||||
def _remove_last(self, variable):
|
||||
if not isinstance(variable, list):
|
||||
return "ERROR:VARIABLE_NOT_LIST"
|
||||
if len(variable)==0:
|
||||
if len(variable) == 0:
|
||||
return variable
|
||||
return variable[:-1]
|
||||
|
||||
@@ -163,32 +161,32 @@ class VariableAssigner(ComponentBase,ABC):
|
||||
return False
|
||||
return isinstance(value, numbers.Number)
|
||||
|
||||
def _add(self,variable,parameter):
|
||||
def _add(self, variable, parameter):
|
||||
if self.is_number(variable) and self.is_number(parameter):
|
||||
return variable + parameter
|
||||
else:
|
||||
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
|
||||
|
||||
def _subtract(self,variable,parameter):
|
||||
def _subtract(self, variable, parameter):
|
||||
if self.is_number(variable) and self.is_number(parameter):
|
||||
return variable - parameter
|
||||
else:
|
||||
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
|
||||
|
||||
def _multiply(self,variable,parameter):
|
||||
def _multiply(self, variable, parameter):
|
||||
if self.is_number(variable) and self.is_number(parameter):
|
||||
return variable * parameter
|
||||
else:
|
||||
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
|
||||
|
||||
def _divide(self,variable,parameter):
|
||||
def _divide(self, variable, parameter):
|
||||
if self.is_number(variable) and self.is_number(parameter):
|
||||
if parameter==0:
|
||||
if parameter == 0:
|
||||
return "ERROR:DIVIDE_BY_ZERO"
|
||||
else:
|
||||
return variable/parameter
|
||||
return variable / parameter
|
||||
else:
|
||||
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
|
||||
return "ERROR:VARIABLE_NOT_NUMBER or PARAMETER_NOT_NUMBER"
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Assign variables from canvas."
|
||||
return "Assign variables from canvas."
|
||||
|
||||
@@ -56,7 +56,7 @@ def normalize_chunker_dsl(dsl: dict) -> dict:
|
||||
for old_name, new_name in COMPONENT_RENAMES.items():
|
||||
prefix = f"{old_name}:"
|
||||
if component_id.startswith(prefix):
|
||||
new_component_id = f"{new_name}:{component_id[len(prefix):]}"
|
||||
new_component_id = f"{new_name}:{component_id[len(prefix) :]}"
|
||||
break
|
||||
component_id_map[component_id] = new_component_id
|
||||
|
||||
@@ -66,12 +66,7 @@ def normalize_chunker_dsl(dsl: dict) -> dict:
|
||||
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
component_id = match.group(2)
|
||||
return (
|
||||
match.group(1)
|
||||
+ component_id_map.get(component_id, component_id)
|
||||
+ match.group(3)
|
||||
+ match.group(4)
|
||||
)
|
||||
return match.group(1) + component_id_map.get(component_id, component_id) + match.group(3) + match.group(4)
|
||||
|
||||
return VARIABLE_REF_PATTERN.sub(repl, text)
|
||||
|
||||
@@ -96,15 +91,9 @@ def normalize_chunker_dsl(dsl: dict) -> dict:
|
||||
obj["component_name"] = COMPONENT_RENAMES.get(component_name, component_name)
|
||||
|
||||
if isinstance(new_component.get("downstream"), list):
|
||||
new_component["downstream"] = [
|
||||
component_id_map.get(component_id, component_id)
|
||||
for component_id in new_component["downstream"]
|
||||
]
|
||||
new_component["downstream"] = [component_id_map.get(component_id, component_id) for component_id in new_component["downstream"]]
|
||||
if isinstance(new_component.get("upstream"), list):
|
||||
new_component["upstream"] = [
|
||||
component_id_map.get(component_id, component_id)
|
||||
for component_id in new_component["upstream"]
|
||||
]
|
||||
new_component["upstream"] = [component_id_map.get(component_id, component_id) for component_id in new_component["upstream"]]
|
||||
|
||||
parent_id = new_component.get("parent_id")
|
||||
if isinstance(parent_id, str):
|
||||
@@ -115,10 +104,7 @@ def normalize_chunker_dsl(dsl: dict) -> dict:
|
||||
normalized["components"] = rewritten_components
|
||||
|
||||
if isinstance(normalized.get("path"), list):
|
||||
normalized["path"] = [
|
||||
component_id_map.get(component_id, component_id)
|
||||
for component_id in normalized["path"]
|
||||
]
|
||||
normalized["path"] = [component_id_map.get(component_id, component_id) for component_id in normalized["path"]]
|
||||
|
||||
graph = normalized.get("graph")
|
||||
if isinstance(graph, dict):
|
||||
|
||||
@@ -1 +1 @@
|
||||
PLUGIN_TYPE_LLM_TOOLS = "llm_tools"
|
||||
PLUGIN_TYPE_LLM_TOOLS = "llm_tools"
|
||||
|
||||
@@ -7,6 +7,7 @@ class BadCalculatorPlugin(LLMToolPlugin):
|
||||
A sample LLM tool plugin, will add two numbers with 100.
|
||||
It only presents for demo purpose. Do not use it in production.
|
||||
"""
|
||||
|
||||
_version_ = "1.0.0"
|
||||
|
||||
@classmethod
|
||||
@@ -17,19 +18,9 @@ class BadCalculatorPlugin(LLMToolPlugin):
|
||||
"description": "A tool to calculate the sum of two numbers (will give wrong answer)",
|
||||
"displayDescription": "$t:bad_calculator.description",
|
||||
"parameters": {
|
||||
"a": {
|
||||
"type": "number",
|
||||
"description": "The first number",
|
||||
"displayDescription": "$t:bad_calculator.params.a",
|
||||
"required": True
|
||||
},
|
||||
"b": {
|
||||
"type": "number",
|
||||
"description": "The second number",
|
||||
"displayDescription": "$t:bad_calculator.params.b",
|
||||
"required": True
|
||||
}
|
||||
}
|
||||
"a": {"type": "number", "description": "The first number", "displayDescription": "$t:bad_calculator.params.a", "required": True},
|
||||
"b": {"type": "number", "description": "The second number", "displayDescription": "$t:bad_calculator.params.b", "required": True},
|
||||
},
|
||||
}
|
||||
|
||||
def invoke(self, a: int, b: int) -> str:
|
||||
|
||||
@@ -38,14 +38,8 @@ def llm_tool_metadata_to_openai_tool(llm_tool_metadata: LLMToolMetadata) -> dict
|
||||
"description": llm_tool_metadata["description"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
k: {
|
||||
"type": p["type"],
|
||||
"description": p["description"]
|
||||
}
|
||||
for k, p in llm_tool_metadata["parameters"].items()
|
||||
},
|
||||
"required": [k for k, p in llm_tool_metadata["parameters"].items() if p["required"]]
|
||||
}
|
||||
}
|
||||
"properties": {k: {"type": p["type"], "description": p["description"]} for k, p in llm_tool_metadata["parameters"].items()},
|
||||
"required": [k for k, p in llm_tool_metadata["parameters"].items() if p["required"]],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -15,10 +15,8 @@ class PluginManager:
|
||||
self._llm_tool_plugins = {}
|
||||
|
||||
def load_plugins(self) -> None:
|
||||
loader = pluginlib.PluginLoader(
|
||||
paths=[str(Path(os.path.dirname(__file__), "embedded_plugins"))]
|
||||
)
|
||||
|
||||
loader = pluginlib.PluginLoader(paths=[str(Path(os.path.dirname(__file__), "embedded_plugins"))])
|
||||
|
||||
for type, plugins in loader.plugins.items():
|
||||
for name, plugin in plugins.items():
|
||||
logging.info(f"Loaded {type} plugin {name} version {plugin.version}")
|
||||
|
||||
@@ -111,7 +111,10 @@ def _load_provider_from_settings() -> None:
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load sandbox provider from settings: {e}")
|
||||
import traceback
|
||||
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
def _load_provider_config_from_settings(provider_type: str) -> Dict[str, Any]:
|
||||
provider_config_settings = SystemSettingsService.get_by_name(f"sandbox.{provider_type}")
|
||||
if not provider_config_settings:
|
||||
@@ -147,12 +150,7 @@ def reload_provider() -> None:
|
||||
_load_provider_from_settings()
|
||||
|
||||
|
||||
def execute_code(
|
||||
code: str,
|
||||
language: str = "python",
|
||||
timeout: int = 30,
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
) -> ExecutionResult:
|
||||
def execute_code(code: str, language: str = "python", timeout: int = 30, arguments: Optional[Dict[str, Any]] = None) -> ExecutionResult:
|
||||
"""
|
||||
Execute code in the configured sandbox.
|
||||
|
||||
@@ -173,9 +171,7 @@ def execute_code(
|
||||
provider_manager = get_provider_manager()
|
||||
|
||||
if not provider_manager.is_configured():
|
||||
raise RuntimeError(
|
||||
"No sandbox provider configured. Please configure sandbox settings in the admin panel."
|
||||
)
|
||||
raise RuntimeError("No sandbox provider configured. Please configure sandbox settings in the admin panel.")
|
||||
|
||||
provider = provider_manager.get_provider()
|
||||
provider_name = provider_manager.get_provider_name() or getattr(provider, "__class__", type(provider)).__name__
|
||||
@@ -192,13 +188,7 @@ def execute_code(
|
||||
|
||||
try:
|
||||
# Execute the code
|
||||
result = provider.execute_code(
|
||||
instance_id=instance.instance_id,
|
||||
code=code,
|
||||
language=language,
|
||||
timeout=timeout,
|
||||
arguments=arguments
|
||||
)
|
||||
result = provider.execute_code(instance_id=instance.instance_id, code=code, language=language, timeout=timeout, arguments=arguments)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -22,4 +22,3 @@ router = APIRouter()
|
||||
router.get("/")(healthz_handler)
|
||||
router.get("/healthz")(healthz_handler)
|
||||
router.post("/run")(run_code_handler)
|
||||
|
||||
|
||||
@@ -230,9 +230,7 @@ async def execute_code(req: CodeExecutionRequest):
|
||||
if returncode != 0:
|
||||
raise RuntimeError(f"Directory creation failed: {stderr}")
|
||||
|
||||
tar_proc = await asyncio.create_subprocess_exec(
|
||||
"tar", "czf", "-", "-C", workdir, code_name, runner_name, str(bundle["args_name"]), stdout=asyncio.subprocess.PIPE
|
||||
)
|
||||
tar_proc = await asyncio.create_subprocess_exec("tar", "czf", "-", "-C", workdir, code_name, runner_name, str(bundle["args_name"]), stdout=asyncio.subprocess.PIPE)
|
||||
tar_stdout, _ = await tar_proc.communicate()
|
||||
|
||||
docker_proc = await asyncio.create_subprocess_exec(
|
||||
@@ -334,8 +332,16 @@ async def _collect_artifacts(container: str, task_id: str, host_workdir: str) ->
|
||||
|
||||
# List files in the artifacts directory inside the container
|
||||
returncode, stdout, _ = await async_run_command(
|
||||
"docker", "exec", container, "find", artifacts_path,
|
||||
"-maxdepth", "1", "-type", "f", timeout=5,
|
||||
"docker",
|
||||
"exec",
|
||||
container,
|
||||
"find",
|
||||
artifacts_path,
|
||||
"-maxdepth",
|
||||
"1",
|
||||
"-type",
|
||||
"f",
|
||||
timeout=5,
|
||||
)
|
||||
if returncode != 0 or not stdout.strip():
|
||||
return []
|
||||
@@ -359,7 +365,14 @@ async def _collect_artifacts(container: str, task_id: str, host_workdir: str) ->
|
||||
|
||||
# Check file size inside the container
|
||||
returncode, size_str, _ = await async_run_command(
|
||||
"docker", "exec", container, "stat", "-c", "%s", file_path, timeout=5,
|
||||
"docker",
|
||||
"exec",
|
||||
container,
|
||||
"stat",
|
||||
"-c",
|
||||
"%s",
|
||||
file_path,
|
||||
timeout=5,
|
||||
)
|
||||
if returncode != 0:
|
||||
logger.warning(f"Failed to stat artifact {fname}")
|
||||
@@ -374,7 +387,12 @@ async def _collect_artifacts(container: str, task_id: str, host_workdir: str) ->
|
||||
|
||||
# Read file content via docker exec (docker cp doesn't work with gVisor tmpfs)
|
||||
returncode, content_b64, stderr = await async_run_command(
|
||||
"docker", "exec", container, "base64", file_path, timeout=30,
|
||||
"docker",
|
||||
"exec",
|
||||
container,
|
||||
"base64",
|
||||
file_path,
|
||||
timeout=30,
|
||||
)
|
||||
if returncode != 0:
|
||||
logger.warning(f"Failed to read artifact {fname}: {stderr}")
|
||||
@@ -382,12 +400,14 @@ async def _collect_artifacts(container: str, task_id: str, host_workdir: str) ->
|
||||
|
||||
content_b64 = content_b64.replace("\n", "").strip()
|
||||
|
||||
items.append(ArtifactItem(
|
||||
name=fname,
|
||||
mime_type=mime_type,
|
||||
size=file_size,
|
||||
content_b64=content_b64,
|
||||
))
|
||||
items.append(
|
||||
ArtifactItem(
|
||||
name=fname,
|
||||
mime_type=mime_type,
|
||||
size=file_size,
|
||||
content_b64=content_b64,
|
||||
)
|
||||
)
|
||||
logger.info(f"Collected artifact: {fname} ({file_size} bytes, {mime_type})")
|
||||
|
||||
return items
|
||||
|
||||
@@ -232,11 +232,7 @@ class AliyunCodeInterpreterProvider(SandboxProvider):
|
||||
# Wrap code to call main() function
|
||||
# Matches self_managed provider behavior: call main(**arguments)
|
||||
args_json = json.dumps(arguments or {})
|
||||
wrapped_code = (
|
||||
build_python_wrapper(code, args_json)
|
||||
if normalized_lang == "python"
|
||||
else build_javascript_wrapper(code, args_json)
|
||||
)
|
||||
wrapped_code = build_python_wrapper(code, args_json) if normalized_lang == "python" else build_javascript_wrapper(code, args_json)
|
||||
logger.debug(f"Aliyun Code Interpreter: Wrapped code (first 200 chars): {wrapped_code[:200]}")
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
@@ -33,6 +33,7 @@ class SandboxProviderConfigError(Exception):
|
||||
@dataclass
|
||||
class SandboxInstance:
|
||||
"""Represents a sandbox execution instance"""
|
||||
|
||||
instance_id: str
|
||||
provider: str
|
||||
status: str # running, stopped, error
|
||||
@@ -46,6 +47,7 @@ class SandboxInstance:
|
||||
@dataclass
|
||||
class ExecutionResult:
|
||||
"""Result of code execution in a sandbox"""
|
||||
|
||||
stdout: str
|
||||
stderr: str
|
||||
exit_code: int
|
||||
@@ -96,14 +98,7 @@ class SandboxProvider(ABC):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def execute_code(
|
||||
self,
|
||||
instance_id: str,
|
||||
code: str,
|
||||
language: str,
|
||||
timeout: int = 10,
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
) -> ExecutionResult:
|
||||
def execute_code(self, instance_id: str, code: str, language: str, timeout: int = 10, arguments: Optional[Dict[str, Any]] = None) -> ExecutionResult:
|
||||
"""
|
||||
Execute code in a sandbox instance.
|
||||
|
||||
|
||||
@@ -97,16 +97,10 @@ class E2BProvider(SandboxProvider):
|
||||
metadata={
|
||||
"language": language,
|
||||
"region": self.region,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
def execute_code(
|
||||
self,
|
||||
instance_id: str,
|
||||
code: str,
|
||||
language: str,
|
||||
timeout: int = 10
|
||||
) -> ExecutionResult:
|
||||
def execute_code(self, instance_id: str, code: str, language: str, timeout: int = 10) -> ExecutionResult:
|
||||
"""
|
||||
Execute code in the E2B instance.
|
||||
|
||||
@@ -130,9 +124,7 @@ class E2BProvider(SandboxProvider):
|
||||
# POST /sandbox/{sandboxID}/execute
|
||||
|
||||
raise RuntimeError(
|
||||
"E2B provider is not yet fully implemented. "
|
||||
"Please use the self-managed provider or implement the E2B API integration. "
|
||||
"See https://github.com/e2b-dev/e2b for API documentation."
|
||||
"E2B provider is not yet fully implemented. Please use the self-managed provider or implement the E2B API integration. See https://github.com/e2b-dev/e2b for API documentation."
|
||||
)
|
||||
|
||||
def destroy_instance(self, instance_id: str) -> bool:
|
||||
@@ -208,7 +200,7 @@ class E2BProvider(SandboxProvider):
|
||||
"min": 5,
|
||||
"max": 300,
|
||||
"description": "API request timeout for code execution",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
def _normalize_language(self, language: str) -> str:
|
||||
|
||||
@@ -49,6 +49,8 @@ LOCAL_PYTHON_THREAD_ENV_VARS = (
|
||||
"BLIS_NUM_THREADS",
|
||||
"VECLIB_MAXIMUM_THREADS",
|
||||
)
|
||||
|
||||
|
||||
class LocalProvider(SandboxProvider):
|
||||
"""
|
||||
Execute code as a local child process.
|
||||
|
||||
@@ -108,17 +108,10 @@ class SelfManagedProvider(SandboxProvider):
|
||||
"language": language,
|
||||
"endpoint": self.endpoint,
|
||||
"pool_size": self.pool_size,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
def execute_code(
|
||||
self,
|
||||
instance_id: str,
|
||||
code: str,
|
||||
language: str,
|
||||
timeout: int = 10,
|
||||
arguments: Optional[Dict[str, Any]] = None
|
||||
) -> ExecutionResult:
|
||||
def execute_code(self, instance_id: str, code: str, language: str, timeout: int = 10, arguments: Optional[Dict[str, Any]] = None) -> ExecutionResult:
|
||||
"""
|
||||
Execute code in the sandbox.
|
||||
|
||||
@@ -144,11 +137,7 @@ class SelfManagedProvider(SandboxProvider):
|
||||
|
||||
# Prepare request
|
||||
code_b64 = base64.b64encode(code.encode("utf-8")).decode("utf-8")
|
||||
payload = {
|
||||
"code_b64": code_b64,
|
||||
"language": normalized_lang,
|
||||
"arguments": arguments or {}
|
||||
}
|
||||
payload = {"code_b64": code_b64, "language": normalized_lang, "arguments": arguments or {}}
|
||||
|
||||
url = f"{self.endpoint}/run"
|
||||
exec_timeout = timeout or self.timeout
|
||||
@@ -156,19 +145,12 @@ class SelfManagedProvider(SandboxProvider):
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
url,
|
||||
json=payload,
|
||||
timeout=exec_timeout,
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
response = requests.post(url, json=payload, timeout=exec_timeout, headers={"Content-Type": "application/json"})
|
||||
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"HTTP {response.status_code}: {response.text}"
|
||||
)
|
||||
raise RuntimeError(f"HTTP {response.status_code}: {response.text}")
|
||||
|
||||
result = response.json()
|
||||
structured_result = result.get("result") or {}
|
||||
@@ -188,14 +170,12 @@ class SelfManagedProvider(SandboxProvider):
|
||||
"result_present": structured_result.get("present", False),
|
||||
"result_value": structured_result.get("value"),
|
||||
"result_type": structured_result.get("type"),
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
except requests.Timeout:
|
||||
execution_time = time.time() - start_time
|
||||
raise TimeoutError(
|
||||
f"Execution timed out after {exec_timeout} seconds"
|
||||
)
|
||||
raise TimeoutError(f"Execution timed out after {exec_timeout} seconds")
|
||||
|
||||
except requests.RequestException as e:
|
||||
raise RuntimeError(f"HTTP request failed: {str(e)}")
|
||||
@@ -388,7 +368,8 @@ class SelfManagedProvider(SandboxProvider):
|
||||
if endpoint:
|
||||
# Check if it's a valid HTTP/HTTPS URL or localhost
|
||||
import re
|
||||
url_pattern = r'^(https?://|http://localhost|http://[\d\.]+:[a-z]+:[/]|http://[\w\.]+:)'
|
||||
|
||||
url_pattern = r"^(https?://|http://localhost|http://[\d\.]+:[a-z]+:[/]|http://[\w\.]+:)"
|
||||
if not re.match(url_pattern, endpoint):
|
||||
return False, f"Invalid endpoint format: {endpoint}. Must start with http:// or https://"
|
||||
|
||||
|
||||
@@ -135,9 +135,7 @@ class SSHProvider(SandboxProvider):
|
||||
timeout=min(self.timeout, 10),
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to create remote artifacts directory: {stderr or stdout or 'unknown error'}"
|
||||
)
|
||||
raise RuntimeError(f"Failed to create remote artifacts directory: {stderr or stdout or 'unknown error'}")
|
||||
except Exception:
|
||||
sftp.close()
|
||||
client.close()
|
||||
@@ -211,9 +209,7 @@ class SSHProvider(SandboxProvider):
|
||||
"status": "ok" if exit_code == 0 else "error",
|
||||
"timeout": exec_timeout,
|
||||
"command": command,
|
||||
"artifacts": self._collect_artifacts(
|
||||
sftp, posixpath.join(remote_work_dir, "artifacts")
|
||||
),
|
||||
"artifacts": self._collect_artifacts(sftp, posixpath.join(remote_work_dir, "artifacts")),
|
||||
"result_present": structured_result.get("present", False),
|
||||
"result_value": structured_result.get("value"),
|
||||
"result_type": structured_result.get("type"),
|
||||
@@ -269,18 +265,13 @@ class SSHProvider(SandboxProvider):
|
||||
timeout=min(self.timeout, 10),
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise SandboxProviderConfigError(
|
||||
f"SSH connectivity check failed on {self.username}@{self.host}:{self.port}: "
|
||||
f"{stderr or 'remote command returned non-zero exit status'}"
|
||||
)
|
||||
raise SandboxProviderConfigError(f"SSH connectivity check failed on {self.username}@{self.host}:{self.port}: {stderr or 'remote command returned non-zero exit status'}")
|
||||
finally:
|
||||
client.close()
|
||||
except SandboxProviderConfigError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise SandboxProviderConfigError(
|
||||
f"Failed to connect to SSH host {self.username}@{self.host}:{self.port}: {exc}"
|
||||
) from exc
|
||||
raise SandboxProviderConfigError(f"Failed to connect to SSH host {self.username}@{self.host}:{self.port}: {exc}") from exc
|
||||
|
||||
def get_supported_languages(self) -> List[str]:
|
||||
return ["python", "javascript", "nodejs"]
|
||||
@@ -470,9 +461,7 @@ class SSHProvider(SandboxProvider):
|
||||
# Match the Go provider's fail-closed posture (see
|
||||
# internal/agent/sandbox/ssh.go::hostKeyCallback).
|
||||
logging.warning("SSH: failed to load configured known_hosts file; refusing connection")
|
||||
raise SandboxProviderConfigError(
|
||||
"Failed to load configured SSH known_hosts file."
|
||||
) from exc
|
||||
raise SandboxProviderConfigError("Failed to load configured SSH known_hosts file.") from exc
|
||||
# Reject unknown hosts: this is the default fail-closed posture
|
||||
# to prevent silent MITM. Operators must either ship a populated
|
||||
# known_hosts file or accept the warning (paramiko will fail the
|
||||
@@ -522,9 +511,7 @@ class SSHProvider(SandboxProvider):
|
||||
except Exception as exc:
|
||||
errors.append(str(exc))
|
||||
|
||||
raise SandboxProviderConfigError(
|
||||
"Failed to load SSH private key. " + "; ".join(error for error in errors if error)
|
||||
)
|
||||
raise SandboxProviderConfigError("Failed to load SSH private key. " + "; ".join(error for error in errors if error))
|
||||
|
||||
def _create_remote_workspace(self, client: paramiko.SSHClient) -> str:
|
||||
base_dir = self.work_dir.rstrip("/") or "/tmp"
|
||||
@@ -535,9 +522,7 @@ class SSHProvider(SandboxProvider):
|
||||
timeout=min(self.timeout, 10),
|
||||
)
|
||||
if exit_code != 0:
|
||||
raise RuntimeError(
|
||||
f"Failed to create remote workspace on {self.host}: {stderr or stdout or 'unknown error'}"
|
||||
)
|
||||
raise RuntimeError(f"Failed to create remote workspace on {self.host}: {stderr or stdout or 'unknown error'}")
|
||||
|
||||
remote_work_dir = stdout.strip().splitlines()[-1] if stdout.strip() else ""
|
||||
if not remote_work_dir:
|
||||
@@ -577,10 +562,7 @@ class SSHProvider(SandboxProvider):
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported language for SSH provider: {language}")
|
||||
|
||||
return (
|
||||
f"cd {shlex.quote(remote_work_dir)} && "
|
||||
f"{shlex.quote(executable)} {shlex.quote(remote_script_path)}"
|
||||
)
|
||||
return f"cd {shlex.quote(remote_work_dir)} && {shlex.quote(executable)} {shlex.quote(remote_script_path)}"
|
||||
|
||||
def _run_remote_command(
|
||||
self,
|
||||
@@ -700,7 +682,5 @@ def _get_paramiko_module():
|
||||
try:
|
||||
import paramiko
|
||||
except ImportError as exc:
|
||||
raise SandboxProviderConfigError(
|
||||
"paramiko is required for the SSH sandbox provider. Install the project dependencies to enable it."
|
||||
) from exc
|
||||
raise SandboxProviderConfigError("paramiko is required for the SSH sandbox provider. Install the project dependencies to enable it.") from exc
|
||||
return paramiko
|
||||
|
||||
@@ -36,7 +36,7 @@ if __name__ == "__main__":
|
||||
|
||||
|
||||
def build_javascript_wrapper(code: str, args_json: str) -> str:
|
||||
return f'''{code}
|
||||
return f"""{code}
|
||||
|
||||
const __ragflowArgs = {args_json};
|
||||
|
||||
@@ -55,7 +55,7 @@ const __ragflowArgs = {args_json};
|
||||
}}
|
||||
console.log('{RESULT_MARKER_PREFIX}' + Buffer.from(payload, 'utf8').toString('base64'));
|
||||
}})();
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
def extract_structured_result(stdout: str) -> tuple[str, dict[str, Any]]:
|
||||
|
||||
@@ -151,7 +151,7 @@ class TestAliyunCodeInterpreterIntegration:
|
||||
""",
|
||||
language="python",
|
||||
timeout=30,
|
||||
arguments={"name": "World", "count": 2}
|
||||
arguments={"name": "World", "count": 2},
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
@@ -211,7 +211,7 @@ class TestAliyunCodeInterpreterIntegration:
|
||||
}""",
|
||||
language="javascript",
|
||||
timeout=30,
|
||||
arguments={"name": "World", "count": 2}
|
||||
arguments={"name": "World", "count": 2},
|
||||
)
|
||||
|
||||
assert result.exit_code == 0
|
||||
|
||||
@@ -32,12 +32,7 @@ class TestSandboxDataclasses:
|
||||
|
||||
def test_sandbox_instance_creation(self):
|
||||
"""Test SandboxInstance dataclass creation."""
|
||||
instance = SandboxInstance(
|
||||
instance_id="test-123",
|
||||
provider="self_managed",
|
||||
status="running",
|
||||
metadata={"language": "python"}
|
||||
)
|
||||
instance = SandboxInstance(instance_id="test-123", provider="self_managed", status="running", metadata={"language": "python"})
|
||||
|
||||
assert instance.instance_id == "test-123"
|
||||
assert instance.provider == "self_managed"
|
||||
@@ -46,24 +41,13 @@ class TestSandboxDataclasses:
|
||||
|
||||
def test_sandbox_instance_default_metadata(self):
|
||||
"""Test SandboxInstance with None metadata."""
|
||||
instance = SandboxInstance(
|
||||
instance_id="test-123",
|
||||
provider="self_managed",
|
||||
status="running",
|
||||
metadata=None
|
||||
)
|
||||
instance = SandboxInstance(instance_id="test-123", provider="self_managed", status="running", metadata=None)
|
||||
|
||||
assert instance.metadata == {}
|
||||
|
||||
def test_execution_result_creation(self):
|
||||
"""Test ExecutionResult dataclass creation."""
|
||||
result = ExecutionResult(
|
||||
stdout="Hello, World!",
|
||||
stderr="",
|
||||
exit_code=0,
|
||||
execution_time=1.5,
|
||||
metadata={"status": "success"}
|
||||
)
|
||||
result = ExecutionResult(stdout="Hello, World!", stderr="", exit_code=0, execution_time=1.5, metadata={"status": "success"})
|
||||
|
||||
assert result.stdout == "Hello, World!"
|
||||
assert result.stderr == ""
|
||||
@@ -73,13 +57,7 @@ class TestSandboxDataclasses:
|
||||
|
||||
def test_execution_result_default_metadata(self):
|
||||
"""Test ExecutionResult with None metadata."""
|
||||
result = ExecutionResult(
|
||||
stdout="output",
|
||||
stderr="error",
|
||||
exit_code=1,
|
||||
execution_time=0.5,
|
||||
metadata=None
|
||||
)
|
||||
result = ExecutionResult(stdout="output", stderr="error", exit_code=1, execution_time=0.5, metadata=None)
|
||||
|
||||
assert result.metadata == {}
|
||||
|
||||
@@ -145,7 +123,7 @@ class TestSelfManagedProvider:
|
||||
assert provider.pool_size == 10
|
||||
assert not provider._initialized
|
||||
|
||||
@patch('requests.get')
|
||||
@patch("requests.get")
|
||||
def test_initialize_success(self, mock_get):
|
||||
"""Test successful initialization."""
|
||||
mock_response = Mock()
|
||||
@@ -153,12 +131,7 @@ class TestSelfManagedProvider:
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
provider = SelfManagedProvider()
|
||||
result = provider.initialize({
|
||||
"endpoint": "http://test-endpoint:9385",
|
||||
"timeout": 60,
|
||||
"max_retries": 5,
|
||||
"pool_size": 20
|
||||
})
|
||||
result = provider.initialize({"endpoint": "http://test-endpoint:9385", "timeout": 60, "max_retries": 5, "pool_size": 20})
|
||||
|
||||
assert result is True
|
||||
assert provider.endpoint == "http://test-endpoint:9385"
|
||||
@@ -168,7 +141,7 @@ class TestSelfManagedProvider:
|
||||
assert provider._initialized
|
||||
mock_get.assert_called_once_with("http://test-endpoint:9385/healthz", timeout=5)
|
||||
|
||||
@patch('requests.get')
|
||||
@patch("requests.get")
|
||||
def test_initialize_failure(self, mock_get):
|
||||
"""Test initialization failure."""
|
||||
mock_get.side_effect = Exception("Connection error")
|
||||
@@ -181,7 +154,7 @@ class TestSelfManagedProvider:
|
||||
|
||||
def test_initialize_default_config(self):
|
||||
"""Test initialization with default config."""
|
||||
with patch('requests.get') as mock_get:
|
||||
with patch("requests.get") as mock_get:
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_get.return_value = mock_response
|
||||
@@ -222,30 +195,18 @@ class TestSelfManagedProvider:
|
||||
with pytest.raises(RuntimeError, match="Provider not initialized"):
|
||||
provider.create_instance("python")
|
||||
|
||||
@patch('requests.post')
|
||||
@patch("requests.post")
|
||||
def test_execute_code_success(self, mock_post):
|
||||
"""Test successful code execution."""
|
||||
mock_response = Mock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
"status": "success",
|
||||
"stdout": '{"result": 42}',
|
||||
"stderr": "",
|
||||
"exit_code": 0,
|
||||
"time_used_ms": 100.0,
|
||||
"memory_used_kb": 1024.0
|
||||
}
|
||||
mock_response.json.return_value = {"status": "success", "stdout": '{"result": 42}', "stderr": "", "exit_code": 0, "time_used_ms": 100.0, "memory_used_kb": 1024.0}
|
||||
mock_post.return_value = mock_response
|
||||
|
||||
provider = SelfManagedProvider()
|
||||
provider._initialized = True
|
||||
|
||||
result = provider.execute_code(
|
||||
instance_id="test-123",
|
||||
code="def main(): return {'result': 42}",
|
||||
language="python",
|
||||
timeout=10
|
||||
)
|
||||
result = provider.execute_code(instance_id="test-123", code="def main(): return {'result': 42}", language="python", timeout=10)
|
||||
|
||||
assert result.stdout == '{"result": 42}'
|
||||
assert result.stderr == ""
|
||||
@@ -254,7 +215,7 @@ class TestSelfManagedProvider:
|
||||
assert result.metadata["status"] == "success"
|
||||
assert result.metadata["instance_id"] == "test-123"
|
||||
|
||||
@patch('requests.post')
|
||||
@patch("requests.post")
|
||||
def test_execute_code_maps_structured_result_into_metadata(self, mock_post):
|
||||
"""Test successful code execution with structured result envelope."""
|
||||
mock_response = Mock()
|
||||
@@ -277,19 +238,14 @@ class TestSelfManagedProvider:
|
||||
provider = SelfManagedProvider()
|
||||
provider._initialized = True
|
||||
|
||||
result = provider.execute_code(
|
||||
instance_id="test-123",
|
||||
code="def main(): return {'items': ['a', 'b']}",
|
||||
language="python",
|
||||
timeout=10
|
||||
)
|
||||
result = provider.execute_code(instance_id="test-123", code="def main(): return {'items': ['a', 'b']}", language="python", timeout=10)
|
||||
|
||||
assert result.stdout == "debug line\n"
|
||||
assert result.metadata["result_present"] is True
|
||||
assert result.metadata["result_value"] == {"items": ["a", "b"]}
|
||||
assert result.metadata["result_type"] == "json"
|
||||
|
||||
@patch('requests.post')
|
||||
@patch("requests.post")
|
||||
def test_execute_code_timeout(self, mock_post):
|
||||
"""Test code execution timeout."""
|
||||
mock_post.side_effect = requests.Timeout()
|
||||
@@ -298,14 +254,9 @@ class TestSelfManagedProvider:
|
||||
provider._initialized = True
|
||||
|
||||
with pytest.raises(TimeoutError, match="Execution timed out"):
|
||||
provider.execute_code(
|
||||
instance_id="test-123",
|
||||
code="while True: pass",
|
||||
language="python",
|
||||
timeout=5
|
||||
)
|
||||
provider.execute_code(instance_id="test-123", code="while True: pass", language="python", timeout=5)
|
||||
|
||||
@patch('requests.post')
|
||||
@patch("requests.post")
|
||||
def test_execute_code_http_error(self, mock_post):
|
||||
"""Test code execution with HTTP error."""
|
||||
mock_response = Mock()
|
||||
@@ -317,22 +268,14 @@ class TestSelfManagedProvider:
|
||||
provider._initialized = True
|
||||
|
||||
with pytest.raises(RuntimeError, match="HTTP 500"):
|
||||
provider.execute_code(
|
||||
instance_id="test-123",
|
||||
code="invalid code",
|
||||
language="python"
|
||||
)
|
||||
provider.execute_code(instance_id="test-123", code="invalid code", language="python")
|
||||
|
||||
def test_execute_code_not_initialized(self):
|
||||
"""Test executing code when provider not initialized."""
|
||||
provider = SelfManagedProvider()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Provider not initialized"):
|
||||
provider.execute_code(
|
||||
instance_id="test-123",
|
||||
code="print('hello')",
|
||||
language="python"
|
||||
)
|
||||
provider.execute_code(instance_id="test-123", code="print('hello')", language="python")
|
||||
|
||||
def test_destroy_instance(self):
|
||||
"""Test destroying an instance (no-op for self-managed)."""
|
||||
@@ -344,7 +287,7 @@ class TestSelfManagedProvider:
|
||||
|
||||
assert result is True
|
||||
|
||||
@patch('requests.get')
|
||||
@patch("requests.get")
|
||||
def test_health_check_success(self, mock_get):
|
||||
"""Test successful health check."""
|
||||
mock_response = Mock()
|
||||
@@ -358,7 +301,7 @@ class TestSelfManagedProvider:
|
||||
assert result is True
|
||||
mock_get.assert_called_once_with("http://localhost:9385/healthz", timeout=5)
|
||||
|
||||
@patch('requests.get')
|
||||
@patch("requests.get")
|
||||
def test_health_check_failure(self, mock_get):
|
||||
"""Test health check failure."""
|
||||
mock_get.side_effect = Exception("Connection error")
|
||||
@@ -439,20 +382,20 @@ class TestProviderInterface:
|
||||
provider = SelfManagedProvider()
|
||||
|
||||
# Check all abstract methods are implemented
|
||||
assert hasattr(provider, 'initialize')
|
||||
assert hasattr(provider, "initialize")
|
||||
assert callable(provider.initialize)
|
||||
|
||||
assert hasattr(provider, 'create_instance')
|
||||
assert hasattr(provider, "create_instance")
|
||||
assert callable(provider.create_instance)
|
||||
|
||||
assert hasattr(provider, 'execute_code')
|
||||
assert hasattr(provider, "execute_code")
|
||||
assert callable(provider.execute_code)
|
||||
|
||||
assert hasattr(provider, 'destroy_instance')
|
||||
assert hasattr(provider, "destroy_instance")
|
||||
assert callable(provider.destroy_instance)
|
||||
|
||||
assert hasattr(provider, 'health_check')
|
||||
assert hasattr(provider, "health_check")
|
||||
assert callable(provider.health_check)
|
||||
|
||||
assert hasattr(provider, 'get_supported_languages')
|
||||
assert hasattr(provider, "get_supported_languages")
|
||||
assert callable(provider.get_supported_languages)
|
||||
|
||||
@@ -76,9 +76,7 @@ def test_python_builtins_import_is_rejected():
|
||||
assert is_safe is False
|
||||
# Pin the specific reason: rejection must come from the new ``builtins``
|
||||
# entry in ``DANGEROUS_IMPORTS``, not from some unrelated parse error.
|
||||
assert any("builtins" in issue for issue, _ in issues), (
|
||||
f"expected an issue mentioning 'builtins', got {issues!r}"
|
||||
)
|
||||
assert any("builtins" in issue for issue, _ in issues), f"expected an issue mentioning 'builtins', got {issues!r}"
|
||||
|
||||
|
||||
def test_python_attribute_eval_call_is_rejected():
|
||||
@@ -94,9 +92,7 @@ def test_python_attribute_eval_call_is_rejected():
|
||||
# not from the ``import builtins`` line above. We assert ``exec`` is in at
|
||||
# least one finding so the test fails if visit_Call's attribute branch is
|
||||
# ever reverted.
|
||||
assert any("exec" in issue for issue, _ in issues), (
|
||||
f"expected an issue mentioning 'exec', got {issues!r}"
|
||||
)
|
||||
assert any("exec" in issue for issue, _ in issues), f"expected an issue mentioning 'exec', got {issues!r}"
|
||||
|
||||
|
||||
def test_javascript_safe_code_still_passes():
|
||||
|
||||
@@ -36,17 +36,14 @@ print("✓ Provider has all required methods")
|
||||
print("\n[3/5] Testing SDK imports...")
|
||||
try:
|
||||
# Check if agentrun SDK is available using importlib
|
||||
if (
|
||||
importlib.util.find_spec("agentrun.sandbox") is None
|
||||
or importlib.util.find_spec("agentrun.utils.config") is None
|
||||
or importlib.util.find_spec("agentrun.utils.exception") is None
|
||||
):
|
||||
if importlib.util.find_spec("agentrun.sandbox") is None or importlib.util.find_spec("agentrun.utils.config") is None or importlib.util.find_spec("agentrun.utils.exception") is None:
|
||||
raise ImportError("agentrun SDK not found")
|
||||
|
||||
# Verify imports work (assign to _ to indicate they're intentionally unused)
|
||||
from agentrun.sandbox import CodeInterpreterSandbox, TemplateType, CodeLanguage
|
||||
from agentrun.utils.config import Config
|
||||
from agentrun.utils.exception import ServerError
|
||||
|
||||
_ = (CodeInterpreterSandbox, TemplateType, CodeLanguage, Config, ServerError)
|
||||
|
||||
print("✓ SDK modules imported successfully")
|
||||
|
||||
275
agent/templates/ingestion_pipeline_audio.json
Normal file
275
agent/templates/ingestion_pipeline_audio.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
335
agent/templates/ingestion_pipeline_email.json
Normal file
335
agent/templates/ingestion_pipeline_email.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
294
agent/templates/ingestion_pipeline_picture.json
Normal file
294
agent/templates/ingestion_pipeline_picture.json
Normal file
File diff suppressed because one or more lines are too long
296
agent/templates/ingestion_pipeline_presentation.json
Normal file
296
agent/templates/ingestion_pipeline_presentation.json
Normal file
File diff suppressed because one or more lines are too long
372
agent/templates/ingestion_pipeline_qa.json
Normal file
372
agent/templates/ingestion_pipeline_qa.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
308
agent/templates/ingestion_pipeline_table.json
Normal file
308
agent/templates/ingestion_pipeline_table.json
Normal file
File diff suppressed because one or more lines are too long
308
agent/templates/ingestion_pipeline_tag.json
Normal file
308
agent/templates/ingestion_pipeline_tag.json
Normal file
File diff suppressed because one or more lines are too long
@@ -18,29 +18,29 @@ import os
|
||||
from agent.canvas import Canvas
|
||||
from common import settings
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
dsl_default_path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)),
|
||||
"dsl_examples",
|
||||
"retrieval_and_generate.json",
|
||||
)
|
||||
parser.add_argument('-s', '--dsl', default=dsl_default_path, help="input dsl", action='store', required=True)
|
||||
parser.add_argument('-t', '--tenant_id', default=False, help="Tenant ID", action='store', required=True)
|
||||
parser.add_argument('-m', '--stream', default=False, help="Stream output", action='store_true', required=False)
|
||||
parser.add_argument("-s", "--dsl", default=dsl_default_path, help="input dsl", action="store", required=True)
|
||||
parser.add_argument("-t", "--tenant_id", default=False, help="Tenant ID", action="store", required=True)
|
||||
parser.add_argument("-m", "--stream", default=False, help="Stream output", action="store_true", required=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
settings.init_settings()
|
||||
canvas = Canvas(open(args.dsl, "r").read(), args.tenant_id)
|
||||
if canvas.get_prologue():
|
||||
print(f"==================== Bot =====================\n> {canvas.get_prologue()}", end='')
|
||||
print(f"==================== Bot =====================\n> {canvas.get_prologue()}", end="")
|
||||
query = ""
|
||||
while True:
|
||||
canvas.reset(True)
|
||||
query = input("\n==================== User =====================\n> ")
|
||||
ans = canvas.run(query=query)
|
||||
print("==================== Bot =====================\n> ", end='')
|
||||
print("==================== Bot =====================\n> ", end="")
|
||||
for ans in canvas.run(query=query):
|
||||
print(ans, end='\n', flush=True)
|
||||
print(ans, end="\n", flush=True)
|
||||
|
||||
print(canvas.path)
|
||||
|
||||
@@ -22,8 +22,9 @@ from typing import Dict, Type
|
||||
_package_path = os.path.dirname(__file__)
|
||||
__all_classes: Dict[str, Type] = {}
|
||||
|
||||
|
||||
def _import_submodules() -> None:
|
||||
for filename in os.listdir(_package_path): # noqa: F821
|
||||
for filename in os.listdir(_package_path): # noqa: F821
|
||||
if filename.startswith("__") or not filename.endswith(".py") or filename.startswith("base"):
|
||||
continue
|
||||
module_name = filename[:-3]
|
||||
@@ -34,15 +35,16 @@ def _import_submodules() -> None:
|
||||
except ImportError as e:
|
||||
print(f"Warning: Failed to import module {module_name}: {str(e)}")
|
||||
|
||||
|
||||
def _extract_classes_from_module(module: ModuleType) -> None:
|
||||
for name, obj in inspect.getmembers(module):
|
||||
if (inspect.isclass(obj) and
|
||||
obj.__module__ == module.__name__ and not name.startswith("_")):
|
||||
if inspect.isclass(obj) and obj.__module__ == module.__name__ and not name.startswith("_"):
|
||||
__all_classes[name] = obj
|
||||
globals()[name] = obj
|
||||
|
||||
|
||||
_import_submodules()
|
||||
|
||||
__all__ = list(__all_classes.keys()) + ["__all_classes"]
|
||||
|
||||
del _package_path, _import_submodules, _extract_classes_from_module
|
||||
del _package_path, _import_submodules, _extract_classes_from_module
|
||||
|
||||
@@ -13,44 +13,85 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from abc import ABC
|
||||
import pandas as pd
|
||||
from agent.component.base import ComponentBase, ComponentParamBase
|
||||
from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
|
||||
from common.connection_utils import timeout
|
||||
|
||||
|
||||
class AkShareParam(ComponentParamBase):
|
||||
class AkShareParam(ToolParamBase):
|
||||
"""
|
||||
Define the AkShare component parameters.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.meta: ToolMeta = {
|
||||
"name": "akshare_stock_news",
|
||||
"description": "AkShare retrieves the latest news articles for a given Chinese A-share stock from East Money (东方财富).",
|
||||
"parameters": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The stock symbol/code to fetch news for, e.g. '600519'.",
|
||||
"default": "{sys.query}",
|
||||
"required": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
super().__init__()
|
||||
self.top_n = 10
|
||||
|
||||
def check(self):
|
||||
self.check_positive_integer(self.top_n, "Top N")
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {"query": {"name": "Stock symbol", "type": "line"}}
|
||||
|
||||
class AkShare(ComponentBase, ABC):
|
||||
|
||||
class AkShare(ToolBase, ABC):
|
||||
component_name = "AkShare"
|
||||
|
||||
def _run(self, history, **kwargs):
|
||||
import akshare as ak
|
||||
ans = self.get_input()
|
||||
ans = ",".join(ans["content"]) if "content" in ans else ""
|
||||
if not ans:
|
||||
return AkShare.be_output("")
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 12)))
|
||||
def _invoke(self, **kwargs):
|
||||
if self.check_if_canceled("AkShare processing"):
|
||||
return
|
||||
|
||||
try:
|
||||
ak_res = []
|
||||
stock_news_em_df = ak.stock_news_em(symbol=ans)
|
||||
stock_news_em_df = stock_news_em_df.head(self._param.top_n)
|
||||
ak_res = [{"content": '<a href="' + i["新闻链接"] + '">' + i["新闻标题"] + '</a>\n 新闻内容: ' + i[
|
||||
"新闻内容"] + " \n发布时间:" + i["发布时间"] + " \n文章来源: " + i["文章来源"]} for index, i in stock_news_em_df.iterrows()]
|
||||
except Exception as e:
|
||||
return AkShare.be_output("**ERROR**: " + str(e))
|
||||
symbol = kwargs.get("query")
|
||||
if not symbol:
|
||||
self.set_output("formalized_content", "")
|
||||
return ""
|
||||
|
||||
if not ak_res:
|
||||
return AkShare.be_output("")
|
||||
last_e = None
|
||||
for _ in range(self._param.max_retries + 1):
|
||||
if self.check_if_canceled("AkShare processing"):
|
||||
return
|
||||
|
||||
return pd.DataFrame(ak_res)
|
||||
try:
|
||||
import akshare as ak
|
||||
|
||||
df = ak.stock_news_em(symbol=symbol).head(self._param.top_n)
|
||||
|
||||
if self.check_if_canceled("AkShare processing"):
|
||||
return
|
||||
|
||||
items = ['<a href="{}">{}</a>\n 新闻内容: {} \n发布时间:{} \n文章来源: {}'.format(i["新闻链接"], i["新闻标题"], i["新闻内容"], i["发布时间"], i["文章来源"]) for _, i in df.iterrows()]
|
||||
res = "\n\n".join(items)
|
||||
self.set_output("formalized_content", res)
|
||||
return res
|
||||
except Exception as e:
|
||||
if self.check_if_canceled("AkShare processing"):
|
||||
return
|
||||
|
||||
last_e = e
|
||||
logging.exception(f"AkShare error: {e}")
|
||||
time.sleep(self._param.delay_after_error)
|
||||
|
||||
if last_e:
|
||||
self.set_output("_ERROR", str(last_e))
|
||||
return f"AkShare error: {last_e}"
|
||||
|
||||
assert False, self.output()
|
||||
|
||||
def thoughts(self) -> str:
|
||||
return "Looking up the latest stock news for: {}".format(self.get_input().get("query", "-_-!"))
|
||||
|
||||
@@ -28,7 +28,7 @@ class ArXivParam(ToolParamBase):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.meta:ToolMeta = {
|
||||
self.meta: ToolMeta = {
|
||||
"name": "arxiv_search",
|
||||
"description": """arXiv is a free distribution service and an open-access archive for nearly 2.4 million scholarly articles in the fields of physics, mathematics, computer science, quantitative biology, quantitative finance, statistics, electrical engineering and systems science, and economics. Materials on this site are not peer-reviewed by arXiv.""",
|
||||
"parameters": {
|
||||
@@ -36,26 +36,20 @@ class ArXivParam(ToolParamBase):
|
||||
"type": "string",
|
||||
"description": "The search keywords to execute with arXiv. The keywords should be the most important words/terms(includes synonyms) from the original request.",
|
||||
"default": "{sys.query}",
|
||||
"required": True
|
||||
"required": True,
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
super().__init__()
|
||||
self.top_n = 12
|
||||
self.sort_by = 'submittedDate'
|
||||
self.sort_by = "submittedDate"
|
||||
|
||||
def check(self):
|
||||
self.check_positive_integer(self.top_n, "Top N")
|
||||
self.check_valid_value(self.sort_by, "ArXiv Search Sort_by",
|
||||
['submittedDate', 'lastUpdatedDate', 'relevance'])
|
||||
self.check_valid_value(self.sort_by, "ArXiv Search Sort_by", ["submittedDate", "lastUpdatedDate", "relevance"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"query": {
|
||||
"name": "Query",
|
||||
"type": "line"
|
||||
}
|
||||
}
|
||||
return {"query": {"name": "Query", "type": "line"}}
|
||||
|
||||
|
||||
class ArXiv(ToolBase, ABC):
|
||||
@@ -71,29 +65,20 @@ class ArXiv(ToolBase, ABC):
|
||||
return ""
|
||||
|
||||
last_e = ""
|
||||
for _ in range(self._param.max_retries+1):
|
||||
for _ in range(self._param.max_retries + 1):
|
||||
if self.check_if_canceled("ArXiv processing"):
|
||||
return
|
||||
|
||||
try:
|
||||
sort_choices = {"relevance": arxiv.SortCriterion.Relevance,
|
||||
"lastUpdatedDate": arxiv.SortCriterion.LastUpdatedDate,
|
||||
'submittedDate': arxiv.SortCriterion.SubmittedDate}
|
||||
sort_choices = {"relevance": arxiv.SortCriterion.Relevance, "lastUpdatedDate": arxiv.SortCriterion.LastUpdatedDate, "submittedDate": arxiv.SortCriterion.SubmittedDate}
|
||||
arxiv_client = arxiv.Client()
|
||||
search = arxiv.Search(
|
||||
query=kwargs["query"],
|
||||
max_results=self._param.top_n,
|
||||
sort_by=sort_choices[self._param.sort_by]
|
||||
)
|
||||
search = arxiv.Search(query=kwargs["query"], max_results=self._param.top_n, sort_by=sort_choices[self._param.sort_by])
|
||||
results = list(arxiv_client.results(search))
|
||||
|
||||
if self.check_if_canceled("ArXiv processing"):
|
||||
return
|
||||
|
||||
self._retrieve_chunks(results,
|
||||
get_title=lambda r: r.title,
|
||||
get_url=lambda r: r.pdf_url,
|
||||
get_content=lambda r: r.summary)
|
||||
self._retrieve_chunks(results, get_title=lambda r: r.title, get_url=lambda r: r.pdf_url, get_content=lambda r: r.summary)
|
||||
return self.output("formalized_content")
|
||||
except Exception as e:
|
||||
if self.check_if_canceled("ArXiv processing"):
|
||||
|
||||
@@ -28,10 +28,9 @@ from common.mcp_tool_call_conn import MCPToolBinding, MCPToolCallSession, ToolCa
|
||||
from timeit import default_timer as timer
|
||||
|
||||
|
||||
|
||||
|
||||
from common.misc_utils import thread_pool_exec
|
||||
|
||||
|
||||
class ToolParameter(TypedDict):
|
||||
type: str
|
||||
description: str
|
||||
@@ -81,7 +80,9 @@ class LLMToolPluginCallSession(ToolCallSession):
|
||||
resp = fallback_output
|
||||
else:
|
||||
resp = fallback_output
|
||||
logging.warning(f"[ToolCall] resp is None, fallback to output name={name} output_keys={list(fallback_output.keys()) if isinstance(fallback_output, dict) else type(fallback_output).__name__}")
|
||||
logging.warning(
|
||||
f"[ToolCall] resp is None, fallback to output name={name} output_keys={list(fallback_output.keys()) if isinstance(fallback_output, dict) else type(fallback_output).__name__}"
|
||||
)
|
||||
except Exception as e:
|
||||
logging.warning(f"[ToolCall] resp is None and output fallback failed name={name} err={e}")
|
||||
|
||||
@@ -96,28 +97,25 @@ class LLMToolPluginCallSession(ToolCallSession):
|
||||
|
||||
class ToolParamBase(ComponentParamBase):
|
||||
def __init__(self):
|
||||
#self.meta:ToolMeta = None
|
||||
# self.meta:ToolMeta = None
|
||||
super().__init__()
|
||||
self._init_inputs()
|
||||
self._init_attr_by_meta()
|
||||
|
||||
def _init_inputs(self):
|
||||
self.inputs = {}
|
||||
for k,p in self.meta["parameters"].items():
|
||||
for k, p in self.meta["parameters"].items():
|
||||
self.inputs[k] = deepcopy(p)
|
||||
|
||||
def _init_attr_by_meta(self):
|
||||
for k,p in self.meta["parameters"].items():
|
||||
for k, p in self.meta["parameters"].items():
|
||||
if not hasattr(self, k):
|
||||
setattr(self, k, p.get("default"))
|
||||
|
||||
def get_meta(self):
|
||||
params = {}
|
||||
for k, p in self.meta["parameters"].items():
|
||||
params[k] = {
|
||||
"type": p["type"],
|
||||
"description": p["description"]
|
||||
}
|
||||
params[k] = {"type": p["type"], "description": p["description"]}
|
||||
if "enum" in p:
|
||||
params[k]["enum"] = p["enum"]
|
||||
|
||||
@@ -129,12 +127,8 @@ class ToolParamBase(ComponentParamBase):
|
||||
"function": {
|
||||
"name": function_name,
|
||||
"description": desc,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": params,
|
||||
"required": [k for k, p in self.meta["parameters"].items() if p["required"]]
|
||||
}
|
||||
}
|
||||
"parameters": {"type": "object", "properties": params, "required": [k for k, p in self.meta["parameters"].items() if p["required"]]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -209,20 +203,8 @@ class ToolBase(ComponentBase):
|
||||
title = get_title(r)
|
||||
url = get_url(r)
|
||||
score = get_score(r) if get_score else 1
|
||||
chunks.append({
|
||||
"chunk_id": id,
|
||||
"content": content,
|
||||
"doc_id": id,
|
||||
"docnm_kwd": title,
|
||||
"similarity": score,
|
||||
"url": url
|
||||
})
|
||||
aggs.append({
|
||||
"doc_name": title,
|
||||
"doc_id": id,
|
||||
"count": 1,
|
||||
"url": url
|
||||
})
|
||||
chunks.append({"chunk_id": id, "content": content, "doc_id": id, "docnm_kwd": title, "similarity": score, "url": url})
|
||||
aggs.append({"doc_name": title, "doc_id": id, "count": 1, "url": url})
|
||||
self._canvas.add_reference(chunks, aggs)
|
||||
self.set_output("formalized_content", "\n".join(kb_prompt({"chunks": chunks, "doc_aggs": aggs}, 200000, True)))
|
||||
|
||||
|
||||
@@ -13,10 +13,13 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import logging
|
||||
import os
|
||||
from abc import ABC
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler
|
||||
from agent.tools.base import ToolParamBase, ToolBase
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
|
||||
from common.connection_utils import timeout
|
||||
|
||||
|
||||
class CrawlerParam(ToolParamBase):
|
||||
@@ -25,6 +28,18 @@ class CrawlerParam(ToolParamBase):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.meta: ToolMeta = {
|
||||
"name": "web_crawler",
|
||||
"description": "This tool can be used to crawl a web page and return its content as HTML, Markdown, or the extracted main text.",
|
||||
"parameters": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The absolute URL (including the http:// or https:// scheme) of the web page to crawl.",
|
||||
"default": "{sys.query}",
|
||||
"required": True,
|
||||
}
|
||||
},
|
||||
}
|
||||
super().__init__()
|
||||
self.proxy = None
|
||||
self.extract_type = "markdown"
|
||||
@@ -32,37 +47,70 @@ class CrawlerParam(ToolParamBase):
|
||||
def check(self):
|
||||
self.check_valid_value(self.extract_type, "Type of content from the crawler", ["html", "markdown", "content"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {"query": {"name": "URL", "type": "line"}}
|
||||
|
||||
|
||||
class Crawler(ToolBase, ABC):
|
||||
component_name = "Crawler"
|
||||
|
||||
def _run(self, history, **kwargs):
|
||||
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
|
||||
def _invoke(self, **kwargs):
|
||||
from common.ssrf_guard import assert_url_is_safe, pin_dns_global
|
||||
|
||||
ans = self.get_input()
|
||||
ans = " - ".join(ans["content"]) if "content" in ans else ""
|
||||
if self.check_if_canceled("Crawler processing"):
|
||||
return
|
||||
|
||||
url = kwargs.get("query")
|
||||
if not url:
|
||||
self.set_output("formalized_content", "")
|
||||
return ""
|
||||
|
||||
try:
|
||||
_ssrf_hostname, _ssrf_ip = assert_url_is_safe(ans)
|
||||
_ssrf_hostname, _ssrf_ip = assert_url_is_safe(url)
|
||||
except ValueError:
|
||||
return Crawler.be_output("URL not valid")
|
||||
msg = "URL not valid"
|
||||
self.set_output("_ERROR", msg)
|
||||
return msg
|
||||
|
||||
try:
|
||||
# pin_dns_global is used (not thread-local) because crawl4ai resolves
|
||||
# DNS in asyncio executor threads that don't share thread-local state.
|
||||
with pin_dns_global(_ssrf_hostname, _ssrf_ip):
|
||||
result = asyncio.run(self.get_web(ans))
|
||||
result = asyncio.run(self.get_web(url))
|
||||
|
||||
return Crawler.be_output(result)
|
||||
if self.check_if_canceled("Crawler processing"):
|
||||
return
|
||||
|
||||
result = result or ""
|
||||
self.set_output("formalized_content", result)
|
||||
return result
|
||||
except Exception as e:
|
||||
return Crawler.be_output(f"An unexpected error occurred: {str(e)}")
|
||||
if self.check_if_canceled("Crawler processing"):
|
||||
return
|
||||
|
||||
logging.exception(f"Crawler error: {e}")
|
||||
msg = f"An unexpected error occurred: {str(e)}"
|
||||
self.set_output("_ERROR", msg)
|
||||
return msg
|
||||
|
||||
async def get_web(self, url):
|
||||
if self.check_if_canceled("Crawler async operation"):
|
||||
return
|
||||
|
||||
proxy = self._param.proxy if self._param.proxy else None
|
||||
async with AsyncWebCrawler(verbose=True, proxy=proxy) as crawler:
|
||||
result = await crawler.arun(url=url, bypass_cache=True)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
verbose=True,
|
||||
proxy_config=proxy,
|
||||
)
|
||||
|
||||
run_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(url=url, config=run_config)
|
||||
|
||||
if self.check_if_canceled("Crawler async operation"):
|
||||
return
|
||||
|
||||
@@ -27,18 +27,53 @@ class DeepLParam(ComponentParamBase):
|
||||
super().__init__()
|
||||
self.auth_key = "xxx"
|
||||
self.parameters = []
|
||||
self.source_lang = 'ZH'
|
||||
self.target_lang = 'EN-GB'
|
||||
self.source_lang = "ZH"
|
||||
self.target_lang = "EN-GB"
|
||||
|
||||
def check(self):
|
||||
self.check_valid_value(self.source_lang, "Source language",
|
||||
['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT',
|
||||
'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR',
|
||||
'UK', 'ZH'])
|
||||
self.check_valid_value(self.target_lang, "Target language",
|
||||
['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU',
|
||||
'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT-BR', 'PT-PT', 'RO', 'RU',
|
||||
'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
|
||||
self.check_valid_value(
|
||||
self.source_lang,
|
||||
"Source language",
|
||||
["AR", "BG", "CS", "DA", "DE", "EL", "EN", "ES", "ET", "FI", "FR", "HU", "ID", "IT", "JA", "KO", "LT", "LV", "NB", "NL", "PL", "PT", "RO", "RU", "SK", "SL", "SV", "TR", "UK", "ZH"],
|
||||
)
|
||||
self.check_valid_value(
|
||||
self.target_lang,
|
||||
"Target language",
|
||||
[
|
||||
"AR",
|
||||
"BG",
|
||||
"CS",
|
||||
"DA",
|
||||
"DE",
|
||||
"EL",
|
||||
"EN-GB",
|
||||
"EN-US",
|
||||
"ES",
|
||||
"ET",
|
||||
"FI",
|
||||
"FR",
|
||||
"HU",
|
||||
"ID",
|
||||
"IT",
|
||||
"JA",
|
||||
"KO",
|
||||
"LT",
|
||||
"LV",
|
||||
"NB",
|
||||
"NL",
|
||||
"PL",
|
||||
"PT-BR",
|
||||
"PT-PT",
|
||||
"RO",
|
||||
"RU",
|
||||
"SK",
|
||||
"SL",
|
||||
"SV",
|
||||
"TR",
|
||||
"UK",
|
||||
"ZH",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class DeepL(ComponentBase, ABC):
|
||||
@@ -57,8 +92,7 @@ class DeepL(ComponentBase, ABC):
|
||||
|
||||
try:
|
||||
translator = deepl.Translator(self._param.auth_key)
|
||||
result = translator.translate_text(ans, source_lang=self._param.source_lang,
|
||||
target_lang=self._param.target_lang)
|
||||
result = translator.translate_text(ans, source_lang=self._param.source_lang, target_lang=self._param.target_lang)
|
||||
|
||||
return DeepL.be_output(result.text)
|
||||
except Exception as e:
|
||||
|
||||
@@ -28,7 +28,7 @@ class DuckDuckGoParam(ToolParamBase):
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.meta:ToolMeta = {
|
||||
self.meta: ToolMeta = {
|
||||
"name": "duckduckgo_search",
|
||||
"description": "DuckDuckGo is a search engine focused on privacy. It offers search capabilities for web pages, images, and provides translation services. DuckDuckGo also features a private AI chat interface, providing users with an AI assistant that prioritizes data protection.",
|
||||
"parameters": {
|
||||
@@ -36,7 +36,7 @@ class DuckDuckGoParam(ToolParamBase):
|
||||
"type": "string",
|
||||
"description": "The search keywords to execute with DuckDuckGo. The keywords should be the most important words/terms(includes synonyms) from the original request.",
|
||||
"default": "{sys.query}",
|
||||
"required": True
|
||||
"required": True,
|
||||
},
|
||||
"channel": {
|
||||
"type": "string",
|
||||
@@ -45,7 +45,7 @@ class DuckDuckGoParam(ToolParamBase):
|
||||
"default": "general",
|
||||
"required": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
super().__init__()
|
||||
self.top_n = 10
|
||||
@@ -56,18 +56,7 @@ class DuckDuckGoParam(ToolParamBase):
|
||||
self.check_valid_value(self.channel, "Web Search or News", ["text", "news"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {
|
||||
"query": {
|
||||
"name": "Query",
|
||||
"type": "line"
|
||||
},
|
||||
"channel": {
|
||||
"name": "Channel",
|
||||
"type": "options",
|
||||
"value": "general",
|
||||
"options": ["general", "news"]
|
||||
}
|
||||
}
|
||||
return {"query": {"name": "Query", "type": "line"}, "channel": {"name": "Channel", "type": "options", "value": "general", "options": ["general", "news"]}}
|
||||
|
||||
|
||||
class DuckDuckGo(ToolBase, ABC):
|
||||
@@ -83,7 +72,7 @@ class DuckDuckGo(ToolBase, ABC):
|
||||
return ""
|
||||
|
||||
last_e = ""
|
||||
for _ in range(self._param.max_retries+1):
|
||||
for _ in range(self._param.max_retries + 1):
|
||||
if self.check_if_canceled("DuckDuckGo processing"):
|
||||
return
|
||||
|
||||
@@ -99,10 +88,7 @@ class DuckDuckGo(ToolBase, ABC):
|
||||
if self.check_if_canceled("DuckDuckGo processing"):
|
||||
return
|
||||
|
||||
self._retrieve_chunks(duck_res,
|
||||
get_title=lambda r: r["title"],
|
||||
get_url=lambda r: r.get("href", r.get("url")),
|
||||
get_content=lambda r: r["body"])
|
||||
self._retrieve_chunks(duck_res, get_title=lambda r: r["title"], get_url=lambda r: r.get("href", r.get("url")), get_content=lambda r: r["body"])
|
||||
self.set_output("json", duck_res)
|
||||
return self.output("formalized_content")
|
||||
else:
|
||||
@@ -116,10 +102,7 @@ class DuckDuckGo(ToolBase, ABC):
|
||||
if self.check_if_canceled("DuckDuckGo processing"):
|
||||
return
|
||||
|
||||
self._retrieve_chunks(duck_res,
|
||||
get_title=lambda r: r["title"],
|
||||
get_url=lambda r: r.get("href", r.get("url")),
|
||||
get_content=lambda r: r["body"])
|
||||
self._retrieve_chunks(duck_res, get_title=lambda r: r["title"], get_url=lambda r: r.get("href", r.get("url")), get_content=lambda r: r["body"])
|
||||
self.set_output("json", duck_res)
|
||||
return self.output("formalized_content")
|
||||
except Exception as e:
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user