fix back release.yml to old version (#16160)

### What problem does this PR solve?

As title

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

.agents/skills/go-naming/SKILL.md

@@ -3,4 +3,4 @@ name: go-naming
 description: Go naming conventions and best practices. Use this skill when working with Go code and need to name packages, files, directories, structs, interfaces, functions, variables, or constants. Provides comprehensive naming guidelines following Go community standards.
 ---
 
-Strictly follow the naming conventions in [rules/named.md](rules/named.md)
+Strictly follow the naming conventions in [rules/named.md](../../rules/named.md)

.dockerignore

@@ -0,0 +1,62 @@
+# RAGFlow .dockerignore
+# Reduces Docker build context sent to the daemon.
+# All excluded items are either rebuilt inside Docker, mounted from
+# infiniflow/ragflow_deps, or are local-only artifacts.
+
+# ── Python virtual environments ─────────────────────────────────────────────
+.venv/
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+.pytest_cache/
+
+# ── Frontend dependencies and build outputs ─────────────────────────────────
+web/node_modules/
+web/dist/
+
+# ── Runtime logs ────────────────────────────────────────────────────────────
+logs/
+*.log
+docker/ragflow-logs/
+
+# ── Docker runtime data ─────────────────────────────────────────────────────
+docker/data/
+docker/oceanbase/
+docker/seekdb/
+
+# ── Go and C++ build outputs ────────────────────────────────────────────────
+internal/cpp/build/
+internal/cpp/cmake-build-release/
+internal/cpp/cmake-build-debug/
+target/
+
+# ── Downloaded dependency artifacts (mounted from infiniflow/ragflow_deps) ──
+chrome-linux64-*
+chromedriver-linux64-*
+tika-server-standard-*.jar
+tika-server-standard-*.jar.md5
+cl100k_base.tiktoken
+libssl*.deb
+uv-*.tar.gz
+huggingface.co/
+nltk_data/
+9b5ad71b2ce5302211f9c61530b329a4922fc6a4
+
+# ── IDE and editor config ──────────────────────────────────────────────────
+.idea/
+.vscode/
+.cursor/
+.trae/
+.DS_Store
+
+# ── Test and coverage artifacts ─────────────────────────────────────────────
+coverage/
+htmlcov/
+.coverage
+.hypothesis/
+.nox/
+
+# ── Docker env (contains secrets) ───────────────────────────────────────────
+docker/.env

.github/workflows/release.yml

@@ -85,7 +85,7 @@ jobs:
       - name: Build and push image
         run: |
           sudo docker login --username infiniflow --password-stdin <<< ${{ secrets.DOCKERHUB_TOKEN }}
-          sudo docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
+          sudo docker build -t infiniflow/ragflow:${RELEASE_TAG} -f Dockerfile .
           sudo docker tag infiniflow/ragflow:${RELEASE_TAG} infiniflow/ragflow:latest
           sudo docker push infiniflow/ragflow:${RELEASE_TAG}
           sudo docker push infiniflow/ragflow:latest

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ on:
   # — pull_request_target workflows use the workflow files from the default branch, and secrets are available.
   # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable.
   pull_request:
-    types: [ synchronize, ready_for_review ]
+    types: [opened, synchronize, reopened, ready_for_review, labeled]
     paths-ignore:
       - 'docs/**'
       - '*.md'
@@ -29,12 +29,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  ragflow_tests:
-    name: ragflow_tests
+  ragflow_preflight:
+    name: ragflow_preflight
     # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
     # https://github.com/orgs/community/discussions/26261
-    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci')) }}
+    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
     runs-on: [ "self-hosted", "ragflow-test" ]
+    outputs:
+      http_api_test_level: ${{ steps.test_level.outputs.http_api_test_level }}
     steps:
       - name: Ensure workspace ownership
         run: |
@@ -97,166 +99,327 @@ jobs:
           version: ">=0.11.x"
           args: "check"
 
-      - name: Check comments of changed Python files
-        if: ${{ false }}
+#      - name: Check comments of changed Python files
+#        if: ${{ false }}
+#        run: |
+#          if [[ ${{ github.event_name }} == 'pull_request' || ${{ github.event_name }} == 'pull_request_target' ]]; then
+#            CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
+#              | grep -E '\.(py)$' || true)
+#
+#            if [ -n "$CHANGED_FILES" ]; then
+#              echo "Check comments of changed Python files with check_comment_ascii.py"
+#
+#              readarray -t files <<< "$CHANGED_FILES"
+#              HAS_ERROR=0
+#
+#              for file in "${files[@]}"; do
+#                if [ -f "$file" ]; then
+#                  if python3 check_comment_ascii.py "$file"; then
+#                    echo "✅ $file"
+#                  else
+#                    echo "❌ $file"
+#                    HAS_ERROR=1
+#                  fi
+#                fi
+#              done
+#
+#              if [ $HAS_ERROR -ne 0 ]; then
+#                exit 1
+#              fi
+#            else
+#              echo "No Python files changed"
+#            fi
+#          fi
+
+      - name: Check gofmt of changed Go files
+        if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
         run: |
-          if [[ ${{ github.event_name }} == 'pull_request' || ${{ github.event_name }} == 'pull_request_target' ]]; then
-            CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
-              | grep -E '\.(py)$' || true)
+          CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
+            | grep -E '\.go$' || true)
 
-            if [ -n "$CHANGED_FILES" ]; then
-              echo "Check comments of changed Python files with check_comment_ascii.py"
-
-              readarray -t files <<< "$CHANGED_FILES"
-              HAS_ERROR=0
-
-              for file in "${files[@]}"; do
-                if [ -f "$file" ]; then
-                  if python3 check_comment_ascii.py "$file"; then
-                    echo "✅ $file"
-                  else
-                    echo "❌ $file"
-                    HAS_ERROR=1
-                  fi
+          if [ -n "$CHANGED_FILES" ]; then
+            echo "Check gofmt of changed Go files"
+            readarray -t files <<< "$CHANGED_FILES"
+            HAS_ERROR=0
+            for file in "${files[@]}"; do
+              if [ -f "$file" ]; then
+                if [ -z "$(gofmt -l "$file")" ]; then
+                  echo "✅ $file"
+                else
+                  echo "❌ $file (run: gofmt -w \"$file\")"
+                  HAS_ERROR=1
                 fi
-              done
-
-              if [ $HAS_ERROR -ne 0 ]; then
-                exit 1
               fi
-            else
-              echo "No Python files changed"
+            done
+
+            if [ $HAS_ERROR -ne 0 ]; then
+              exit 1
             fi
+          else
+            echo "No Go files changed"
           fi
 
-      - name: Build ragflow go server
+      - name: Set test level
+        id: test_level
         run: |
-          BUILDER_CONTAINER=ragflow_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV}
-          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple -v ${PWD}:/ragflow -v ${PWD}/internal/cpp/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu22_clang20
-          sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /ragflow && ./build.sh --cpp"
-          ./build.sh --go
-          if [[ -n "${BUILDER_CONTAINER}" ]]; then
-            sudo docker rm -f -v "${BUILDER_CONTAINER}"
-          fi
-
-      - name: Build ragflow:nightly
-        run: |
-          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
-          RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID}
-          echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> ${GITHUB_ENV}
-          sudo docker pull ubuntu:24.04
-          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
+          set -euo pipefail
           if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
             export HTTP_API_TEST_LEVEL=p3
           else
             export HTTP_API_TEST_LEVEL=p2
           fi
           echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_ENV}
-          echo "RAGFLOW_CONTAINER=${GITHUB_RUN_ID}-ragflow-cpu-1" >> ${GITHUB_ENV}
+          echo "http_api_test_level=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_OUTPUT}
+
+      - name: Prepare Python test environment
+        run: |
+          uv sync --python 3.13 --group test --frozen
+          uv pip install -e sdk/python
 
       - name: Run unit test
         run: |
-          uv sync --python 3.12 --group test --frozen
           source .venv/bin/activate
           which pytest || echo "pytest not in PATH"
           echo "Start to run unit test"
           python3 run_tests.py -i
 
+
+
+  ragflow_tests_infinity:
+    name: ragflow_tests_infinity
+    needs: ragflow_preflight
+    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
+    runs-on: [ "self-hosted", "ragflow-test" ]
+    env:
+      DOC_ENGINE: infinity
+      RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity
+      HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
+    steps:
+      - name: Ensure workspace ownership
+        run: |
+          echo "Workflow triggered by ${{ github.event_name }}"
+          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
+
+      - name: Check out code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build ragflow go server
+        run: |
+          set -euo pipefail
+          BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          cleanup_builder() {
+            if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
+              sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
+            fi
+          }
+          trap cleanup_builder EXIT
+
+          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
+            -e TZ="${TZ}" \
+            -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -v "${PWD}:/ragflow" \
+            -v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
+            infiniflow/infinity_builder:ubuntu22_clang20
+          sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
+          ./build.sh --go
+
+      - name: Build ragflow:nightly
+        run: |
+          set -euo pipefail
+          sudo docker pull ubuntu:24.04
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
+
+      - name: Prepare Python test environment
+        run: |
+          uv sync --python 3.13 --group test --frozen
+          uv pip install -e sdk/python
+
       - name: Prepare function test environment
         working-directory: docker
         run: |
+          set -euo pipefail
+          # install ss
+          sudo apt update && sudo apt install -y iproute2
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
+          COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
+          echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
+          echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
+          ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
+          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
+          rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
+
           # Determine runner number (default to 1 if not found)
                RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
                RUNNER_NUM=${RUNNER_NUM:-1}
 
-          # Compute port numbers using bash arithmetic
-               ES_PORT=$((1200 + RUNNER_NUM * 10))
-               OS_PORT=$((1201 + RUNNER_NUM * 10))
-               INFINITY_THRIFT_PORT=$((23817 + RUNNER_NUM * 10))
-               INFINITY_HTTP_PORT=$((23820 + RUNNER_NUM * 10))
-               INFINITY_PSQL_PORT=$((5432 + RUNNER_NUM * 10))
-               EXPOSE_MYSQL_PORT=$((5455 + RUNNER_NUM * 10))
-               MINIO_PORT=$((9000 + RUNNER_NUM * 10))
-               MINIO_CONSOLE_PORT=$((9001 + RUNNER_NUM * 10))
-               REDIS_PORT=$((6379 + RUNNER_NUM * 10))
-               TEI_PORT=$((6380 + RUNNER_NUM * 10))
-               KIBANA_PORT=$((6601 + RUNNER_NUM * 10))
-               SVR_HTTP_PORT=$((9380 + RUNNER_NUM * 10))
-               ADMIN_SVR_HTTP_PORT=$((9381 + RUNNER_NUM * 10))
-               SVR_MCP_PORT=$((9382 + RUNNER_NUM * 10))
-               GO_HTTP_PORT=$((9384 + RUNNER_NUM * 10))
-               GO_ADMIN_PORT=$((9383 + RUNNER_NUM * 10))
-               SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + RUNNER_NUM * 10))
-               SVR_WEB_HTTP_PORT=$((80 + RUNNER_NUM * 10))
-               SVR_WEB_HTTPS_PORT=$((443 + RUNNER_NUM * 10))
+          # Engine-specific offset partitions keep concurrent engine jobs from
+          # choosing the same host ports when they land on the same self-hosted runner.
+          # A lock plus reservation file closes the check/start race between parallel jobs.
+               PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
+               PARTITION_SIZE=6000
+               case "${DOC_ENGINE}" in
+                 elasticsearch) PARTITION_BASE=1000 ;;
+                 infinity) PARTITION_BASE=31000 ;;
+                 *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
+               esac
+               PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks
+               mkdir -p "${PORT_LOCK_DIR}"
 
-          # Persist computed ports into .env so docker-compose uses the correct host bindings
-               echo "" >> .env
-               echo -e "ES_PORT=${ES_PORT}" >> .env
-               echo -e "OS_PORT=${OS_PORT}" >> .env
-               echo -e "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}" >> .env
-               echo -e "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}" >> .env
-               echo -e "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}" >> .env
-               echo -e "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}" >> .env
-               echo -e "MINIO_PORT=${MINIO_PORT}" >> .env
-               echo -e "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}" >> .env
-               echo -e "REDIS_PORT=${REDIS_PORT}" >> .env
-               echo -e "TEI_PORT=${TEI_PORT}" >> .env
-               echo -e "KIBANA_PORT=${KIBANA_PORT}" >> .env
-               echo -e "SVR_HTTP_PORT=${SVR_HTTP_PORT}" >> .env
-               echo -e "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}" >> .env
-               echo -e "SVR_MCP_PORT=${SVR_MCP_PORT}" >> .env
-               echo -e "GO_HTTP_PORT=${GO_HTTP_PORT}" >> .env
-               echo -e "GO_ADMIN_PORT=${GO_ADMIN_PORT}" >> .env
-               echo -e "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}" >> .env
-               echo -e "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}" >> .env
-               echo -e "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}" >> .env
-               
-               echo -e "COMPOSE_PROFILES=\${COMPOSE_PROFILES},tei-cpu" >> .env
-               echo -e "TEI_MODEL=BAAI/bge-small-en-v1.5" >> .env
-               echo -e "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> .env
+               port_offset_available() {
+                 local offset=$1
+                 local base port
+                 for base in "${PORT_BASES[@]}"; do
+                   port=$((base + offset))
+                   if ss -ltnH "sport = :${port}" | grep -q .; then
+                     return 1
+                   fi
+                 done
+                 return 0
+               }
+
+               cleanup_stale_port_locks() {
+                 local now stale_after lock lock_ts
+                 now=$(date -u +%s)
+                 stale_after=$((6 * 60 * 60))
+                 for lock in "${PORT_LOCK_DIR}"/*.lock; do
+                   [[ -e "${lock}" ]] || continue
+                   lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true)
+                   if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then
+                     rm -f "${lock}"
+                   fi
+                 done
+               }
+
+               reserve_port_offset() {
+                 local attempt candidate reservation
+                 cleanup_stale_port_locks
+                 for attempt in $(seq 0 59); do
+                   candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
+                   reservation="${PORT_LOCK_DIR}/${candidate}.lock"
+                   if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then
+                     if port_offset_available "${candidate}"; then
+                       PORT_OFFSET=${candidate}
+                       PORT_RESERVATION=${reservation}
+                       return 0
+                     fi
+                     rm -f "${reservation}"
+                   fi
+                 done
+                 return 1
+               }
+
+               if ! reserve_port_offset; then
+                 echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
+                 exit 1
+               fi
+               echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
+               echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
+               ES_PORT=$((1200 + PORT_OFFSET))
+               OS_PORT=$((1201 + PORT_OFFSET))
+               INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
+               INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
+               INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
+               EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
+               MINIO_PORT=$((9000 + PORT_OFFSET))
+               MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
+               REDIS_PORT=$((6379 + PORT_OFFSET))
+               NATS_PORT=$((4222 + PORT_OFFSET))
+               TEI_PORT=$((6380 + PORT_OFFSET))
+               KIBANA_PORT=$((6601 + PORT_OFFSET))
+               SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
+               ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
+               SVR_MCP_PORT=$((9382 + PORT_OFFSET))
+               GO_HTTP_PORT=$((9384 + PORT_OFFSET))
+               GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
+               SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
+               SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
+               SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
+
+          # Persist computed ports into .env so docker-compose uses the correct host bindings.
+          # Remove previous CI overrides first; docker compose uses the last duplicate key.
+               sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
+               {
+                 echo ""
+                 echo "ES_PORT=${ES_PORT}"
+                 echo "OS_PORT=${OS_PORT}"
+                 echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
+                 echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
+                 echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
+                 echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
+                 echo "MINIO_PORT=${MINIO_PORT}"
+                 echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
+                 echo "REDIS_PORT=${REDIS_PORT}"
+                 echo "NATS_PORT=${NATS_PORT}"
+                 echo "TEI_PORT=${TEI_PORT}"
+                 echo "KIBANA_PORT=${KIBANA_PORT}"
+                 echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
+                 echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
+                 echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
+                 echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
+                 echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
+                 echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
+                 echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
+                 echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
+                 echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu"
+                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
+                 echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
+                 echo "DOC_ENGINE=${DOC_ENGINE}"
+               } >> .env
                echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
 
                # Patch entrypoint.sh for coverage
                sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\   echo "Ensuring coverage is installed..."\n      "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n     export COVERAGE_FILE=/ragflow/logs/.coverage\n        echo "Starting ragflow_server with coverage..."\n        "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
-               cd ..
-               uv sync --python 3.12 --group test --frozen && uv pip install -e sdk/python
 
 
       - name: Start ragflow:nightly for Infinity
         run: |
-          sed -i 's/^DOC_ENGINE=.*$/DOC_ENGINE=infinity/' docker/.env
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
 
       - name: Run sdk tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
           echo "Start to run test sdk on Infinity"
           source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log
 
-      - name: Run web api tests against Infinity
+      - name: Run New RESTFUL api tests against Infinity
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
-          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api/test_chunk_feedback 2>&1 | tee infinity_web_api_test.log
-
-      - name: Run http api tests against Infinity
-        run: |
-          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
-            sleep 5
-          done
-          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee infinity_http_api_test.log
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
+          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee infinity_restful_api_test.log
 
       - name: RAGFlow CLI retrieval test Infinity
         env:
@@ -318,10 +481,20 @@ jobs:
           ADMIN_HOST="${USER_HOST}"
           ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
 
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
 
           admin_ready=0
           for i in $(seq 1 30); do
@@ -372,7 +545,7 @@ jobs:
           else
             echo "ragflow_server.py not found!"
           fi
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} stop
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
 
       - name: Generate server coverage report Infinity
         if: ${{ !cancelled() }}
@@ -415,41 +588,252 @@ jobs:
         if: always()  # always run this step even if previous steps failed
         run: |
           # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
-          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
+          if [[ -n ${RAGFLOW_IMAGE} ]]; then
+            sudo docker rmi -f ${RAGFLOW_IMAGE}
+          fi
+          if [[ -n ${PORT_RESERVATION:-} ]]; then
+            rm -f "${PORT_RESERVATION}"
+          fi
+
+
+
+  ragflow_tests_elasticsearch:
+    name: ragflow_tests_elasticsearch
+    needs: ragflow_preflight
+    if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
+    runs-on: [ "self-hosted", "ragflow-test" ]
+    env:
+      DOC_ENGINE: elasticsearch
+      RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch
+      HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
+    steps:
+      - name: Ensure workspace ownership
+        run: |
+          echo "Workflow triggered by ${{ github.event_name }}"
+          echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
+
+      - name: Check out code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Build ragflow go server
+        run: |
+          set -euo pipefail
+          BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
+          cleanup_builder() {
+            if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
+              sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
+            fi
+          }
+          trap cleanup_builder EXIT
+
+          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
+            -e TZ="${TZ}" \
+            -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
+            -v "${PWD}:/ragflow" \
+            -v "${PWD}/internal/cpp/resource:/usr/share/infinity/resource" \
+            infiniflow/infinity_builder:ubuntu22_clang20
+          sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
+          ./build.sh --go
+
+      - name: Build ragflow:nightly
+        run: |
+          set -euo pipefail
+          sudo docker pull ubuntu:24.04
+          sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} .
+
+      - name: Prepare Python test environment
+        run: |
+          uv sync --python 3.13 --group test --frozen
+          uv pip install -e sdk/python
+
+      - name: Prepare function test environment
+        working-directory: docker
+        run: |
+          set -euo pipefail
+          # install ss
+          sudo apt update && sudo apt install -y iproute2
+          RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
+          COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
+          echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
+          echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
+          ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
+          echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
+          rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
+
+          # Determine runner number (default to 1 if not found)
+               RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
+               RUNNER_NUM=${RUNNER_NUM:-1}
+
+          # Engine-specific offset partitions keep concurrent engine jobs from
+          # choosing the same host ports when they land on the same self-hosted runner.
+          # A lock plus reservation file closes the check/start race between parallel jobs.
+               PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
+               PARTITION_SIZE=6000
+               case "${DOC_ENGINE}" in
+                 elasticsearch) PARTITION_BASE=1000 ;;
+                 infinity) PARTITION_BASE=31000 ;;
+                 *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
+               esac
+               PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks
+               mkdir -p "${PORT_LOCK_DIR}"
+
+               port_offset_available() {
+                 local offset=$1
+                 local base port
+                 for base in "${PORT_BASES[@]}"; do
+                   port=$((base + offset))
+                   if ss -ltnH "sport = :${port}" | grep -q .; then
+                     return 1
+                   fi
+                 done
+                 return 0
+               }
+
+               cleanup_stale_port_locks() {
+                 local now stale_after lock lock_ts
+                 now=$(date -u +%s)
+                 stale_after=$((6 * 60 * 60))
+                 for lock in "${PORT_LOCK_DIR}"/*.lock; do
+                   [[ -e "${lock}" ]] || continue
+                   lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true)
+                   if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then
+                     rm -f "${lock}"
+                   fi
+                 done
+               }
+
+               reserve_port_offset() {
+                 local attempt candidate reservation
+                 cleanup_stale_port_locks
+                 for attempt in $(seq 0 59); do
+                   candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
+                   reservation="${PORT_LOCK_DIR}/${candidate}.lock"
+                   if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then
+                     if port_offset_available "${candidate}"; then
+                       PORT_OFFSET=${candidate}
+                       PORT_RESERVATION=${reservation}
+                       return 0
+                     fi
+                     rm -f "${reservation}"
+                   fi
+                 done
+                 return 1
+               }
+
+               if ! reserve_port_offset; then
+                 echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
+                 exit 1
+               fi
+               echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
+               echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
+               ES_PORT=$((1200 + PORT_OFFSET))
+               OS_PORT=$((1201 + PORT_OFFSET))
+               INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
+               INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
+               INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
+               EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
+               MINIO_PORT=$((9000 + PORT_OFFSET))
+               MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
+               REDIS_PORT=$((6379 + PORT_OFFSET))
+               NATS_PORT=$((4222 + PORT_OFFSET))
+               TEI_PORT=$((6380 + PORT_OFFSET))
+               KIBANA_PORT=$((6601 + PORT_OFFSET))
+               SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
+               ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
+               SVR_MCP_PORT=$((9382 + PORT_OFFSET))
+               GO_HTTP_PORT=$((9384 + PORT_OFFSET))
+               GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
+               SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
+               SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
+               SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
+
+          # Persist computed ports into .env so docker-compose uses the correct host bindings.
+          # Remove previous CI overrides first; docker compose uses the last duplicate key.
+               sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
+               {
+                 echo ""
+                 echo "ES_PORT=${ES_PORT}"
+                 echo "OS_PORT=${OS_PORT}"
+                 echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
+                 echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
+                 echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
+                 echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
+                 echo "MINIO_PORT=${MINIO_PORT}"
+                 echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
+                 echo "REDIS_PORT=${REDIS_PORT}"
+                 echo "NATS_PORT=${NATS_PORT}"
+                 echo "TEI_PORT=${TEI_PORT}"
+                 echo "KIBANA_PORT=${KIBANA_PORT}"
+                 echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
+                 echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
+                 echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
+                 echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
+                 echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
+                 echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
+                 echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
+                 echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
+                 echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu"
+                 echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
+                 echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
+                 echo "DOC_ENGINE=${DOC_ENGINE}"
+               } >> .env
+               echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
+
+               # Patch entrypoint.sh for coverage
+               sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\   echo "Ensuring coverage is installed..."\n      "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n     export COVERAGE_FILE=/ragflow/logs/.coverage\n        echo "Starting ragflow_server with coverage..."\n        "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
+
 
       - name: Start ragflow:nightly for Elasticsearch
         run: |
-          sed -i 's/^DOC_ENGINE=.*$/DOC_ENGINE=elasticsearch/' docker/.env
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} up -d
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
 
       - name: Run sdk tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
           echo "Start to run test sdk on Elasticsearch"
-          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-es-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log
 
-      - name: Run web api tests against Elasticsearch
+      - name: Run New RESTFUL api tests against Elasticsearch
         run: |
           export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
-          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_web_api 2>&1 | tee es_web_api_test.log
-
-      - name: Run http api tests against Elasticsearch
-        run: |
-          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
-            sleep 5
-          done
-          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee es_http_api_test.log
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee es_restful_api_test.log
 
       - name: RAGFlow CLI retrieval test Elasticsearch
         env:
@@ -511,10 +895,20 @@ jobs:
           ADMIN_HOST="${USER_HOST}"
           ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
 
-          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null 2>&1; do
-            echo "Waiting for service to be available... (last exit code: $?)"
+          svc_ready=0
+          for i in $(seq 1 60); do
+            if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
+              svc_ready=1
+              break
+            fi
+            echo "Waiting for service to be available... ($i/60)"
             sleep 5
           done
+          if [ "$svc_ready" -ne 1 ]; then
+            echo "Service did not become ready after 5 minutes. Docker logs:"
+            sudo docker logs ${RAGFLOW_CONTAINER}
+            exit 1
+          fi
 
           admin_ready=0
           for i in $(seq 1 30); do
@@ -565,7 +959,7 @@ jobs:
           else
             echo "ragflow_server.py not found!"
           fi
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} stop
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
 
       - name: Generate server coverage report Elasticsearch
         if: ${{ !cancelled() }}
@@ -587,7 +981,7 @@ jobs:
           else
             echo ".coverage file not found!"
           fi
-          
+
       - name: Collect ragflow log Elasticsearch
         if: ${{ !cancelled() }}
         run: |
@@ -603,8 +997,11 @@ jobs:
         if: always()  # always run this step even if previous steps failed
         run: |
           # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
-          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
+          sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
           if [[ -n ${RAGFLOW_IMAGE} ]]; then
             sudo docker rmi -f ${RAGFLOW_IMAGE}
           fi
+          if [[ -n ${PORT_RESERVATION:-} ]]; then
+            rm -f "${PORT_RESERVATION}"
+          fi

.gitignore

@@ -21,6 +21,7 @@ Cargo.lock
 
 .idea/
 .vscode/
+.cursor/settings.json
 
 # Exclude Mac generated files
 .DS_Store
@@ -136,6 +137,9 @@ web_modules/
 # Output of 'npm pack'
 *.tgz
 
+# Claude Code plans / state — local-only artifacts
+.claude/
+
 # Yarn Integrity file
 .yarn-integrity
 
@@ -231,3 +235,9 @@ internal/cpp/cmake-build-debug/
 # Go server build output
 bin/*
 !bin/.gitkeep
+.claude/settings.local.json
+
+.run/
+# Local agent tooling state (per-developer; not for commit)
+.omc/
+.marscode/

.rooignore

@@ -0,0 +1,85 @@
+# .rooignore for RAGFlow
+# Purpose: reduce indexing noise, token waste, and accidental reads of generated files
+
+# Git / platform
+.git/
+.github/
+
+# IDE / local editor
+.idea/
+.vscode/
+.trae/
+
+# Python caches / build artifacts
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.hypothesis/
+.coverage
+*.egg-info/
+ragflow.egg-info/
+sdk/python/ragflow_sdk.egg-info/
+sdk/python/build/
+sdk/python/dist/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Node / frontend dependencies and build output
+node_modules/
+web/node_modules/
+web/dist/
+web/build/
+web/.cache/
+*.tsbuildinfo
+
+# Logs / runtime artifacts
+logs/
+docker/ragflow-logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# Large local dependency artifacts
+libssl*.deb
+tika-server*.jar*
+cl100k_base.tiktoken
+chrome*
+huggingface.co/
+nltk_data/
+uv-x86_64*.tar.gz
+uv-aarch64*.tar.gz
+
+# Temp / data / local storage
+tmp/
+cache/
+backup/
+docker/data/
+docker/oceanbase/conf
+docker/oceanbase/data
+docker/seekdb
+
+# Native / compiled build dirs
+target/
+bin/
+internal/cpp/build/
+internal/cpp/cmake-build-release/
+internal/cpp/cmake-build-debug/
+
+# Optional: skip tests and docs from indexing
+# test/
+# tests/
+# docs/
+
+# Ignore Roo's own config file
+.rooignore

AGENTS.md

@@ -34,8 +34,8 @@ The project uses **uv** for dependency management.
 
 1. **Setup Environment**:
    ```bash
-   uv sync --python 3.12 --all-extras
-   uv run download_deps.py
+   uv sync --python 3.13 --all-extras
+   uv run python3 download_deps.py
    ```
 
 2. **Run Server**:

CLAUDE.md

@@ -6,44 +6,62 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 
 RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on deep document understanding. It's a full-stack application with:
 
-- Python backend (Flask-based API server)
+- Python backend (Quart-based async API server — Quart is the async reimplementation of Flask)
 - React/TypeScript frontend (built with vitejs)
-- Microservices architecture with Docker deployment
-- Multiple data stores (MySQL, Elasticsearch/Infinity, Redis, MinIO)
+- Background task executor workers (separate Python processes, Redis-queue-driven)
+- Peewee ORM for database models (not SQLAlchemy)
+- Multiple data stores (MySQL/PostgreSQL, Elasticsearch/Infinity/OpenSearch/OceanBase, Redis, MinIO)
 
 ## Architecture
 
-### Backend (`/api/`)
+### Runtime Architecture
 
-- **Main Server**: `api/ragflow_server.py` - Flask application entry point
-- **Apps**: Modular Flask blueprints in `api/apps/` for different functionalities:
-  - `kb_app.py` - Knowledge base management
-  - `dialog_app.py` - Chat/conversation handling
-  - `document_app.py` - Document processing
-  - `canvas_app.py` - Agent workflow canvas
-  - `file_app.py` - File upload/management
-- **Services**: Business logic in `api/db/services/`
-- **Models**: Database models in `api/db/db_models.py`
+RAGFlow runs as **two separate Python process types**, orchestrated by `docker/launch_backend_service.sh`:
+
+- **API Server** (`api/ragflow_server.py`): Quart-based async HTTP server
+- **Task Executors** (`rag/svr/task_executor.py`): Background workers processing documents from Redis streams. Multiple instances run in parallel (controlled by `WS` env var). Each consumes from priority-ordered Redis streams (`te.1.common`, `te.0.common`), using consumer groups for load distribution.
+
+Key consequence: task executors import a different code surface than the API server, so always check which process a module is meant for.
+
+### Backend API (`/api/`)
+
+- **App factory**: `api/apps/__init__.py` — creates the Quart app, configures auth (`login_required` decorator, JWT + API token + session fallback), and dynamically discovers/registers blueprints
+- **Two API coexisting patterns**:
+  - **RESTful APIs** in `api/apps/restful_apis/` — newer pattern with Pydantic request validation, service layer in `api/apps/services/`, routes registered under `/api/v1`
+  - **Legacy APIs** in `api/apps/*_app.py` — older pattern using `@validate_request()`, routes registered under `/v1/<page_name>`
+  - **SDK APIs** in `api/apps/sdk/` — registered under `/v1/`
+- **Services**: `api/db/services/` — business logic wrapping Peewee model operations. `api/apps/services/` — service layer for the RESTful APIs
+- **Models**: `api/db/db_models.py` — Peewee ORM models with pooled MySQL/PostgreSQL connections, custom `JSONField`/`ListField` types, retry logic on connection loss
 
 ### Core Processing (`/rag/`)
 
-- **Document Processing**: `deepdoc/` - PDF parsing, OCR, layout analysis
-- **LLM Integration**: `rag/llm/` - Model abstractions for chat, embedding, reranking
-- **RAG Pipeline**: `rag/flow/` - Chunking, parsing, tokenization
-- **Graph RAG**: `rag/graphrag/` - Knowledge graph construction and querying
+- **Document ingestion pipeline**: `rag/flow/pipeline.py` — `Pipeline` (extends `agent.canvas.Graph`) orchestrates the ingestion DAG. Components: File (fetches binary from storage), Parser (dispatches to `deepdoc.parser` based on file type), TokenChunker/TitleChunker (splits into chunks), Tokenizer (computes full-text tokens + embedding vectors), Extractor (LLM-based extraction). Data flows via Pydantic `*FromUpstream` schemas.
+- **Document parsing**: `deepdoc/` — PDF parsing (vision-based OCR, layout analysis, table structure recognition) and format-specific parsers (DOCX, XLSX, PPT, Markdown, HTML, images). All parsers normalize to a common structure (list of bbox dicts for PDFs, `{text, doc_type_kwd}` for others).
+- **LLM Integration**: `rag/llm/` — factory pattern with runtime class discovery. `chat_model.py` (30+ providers via OpenAI SDK and LiteLLM wrappers), `embedding_model.py`, `rerank_model.py`, `cv_model.py` (image-to-text), `sequence2txt_model.py` (ASR), `tts_model.py`. Use `LLMBundle` (from `api.db.services.llm_service`) as the unified interface.
+- **Graph RAG**: `rag/graphrag/` — multi-phase pipeline: per-document subgraph extraction (LLM or spaCy NER), Leiden community detection, entity resolution, community summarization. Entities/relations/reports are indexed as chunks alongside regular text chunks, differentiated by `knowledge_graph_kwd`.
+- **Search**: `rag/nlp/search.py` — `Dealer` class combines vector similarity + BM25 + re-ranking. `KGSearch` extends it for graph-aware retrieval (entity resolution, n-hop enrichment).
 
 ### Agent System (`/agent/`)
 
-- **Components**: Modular workflow components (LLM, retrieval, categorize, etc.)
-- **Templates**: Pre-built agent workflows in `agent/templates/`
-- **Tools**: External API integrations (Tavily, Wikipedia, SQL execution, etc.)
+- **Execution engine**: `agent/canvas.py` — `Canvas` (extends `Graph`) executes the DAG. Components are run in topological order via `_run_batch`, each receiving upstream outputs as kwargs. Control-flow components (`Categorize`, `Switch`, `Iteration`, `Loop`) dynamically modify the execution path.
+- **Component base**: `agent/component/base.py` — `ComponentBase` with `invoke(**kwargs)` / `invoke_async(**kwargs)` lifecycle. Variable references (`{component_id@output_var}` or `{sys.query}`) are resolved from the canvas graph at runtime.
+- **Components**: Modular workflow components in `agent/component/` — Begin, LLM, Agent (tool-calling LLM), Categorize, Switch, Iteration, Loop, Message, Invoke (HTTP), and data manipulation nodes. Auto-discovered by `__init__.py`.
+- **Templates**: Pre-built agent workflows as JSON DSL files in `agent/templates/`. Each contains a complete `components` DAG, `path`, and `globals`.
+- **Tools**: `agent/tools/` — Retrieval, web search (DuckDuckGo, Google, Tavily, SearXNG), academic search (ArXiv, PubMed, Google Scholar, Wikipedia), code execution, SQL execution, email, GitHub, finance data, translation, weather. Tools implement `ToolBase` (extends `ComponentBase`) and produce OpenAI-compatible function descriptors.
+- **Plugins**: `agent/plugin/` — plugin system using `pluginlib` for loading external LLM tool plugins from `embedded_plugins/`.
 
 ### Frontend (`/web/`)
 
 - React/TypeScript with vitejs framework
-- shadcn/ui components
-- State management with Zustand
-- Tailwind CSS for styling
+- shadcn/ui components (Radix UI primitives + Tailwind CSS)
+- `@tanstack/react-query` for server state (cache keys, mutations, invalidation)
+- Zustand for local state (primarily agent canvas graph store)
+- `react-router` v7 with lazy-loaded pages
+- `react-i18next` for i18n (17 languages)
+- Axios for HTTP with a layered pattern: endpoint definitions (`utils/api.ts`) → HTTP client (`utils/next-request.ts`) → service layer (`services/`) → query hooks (`hooks/use-*-request.ts`) → components
+- `@xyflow/react` for the agent workflow canvas
+- `react-hook-form` + `zod` for form validation
+- Two API proxy prefixes: `webAPI = '/v1'` (legacy) and `restAPIv1 = '/api/v1'` (RESTful)
 
 ## Common Development Commands
 
@@ -51,8 +69,8 @@ RAGFlow is an open-source RAG (Retrieval-Augmented Generation) engine based on d
 
 ```bash
 # Install Python dependencies
-uv sync --python 3.12 --all-extras
-uv run download_deps.py
+uv sync --python 3.13 --all-extras
+uv run python3 download_deps.py
 pre-commit install
 
 # Start dependent services
@@ -118,7 +136,7 @@ RAGFlow supports switching between Elasticsearch (default) and Infinity:
 
 ## Development Environment Requirements
 
-- Python 3.10-3.12
+- Python 3.10-3.13
 - Node.js >=18.20.4
 - Docker & Docker Compose
 - uv package manager

Dockerfile

@@ -43,7 +43,8 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     chmod 1777 /tmp && \
     apt update && \
     apt install -y \
-    build-essential libglib2.0-0 libglx-mesa0 libgl1 pkg-config libicu-dev libgdiplus default-jdk libatk-bridge2.0-0 libpython3-dev libgtk-4-1 libnss3 xdg-utils libgbm-dev libjemalloc-dev gnupg unzip curl wget git vim less ghostscript pandoc texlive texlive-latex-extra texlive-xetex texlive-lang-chinese fonts-freefont-ttf fonts-noto-cjk postgresql-client
+    libglib2.0-0 libglx-mesa0 libgl1 pkg-config libgdiplus default-jdk libatk-bridge2.0-0 libgtk-4-1 libnss3 xdg-utils libjemalloc-dev gnupg unzip curl wget git vim less ghostscript pandoc texlive texlive-latex-extra texlive-xetex texlive-lang-chinese fonts-freefont-ttf fonts-noto-cjk postgresql-client && \
+    rm -rf /var/lib/apt/lists/*
 
 # Download resource from GitHub to /usr/share/infinity
 RUN mkdir -p /usr/share/infinity/resource && \
@@ -55,14 +56,15 @@ RUN mkdir -p /usr/share/infinity/resource && \
     cp -r /tmp/resource/* /usr/share/infinity/resource && \
     rm -rf /tmp/resource
 
-ARG NGINX_VERSION=1.29.5-1~noble
+ARG NGINX_VERSION=1.31.0-1~noble
 RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     mkdir -p /etc/apt/keyrings && \
     curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://nginx.org/keys/nginx_signing.key | gpg --dearmor -o /etc/apt/keyrings/nginx-archive-keyring.gpg && \
     echo "deb [signed-by=/etc/apt/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/mainline/ubuntu/ noble nginx" > /etc/apt/sources.list.d/nginx.list && \
     apt -o Acquire::Retries=5 update && \
     apt -o Acquire::Retries=5 install -y nginx=${NGINX_VERSION} && \
-    apt-mark hold nginx
+    apt-mark hold nginx && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install uv
 RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps \
@@ -78,7 +80,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/deps
     tar xzf "/deps/uv-${uv_arch}-unknown-linux-gnu.tar.gz" \
     && cp "uv-${uv_arch}-unknown-linux-gnu/"* /usr/local/bin/ \
     && rm -rf "uv-${uv_arch}-unknown-linux-gnu" \
-    && uv python install 3.12
+    && uv python install 3.13
 
 ENV PYTHONDONTWRITEBYTECODE=1 DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 \
     UV_HTTP_TIMEOUT=200 \
@@ -91,7 +93,8 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     apt purge -y nodejs npm && \
     apt autoremove -y && \
     apt update && \
-    apt install -y nodejs
+    apt install -y nodejs && \
+    rm -rf /var/lib/apt/lists/*
 
 # Add msssql ODBC driver
 # macOS ARM64 environment, install msodbcsql18.
@@ -107,7 +110,8 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
     else \
         # x86_64 or others \
         ACCEPT_EULA=Y apt install -y unixodbc-dev msodbcsql17; \
-    fi || \
+    fi && \
+    rm -rf /var/lib/apt/lists/* || \
     { echo "Failed to install ODBC driver"; exit 1; }
 
 
@@ -136,26 +140,54 @@ USER root
 
 WORKDIR /ragflow
 
+# Install build-only dependencies for compiling Python C extensions.
+# These are not inherited from base to keep the production image smaller.
+RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
+    apt update && \
+    apt install -y build-essential libpython3-dev libicu-dev libgbm-dev && \
+    rm -rf /var/lib/apt/lists/*
+
 # install dependencies from uv.lock file
 COPY pyproject.toml uv.lock ./
 
 # https://github.com/astral-sh/uv/issues/10462
 # uv records index url into uv.lock but doesn't failover among multiple indexes
+# Also rewrite pypi.tuna.tsinghua.edu.cn to mirrors.aliyun.com/pypi so locks
+# that were resolved against the Tsinghua mirror (e.g. when UV_INDEX pointed
+# there) get normalized to the Aliyun mirror in NEED_MIRROR=1 builds. Without
+# this, stale Tsinghua URLs slip through and `uv sync --frozen` 404s on
+# packages that the Tsinghua mirror no longer carries.
 RUN --mount=type=cache,id=ragflow_uv,target=/root/.cache/uv,sharing=locked \
     if [ "$NEED_MIRROR" == "1" ]; then \
         sed -i 's|pypi.org|mirrors.aliyun.com/pypi|g' uv.lock; \
+        sed -i 's|pypi.tuna.tsinghua.edu.cn|mirrors.aliyun.com/pypi|g' uv.lock; \
     else \
         sed -i 's|mirrors.aliyun.com/pypi|pypi.org|g' uv.lock; \
+        sed -i 's|pypi.tuna.tsinghua.edu.cn|pypi.org|g' uv.lock; \
+        sed -i 's|gitee.com|github.com|g' uv.lock; \
     fi; \
-    uv sync --python 3.12 --frozen && \
+    # --refresh-package litellm forces a re-download of litellm from the
+    # (post-sed) URLs in uv.lock even if BuildKit's persistent uv cache mount
+    # holds a stale wheel from a previous build. litellm 1.88.x has had
+    # multiple internal ImportError issues (1.88.1 missing
+    # DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, 1.88.0 wheel pulled via
+    # some proxies missing RedisPipelineLpopOperation) — always re-fetching
+    # the locked version avoids serving a half-broken cached copy.
+    uv sync --python 3.13 --frozen --refresh-package litellm && \
     # Ensure pip is available in the venv for runtime package installation (fixes #12651)
     .venv/bin/python3 -m ensurepip --upgrade
 
+# Install frontend dependencies — depends only on package manifests so
+# web source / docs changes don't invalidate this layer.
+COPY web/package.json web/package-lock.json web/.npmrc ./web/
+RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
+    cd web && NODE_OPTIONS="--max-old-space-size=8192" npm install
+
+# Copy full web source and docs for the frontend build.
 COPY web web
 COPY docs docs
 RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
-    cd web && NODE_OPTIONS="--max-old-space-size=8192" npm install && \
-    NODE_OPTIONS="--max-old-space-size=8192" VITE_BUILD_SOURCEMAP=false VITE_MINIFY=esbuild npm run build
+    cd web && NODE_OPTIONS="--max-old-space-size=8192" VITE_BUILD_SOURCEMAP=false VITE_MINIFY=esbuild npm run build
 
 COPY .git /ragflow/.git
 
@@ -177,7 +209,6 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 ENV PYTHONPATH=/ragflow/
 
-COPY web web
 COPY admin admin
 COPY api api
 COPY conf conf
@@ -189,6 +220,7 @@ COPY mcp mcp
 COPY common common
 COPY memory memory
 COPY bin bin
+COPY tools/scripts tools/scripts
 
 COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
 COPY docker/entrypoint.sh ./

README.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 Table of Contents</b></summary>
 
 - 💡 [What is RAGFlow?](#-what-is-ragflow)
-- 🎮 [Demo](#-demo)
+- 🎮 [Get Started](#-get-started)
 - 📌 [Latest Updates](#-latest-updates)
 - 🌟 [Key Features](#-key-features)
 - 🔎 [System Architecture](#-system-architecture)
-- 🎬 [Get Started](#-get-started)
+- 🎬 [Self-Hosting](#-self-hosting)
 - 🔧 [Configurations](#-configurations)
 - 🔧 [Build a Docker image](#-build-a-docker-image)
 - 🔨 [Launch service from source for development](#-launch-service-from-source-for-development)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/) is a leading open-source Retrieval-Augmented Generation ([RAG](https://ragflow.io/basics/what-is-rag)) engine that fuses cutting-edge RAG with Agent capabilities to create a superior context layer for LLMs. It offers a streamlined RAG workflow adaptable to enterprises of any scale. Powered by a converged [context engine](https://ragflow.io/basics/what-is-agent-context-engine) and pre-built agent templates, RAGFlow enables developers to transform complex data into high-fidelity, production-ready AI systems with exceptional efficiency and precision.
 
-## 🎮 Demo
+## 🎮 Get Started
 
-Try our demo at [https://cloud.ragflow.io](https://cloud.ragflow.io).
+Try our cloud service at [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,6 +87,8 @@ Try our demo at [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 ## 🔥 Latest Updates
 
+- 2026-06-15 Support multiple chat channels such as Feishu, Discord, Telegram, Line, etc.
+- 2026-04-24 Supports DeepSeek v4.
 - 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — Provides an official skill for accessing RAGFlow datasets via OpenClaw.
 - 2025-12-26 Supports 'Memory' for AI agent.
 - 2025-11-19 Supports Gemini 3 Pro.
@@ -97,7 +98,6 @@ Try our demo at [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - 2025-08-08 Supports OpenAI's latest GPT-5 series models.
 - 2025-08-01 Supports agentic workflow and MCP.
 - 2025-05-23 Adds a Python/JavaScript code executor component to Agent.
-- 2025-05-05 Supports cross-language query.
 - 2025-03-19 Supports using a multi-modal model to make sense of images within PDF or DOCX files.
 
 ## 🎉 Stay Tuned
@@ -144,7 +144,7 @@ releases! 🌟
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 Get Started
+## 🎬 Self-Hosting
 
 ### 📝 Prerequisites
 
@@ -152,6 +152,7 @@ releases! 🌟
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): Required only if you intend to use the code executor (sandbox) feature of RAGFlow.
 
 > [!TIP]
@@ -192,12 +193,12 @@ releases! 🌟
 > All Docker images are built for x86 platforms. We don't currently offer Docker images for ARM64.
 > If you are on an ARM64 platform, follow [this guide](https://ragflow.io/docs/dev/build_docker_image) to build a Docker image compatible with your system.
 
-> The command below downloads the `v0.25.0` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.25.0`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
+> The command below downloads the `v0.26.1` edition of the RAGFlow Docker image. See the following table for descriptions of different RAGFlow editions. To download a RAGFlow edition different from `v0.26.1`, update the `RAGFLOW_IMAGE` variable accordingly in **docker/.env** before using `docker compose` to start the server.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
    # This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
 
@@ -328,7 +329,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -405,7 +406,7 @@ See the [RAGFlow Roadmap 2026](https://github.com/infiniflow/ragflow/issues/1224
 ## 🏄 Community
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contributing

README_ar.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DBEDFA"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 جدول المحتويات</b></summary>
 
 - 💡 [ما هو RAGFlow؟](#-what-is-ragflow)
-- 🎮 [Demo](#-demo)
+- 🎮 [ابدأ](#-get-started)
 - 📌 [آخر التحديثات](#-latest-updates)
 - 🌟 [الميزات الرئيسية](#-key-features)
 - 🔎 [بنية النظام](#-system-architecture)
-- 🎬 [ابدأ](#-get-started)
+- 🎬 [الاستضافة الذاتية](#-self-hosting)
 - 🔧 [التكوينات](#-configurations)
 - 🔧 [إنشاء صورة Docker](#-build-a-docker-image)
 - 🔨 [إطلاق الخدمة من المصدر للتطوير](#-launch-service-from-source-for-development)
@@ -77,7 +76,7 @@
 
 يُعد مشروع [RAGFlow](https://ragflow.io/) محركًا رائدًا ومفتوح المصدر للاسترجاع المعزز بالتوليد (<bdi dir="ltr">RAG</bdi>)، ويجمع أحدث تقنيات <bdi dir="ltr">RAG</bdi> مع قدرات الوكلاء لبناء طبقة سياق متقدمة لنماذج <bdi dir="ltr">LLMs</bdi>. يوفّر سير عمل <bdi dir="ltr">RAG</bdi> مبسّطًا وقابلًا للتكيّف مع المؤسسات بمختلف أحجامها. وبالاعتماد على [محرك سياق موحّد](https://ragflow.io/basics/what-is-agent-context-engine) وقوالب وكلاء جاهزة، يتيح <bdi dir="ltr">RAGFlow</bdi> للمطورين تحويل البيانات المعقّدة إلى أنظمة <bdi dir="ltr">AI</bdi> عالية الدقة وجاهزة للإنتاج بكفاءة وموثوقية.
 
-## 🎮 Demo
+## 🎮 ابدأ
 
 جرّب النسخة التجريبية على [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
@@ -88,8 +87,10 @@
 
 ## 🔥 آخر التحديثات
 
-- 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — توفر مهارة رسمية للوصول إلى مجموعات بيانات RAGFlow عبر OpenClaw.
-- 2025-12-26 يدعم ميزة "Memory" لوكلاء الذكاء الاصطناعي.
+- 15-06-2026 يدعم قنوات دردشة متعددة مثل Feishu و Discord و Telegram و Line وما إلى ذلك.
+- 24-04-2026 يدعم DeepSeek v4.
+- 24-03-2026 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — توفر مهارة رسمية للوصول إلى مجموعات بيانات RAGFlow عبر OpenClaw.
+- 26-12-2025 يدعم ميزة "Memory" لوكلاء الذكاء الاصطناعي.
 - 11-11-2025 يدعم Gemini 3 Pro.
 - 12-11-2025 يدعم مزامنة البيانات من Confluence، S3، Notion، Discord، Google Drive.
 - 23-10-2025 يدعم MinerU وDocling كطرق لتحليل المستندات.
@@ -97,7 +98,6 @@
 - 08-08-2025 يدعم أحدث موديلات سلسلة OpenAI.
 - 01-08-2025 يدعم سير العمل الوكيل وMCP.
 - 23-05-2025 تمت إضافة مكون منفذ كود Python/JavaScript إلى Agent.
-- 05-05-2025 يدعم الاستعلام بين اللغات.
 - 19-03-2025 يدعم استخدام نموذج متعدد الوسائط لفهم الصور داخل ملفات PDF أو DOCX.
 
 ## 🎉 تابعونا
@@ -144,7 +144,7 @@
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 ابدأ
+## 🎬 الاستضافة الذاتية
 
 ### 📝 المتطلبات الأساسية
 
@@ -152,6 +152,7 @@
 - الرام >= 16 جيجا
 - القرص >= 50 جيجا بايت
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- بايثون >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): مطلوب فقط إذا كنت تنوي استخدام ميزة منفذ التعليمات البرمجية (وضع الحماية) لـ RAGFlow.
 
 > [!TIP]
@@ -192,12 +193,12 @@
 > جميع الصور Docker مصممة لمنصات x86. لا نعرض حاليًا صور Docker لـ ARM64.
 > إذا كنت تستخدم نظامًا أساسيًا ARM64، فاتبع [هذا الدليل](https://ragflow.io/docs/dev/build_docker_image) لإنشاء صورة Docker متوافقة مع نظامك.
 
-> يقوم الأمر أدناه بتنزيل إصدار `v0.25.0` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.25.0`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
+> يقوم الأمر أدناه بتنزيل إصدار `v0.26.1` من الصورة RAGFlow Docker. راجع الجدول التالي للحصول على أوصاف لإصدارات RAGFlow المختلفة. لتنزيل إصدار RAGFlow مختلف عن `v0.26.1`، قم بتحديث المتغير `RAGFLOW_IMAGE` وفقًا لذلك في **docker/.env** قبل استخدام `docker compose` لبدء تشغيل الخادم.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
    # This step ensures the **entrypoint.sh** file in the code matches the Docker image version.
 
@@ -328,7 +329,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -405,7 +406,7 @@ docker build --platform linux/amd64 \
 ## 🏄 المجتمع
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [مناقشات جيثب](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 المساهمة

README_fr.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DBEDFA"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DBEDFA"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="suivre sur X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Badge statique" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Badge statique" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Dernière%20version" alt="Dernière version">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Documentation</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Démo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 Table des matières</b></summary>
 
 - 💡 [Qu'est-ce que RAGFlow?](#-quest-ce-que-ragflow)
-- 🎮 [Démo](#-démo)
+- 🎮 [Démarrage](#-démarrage)
 - 📌 [Dernières mises à jour](#-dernières-mises-à-jour)
 - 🌟 [Fonctionnalités clés](#-fonctionnalités-clés)
 - 🔎 [Architecture du système](#-architecture-du-système)
-- 🎬 [Démarrage](#-démarrage)
+- 🎬 [Auto-hébergement](#-auto-hébergement)
 - 🔧 [Configurations](#-configurations)
 - 🔧 [Construire une image Docker](#-construire-une-image-docker)
 - 🔨 [Lancer le service depuis les sources pour le développement](#-lancer-le-service-depuis-les-sources-pour-le-développement)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/) est un moteur de [RAG](https://ragflow.io/basics/what-is-rag) (Retrieval-Augmented Generation) open-source de premier plan qui fusionne les technologies RAG de pointe avec des capacités Agent pour créer une couche de contexte supérieure pour les LLM. Il offre un flux de travail RAG rationalisé, adaptable aux entreprises de toute taille. Alimenté par un [moteur de contexte](https://ragflow.io/basics/what-is-agent-context-engine) convergent et des modèles d'agents préconstruits, RAGFlow permet aux développeurs de transformer des données complexes en systèmes d'IA haute-fidélité, prêts pour la production, avec une efficacité et une précision exceptionnelles.
 
-## 🎮 Démo
+## 🎮 Démarrage
 
-Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
+Essayez notre service cloud sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,6 +87,8 @@ Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 ## 🔥 Dernières mises à jour
 
+- 15-06-2026 Prise en charge de plusieurs canaux de discussion tels que Feishu, Discord, Telegram, Line, etc.
+- 24-04-2026 Prise en charge de DeepSeek v4.
 - 24-03-2026 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — Fournit un skill officiel pour accéder aux datasets RAGFlow via OpenClaw.
 - 26-12-2025 Prise en charge de la « Mémoire » pour l'agent IA.
 - 19-11-2025 Prise en charge de Gemini 3 Pro.
@@ -97,7 +98,6 @@ Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - 08-08-2025 Prise en charge des derniers modèles de la série GPT-5 d'OpenAI.
 - 01-08-2025 Prise en charge du flux de travail agentique et de MCP.
 - 23-05-2025 Ajout d'un composant exécuteur de code Python/JavaScript à l'Agent.
-- 05-05-2025 Prise en charge des requêtes inter-langues.
 - 19-03-2025 Prise en charge de l'utilisation d'un modèle multi-modal pour analyser les images dans les fichiers PDF ou DOCX.
 
 ## 🎉 Restez informé
@@ -142,7 +142,7 @@ Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 Démarrage
+## 🎬 Auto-hébergement
 
 ### 📝 Prérequis
 
@@ -150,6 +150,7 @@ Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - RAM >= 16 Go
 - Disque >= 50 Go
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/) : Requis uniquement si vous souhaitez utiliser la fonctionnalité d'exécuteur de code (sandbox) de RAGFlow.
 
 > [!TIP]
@@ -189,12 +190,12 @@ Essayez notre démo sur [https://cloud.ragflow.io](https://cloud.ragflow.io).
 > Toutes les images Docker sont construites pour les plateformes x86. Nous ne proposons pas actuellement d'images Docker pour ARM64.
 > Si vous êtes sur une plateforme ARM64, suivez [ce guide](https://ragflow.io/docs/dev/build_docker_image) pour construire une image Docker compatible avec votre système.
 
-> La commande ci-dessous télécharge l'édition `v0.25.0` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.25.0`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
+> La commande ci-dessous télécharge l'édition `v0.26.1` de l'image Docker RAGFlow. Consultez le tableau suivant pour les descriptions des différentes éditions de RAGFlow. Pour télécharger une édition de RAGFlow différente de `v0.26.1`, mettez à jour la variable `RAGFLOW_IMAGE` dans **docker/.env** avant d'utiliser `docker compose` pour démarrer le serveur.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Optionnel : utiliser un tag stable (voir les versions : https://github.com/infiniflow/ragflow/releases)
    # Cette étape garantit que le fichier **entrypoint.sh** dans le code correspond à la version de l'image Docker.
 
@@ -319,7 +320,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -396,7 +397,7 @@ Voir la [Feuille de route RAGFlow 2026](https://github.com/infiniflow/ragflow/is
 ## 🏄 Communauté
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contribuer

README_id.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DBEDFA"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="Ikuti di X (Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Lencana Daring" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Lencana Daring" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Rilis%20Terbaru" alt="Rilis Terbaru">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Dokumentasi</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Peta Jalan</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 Daftar Isi </b> </summary>
 
 - 💡 [Apa Itu RAGFlow?](#-apa-itu-ragflow)
-- 🎮 [Demo](#-demo)
+- 🎮 [Mulai](#-mulai)
 - 📌 [Pembaruan Terbaru](#-pembaruan-terbaru)
 - 🌟 [Fitur Utama](#-fitur-utama)
 - 🔎 [Arsitektur Sistem](#-arsitektur-sistem)
-- 🎬 [Mulai](#-mulai)
+- 🎬 [Pengelolaan Mandiri](#-pengelolaan-mandiri)
 - 🔧 [Konfigurasi](#-konfigurasi)
 - 🔧 [Membangun Image Docker](#-membangun-docker-image)
 - 🔨 [Meluncurkan aplikasi dari Sumber untuk Pengembangan](#-meluncurkan-aplikasi-dari-sumber-untuk-pengembangan)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/) adalah mesin [RAG](https://ragflow.io/basics/what-is-rag) (Retrieval-Augmented Generation) open-source terkemuka yang mengintegrasikan teknologi RAG mutakhir dengan kemampuan Agent untuk menciptakan lapisan kontekstual superior bagi LLM. Menyediakan alur kerja RAG yang efisien dan dapat diadaptasi untuk perusahaan segala skala. Didukung oleh mesin konteks terkonvergensi dan template Agent yang telah dipra-bangun, RAGFlow memungkinkan pengembang mengubah data kompleks menjadi sistem AI kesetiaan-tinggi dan siap-produksi dengan efisiensi dan presisi yang luar biasa.
 
-## 🎮 Demo
+## 🎮 Mulai
 
-Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
+Coba layanan cloud kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,6 +87,8 @@ Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 ## 🔥 Pembaruan Terbaru
 
+- 2026-06-15 Mendukung berbagai saluran obrolan seperti Feishu, Discord, Telegram, Line, dll.
+- 2026-04-24 Mendukung DeepSeek v4.
 - 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — Menyediakan skill resmi untuk mengakses dataset RAGFlow melalui OpenClaw.
 - 2025-12-26 Mendukung 'Memori' untuk agen AI.
 - 2025-11-19 Mendukung Gemini 3 Pro.
@@ -97,10 +98,7 @@ Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - 2025-08-08 Mendukung model seri GPT-5 terbaru dari OpenAI.
 - 2025-08-01 Mendukung alur kerja agen dan MCP.
 - 2025-05-23 Menambahkan komponen pelaksana kode Python/JS ke Agen.
-- 2025-05-05 Mendukung kueri lintas bahasa.
 - 2025-03-19 Mendukung penggunaan model multi-modal untuk memahami gambar di dalam file PDF atau DOCX.
-- 2024-12-18 Meningkatkan model Analisis Tata Letak Dokumen di DeepDoc.
-- 2024-08-22 Dukungan untuk teks ke pernyataan SQL melalui RAG.
 
 ## 🎉 Tetap Terkini
 
@@ -144,7 +142,7 @@ Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 Mulai
+## 🎬 Pengelolaan Mandiri
 
 ### 📝 Prasyarat
 
@@ -152,6 +150,7 @@ Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): Hanya diperlukan jika Anda ingin menggunakan fitur eksekutor kode (sandbox) dari RAGFlow.
 
 > [!TIP]
@@ -192,12 +191,12 @@ Coba demo kami di [https://cloud.ragflow.io](https://cloud.ragflow.io).
 > Semua gambar Docker dibangun untuk platform x86. Saat ini, kami tidak menawarkan gambar Docker untuk ARM64.
 > Jika Anda menggunakan platform ARM64, [silakan gunakan panduan ini untuk membangun gambar Docker yang kompatibel dengan sistem Anda](https://ragflow.io/docs/dev/build_docker_image).
 
-> Perintah di bawah ini mengunduh edisi v0.25.0 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.25.0, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
+> Perintah di bawah ini mengunduh edisi v0.26.1 dari gambar Docker RAGFlow. Silakan merujuk ke tabel berikut untuk deskripsi berbagai edisi RAGFlow. Untuk mengunduh edisi RAGFlow yang berbeda dari v0.26.1, perbarui variabel RAGFLOW_IMAGE di docker/.env sebelum menggunakan docker compose untuk memulai server.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Opsional: gunakan tag stabil (lihat releases: https://github.com/infiniflow/ragflow/releases)
    # This steps ensures the **entrypoint.sh** file in the code matches the Docker image version.
 
@@ -302,7 +301,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -377,7 +376,7 @@ Lihat [Roadmap RAGFlow 2026](https://github.com/infiniflow/ragflow/issues/12241)
 ## 🏄 Komunitas
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Kontribusi

README_ja.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體中文版自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DBEDFA"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,9 +57,9 @@
 
 [RAGFlow](https://ragflow.io/) は、先進的な[RAG](https://ragflow.io/basics/what-is-rag)（Retrieval-Augmented Generation）技術と Agent 機能を融合し、大規模言語モデル（LLM）に優れたコンテキスト層を構築する最先端のオープンソース RAG エンジンです。あらゆる規模の企業に対応可能な合理化された RAG ワークフローを提供し、統合型[コンテキストエンジン](https://ragflow.io/basics/what-is-agent-context-engine)と事前構築されたAgentテンプレートにより、開発者が複雑なデータを驚異的な効率性と精度で高精細なプロダクションレディAIシステムへ変換することを可能にします。
 
-## 🎮 Demo
+## 🎮 はじめに
 
-デモをお試しください：[https://cloud.ragflow.io](https://cloud.ragflow.io)。
+当社のクラウドサービスをぜひお試しください：[https://cloud.ragflow.io](https://cloud.ragflow.io)。
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -69,6 +68,8 @@
 
 ## 🔥 最新情報
 
+- 2026-06-15 Feishu、Discord、Telegram、Lineなどの複数のチャットチャンネルをサポートします。
+- 2026-04-24 DeepSeek v4 をサポート。
 - 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — OpenClaw経由でRAGFlowデータセットにアクセスする公式スキルを提供。
 - 2025-12-26 AIエージェントの「メモリ」機能をサポート。
 - 2025-11-19 Gemini 3 Proをサポートしています。
@@ -78,10 +79,8 @@
 - 2025-08-08 OpenAI の最新 GPT-5 シリーズモデルをサポートします。
 - 2025-08-01 エージェントワークフローとMCPをサポート。
 - 2025-05-23 エージェントに Python/JS コードエグゼキュータコンポーネントを追加しました。
-- 2025-05-05 言語間クエリをサポートしました。
 - 2025-03-19 PDFまたはDOCXファイル内の画像を理解するために、多モーダルモデルを使用することをサポートします。
-- 2024-12-18 DeepDoc のドキュメント レイアウト分析モデルをアップグレードします。
-- 2024-08-22 RAG を介して SQL ステートメントへのテキストをサポートします。
+
 
 ## 🎉 続きを楽しみに
 
@@ -125,7 +124,7 @@
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 初期設定
+## 🎬 セルフホスティング
 
 ### 📝 必要条件
 
@@ -133,6 +132,7 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlowのコード実行（サンドボックス）機能を利用する場合のみ必要です。
 
 > [!TIP]
@@ -172,12 +172,12 @@
 > 現在、公式に提供されているすべての Docker イメージは x86 アーキテクチャ向けにビルドされており、ARM64 用の Docker イメージは提供されていません。
 > ARM64 アーキテクチャのオペレーティングシステムを使用している場合は、[このドキュメント](https://ragflow.io/docs/dev/build_docker_image)を参照して Docker イメージを自分でビルドしてください。
 
-> 以下のコマンドは、RAGFlow Docker イメージの v0.25.0 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.25.0 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
+> 以下のコマンドは、RAGFlow Docker イメージの v0.26.1 エディションをダウンロードします。異なる RAGFlow エディションの説明については、以下の表を参照してください。v0.26.1 とは異なるエディションをダウンロードするには、docker/.env ファイルの RAGFLOW_IMAGE 変数を適宜更新し、docker compose を使用してサーバーを起動してください。
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # 任意: 安定版タグを利用 (一覧: https://github.com/infiniflow/ragflow/releases)
    # この手順は、コード内の entrypoint.sh ファイルが Docker イメージのバージョンと一致していることを確認します。
 
@@ -302,7 +302,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -377,7 +377,7 @@ docker build --platform linux/amd64 \
 ## 🏄 コミュニティ
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 コントリビュート

README_ko.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DBEDFA"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -59,9 +58,9 @@
 
 [RAGFlow](https://ragflow.io/) 는 최첨단 [RAG](https://ragflow.io/basics/what-is-rag)(Retrieval-Augmented Generation)와 Agent 기능을 융합하여 대규모 언어 모델(LLM)을 위한 우수한 컨텍스트 계층을 생성하는 선도적인 오픈소스 RAG 엔진입니다. 모든 규모의 기업에 적용 가능한 효율적인 RAG 워크플로를 제공하며, 통합 [컨텍스트 엔진](https://ragflow.io/basics/what-is-agent-context-engine)과 사전 구축된 Agent 템플릿을 통해 개발자들이 복잡한 데이터를 예외적인 효율성과 정밀도로 고급 구현도의 프로덕션 준비 완료 AI 시스템으로 변환할 수 있도록 지원합니다.
 
-## 🎮 데모
+## 🎮 시작하기
 
-데모를 [https://cloud.ragflow.io](https://cloud.ragflow.io)에서 실행해 보세요.
+[https://cloud.ragflow.io](https://cloud.ragflow.io)에서 저희 클라우드 서비스를 이용해 보세요.
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -70,6 +69,8 @@
 
 ## 🔥 업데이트
 
+- 2026-06-15 Feishu, Discord, Telegram, Line 등 다양한 채팅 채널을 지원합니다.
+- 2026-04-24 DeepSeek v4를 지원합니다.
 - 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — OpenClaw를 통해 RAGFlow 데이터셋에 접근하는 공식 스킬 제공.
 - 2025-12-26 AI 에이전트의 '메모리' 기능 지원.
 - 2025-11-19 Gemini 3 Pro를 지원합니다.
@@ -79,10 +80,8 @@
 - 2025-08-08 OpenAI의 최신 GPT-5 시리즈 모델을 지원합니다.
 - 2025-08-01 에이전트 워크플로우와 MCP를 지원합니다.
 - 2025-05-23 Agent에 Python/JS 코드 실행기 구성 요소를 추가합니다.
-- 2025-05-05 언어 간 쿼리를 지원합니다.
 - 2025-03-19 PDF 또는 DOCX 파일 내의 이미지를 이해하기 위해 다중 모드 모델을 사용하는 것을 지원합니다.
-- 2024-12-18 DeepDoc의 문서 레이아웃 분석 모델 업그레이드.
-- 2024-08-22 RAG를 통해 SQL 문에 텍스트를 지원합니다.
+
 
 ## 🎉 계속 지켜봐 주세요
 
@@ -126,7 +125,7 @@
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 시작하기
+## 🎬 자체 호스팅
 
 ### 📝 사전 준비 사항
 
@@ -134,6 +133,7 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): RAGFlow의 코드 실행기(샌드박스) 기능을 사용하려는 경우에만 필요합니다.
 
 > [!TIP]
@@ -174,12 +174,12 @@
 > 모든 Docker 이미지는 x86 플랫폼을 위해 빌드되었습니다. 우리는 현재 ARM64 플랫폼을 위한 Docker 이미지를 제공하지 않습니다.
 > ARM64 플랫폼을 사용 중이라면, [시스템과 호환되는 Docker 이미지를 빌드하려면 이 가이드를 사용해 주세요](https://ragflow.io/docs/dev/build_docker_image).
 
-   > 아래 명령어는 RAGFlow Docker 이미지의 v0.25.0 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.25.0과 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
+   > 아래 명령어는 RAGFlow Docker 이미지의 v0.26.1 버전을 다운로드합니다. 다양한 RAGFlow 버전에 대한 설명은 다음 표를 참조하십시오. v0.26.1와 다른 RAGFlow 버전을 다운로드하려면, docker/.env 파일에서 RAGFLOW_IMAGE 변수를 적절히 업데이트한 후 docker compose를 사용하여 서버를 시작하십시오.
 
    ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Optional: use a stable tag (see releases: https://github.com/infiniflow/ragflow/releases)
    # 이 단계는 코드의 entrypoint.sh 파일이 Docker 이미지 버전과 일치하도록 보장합니다.
 
@@ -297,7 +297,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -381,7 +381,7 @@ docker build --platform linux/amd64 \
 ## 🏄 커뮤니티
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 컨트리뷰션

README_pt_br.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DBEDFA"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="seguir no X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Badge Estático" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Badge Estático" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Última%20Relese" alt="Última Versão">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Documentação</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 Índice</b></summary>
 
 - 💡 [O que é o RAGFlow?](#-o-que-é-o-ragflow)
-- 🎮 [Demo](#-demo)
+- 🎮 [Primeiros Passos](#-primeiros-passos)
 - 📌 [Últimas Atualizações](#-últimas-atualizações)
 - 🌟 [Principais Funcionalidades](#-principais-funcionalidades)
 - 🔎 [Arquitetura do Sistema](#-arquitetura-do-sistema)
-- 🎬 [Primeiros Passos](#-primeiros-passos)
+- 🎬 [Auto-hospedagem](#-auto-hospedagem)
 - 🔧 [Configurações](#-configurações)
 - 🔧 [Construir uma imagem docker sem incorporar modelos](#-construir-uma-imagem-docker-sem-incorporar-modelos)
 - 🔧 [Construir uma imagem docker incluindo modelos](#-construir-uma-imagem-docker-incluindo-modelos)
@@ -78,9 +77,9 @@
 
 [RAGFlow](https://ragflow.io/) é um mecanismo de [RAG](https://ragflow.io/basics/what-is-rag) (Retrieval-Augmented Generation) open-source líder que fusiona tecnologias RAG de ponta com funcionalidades Agent para criar uma camada contextual superior para LLMs. Oferece um fluxo de trabalho RAG otimizado adaptável a empresas de qualquer escala. Alimentado por [um motor de contexto](https://ragflow.io/basics/what-is-agent-context-engine) convergente e modelos Agent pré-construídos, o RAGFlow permite que desenvolvedores transformem dados complexos em sistemas de IA de alta fidelidade e pronto para produção com excepcional eficiência e precisão.
 
-## 🎮 Demo
+## 🎮 Primeiros Passos
 
-Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
+Experimente o nosso serviço na nuvem em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -89,6 +88,8 @@ Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 
 ## 🔥 Últimas Atualizações
 
+- 15-06-2026 Suporte a múltiplos canais de chat, como Feishu, Discord, Telegram, Line, etc..
+- 24-04-2026 Suporta DeepSeek v4.
 - 24-03-2026 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — Fornece um skill oficial para acessar datasets do RAGFlow via OpenClaw.
 - 26-12-2025 Suporte à função 'Memória' para agentes de IA.
 - 19-11-2025 Suporta Gemini 3 Pro.
@@ -98,10 +99,7 @@ Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - 08-08-2025 Suporta a mais recente série GPT-5 da OpenAI.
 - 01-08-2025 Suporta fluxo de trabalho agente e MCP.
 - 23-05-2025 Adicione o componente executor de código Python/JS ao Agente.
-- 05-05-2025 Suporte a consultas entre idiomas.
 - 19-03-2025 Suporta o uso de um modelo multi-modal para entender imagens dentro de arquivos PDF ou DOCX.
-- 18-12-2024 Atualiza o modelo de Análise de Layout de Documentos no DeepDoc.
-- 22-08-2024 Suporta conversão de texto para comandos SQL via RAG.
 
 ## 🎉 Fique Ligado
 
@@ -145,7 +143,7 @@ Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 Primeiros Passos
+## 🎬 Auto-hospedagem
 
 ### 📝 Pré-requisitos
 
@@ -153,6 +151,7 @@ Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 - RAM >= 16 GB
 - Disco >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): Necessário apenas se você pretende usar o recurso de executor de código (sandbox) do RAGFlow.
 
 > [!TIP]
@@ -192,12 +191,12 @@ Experimente nossa demo em [https://cloud.ragflow.io](https://cloud.ragflow.io).
 > Todas as imagens Docker são construídas para plataformas x86. Atualmente, não oferecemos imagens Docker para ARM64.
 > Se você estiver usando uma plataforma ARM64, por favor, utilize [este guia](https://ragflow.io/docs/dev/build_docker_image) para construir uma imagem Docker compatível com o seu sistema.
 
-    > O comando abaixo baixa a edição`v0.25.0` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.25.0`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
+    > O comando abaixo baixa a edição`v0.26.1` da imagem Docker do RAGFlow. Consulte a tabela a seguir para descrições de diferentes edições do RAGFlow. Para baixar uma edição do RAGFlow diferente da `v0.26.1`, atualize a variável `RAGFLOW_IMAGE` conforme necessário no **docker/.env** antes de usar `docker compose` para iniciar o servidor.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # Opcional: use uma tag estável (veja releases: https://github.com/infiniflow/ragflow/releases)
    # Esta etapa garante que o arquivo entrypoint.sh no código corresponda à versão da imagem do Docker.
 
@@ -319,7 +318,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # instala os módulos Python dependentes do RAGFlow
+   uv sync --python 3.13 # instala os módulos Python dependentes do RAGFlow
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -394,7 +393,7 @@ Veja o [RAGFlow Roadmap 2026](https://github.com/infiniflow/ragflow/issues/12241
 ## 🏄 Comunidade
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Contribuindo

README_tr.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DBEDFA"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="X(Twitter)'da takip et">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Çevrimiçi Demo" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Çevrimiçi Demo" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Son%20Sürüm" alt="Son Sürüm">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Dokümantasyon</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Yol Haritası</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> 
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 İçindekiler</b></summary>
 
 - 💡 [RAGFlow Nedir?](#-ragflow-nedir)
-- 🎮 [Demo](#-demo)
+- 🎮 [Başlarken](#-başlarken)
 - 📌 [Son Güncellemeler](#-son-güncellemeler)
 - 🌟 [Temel Özellikler](#-temel-özellikler)
 - 🔎 [Sistem Mimarisi](#-sistem-mimarisi)
-- 🎬 [Başlarken](#-başlarken)
+- 🎬 [Kendi Sunucusunda Barındırma](#-kendi-sunucusunda-barındırma)
 - 🔧 [Yapılandırmalar](#-yapılandırmalar)
 - 🔧 [Docker İmajı Oluşturma](#-docker-i̇majı-oluşturma)
 - 🔨 [Geliştirme İçin Kaynaktan Hizmet Başlatma](#-geliştirme-i̇çin-kaynaktan-hizmet-başlatma)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/), derin doküman anlayışına dayalı, açık kaynaklı ve öncü bir Artırılmış Üretim ile Bilgi Erişimi ([RAG](https://ragflow.io/basics/what-is-rag)) motorudur. En son RAG teknolojisini Ajan yetenekleriyle birleştirerek LLM'ler için üstün bir bağlam katmanı oluşturur. Her ölçekteki kuruluşa uyarlanabilir, kolaylaştırılmış bir RAG iş akışı sunar. Yakınsanmış bir [bağlam motoru](https://ragflow.io/basics/what-is-agent-context-engine) ve hazır ajan şablonlarıyla donatılmış RAGFlow, geliştiricilerin karmaşık verileri yüksek doğrulukta, üretime hazır yapay zeka sistemlerine olağanüstü verimlilik ve hassasiyetle dönüştürmesini sağlar.
 
-## 🎮 Demo
+## 🎮 Başlarken
 
-Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyebilirsiniz.
+Bulut hizmetimizi [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyin.
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,6 +87,8 @@ Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyeb
 
 ## 🔥 Son Güncellemeler
 
+- 2026-06-15 Feishu, Discord, Telegram, Line vb. gibi birden fazla sohbet kanalını destekleyin.
+- 2026-04-24 DeepSeek v4 desteği.
 - 2026-03-24 [RAGFlow Skill on OpenClaw](https://clawhub.ai/yingfeng/ragflow-skill) — OpenClaw üzerinden RAGFlow veri setlerine erişmek için resmi bir skill sağlar.
 - 2025-12-26 Yapay zeka ajanı için 'Bellek' desteği eklendi.
 - 2025-11-19 Gemini 3 Pro desteği eklendi.
@@ -97,7 +98,6 @@ Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyeb
 - 2025-08-08 OpenAI'ın en yeni GPT-5 serisi modelleri için destek eklendi.
 - 2025-08-01 Ajanlı iş akışı ve MCP desteği eklendi.
 - 2025-05-23 Ajana Python/JavaScript kod çalıştırıcı bileşeni eklendi.
-- 2025-05-05 Diller arası sorgu desteği eklendi.
 - 2025-03-19 PDF veya DOCX dosyalarındaki görselleri yorumlamak için çok modlu model desteği eklendi.
 
 ## 🎉 Bizi Takip Edin
@@ -142,7 +142,7 @@ Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyeb
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 Başlarken
+## 🎬 Kendi Sunucusunda Barındırma
 
 ### 📝 Ön Koşullar
 
@@ -150,6 +150,7 @@ Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyeb
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): Yalnızca RAGFlow'un kod çalıştırıcı (sandbox) özelliğini kullanmayı planlıyorsanız gereklidir.
 
 > [!TIP]
@@ -190,12 +191,12 @@ Demomuzu [https://cloud.ragflow.io](https://cloud.ragflow.io) adresinden deneyeb
 > Tüm Docker imajları x86 platformları için oluşturulmuştur. Şu anda ARM64 için Docker imajı sunmuyoruz.
 > ARM64 platformundaysanız, sisteminizle uyumlu bir Docker imajı oluşturmak için [bu kılavuzu](https://ragflow.io/docs/dev/build_docker_image) takip edin.
 
-> Aşağıdaki komut RAGFlow Docker imajının `v0.25.0` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.25.0` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
+> Aşağıdaki komut RAGFlow Docker imajının `v0.26.1` sürümünü indirir. Farklı RAGFlow sürümleri için aşağıdaki tabloya bakın. `v0.26.1` dışında bir sürüm indirmek için, `docker compose` ile sunucuyu başlatmadan önce **docker/.env** dosyasındaki `RAGFLOW_IMAGE` değişkenini güncelleyin.
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # İsteğe bağlı: Kararlı bir etiket kullanın (sürümler: https://github.com/infiniflow/ragflow/releases)
    # Bu adım, koddaki **entrypoint.sh** dosyasının Docker imaj sürümüyle eşleşmesini sağlar.
 
@@ -323,7 +324,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # RAGFlow'un bağımlı Python modüllerini yükler
+   uv sync --python 3.13 # RAGFlow'un bağımlı Python modüllerini yükler
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -400,7 +401,7 @@ docker build --platform linux/amd64 \
 ## 🏄 Topluluk
 
 - [Discord](https://discord.gg/NjYzJD3GM3)
-- [Twitter](https://twitter.com/infiniflowai)
+- [X](https://x.com/infiniflowai)
 - [GitHub Tartışmalar](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 Katkıda Bulunma

README_tzh.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DBEDFA"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 目錄</b></summary>
 
 - 💡 [RAGFlow 是什麼？](#-RAGFlow-是什麼)
-- 🎮 [Demo-試用](#-demo-試用)
+- 🎮 [快速開始](#-快速開始)
 - 📌 [近期更新](#-近期更新)
 - 🌟 [主要功能](#-主要功能)
 - 🔎 [系統架構](#-系統架構)
-- 🎬 [快速開始](#-快速開始)
+- 🎬 [自行架設](#-自行架設)
 - 🔧 [系統配置](#-系統配置)
 - 🔨 [以原始碼啟動服務](#-以原始碼啟動服務)
 - 📚 [技術文檔](#-技術文檔)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/) 是一款領先的開源 [RAG](https://ragflow.io/basics/what-is-rag)（Retrieval-Augmented Generation）引擎，通過融合前沿的 RAG 技術與 Agent 能力，為大型語言模型提供卓越的上下文層。它提供可適配任意規模企業的端到端 RAG 工作流，憑藉融合式[上下文引擎](https://ragflow.io/basics/what-is-agent-context-engine)與預置的 Agent 模板，助力開發者以極致效率與精度將複雜數據轉化為高可信、生產級的人工智能系統。
 
-## 🎮 Demo 試用
+## 🎮 快速開始
 
-請登入網址 [https://cloud.ragflow.io](https://cloud.ragflow.io) 試用 demo。
+請登入網址 [https://cloud.ragflow.io](https://cloud.ragflow.io) 試用雲服務。
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,6 +87,8 @@
 
 ## 🔥 近期更新
 
+- 2026-06-15 支援飛書、Discord、Telegram、Line 等多種聊天管道。
+- 2026-04-24 支援 DeepSeek v4 版本。
 - 2026-03-24 發布 [RAGFlow 官方 Skill](https://clawhub.ai/yingfeng/ragflow-skill) — 提供官方 Skill 以透過 OpenClaw 訪問 RAGFlow 數據集。
 - 2025-12-26 支援AI代理的「記憶」功能。
 - 2025-11-19 支援 Gemini 3 Pro。
@@ -97,10 +98,8 @@
 - 2025-08-08 支援 OpenAI 最新的 GPT-5 系列模型。
 - 2025-08-01 支援 agentic workflow 和 MCP。
 - 2025-05-23 為 Agent 新增 Python/JS 程式碼執行器元件。
-- 2025-05-05 支援跨語言查詢。
 - 2025-03-19 PDF和DOCX中的圖支持用多模態大模型去解析得到描述。
-- 2024-12-18 升級了 DeepDoc 的文檔佈局分析模型。
-- 2024-08-22 支援用 RAG 技術實現從自然語言到 SQL 語句的轉換。
+
 
 ## 🎉 關注項目
 
@@ -144,7 +143,7 @@
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 快速開始
+## 🎬 自行架設
 
 ### 📝 前提條件
 
@@ -152,6 +151,7 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): 僅在您打算使用 RAGFlow 的代碼執行器（沙箱）功能時才需要安裝。
 
 > [!TIP]
@@ -191,12 +191,12 @@
 > 所有 Docker 映像檔都是為 x86 平台建置的。目前，我們不提供 ARM64 平台的 Docker 映像檔。
 > 如果您使用的是 ARM64 平台，請使用 [這份指南](https://ragflow.io/docs/dev/build_docker_image) 來建置適合您系統的 Docker 映像檔。
 
-> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.25.0`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.25.0` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
+> 執行以下指令會自動下載 RAGFlow Docker 映像 `v0.26.1`。請參考下表查看不同 Docker 發行版的說明。如需下載不同於 `v0.26.1` 的 Docker 映像，請在執行 `docker compose` 啟動服務之前先更新 **docker/.env** 檔案內的 `RAGFLOW_IMAGE` 變數。
 
 ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # 可選：使用穩定版標籤（查看發佈：https://github.com/infiniflow/ragflow/releases）
    # 此步驟確保程式碼中的 entrypoint.sh 檔案與 Docker 映像版本一致。
 
@@ -329,7 +329,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -407,8 +407,8 @@ docker build --platform linux/amd64 \
 
 ## 🏄 開源社群
 
-- [Discord](https://discord.gg/zd4qPW6t)
-- [Twitter](https://twitter.com/infiniflowai)
+- [Discord](https://discord.gg/NjYzJD3GM3)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 貢獻指南

README_zh.md

@@ -10,9 +10,9 @@
   <a href="./README_tzh.md"><img alt="繁體版中文自述文件" src="https://img.shields.io/badge/繁體中文-DFE0E5"></a>
   <a href="./README_ja.md"><img alt="日本語のREADME" src="https://img.shields.io/badge/日本語-DFE0E5"></a>
   <a href="./README_ko.md"><img alt="한국어" src="https://img.shields.io/badge/한국어-DFE0E5"></a>
+  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_id.md"><img alt="Bahasa Indonesia" src="https://img.shields.io/badge/Bahasa Indonesia-DFE0E5"></a>
   <a href="./README_pt_br.md"><img alt="Português(Brasil)" src="https://img.shields.io/badge/Português(Brasil)-DFE0E5"></a>
-  <a href="./README_fr.md"><img alt="README en Français" src="https://img.shields.io/badge/Français-DFE0E5"></a>
   <a href="./README_ar.md"><img alt="README in Arabic" src="https://img.shields.io/badge/Arabic-DFE0E5"></a>
   <a href="./README_tr.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-DFE0E5"></a>
 </p>
@@ -22,10 +22,10 @@
         <img src="https://img.shields.io/twitter/follow/infiniflow?logo=X&color=%20%23f5f5f5" alt="follow on X(Twitter)">
     </a>
     <a href="https://cloud.ragflow.io" target="_blank">
-        <img alt="Static Badge" src="https://img.shields.io/badge/Online-Demo-4e6b99">
+        <img alt="Static Badge" src="https://img.shields.io/badge/Get-Started-4e6b99">
     </a>
     <a href="https://hub.docker.com/r/infiniflow/ragflow" target="_blank">
-        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.25.0">
+        <img src="https://img.shields.io/docker/pulls/infiniflow/ragflow?label=Docker%20Pulls&color=0db7ed&logo=docker&logoColor=white&style=flat-square" alt="docker pull infiniflow/ragflow:v0.26.1">
     </a>
     <a href="https://github.com/infiniflow/ragflow/releases/latest">
         <img src="https://img.shields.io/github/v/release/infiniflow/ragflow?color=blue&label=Latest%20Release" alt="Latest Release">
@@ -39,11 +39,10 @@
 </p>
 
 <h4 align="center">
+  <a href="https://cloud.ragflow.io">Cloud</a> |
   <a href="https://ragflow.io/docs/dev/">Document</a> |
   <a href="https://github.com/infiniflow/ragflow/issues/12241">Roadmap</a> |
-  <a href="https://twitter.com/infiniflowai">Twitter</a> |
-  <a href="https://discord.gg/NjYzJD3GM3">Discord</a> |
-  <a href="https://cloud.ragflow.io">Demo</a>
+  <a href="https://discord.gg/NjYzJD3GM3">Discord</a>
 </h4>
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
@@ -58,11 +57,11 @@
 <summary><b>📕 目录</b></summary>
 
 - 💡 [RAGFlow 是什么？](#-RAGFlow-是什么)
-- 🎮 [Demo](#-demo)
+- 🎮 [快速开始](#-快速开始)
 - 📌 [近期更新](#-近期更新)
 - 🌟 [主要功能](#-主要功能)
 - 🔎 [系统架构](#-系统架构)
-- 🎬 [快速开始](#-快速开始)
+- 🎬 [自主托管](#-自主托管)
 - 🔧 [系统配置](#-系统配置)
 - 🔨 [以源代码启动服务](#-以源代码启动服务)
 - 📚 [技术文档](#-技术文档)
@@ -77,9 +76,9 @@
 
 [RAGFlow](https://ragflow.io/) 是一款领先的开源检索增强生成（[RAG](https://ragflow.io/basics/what-is-rag)）引擎，通过融合前沿的 RAG 技术与 Agent 能力，为大型语言模型提供卓越的上下文层。它提供可适配任意规模企业的端到端 RAG 工作流，凭借融合式[上下文引擎](https://ragflow.io/basics/what-is-agent-context-engine)与预置的 Agent 模板，助力开发者以极致效率与精度将复杂数据转化为高可信、生产级的人工智能系统。
 
-## 🎮 Demo 试用
+## 🎮 快速开始
 
-请登录网址 [https://cloud.ragflow.io](https://cloud.ragflow.io) 试用 demo。
+请登录网址 [https://cloud.ragflow.io](https://cloud.ragflow.io) 体验云服务。
 
 <div align="center" style="margin-top:20px;margin-bottom:20px;">
 <img src="https://raw.githubusercontent.com/infiniflow/ragflow-docs/refs/heads/image/image/chunking.gif" width="1200"/>
@@ -88,8 +87,10 @@
 
 ## 🔥 近期更新
 
+- 2026-06-15 支持飞书、Discord、Telegram、Line 等多种聊天渠道。
+- 2026-04-24 支持 DeepSeek v4.
 - 2026-03-24 发布 [RAGFlow 官方 Skill](https://clawhub.ai/yingfeng/ragflow-skill) — 提供官方 Skill 以通过 OpenClaw 访问 RAGFlow 数据集。
-- 2025-12-26 支持AI代理的"记忆"功能。
+- 2025-12-26 支持 AI 代理的"记忆"功能。
 - 2025-11-19 支持 Gemini 3 Pro。
 - 2025-11-12 支持从 Confluence、S3、Notion、Discord、Google Drive 进行数据同步。
 - 2025-10-23 支持 MinerU 和 Docling 作为文档解析方法。
@@ -97,10 +98,8 @@
 - 2025-08-08 支持 OpenAI 最新的 GPT-5 系列模型。
 - 2025-08-01 支持 agentic workflow 和 MCP。
 - 2025-05-23 Agent 新增 Python/JS 代码执行器组件。
-- 2025-05-05 支持跨语言查询。
 - 2025-03-19 PDF 和 DOCX 中的图支持用多模态大模型去解析得到描述。
-- 2024-12-18 升级了 DeepDoc 的文档布局分析模型。
-- 2024-08-22 支持用 RAG 技术实现从自然语言到 SQL 语句的转换。
+
 
 ## 🎉 关注项目
 
@@ -144,7 +143,7 @@
 <img src="https://github.com/user-attachments/assets/31b0dd6f-ca4f-445a-9457-70cb44a381b2" width="1000"/>
 </div>
 
-## 🎬 快速开始
+## 🎬 自主托管
 
 ### 📝 前提条件
 
@@ -152,6 +151,7 @@
 - RAM >= 16 GB
 - Disk >= 50 GB
 - Docker >= 24.0.0 & Docker Compose >= v2.26.1
+- Python >= 3.13
 - [gVisor](https://gvisor.dev/docs/user_guide/install/): 仅在你打算使用 RAGFlow 的代码执行器（沙箱）功能时才需要安装。
 
 > [!TIP]
@@ -192,12 +192,12 @@
 > 请注意，目前官方提供的所有 Docker 镜像均基于 x86 架构构建，并不提供基于 ARM64 的 Docker 镜像。
 > 如果你的操作系统是 ARM64 架构，请参考[这篇文档](https://ragflow.io/docs/dev/build_docker_image)自行构建 Docker 镜像。
 
-   > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.25.0`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.25.0` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
+   > 运行以下命令会自动下载 RAGFlow Docker 镜像 `v0.26.1`。请参考下表查看不同 Docker 发行版的描述。如需下载不同于 `v0.26.1` 的 Docker 镜像，请在运行 `docker compose` 启动服务之前先更新 **docker/.env** 文件内的 `RAGFLOW_IMAGE` 变量。
 
    ```bash
    $ cd ragflow/docker
 
-   # git checkout v0.25.0
+   # git checkout v0.26.1
    # 可选：使用稳定版本标签（查看发布：https://github.com/infiniflow/ragflow/releases）
    # 这一步确保代码中的 entrypoint.sh 文件与 Docker 镜像的版本保持一致。
 
@@ -329,7 +329,7 @@ docker build --platform linux/amd64 \
    ```bash
    git clone https://github.com/infiniflow/ragflow.git
    cd ragflow/
-   uv sync --python 3.12 # install RAGFlow dependent python modules
+   uv sync --python 3.13 # install RAGFlow dependent python modules
    uv run python3 download_deps.py
    pre-commit install
    ```
@@ -410,8 +410,8 @@ docker build --platform linux/amd64 \
 
 ## 🏄 开源社区
 
-- [Discord](https://discord.gg/zd4qPW6t)
-- [Twitter](https://twitter.com/infiniflowai)
+- [Discord](https://discord.gg/NjYzJD3GM3)
+- [X](https://x.com/infiniflowai)
 - [GitHub Discussions](https://github.com/orgs/infiniflow/discussions)
 
 ## 🙌 贡献指南

admin/client/README.md

@@ -48,7 +48,7 @@ It consists of a server-side Service and a command-line client (CLI), both imple
 1.  Ensure the Admin Service is running.
 2.  Install ragflow-cli.
     ```bash
-    pip install ragflow-cli==0.25.0
+    pip install ragflow-cli==0.26.1
     ```
 3.  Launch the CLI client:
     ```bash

admin/client/parser.py

@@ -264,7 +264,7 @@ generate_key: GENERATE KEY FOR USER quoted_string ";"
 list_keys: LIST KEYS OF quoted_string ";"
 drop_key: DROP KEY quoted_string OF quoted_string ";"
 
-set_variable: SET VAR identifier identifier ";"
+set_variable: SET VAR identifier variable_value ";"
 show_variable: SHOW VAR identifier ";"
 list_variables: LIST VARS ";"
 list_configs: LIST CONFIGS ";"
@@ -378,6 +378,7 @@ update_chunk: UPDATE CHUNK quoted_string OF DATASET quoted_string SET quoted_str
 identifier_list: identifier (COMMA identifier)*
 
 identifier: WORD
+variable_value: WORD | NUMBER | QUOTED_STRING
 quoted_string: QUOTED_STRING
 status: ON | WORD
 

admin/client/pyproject.toml

@@ -1,11 +1,11 @@
 [project]
 name = "ragflow-cli"
-version = "0.25.0"
+version = "0.26.1"
 description = "Admin Service's client of [RAGFlow](https://github.com/infiniflow/ragflow). The Admin Service provides user management and system monitoring. "
 authors = [{ name = "Lynn", email = "lynn_inf@hotmail.com" }]
 license = { text = "Apache License, Version 2.0" }
 readme = "README.md"
-requires-python = ">=3.12,<3.15"
+requires-python = ">=3.13,<3.14"
 dependencies = [
     "requests>=2.30.0,<3.0.0",
     "beartype>=0.20.0,<1.0.0",

admin/client/ragflow_client.py

@@ -43,6 +43,12 @@ def encrypt(input_string):
     return base64.b64encode(cipher_text).decode("utf-8")
 
 
+def _strip_tree_value(value):
+    if isinstance(value, Tree):
+        value = value.children[0]
+    return str(value).strip("'\"")
+
+
 class RAGFlowClient:
     def __init__(self, http_client: HttpClient, server_type: str):
         self.http_client = http_client
@@ -526,10 +532,8 @@ class RAGFlowClient:
         if self.server_type != "admin":
             print("This command is only allowed in ADMIN mode")
 
-        var_name_tree: Tree = command["var_name"]
-        var_name = var_name_tree.children[0].strip("'\"")
-        var_value_tree: Tree = command["var_value"]
-        var_value = var_value_tree.children[0].strip("'\"")
+        var_name = _strip_tree_value(command["var_name"])
+        var_value = _strip_tree_value(command["var_value"])
         response = self.http_client.request("PUT", "/admin/variables",
                                             json_body={"var_name": var_name, "var_value": var_value}, use_api_base=True,
                                             auth_kind="admin")
@@ -544,8 +548,7 @@ class RAGFlowClient:
         if self.server_type != "admin":
             print("This command is only allowed in ADMIN mode")
 
-        var_name_tree: Tree = command["var_name"]
-        var_name = var_name_tree.children[0].strip("'\"")
+        var_name = _strip_tree_value(command["var_name"])
         response = self.http_client.request(method="GET", path="/admin/variables", json_body={"var_name": var_name},
                                             use_api_base=True, auth_kind="admin")
         res_json = response.json()
@@ -1215,12 +1218,12 @@ class RAGFlowClient:
         # Prepare payload for completion API
         # Note: stream parameter is not sent, server defaults to stream=True
         payload = {
-            "conversation_id": session_id,
+            "session_id": session_id,
             "messages": [{"role": "user", "content": message}]
         }
 
-        response = self.http_client.request("POST", "/conversation/completion", json_body=payload,
-                                            use_api_base=False, auth_kind="web", stream=True)
+        response = self.http_client.request("POST", "/chat/completions", json_body=payload,
+                                            use_api_base=True, auth_kind="web", stream=True)
 
         if response.status_code != 200:
             print(f"Fail to chat on session, status code: {response.status_code}")
@@ -1325,7 +1328,7 @@ class RAGFlowClient:
             print(f"Documents {document_names} not found in {dataset_name}")
 
         payload = {"doc_ids": document_ids, "run": 1}
-        response = self.http_client.request("POST", "/document/run", json_body=payload, use_api_base=False,
+        response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True,
                                             auth_kind="web")
         res_json = response.json()
         if response.status_code == 200 and res_json["code"] == 0:
@@ -1351,7 +1354,7 @@ class RAGFlowClient:
             document_ids.append(doc["id"])
 
         payload = {"doc_ids": document_ids, "run": 1}
-        response = self.http_client.request("POST", "/document/run", json_body=payload, use_api_base=False,
+        response = self.http_client.request("POST", "/documents/ingest", json_body=payload, use_api_base=True,
                                             auth_kind="web")
         res_json = response.json()
         if response.status_code == 200 and res_json["code"] == 0:

admin/client/user.py

@@ -41,7 +41,7 @@ def encrypt_password(password_plain: str) -> str:
             return base64.b64encode(encrypted_password).decode('utf-8')
     except Exception as exc:
         raise AuthException(
-            "Password encryption unavailable; install pycryptodomex (uv sync --python 3.12 --group test)."
+            "Password encryption unavailable; install pycryptodomex (uv sync --python 3.13 --group test)."
         ) from exc
     return crypt(password_plain)
 

admin/client/uv.lock

@@ -1,6 +1,6 @@
 version = 1
 revision = 3
-requires-python = ">=3.12, <3.15"
+requires-python = "==3.13.*"
 
 [[package]]
 name = "beartype"
@@ -26,22 +26,6 @@ version = "3.4.4"
 source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" }
 wheels = [
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" },
@@ -58,22 +42,6 @@ wheels = [
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" },
 ]
 
@@ -188,20 +156,20 @@ wheels = [
 
 [[package]]
 name = "ragflow-cli"
-version = "0.25.0"
+version = "0.26.1"
 source = { virtual = "." }
 dependencies = [
     { name = "beartype" },
     { name = "lark" },
     { name = "pycryptodomex" },
     { name = "requests" },
+    { name = "requests-toolbelt" },
 ]
 
 [package.dev-dependencies]
 test = [
     { name = "pytest" },
     { name = "requests" },
-    { name = "requests-toolbelt" },
 ]
 
 [package.metadata]
@@ -210,13 +178,13 @@ requires-dist = [
     { name = "lark", specifier = ">=1.1.0" },
     { name = "pycryptodomex", specifier = ">=3.10.0" },
     { name = "requests", specifier = ">=2.30.0,<3.0.0" },
+    { name = "requests-toolbelt", specifier = ">=1.0.0" },
 ]
 
 [package.metadata.requires-dev]
 test = [
     { name = "pytest", specifier = ">=8.3.5" },
     { name = "requests", specifier = ">=2.32.3" },
-    { name = "requests-toolbelt", specifier = ">=1.0.0" },
 ]
 
 [[package]]

admin/server/auth.py

@@ -58,7 +58,7 @@ def setup_auth(login_manager):
                     return None
 
                 # Decode JWT to get the UUID access_token
-                jwt = Serializer(secret_key=settings.SECRET_KEY)
+                jwt = Serializer(secret_key=settings.get_secret_key())
                 access_token = str(jwt.loads(jwt_token))
 
                 if not access_token or not access_token.strip():
@@ -115,8 +115,6 @@ def init_default_admin():
 
 
 def add_tenant_for_admin(user_info: dict, role: str):
-    from api.db.services.tenant_llm_service import TenantLLMService
-    from api.db.services.llm_service import get_init_tenant_llm
 
     tenant = {
         "id": user_info["id"],
@@ -135,10 +133,10 @@ def add_tenant_for_admin(user_info: dict, role: str):
         "role": role
     }
 
-    tenant_llm = get_init_tenant_llm(user_info["id"])
+    # tenant_llm = get_init_tenant_llm(user_info["id"])
     TenantService.insert(**tenant)
     UserTenantService.insert(**usr_tenant)
-    TenantLLMService.insert_many(tenant_llm)
+    # TenantLLMService.insert_many(tenant_llm)
     logging.info(
         f"Added tenant for email: {user_info['email']}, A default tenant has been set; changing the default models after login is strongly recommended.")
 

admin/server/routes.py

@@ -421,7 +421,7 @@ def get_user_permission(user_name: str):
 def set_variable():
     try:
         data = request.get_json()
-        if not data and "var_name" not in data:
+        if not data or "var_name" not in data:
             return error_response("Var name is required", 400)
 
         if "var_value" not in data:
@@ -449,7 +449,7 @@ def get_variable():
 
         # get var
         data = request.get_json()
-        if not data and "var_name" not in data:
+        if not data or "var_name" not in data:
             return error_response("Var name is required", 400)
         var_name: str = data["var_name"]
         res = SettingsMgr.get_by_name(var_name)

admin/server/services.py

@@ -330,36 +330,65 @@ class ServiceMgr:
 
 
 class SettingsMgr:
+    @staticmethod
+    def _format_setting(setting):
+        return {
+            "data_type": setting.data_type,
+            "name": setting.name,
+            "setting_type": "config",
+            "value": setting.value,
+        }
+
+    @staticmethod
+    def _validate_value(name: str, data_type: str, value: str):
+        data_type = data_type.lower()
+        value = str(value)
+        if data_type == "string":
+            return
+        if data_type == "integer":
+            try:
+                int(value)
+            except ValueError:
+                raise AdminException(f"Invalid integer value for {name}: {value}")
+            return
+        if data_type in {"bool", "boolean"}:
+            if value not in {"true", "false"}:
+                raise AdminException(f"Invalid bool value for {name}: expected true or false")
+            return
+        if data_type == "json":
+            try:
+                json.loads(value)
+            except json.JSONDecodeError:
+                raise AdminException(f"Invalid JSON value for {name}")
+            return
+        raise AdminException(f"Unsupported data type for {name}: {data_type}")
+
+    @staticmethod
+    def _infer_data_type(name: str):
+        if name.startswith("sandbox."):
+            return "json"
+        if name.endswith(".enabled"):
+            return "bool"
+        return "string"
+
     @staticmethod
     def get_all():
-        settings = SystemSettingsService.get_all()
+        settings = SystemSettingsService.get_all(reverse=False, order_by="name")
         result = []
         for setting in settings:
-            result.append(
-                {
-                    "name": setting.name,
-                    "source": setting.source,
-                    "data_type": setting.data_type,
-                    "value": setting.value,
-                }
-            )
+            result.append(SettingsMgr._format_setting(setting))
         return result
 
     @staticmethod
     def get_by_name(name: str):
         settings = SystemSettingsService.get_by_name(name)
         if len(settings) == 0:
-            raise AdminException(f"Can't get setting: {name}")
+            settings = SystemSettingsService.get_by_name_prefix(name)
+            if len(settings) == 0:
+                raise AdminException(f"Can't get setting: {name}")
         result = []
         for setting in settings:
-            result.append(
-                {
-                    "name": setting.name,
-                    "source": setting.source,
-                    "data_type": setting.data_type,
-                    "value": setting.value,
-                }
-            )
+            result.append(SettingsMgr._format_setting(setting))
         return result
 
     @staticmethod
@@ -367,6 +396,7 @@ class SettingsMgr:
         settings = SystemSettingsService.get_by_name(name)
         if len(settings) == 1:
             setting = settings[0]
+            SettingsMgr._validate_value(name, setting.data_type, value)
             setting.value = value
             setting_dict = setting.to_dict()
             SystemSettingsService.update_by_name(name, setting_dict)
@@ -376,12 +406,8 @@ class SettingsMgr:
             # Create new setting if it doesn't exist
 
             # Determine data_type based on name and value
-            if name.startswith("sandbox."):
-                data_type = "json"
-            elif name.endswith(".enabled"):
-                data_type = "boolean"
-            else:
-                data_type = "string"
+            data_type = SettingsMgr._infer_data_type(name)
+            SettingsMgr._validate_value(name, data_type, value)
 
             new_setting = {
                 "name": name,
@@ -431,11 +457,21 @@ class SandboxMgr:
 
     # Provider registry with metadata
     PROVIDER_REGISTRY = {
+        "local": {
+            "name": "Local",
+            "description": "Execute code directly on the current host process.",
+            "tags": ["local", "host", "minimal"],
+        },
         "self_managed": {
             "name": "Self-Managed",
             "description": "On-premise deployment using Daytona/Docker",
             "tags": ["self-hosted", "low-latency", "secure"],
         },
+        "ssh": {
+            "name": "SSH",
+            "description": "Execute code on a remote machine over SSH.",
+            "tags": ["remote", "ssh", "custom-runtime"],
+        },
         "aliyun_codeinterpreter": {
             "name": "Aliyun Code Interpreter",
             "description": "Aliyun Function Compute Code Interpreter - Code execution in serverless microVMs",
@@ -463,13 +499,17 @@ class SandboxMgr:
     def get_provider_config_schema(provider_id: str):
         """Get configuration schema for a specific provider."""
         from agent.sandbox.providers import (
+            LocalProvider,
             SelfManagedProvider,
+            SSHProvider,
             AliyunCodeInterpreterProvider,
             E2BProvider,
         )
 
         schemas = {
+            "local": LocalProvider.get_config_schema(),
             "self_managed": SelfManagedProvider.get_config_schema(),
+            "ssh": SSHProvider.get_config_schema(),
             "aliyun_codeinterpreter": AliyunCodeInterpreterProvider.get_config_schema(),
             "e2b": E2BProvider.get_config_schema(),
         }
@@ -486,7 +526,6 @@ class SandboxMgr:
             # Get active provider type
             provider_type_settings = SystemSettingsService.get_by_name("sandbox.provider_type")
             if not provider_type_settings:
-                # Return default config if not set
                 provider_type = "self_managed"
             else:
                 provider_type = provider_type_settings[0].value
@@ -501,6 +540,15 @@ class SandboxMgr:
                 except json.JSONDecodeError:
                     provider_config = {}
 
+            if not provider_config:
+                schema = SandboxMgr.get_provider_config_schema(provider_type)
+                provider_config = {}
+                for field_name, field_schema in schema.items():
+                    if field_schema.get("readonly"):
+                        continue
+                    if field_schema.get("default") is not None:
+                        provider_config[field_name] = field_schema["default"]
+
             return {
                 "provider_type": provider_type,
                 "config": provider_config,
@@ -524,7 +572,9 @@ class SandboxMgr:
             Dictionary with updated provider_type and config
         """
         from agent.sandbox.providers import (
+            LocalProvider,
             SelfManagedProvider,
+            SSHProvider,
             AliyunCodeInterpreterProvider,
             E2BProvider,
         )
@@ -551,7 +601,7 @@ class SandboxMgr:
                     elif field_type == "string":
                         if not isinstance(config[field_name], str):
                             raise AdminException(f"Field '{field_name}' must be a string")
-                    elif field_type == "bool":
+                    elif field_type == "boolean":
                         if not isinstance(config[field_name], bool):
                             raise AdminException(f"Field '{field_name}' must be a boolean")
 
@@ -566,7 +616,9 @@ class SandboxMgr:
 
             # Provider-specific custom validation
             provider_classes = {
+                "local": LocalProvider,
                 "self_managed": SelfManagedProvider,
+                "ssh": SSHProvider,
                 "aliyun_codeinterpreter": AliyunCodeInterpreterProvider,
                 "e2b": E2BProvider,
             }
@@ -582,6 +634,8 @@ class SandboxMgr:
             # Always update the provider config
             config_json = json.dumps(config)
             SettingsMgr.update_by_name(f"sandbox.{provider_type}", config_json)
+            from agent.sandbox.client import reload_provider
+            reload_provider()
 
             return {"provider_type": provider_type, "config": config}
         except AdminException:
@@ -608,14 +662,18 @@ class SandboxMgr:
         """
         try:
             from agent.sandbox.providers import (
+                LocalProvider,
                 SelfManagedProvider,
+                SSHProvider,
                 AliyunCodeInterpreterProvider,
                 E2BProvider,
             )
 
             # Instantiate provider based on type
             provider_classes = {
+                "local": LocalProvider,
                 "self_managed": SelfManagedProvider,
+                "ssh": SSHProvider,
                 "aliyun_codeinterpreter": AliyunCodeInterpreterProvider,
                 "e2b": E2BProvider,
             }
@@ -631,59 +689,40 @@ class SandboxMgr:
 
             # Create a temporary sandbox instance for testing
             instance = provider.create_instance(template="python")
+            if not instance:
+                raise AdminException("Failed to create sandbox instance.")
 
-            if not instance or instance.status != "READY":
-                raise AdminException(f"Failed to create sandbox instance. Status: {instance.status if instance else 'None'}")
-
-            # Simple test code that exercises basic Python functionality
-            test_code = """
-# Test basic Python functionality
-import sys
+            try:
+                # Keep the probe close to the original coverage, but avoid
+                # `sys` because the sandbox security analyzer blocks it.
+                test_code = """
 import json
 import math
 
-print("Python version:", sys.version)
-print("Platform:", sys.platform)
 
-# Test basic calculations
-result = 2 + 2
-print(f"2 + 2 = {result}")
-
-# Test JSON operations
-data = {"test": "data", "value": 123}
-print(f"JSON dump: {json.dumps(data)}")
-
-# Test math operations
-print(f"Math.sqrt(16) = {math.sqrt(16)}")
-
-# Test error handling
-try:
-    x = 1 / 1
-    print("Division test: OK")
-except Exception as e:
-    print(f"Error: {e}")
-
-# Return success indicator
-print("TEST_PASSED")
+def main() -> dict:
+    left = 2
+    right = 2
+    print(f"2 + 2 = {left + right}")
+    print(f"JSON dump: {json.dumps({'test': 'data', 'value': 123})}")
+    print(f"Math.sqrt(16) = {math.sqrt(16)}")
+    print("TEST_PASSED")
+    return {"ok": True, "provider_test": "TEST_PASSED"}
 """
 
-            # Execute test code with timeout
-            execution_result = provider.execute_code(
-                instance_id=instance.instance_id,
-                code=test_code,
-                language="python",
-                timeout=10  # 10 seconds timeout
-            )
-
-            # Clean up the test instance (if provider supports it)
-            try:
-                if hasattr(provider, 'terminate_instance'):
-                    provider.terminate_instance(instance.instance_id)
+                # Execute test code with timeout
+                execution_result = provider.execute_code(
+                    instance_id=instance.instance_id,
+                    code=test_code,
+                    language="python",
+                    timeout=10,
+                )
+            finally:
+                try:
+                    provider.destroy_instance(instance.instance_id)
                     logging.info(f"Cleaned up test instance {instance.instance_id}")
-                else:
-                    logging.warning(f"Provider {provider_type} does not support terminate_instance, test instance may leak")
-            except Exception as cleanup_error:
-                logging.warning(f"Failed to cleanup test instance {instance.instance_id}: {cleanup_error}")
+                except Exception as cleanup_error:
+                    logging.warning(f"Failed to cleanup test instance {instance.instance_id}: {cleanup_error}")
 
             # Build detailed result message
             success = execution_result.exit_code == 0 and "TEST_PASSED" in execution_result.stdout

agent/canvas.py

@@ -17,7 +17,6 @@ import asyncio
 import base64
 import datetime
 import inspect
-import binascii
 import json
 import logging
 import re
@@ -39,6 +38,7 @@ from common.misc_utils import get_uuid, hash_str2int
 from common.exceptions import TaskCanceledException
 from rag.prompts.generator import chunks_format
 from rag.utils.redis_conn import REDIS_CONN
+from rag.utils.tts_cache import synthesize_with_cache
 
 class Graph:
     """
@@ -263,7 +263,7 @@ class Graph:
         keys = path.split('.')
         if not path:
             return value
-        for key in keys:
+        for key in keys[:-1]:
             if key not in cur or not isinstance(cur[key], dict):
                 cur[key] = {}
             cur = cur[key]
@@ -329,6 +329,11 @@ class Canvas(Graph):
         self.dsl["memory"] = self.memory
         return super().__str__()
 
+    def clear_history(self):
+        self.history = []
+        if isinstance(self.globals.get("sys.history"), list):
+            self.globals["sys.history"] = []
+
     def reset(self, mem=False):
         super().reset()
         if not mem:
@@ -354,23 +359,21 @@ class Canvas(Graph):
                 key = k[4:]
                 if key in self.variables:
                     variable = self.variables[key]
-                    if variable["type"] == "string":
-                        self.globals[k] = ""
-                        variable["value"] = ""
-                    elif variable["type"] == "number":
-                        self.globals[k] = 0
-                        variable["value"] = 0
-                    elif variable["type"] == "boolean":
-                        self.globals[k] = False
-                        variable["value"] = False
-                    elif variable["type"] == "object":
-                        self.globals[k] = {}
-                        variable["value"] = {}
-                    elif variable["type"].startswith("array"):
-                        self.globals[k] = []
-                        variable["value"] = []
+                    value = variable.get("value")
+                    if value is not None:
+                        self.globals[k] = value
                     else:
-                        self.globals[k] = ""
+                        var_type = variable.get("type", "")
+                        if var_type == "number":
+                            self.globals[k] = 0
+                        elif var_type == "boolean":
+                            self.globals[k] = False
+                        elif var_type == "object":
+                            self.globals[k] = {}
+                        elif var_type.startswith("array"):
+                            self.globals[k] = []
+                        else:  # "string" or unknown
+                            self.globals[k] = ""
                 else:
                     self.globals[k] = ""
 
@@ -381,8 +384,10 @@ class Canvas(Graph):
         self.message_id = get_uuid()
         created_at = int(time.time())
         self.add_user_input(kwargs.get("query"))
+        path_set = set(self.path)
         for k, cpn in self.components.items():
-            self.components[k]["obj"].reset(True)
+            if k in path_set:
+                self.components[k]["obj"].reset(True)
 
         if kwargs.get("webhook_payload"):
             for k, cpn in self.components.items():
@@ -402,7 +407,7 @@ class Canvas(Graph):
                 break
 
         for k in kwargs.keys():
-            if k in ["query", "user_id", "files"] and kwargs[k]:
+            if k in ["query", "user_id", "files", "chat_template_kwargs"] and kwargs[k]:
                 if k == "files":
                     self.globals[f"sys.{k}"] = await self.get_files_async(kwargs[k], layout_recognize)
                 else:
@@ -714,14 +719,7 @@ class Canvas(Graph):
         text = clean_tts_text(text)
         if not text:
             return None
-        bin = b""
-        try:
-            for chunk in tts_mdl.tts(text):
-                bin += chunk
-        except Exception as e:
-            logging.error(f"TTS failed: {e}, text={text!r}")
-            return None
-        return binascii.hexlify(bin).decode("utf-8")
+        return synthesize_with_cache(tts_mdl, text)
 
     def get_history(self, window_size):
         convs = []

agent/component/agent_with_tools.py

@@ -27,12 +27,11 @@ import json_repair
 
 from agent.component.llm import LLM, LLMParam
 from agent.tools.base import LLMToolPluginCallSession, ToolBase, ToolMeta, ToolParamBase
-from api.db.joint_services.tenant_model_service import get_model_config_by_type_and_name
+from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
 from api.db.services.llm_service import LLMBundle
 from api.db.services.mcp_server_service import MCPServerService
-from api.db.services.tenant_llm_service import TenantLLMService
 from common.connection_utils import timeout
-from common.mcp_tool_call_conn import MCPToolCallSession, mcp_tool_metadata_to_openai_tool
+from common.mcp_tool_call_conn import MCPToolBinding, MCPToolCallSession, mcp_tool_metadata_to_openai_tool
 from rag.prompts.generator import citation_plus, citation_prompt, full_question, kb_prompt, message_fit_in, structured_output_prompt
 
 
@@ -81,7 +80,9 @@ class Agent(LLM, ToolBase):
             original_name = cpn.get_meta()["function"]["name"]
             indexed_name = f"{original_name}_{idx}"
             self.tools[indexed_name] = cpn
-        chat_model_config = get_model_config_by_type_and_name(self._canvas.get_tenant_id(), TenantLLMService.llm_id2llm_type(self._param.llm_id), self._param.llm_id)
+        model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
+        model_type = "chat" if "chat" in model_types else model_types[0]
+        chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
         self.chat_mdl = LLMBundle(
             self._canvas.get_tenant_id(),
             chat_model_config,
@@ -97,13 +98,16 @@ class Agent(LLM, ToolBase):
             indexed_meta["function"]["name"] = indexed_name
             self.tool_meta.append(indexed_meta)
 
+        tool_idx = len(self.tools)
         for mcp in self._param.mcp:
             _, mcp_server = MCPServerService.get_by_id(mcp["mcp_id"])
             custom_header = self._param.custom_header
             tool_call_session = MCPToolCallSession(mcp_server, mcp_server.variables, custom_header)
             for tnm, meta in mcp["tools"].items():
-                self.tool_meta.append(mcp_tool_metadata_to_openai_tool(meta))
-                self.tools[tnm] = tool_call_session
+                indexed_name = f"{tnm}_{tool_idx}"
+                tool_idx += 1
+                self.tool_meta.append(mcp_tool_metadata_to_openai_tool(meta, function_name=indexed_name))
+                self.tools[indexed_name] = MCPToolBinding(tool_call_session, tnm)
         self.callback = partial(self._canvas.tool_use_callback, id)
         self.toolcall_session = LLMToolPluginCallSession(self.tools, self.callback)
         if self.tool_meta:
@@ -145,7 +149,8 @@ class Agent(LLM, ToolBase):
         self._param.function_name = self._id.split("-->")[-1]
         m = super().get_meta()
         if hasattr(self._param, "user_prompt") and self._param.user_prompt:
-            m["function"]["parameters"]["properties"]["user_prompt"] = self._param.user_prompt
+            # Keep the JSON schema valid; user_prompt is a string field, not a schema node.
+            m["function"]["parameters"]["properties"]["user_prompt"]["default"] = self._param.user_prompt
         return m
 
     def get_input_form(self) -> dict[str, dict]:
@@ -276,10 +281,13 @@ class Agent(LLM, ToolBase):
                 return
             if delta.find("**ERROR**") >= 0:
                 if self.get_exception_default_value():
-                    self.set_output("content", self.get_exception_default_value())
-                    yield self.get_exception_default_value()
+                    fallback = self.get_exception_default_value()
+                    self.set_output("content", fallback)
+                    yield fallback
                 else:
                     self.set_output("_ERROR", delta)
+                    self.set_output("content", delta)
+                    yield delta
                 return
             if not need2cite or cited:
                 yield delta

agent/component/base.py

@@ -366,6 +366,7 @@ class ComponentBase(ABC):
     component_name: str
     thread_limiter = asyncio.Semaphore(int(os.environ.get("MAX_CONCURRENT_CHATS", 10)))
     variable_ref_patt = r"\{* *\{([a-zA-Z:0-9]+@[A-Za-z0-9_.-]+|sys\.[A-Za-z0-9_.]+|env\.[A-Za-z0-9_.]+)\} *\}*"
+    iteration_alias_patt = r"\{* *\{(item|index|result)\} *\}*"
 
     def __str__(self):
         """
@@ -486,6 +487,10 @@ class ComponentBase(ABC):
                 continue
             if isinstance(v, str) and self._canvas.is_reff(v):
                 self.set_input_value(var, self._canvas.get_variable_value(v))
+            elif isinstance(v, str) and re.search(self.variable_ref_patt, v):
+                elements = self.get_input_elements_from_text(v)
+                kv = {k: e.get('value', '') for k, e in elements.items()}
+                self.set_input_value(var, self.string_format(v, kv))
             else:
                 self.set_input_value(var, v)
             res[var] = self.get_input_value(var)
@@ -497,6 +502,23 @@ class ComponentBase(ABC):
 
         return {var: self.get_input_value(var) for var, o in self.get_input_elements().items()}
 
+    def _resolve_iteration_alias_ref(self, exp: str) -> str | None:
+        if exp not in {"item", "index", "result"}:
+            return None
+
+        parent = self.get_parent()
+        if not parent or parent.component_name.lower() != "iteration":
+            return None
+
+        for cid, cpn in self._canvas.components.items():
+            if cpn.get("parent_id") != parent._id:
+                continue
+            if cpn["obj"].component_name.lower() != "iterationitem":
+                continue
+            return f"{cid}@{exp}"
+
+        return None
+
     def get_input_elements_from_text(self, txt: str) -> dict[str, dict[str, str]]:
         res = {}
         for r in re.finditer(self.variable_ref_patt, txt, flags=re.IGNORECASE | re.DOTALL):
@@ -508,6 +530,20 @@ class ComponentBase(ABC):
                 "_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references") if cpn_id else None,
                 "_cpn_id": cpn_id
             }
+        for r in re.finditer(self.iteration_alias_patt, txt, flags=re.IGNORECASE | re.DOTALL):
+            exp = r.group(1)
+            if exp in res:
+                continue
+            ref = self._resolve_iteration_alias_ref(exp)
+            if not ref:
+                continue
+            cpn_id, var_nm = ref.split("@", 1)
+            res[exp] = {
+                "name": (self._canvas.get_component_name(cpn_id) + f"@{var_nm}"),
+                "value": self._canvas.get_variable_value(ref),
+                "_retrieval": self._canvas.get_variable_value(f"{cpn_id}@_references"),
+                "_cpn_id": cpn_id
+            }
         return res
 
     def get_input_elements(self) -> dict[str, Any]:

agent/component/browser.py

@@ -0,0 +1,730 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import asyncio
+import hashlib
+import inspect
+import json
+import logging
+import os
+import re
+import shutil
+import tempfile
+from abc import ABC
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import unquote, urlparse
+from urllib.request import Request, urlopen
+
+from agent.component.base import ComponentBase
+from agent.component.llm import LLMParam
+from api.db import FileType
+from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
+from api.db.services import duplicate_name
+from api.db.services.file_service import FileService
+from api.utils.file_utils import filename_type
+from common import settings
+from common.connection_utils import timeout
+from common.misc_utils import get_uuid
+from rag.llm import FACTORY_DEFAULT_BASE_URL
+
+
+class BrowserParam(LLMParam):
+    """
+    Parameters for Browser node.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.prompts = "{sys.query}"
+        self.max_steps = 30
+        self.headless = True
+        self.enable_default_extensions = False
+        self.chromium_sandbox = False
+        # Reuse browser profile across runs of the same agent node by default.
+        self.persist_session = True
+        self.upload_sources = []
+        self.outputs = {
+            "content": {"type": "string", "value": ""},
+            "downloaded_files": {"type": "Array<Object>", "value": []},
+        }
+
+    def check(self):
+        self.check_empty(self.llm_id, "[Browser] LLM")
+        self.check_positive_integer(self.max_steps, "[Browser] Max steps")
+        self.check_boolean(self.headless, "[Browser] Headless")
+        self.check_boolean(self.enable_default_extensions, "[Browser] Enable default extensions")
+        self.check_boolean(self.chromium_sandbox, "[Browser] Chromium sandbox")
+        self.check_boolean(self.persist_session, "[Browser] Persist session")
+        self.check_empty(self.prompts, "[Browser] Prompts")
+        return True
+
+    def get_input_form(self) -> dict[str, dict]:
+        return {
+            "prompts": {"type": "text", "name": "Prompts"},
+            "upload_sources": {"type": "line", "name": "Upload sources"},
+        }
+
+
+class Browser(ComponentBase, ABC):
+    component_name = "Browser"
+
+    def _prepare_input_values(self):
+        for key, meta in self.get_input_elements().items():
+            val = meta.get("value")
+            if val is None:
+                val = ""
+            elif not isinstance(val, str):
+                val = json.dumps(val, ensure_ascii=False)
+            self.set_input_value(key, val)
+
+    def get_input_elements(self) -> dict[str, dict]:
+        text_parts = [
+            str(self._param.prompts or ""),
+            json.dumps(self._param.upload_sources, ensure_ascii=False),
+        ]
+        return self.get_input_elements_from_text("\n".join(text_parts))
+
+    def _resolve_param_value(self, value: Any) -> Any:
+        if isinstance(value, str):
+            direct_ref = value.strip()
+            if direct_ref.startswith("{") and direct_ref.endswith("}") and self._canvas.is_reff(direct_ref):
+                return self._canvas.get_variable_value(direct_ref)
+            return value
+        return value
+
+    def _extract_ids(self, value: Any) -> list[str]:
+        ids: list[str] = []
+        value = self._resolve_param_value(value)
+
+        def collect(item: Any):
+            if item is None:
+                return
+            if isinstance(item, str):
+                token = item.strip()
+                if not token:
+                    return
+                if token.startswith("{") and token.endswith("}") and self._canvas.is_reff(token):
+                    collect(self._canvas.get_variable_value(token))
+                    return
+                if token.startswith("[") and token.endswith("]"):
+                    try:
+                        parsed = json.loads(token)
+                        collect(parsed)
+                        return
+                    except Exception:
+                        pass
+                if self._is_http_url(token):
+                    ids.append(token)
+                    return
+                if "," in token:
+                    for part in token.split(","):
+                        collect(part)
+                    return
+                ids.append(token)
+                return
+            if isinstance(item, dict):
+                for k in ("file_id", "id", "url", "value"):
+                    if k in item:
+                        collect(item[k])
+                        return
+                for v in item.values():
+                    collect(v)
+                return
+            if isinstance(item, (list, tuple, set)):
+                for v in item:
+                    collect(v)
+                return
+            token = str(item).strip()
+            if token:
+                ids.append(token)
+
+        collect(value)
+        deduped: list[str] = []
+        visited = set()
+        for item in ids:
+            if item in visited:
+                continue
+            visited.add(item)
+            deduped.append(item)
+        return deduped
+
+    @staticmethod
+    def _is_http_url(value: str) -> bool:
+        token = str(value or "").strip()
+        if not token:
+            return False
+        parsed = urlparse(token)
+        return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+    @staticmethod
+    def _extract_url_filename(url: str, headers: Any) -> str:
+        content_disposition = str(getattr(headers, "get", lambda *_args, **_kwargs: "")("Content-Disposition", "") or "")
+        if content_disposition:
+            # Prefer RFC 5987 encoded filename*=UTF-8''... when present.
+            m = re.search(r"filename\*\s*=\s*(?:UTF-8''|utf-8'')?([^;]+)", content_disposition)
+            if m:
+                name = unquote(m.group(1).strip().strip('"'))
+                if name:
+                    return os.path.basename(name)
+            m = re.search(r'filename\s*=\s*"([^"]+)"', content_disposition)
+            if m:
+                name = m.group(1).strip()
+                if name:
+                    return os.path.basename(name)
+            m = re.search(r"filename\s*=\s*([^;]+)", content_disposition)
+            if m:
+                name = m.group(1).strip().strip('"')
+                if name:
+                    return os.path.basename(name)
+
+        parsed = urlparse(url)
+        raw_name = os.path.basename(parsed.path or "")
+        name = unquote(raw_name).strip()
+        if name:
+            return name
+        return f"url_file_{get_uuid()[:8]}.bin"
+
+    @staticmethod
+    def _resolve_upload_url_max_bytes() -> int:
+        raw = str(os.getenv("RAGFLOW_BROWSER_UPLOAD_URL_MAX_BYTES", "") or "").strip()
+        default_max_bytes = 100 * 1024 * 1024
+        if not raw:
+            return default_max_bytes
+        try:
+            parsed = int(raw)
+            return parsed if parsed > 0 else default_max_bytes
+        except (TypeError, ValueError):
+            return default_max_bytes
+
+    @staticmethod
+    def _restore_env_var(key: str, value: str | None):
+        if value is None:
+            os.environ.pop(key, None)
+            return
+        os.environ[key] = value
+
+    def _prepare_upload_url_file(self, url: str, upload_dir: str) -> dict[str, Any] | None:
+        max_bytes = self._resolve_upload_url_max_bytes()
+        local_path = ""
+        local_name = ""
+        total_size = 0
+        try:
+            req = Request(url, headers={"User-Agent": "RAGFlow-Browser-Node/1.0"})
+            with urlopen(req, timeout=30) as response:
+                local_name = self._extract_url_filename(url, response.headers)
+
+                local_path = os.path.join(upload_dir, local_name)
+                index = 1
+                while os.path.exists(local_path):
+                    stem, ext = os.path.splitext(local_name)
+                    local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")
+                    index += 1
+
+                with open(local_path, "wb") as f:
+                    while True:
+                        chunk = response.read(1024 * 1024)
+                        if not chunk:
+                            break
+                        total_size += len(chunk)
+                        if total_size > max_bytes:
+                            raise ValueError(f"upload url file exceeds max size limit: {max_bytes}")
+                        f.write(chunk)
+        except (HTTPError, URLError, OSError, TimeoutError, ValueError) as e:
+            if local_path and os.path.exists(local_path):
+                try:
+                    os.remove(local_path)
+                except OSError:
+                    pass
+            logging.warning("Browser failed to fetch upload url. url=%s, error=%s", url, e)
+            return None
+
+        if total_size <= 0:
+            if local_path and os.path.exists(local_path):
+                try:
+                    os.remove(local_path)
+                except OSError:
+                    pass
+            logging.warning("Browser upload url returned empty content: %s", url)
+            return None
+
+        return {
+            "file_id": "",
+            "name": local_name,
+            "size": total_size,
+            "local_path": local_path,
+            "source_url": url,
+        }
+
+    def _resolve_text(self, raw_text: Any) -> str:
+        text = str(self._resolve_param_value(raw_text) or "")
+        vars_map = self.get_input_elements_from_text(text)
+        kv = {}
+        for key, meta in vars_map.items():
+            val = meta.get("value", "")
+            if isinstance(val, str):
+                kv[key] = val
+            else:
+                kv[key] = json.dumps(val, ensure_ascii=False)
+        return self.string_format(text, kv)
+
+    @staticmethod
+    def _as_model_config_dict(cfg_obj: Any) -> dict[str, Any]:
+        if cfg_obj is None:
+            return {}
+        if isinstance(cfg_obj, dict):
+            return cfg_obj
+        if hasattr(cfg_obj, "to_dict") and callable(cfg_obj.to_dict):
+            try:
+                result = cfg_obj.to_dict()
+                return result if isinstance(result, dict) else {}
+            except (AttributeError, TypeError, ValueError):
+                return {}
+        result = {}
+        for key in ("model", "model_name", "llm_name", "llm_factory", "api_key", "base_url", "api_base", "temperature"):
+            val = getattr(cfg_obj, key, None)
+            if val not in (None, ""):
+                result[key] = val
+        return result
+
+    @staticmethod
+    def _error_chain(exc: Exception) -> str:
+        parts = []
+        cur = exc
+        depth = 0
+        while cur is not None and depth < 6:
+            parts.append(f"{type(cur).__name__}: {cur}")
+            cur = cur.__cause__ or cur.__context__
+            depth += 1
+        return " <- ".join(parts)
+
+    @staticmethod
+    def _resolve_browser_executable() -> str:
+        explicit_candidates = [
+            os.getenv("BROWSER_USE_EXECUTABLE_PATH", "").strip(),
+            os.getenv("BROWSER_USE_BROWSER_BINARY_PATH", "").strip(),
+            os.getenv("BROWSER_USE_CHROME_BINARY_PATH", "").strip(),
+        ]
+        for explicit in explicit_candidates:
+            if explicit and os.path.isfile(explicit) and os.access(explicit, os.X_OK):
+                return explicit
+        candidates = [
+            "/opt/chrome/chrome",
+            "/usr/local/bin/chrome",
+            "/usr/local/bin/google-chrome",
+            "/usr/bin/google-chrome",
+            "/usr/bin/google-chrome-stable",
+            "/usr/bin/chromium",
+            "/usr/bin/chromium-browser",
+        ]
+        for path in candidates:
+            if os.path.isfile(path) and os.access(path, os.X_OK):
+                return path
+        for cmd in ("chrome", "google-chrome", "google-chrome-stable", "chromium", "chromium-browser"):
+            path = shutil.which(cmd)
+            if path and os.path.isfile(path) and os.access(path, os.X_OK):
+                return path
+        return ""
+
+    @staticmethod
+    def _normalize_model_name(model: Any) -> str:
+        name = str(model or "").strip()
+        if not name:
+            return ""
+        if name.startswith("bu-") or name.startswith("browser-use/"):
+            return name
+        if "@" in name:
+            # RAGFlow model aliases may include provider suffix, e.g. qwen3.5-flash@Tongyi-Qianwen.
+            # browser-use OpenAI-compatible adapters need the pure model name.
+            name = name.split("@", 1)[0].strip()
+        return name
+
+    @staticmethod
+    def _safe_path_segment(value: Any) -> str:
+        token = str(value or "").strip()
+        if not token:
+            return "unknown"
+        token = re.sub(r"[^A-Za-z0-9._-]+", "_", token)
+        return token.strip("._-") or "unknown"
+
+    def _resolve_persistent_profile_dir(self) -> str:
+        root = os.path.join(tempfile.gettempdir(), "ragflow_browser_use_profiles")
+        tenant = self._safe_path_segment(self._canvas.get_tenant_id())
+        raw_canvas_id = getattr(self._canvas, "_id", "")
+        if not raw_canvas_id:
+            graph_text = json.dumps(
+                self._canvas.dsl.get("graph", {}),
+                sort_keys=True,
+                ensure_ascii=False,
+            )
+            raw_canvas_id = (
+                f"dsl_{hashlib.sha1(graph_text.encode('utf-8')).hexdigest()[:12]}"
+            )
+        canvas_id = self._safe_path_segment(raw_canvas_id)
+        node_id = self._safe_path_segment(self._id)
+        return os.path.join(root, tenant, canvas_id, node_id)
+
+    def _should_persist_session(self) -> bool:
+        return bool(self._param.persist_session)
+
+    def _infer_provider_name(self, cfg: dict[str, Any]) -> str:
+        provider = str(cfg.get("llm_factory") or "").strip()
+        if provider:
+            return provider
+        llm_id = str(self._param.llm_id or "")
+        if "@" in llm_id:
+            return llm_id.split("@", 1)[1].strip()
+        return ""
+
+    def _resolve_openai_compatible_base_url(self, cfg: dict[str, Any]) -> str:
+        explicit = str(cfg.get("base_url") or cfg.get("api_base") or "").strip()
+        if explicit:
+            return explicit
+
+        provider = self._infer_provider_name(cfg)
+        fallback = str(FACTORY_DEFAULT_BASE_URL.get(provider, "")).strip()
+        return fallback if fallback else ""
+
+    def _build_browser_llm(self):
+        from browser_use.llm import ChatBrowserUse, ChatOpenAI
+
+        chat_model_config = get_model_config_from_provider_instance(
+            self._canvas.get_tenant_id(),
+            get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id),
+            self._param.llm_id,
+        )
+        cfg = self._as_model_config_dict(chat_model_config)
+        model_name = self._normalize_model_name(cfg.get("model_name") or cfg.get("model") or self._param.llm_id)
+        if not model_name:
+            raise ValueError(f"Invalid model config for Browser llm_id={self._param.llm_id}")
+        base_url = self._resolve_openai_compatible_base_url(cfg)
+
+        # ChatBrowserUse only supports bu-* models. For tenant models, use OpenAI-compatible adapter.
+        if model_name.startswith("bu-") or model_name.startswith("browser-use/"):
+            llm_kwargs = {
+                "model": model_name,
+                "api_key": cfg.get("api_key"),
+                "base_url": base_url,
+                "temperature": self._param.temperature,
+                "max_retries": self._param.max_retries,
+            }
+            llm_kwargs = {k: v for k, v in llm_kwargs.items() if v not in (None, "")}
+            return ChatBrowserUse(**llm_kwargs)
+
+        # browser-use Agent defaults to json_schema response_format and may use tool_choice via
+        # ChatDeepSeek. Many providers (e.g. DeepSeek thinking models) reject both. Use ChatOpenAI
+        # with schema-in-prompt and without forced structured output on the first run.
+        llm_kwargs = {
+            "model": model_name,
+            "api_key": cfg.get("api_key"),
+            "base_url": base_url,
+            "temperature": self._param.temperature,
+            "max_retries": self._param.max_retries,
+            "add_schema_to_system_prompt": True,
+            "dont_force_structured_output": True,
+        }
+        llm_kwargs = {k: v for k, v in llm_kwargs.items() if v not in (None, "")}
+        return ChatOpenAI(**llm_kwargs)
+
+    async def _run_browser_use_async(
+        self,
+        task_text: str,
+        download_dir: str,
+        available_file_paths: list[str] | None = None,
+        profile_dir: str | None = None,
+    ):
+        from browser_use import Agent as BrowserUseAgent, Browser as BrowserUseBrowser
+
+        llm = self._build_browser_llm()
+        # NOTE:
+        # _invoke() uses asyncio.run(), which creates a fresh event loop per task run.
+        # Reusing a Browser object created by a previous loop can deadlock/timestamp out
+        # in browser-use watchdog handlers on subsequent runs.
+        # We keep persistent user_data_dir for session continuity, but we do not keep
+        # browser instances alive across runs.
+        available_file_paths = available_file_paths or []
+        agent_kwargs: dict[str, Any] = {
+            "task": task_text,
+            "llm": llm,
+            "available_file_paths": available_file_paths,
+        }
+        browser_obj = None
+        previous_disable_extensions = os.environ.get("BROWSER_USE_DISABLE_EXTENSIONS")
+        previous_browser_binary_path = os.environ.get("BROWSER_USE_BROWSER_BINARY_PATH")
+
+        try:
+            enable_default_extensions = bool(self._param.enable_default_extensions)
+            if not enable_default_extensions:
+                os.environ["BROWSER_USE_DISABLE_EXTENSIONS"] = "1"
+            else:
+                os.environ.pop("BROWSER_USE_DISABLE_EXTENSIONS", None)
+
+            executable_path = self._resolve_browser_executable()
+            browser_kwargs = {
+                "headless": self._param.headless,
+                "downloads_path": download_dir,
+                # Docker often runs as root without user namespaces; disable sandbox by default.
+                "chromium_sandbox": bool(self._param.chromium_sandbox),
+                # Disable runtime extension download by default for intranet/offline environments.
+                # Enable only when explicitly required and extensions are pre-cached.
+                "enable_default_extensions": enable_default_extensions,
+            }
+            if executable_path:
+                browser_kwargs["executable_path"] = executable_path
+                # Keep browser-use watchdog fallback in sync with our resolved path.
+                os.environ["BROWSER_USE_BROWSER_BINARY_PATH"] = executable_path
+            else:
+                logging.warning(
+                    "Browser no local browser executable found. "
+                    "Set BROWSER_USE_EXECUTABLE_PATH or preinstall chromium in image to avoid runtime playwright install."
+                )
+            if profile_dir:
+                browser_kwargs["user_data_dir"] = profile_dir
+                # browser-use expects profile_directory to be a profile name
+                # such as "Default" / "Profile 1", not an absolute path.
+                browser_kwargs["profile_directory"] = "Default"
+
+            browser_obj = BrowserUseBrowser(**browser_kwargs)
+            agent_kwargs["browser"] = browser_obj
+        except (OSError, RuntimeError, TypeError, ValueError) as e:
+            logging.warning("Browser browser context customization skipped: %s", e)
+
+        agent = BrowserUseAgent(**agent_kwargs)
+
+        history = None
+        run_fn = getattr(agent, "run", None)
+        if run_fn is None:
+            raise RuntimeError("browser-use Agent does not provide run().")
+
+        run_kwargs = {"max_steps": self._param.max_steps}
+        try:
+            if inspect.iscoroutinefunction(run_fn):
+                history = await run_fn(**run_kwargs)
+            else:
+                history = await asyncio.to_thread(run_fn, **run_kwargs)
+        except Exception as e:
+            logging.error("Browser agent.run failed. error_chain=%s", self._error_chain(e))
+            logging.exception("Browser agent.run traceback")
+            raise
+        finally:
+            if browser_obj:
+                close_fn = getattr(browser_obj, "close", None)
+                if close_fn:
+                    try:
+                        if inspect.iscoroutinefunction(close_fn):
+                            await close_fn()
+                        else:
+                            await asyncio.to_thread(close_fn)
+                    except Exception as close_err:
+                        logging.warning("Browser failed to close browser object cleanly: %s", close_err)
+            self._restore_env_var("BROWSER_USE_DISABLE_EXTENSIONS", previous_disable_extensions)
+            self._restore_env_var("BROWSER_USE_BROWSER_BINARY_PATH", previous_browser_binary_path)
+
+        return history
+
+    def _prepare_upload_files(self, upload_dir: str) -> list[dict[str, Any]]:
+        upload_refs = self._extract_ids(self._param.upload_sources)
+        prepared = []
+        for file_ref in upload_refs:
+            if self._is_http_url(file_ref):
+                prepared_url_file = self._prepare_upload_url_file(file_ref, upload_dir)
+                if prepared_url_file:
+                    prepared.append(prepared_url_file)
+                continue
+
+            file_id = file_ref
+            exists, file = FileService.get_by_id(file_id)
+            if not exists:
+                logging.warning("Browser upload file_id not found: %s", file_id)
+                continue
+            try:
+                blob = settings.STORAGE_IMPL.get(file.parent_id, file.location)
+                if not blob:
+                    logging.warning("Browser upload blob not found: %s", file_id)
+                    continue
+                local_name = os.path.basename(file.location) if file.location else (file.name or f"{file_id}.bin")
+                local_path = os.path.join(upload_dir, local_name)
+                index = 1
+                while os.path.exists(local_path):
+                    stem, ext = os.path.splitext(local_name)
+                    local_path = os.path.join(upload_dir, f"{stem}_{index}{ext}")
+                    index += 1
+                with open(local_path, "wb") as f:
+                    f.write(blob)
+            except OSError as e:
+                logging.warning("Browser failed to prepare upload file. file_id=%s, error=%s", file_id, e)
+                continue
+            except Exception as e:
+                logging.warning("Browser failed to fetch upload blob. file_id=%s, error=%s", file_id, e)
+                continue
+            prepared.append(
+                {
+                    "file_id": file.id,
+                    "name": file.name,
+                    "size": file.size,
+                    "local_path": local_path,
+                }
+            )
+        return prepared
+
+    def _save_downloads(self, download_dir: str, parent_id: str) -> list[dict[str, Any]]:
+        downloaded_files: list[dict[str, Any]] = []
+        exists, folder = FileService.get_by_id(parent_id)
+        if not exists or folder.type != FileType.FOLDER.value:
+            raise ValueError(f"RAGFlow target folder does not exist or is not a folder: {parent_id}")
+        tenant_id = self._canvas.get_tenant_id()
+        storage_put = settings.STORAGE_IMPL.put
+        storage_rm = getattr(settings.STORAGE_IMPL, "rm", None)
+        insert_file = FileService.insert
+
+        for path in Path(download_dir).rglob("*"):
+            if not path.is_file():
+                continue
+            try:
+                if path.stat().st_size <= 0:
+                    continue
+                blob = path.read_bytes()
+            except OSError as e:
+                logging.warning("Browser failed to read downloaded file. path=%s, error=%s", path, e)
+                continue
+            if not blob:
+                continue
+            display_name = ""
+            blob_stored = False
+            try:
+                display_name = duplicate_name(FileService.query, name=path.name, parent_id=parent_id)
+                storage_put(parent_id, display_name, blob)
+                blob_stored = True
+                file_data = {
+                    "id": get_uuid(),
+                    "parent_id": parent_id,
+                    "tenant_id": tenant_id,
+                    "created_by": tenant_id,
+                    "type": filename_type(display_name),
+                    "name": display_name,
+                    "location": display_name,
+                    "size": len(blob),
+                }
+                inserted = insert_file(file_data)
+                downloaded_files.append(
+                    {
+                        "file_id": inserted.id,
+                        "name": inserted.name,
+                        "size": inserted.size,
+                        "parent_id": inserted.parent_id,
+                    }
+                )
+            except Exception as e:
+                if blob_stored and callable(storage_rm):
+                    try:
+                        storage_rm(parent_id, display_name)
+                    except Exception as rollback_err:
+                        logging.warning(
+                            "Browser rollback stored download failed. path=%s, parent_id=%s, display_name=%s, error=%s",
+                            path,
+                            parent_id,
+                            display_name,
+                            rollback_err,
+                        )
+                logging.error(
+                    "Browser failed to save download. path=%s, tenant_id=%s, parent_id=%s, display_name=%s, error=%s",
+                    path,
+                    tenant_id,
+                    parent_id,
+                    display_name,
+                    e,
+                )
+                continue
+        return downloaded_files
+
+    @staticmethod
+    def _extract_history_text(history: Any) -> str:
+        if history is None:
+            return ""
+
+        def pick_final_result(value: Any) -> str:
+            if value is None:
+                return ""
+            if isinstance(value, str):
+                return value.strip()
+            if isinstance(value, (int, float, bool)):
+                return str(value)
+            return ""
+
+        # Only trust browser-use's explicit final_result API/property.
+        final_result_fn = getattr(history, "final_result", None)
+        if callable(final_result_fn):
+            try:
+                final_result_value = final_result_fn()
+                return pick_final_result(final_result_value)
+            except Exception:
+                return ""
+        return pick_final_result(final_result_fn)
+
+    @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 20 * 60)))
+    def _invoke(self, **kwargs):
+        profile_dir = None
+        persist_session = self._should_persist_session()
+        try:
+            self._prepare_input_values()
+            user_prompt = self._resolve_text(kwargs.get("prompts", self._param.prompts))
+            with tempfile.TemporaryDirectory(prefix="browser_use_upload_") as upload_dir, tempfile.TemporaryDirectory(
+                prefix="browser_use_download_"
+            ) as download_dir:
+                uploaded_files = self._prepare_upload_files(upload_dir)
+
+                upload_lines = [
+                    f"- file_id={item['file_id']}, name={item['name']}, local_path={item['local_path']}"
+                    for item in uploaded_files
+                ]
+                task_text = user_prompt
+                if upload_lines:
+                    task_text += (
+                        "\n\nYou can upload files from these local paths when operating web pages:\n"
+                        + "\n".join(upload_lines)
+                    )
+
+                upload_local_paths = [item.get("local_path", "") for item in uploaded_files if item.get("local_path")]
+                if persist_session:
+                    profile_dir = self._resolve_persistent_profile_dir()
+                    os.makedirs(profile_dir, exist_ok=True)
+                else:
+                    try:
+                        profile_dir = tempfile.mkdtemp(prefix="browser_use_profile_")
+                    except OSError:
+                        profile_dir = None
+                history = asyncio.run(
+                    self._run_browser_use_async(
+                        task_text, download_dir, upload_local_paths, profile_dir
+                    )
+                )
+                target_dir_id = FileService.get_root_folder(self._canvas.get_tenant_id())["id"]
+                downloaded_files = self._save_downloads(download_dir, target_dir_id)
+
+                self.set_output("content", self._extract_history_text(history))
+                self.set_output("downloaded_files", downloaded_files)
+                return self.output()
+        except Exception as e:
+            logging.exception("Browser invoke failed")
+            self.set_output("_ERROR", str(e))
+            return self.output()
+        finally:
+            if profile_dir and not persist_session:
+                shutil.rmtree(profile_dir, ignore_errors=True)
+
+    def thoughts(self) -> str:
+        return "Planning and executing browser actions..."

agent/component/categorize.py

@@ -21,7 +21,7 @@ from abc import ABC
 
 from common.constants import LLMType
 from api.db.services.llm_service import LLMBundle
-from api.db.joint_services.tenant_model_service import get_model_config_by_type_and_name
+from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance
 from agent.component.llm import LLMParam, LLM
 from common.connection_utils import timeout
 from rag.llm.chat_model import ERROR_PREFIX
@@ -40,7 +40,8 @@ class CategorizeParam(LLMParam):
         self.update_prompt()
 
     def check(self):
-        self.check_positive_integer(self.message_history_window_size, "[Categorize] Message window size > 0")
+        if not isinstance(self.message_history_window_size, int) or self.message_history_window_size < 0:
+            raise ValueError("[Categorize] Message window size cannot be negative")
         self.check_empty(self.category_description, "[Categorize] Category examples")
         for k, v in self.category_description.items():
             if not k:
@@ -123,7 +124,7 @@ class Categorize(LLM, ABC):
         msg[-1]["content"] = query_value
         self.set_input_value(query_key, msg[-1]["content"])
         self._param.update_prompt()
-        chat_model_config = get_model_config_by_type_and_name(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
+        chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), LLMType.CHAT, self._param.llm_id)
         chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config)
 
         user_prompt = """

agent/component/data_operations.py

@@ -73,7 +73,7 @@ class DataOperations(ComponentBase,ABC):
                 continue
         if self._param.operations == "select_keys":
             self._select_keys()
-        elif self._param.operations == "recursive_eval":
+        elif self._param.operations == "literal_eval":
             self._literal_eval()
         elif self._param.operations == "combine":
             self._combine()

agent/component/docs_generator.py

@@ -1,3 +1,4 @@
+import base64
 import logging
 import json
 import os
@@ -48,8 +49,13 @@ class DocGeneratorParam(ComponentParamBase):
         self.watermark_text = ""
         self.add_page_numbers = True
         self.add_timestamp = True
+        self.include_download_info_in_content = False
         self.font_size = 12
         self.outputs = {
+            "doc_id": {"value": "", "type": "string"},
+            "filename": {"value": "", "type": "string"},
+            "mime_type": {"value": "", "type": "string"},
+            "size": {"value": 0, "type": "number"},
             "download": {"value": "", "type": "string"},
         }
 
@@ -113,6 +119,7 @@ class DocGenerator(Message, ABC):
                     raise Exception("Document file is empty")
 
                 file_size = len(file_bytes)
+                file_base64 = base64.b64encode(file_bytes).decode("utf-8")
                 doc_id = get_uuid()
                 settings.STORAGE_IMPL.put(self._canvas.get_tenant_id(), doc_id, file_bytes)
 
@@ -128,7 +135,13 @@ class DocGenerator(Message, ABC):
                     "filename": filename,
                     "mime_type": mime_type,
                     "size": file_size,
+                    "base64": file_base64,
+                    "include_download_info_in_content": self._param.include_download_info_in_content,
                 }
+                self.set_output("doc_id", doc_id)
+                self.set_output("filename", filename)
+                self.set_output("mime_type", mime_type)
+                self.set_output("size", file_size)
                 self.set_output("download", json.dumps(download_info))
                 return download_info
 

agent/component/invoke.py

@@ -179,10 +179,7 @@ class Invoke(ComponentBase, ABC):
         if not isinstance(headers, dict):
             raise ValueError("Invoke headers must be a JSON object.")
 
-        return {
-            key: self._resolve_header_text(value, kwargs) if isinstance(value, str) else value
-            for key, value in headers.items()
-        }
+        return {key: self._resolve_header_text(value, kwargs) if isinstance(value, str) else value for key, value in headers.items()}
 
     def _build_proxies(self) -> dict | None:
         if not re.sub(r"https?:?/?/?", "", self._param.proxy):
@@ -215,7 +212,7 @@ class Invoke(ComponentBase, ABC):
         # HtmlParser keeps the Invoke output text-focused when the endpoint returns HTML.
         sections = HtmlParser()(None, response.content)
         return "\n".join(sections)
-    
+
     @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3)))
     def _invoke(self, **kwargs):
         if self.check_if_canceled("Invoke processing"):

agent/component/iterationitem.py

@@ -54,7 +54,11 @@ class IterationItem(ComponentBase, ABC):
         if self.check_if_canceled("IterationItem processing"):
             return
 
-        self.set_output("item", arr[self._idx])
+        current_item = arr[self._idx]
+        self.set_output("item", current_item)
+        # Keep `result` as a compatibility alias because existing DSL examples
+        # and downstream references may still consume IterationItem via `@result`.
+        self.set_output("result", current_item)
         self.set_output("index", self._idx)
 
         self._idx += 1

agent/component/list_operations.py

@@ -10,8 +10,9 @@ class ListOperationsParam(ComponentParamBase):
     def __init__(self):
         super().__init__()
         self.query = ""
-        self.operations = "topN"
-        self.n=0
+        self.operations = "nth"
+        self.n = 0
+        self.strict = False
         self.sort_method = "asc"
         self.filter = {
             "operator": "=",
@@ -34,7 +35,11 @@ class ListOperationsParam(ComponentParamBase):
     
     def check(self):
         self.check_empty(self.query, "query")
-        self.check_valid_value(self.operations, "Support operations", ["topN","head","tail","filter","sort","drop_duplicates"])
+        self.check_valid_value(
+            self.operations,
+            "Support operations",
+            ["nth", "head", "tail", "filter", "sort", "drop_duplicates"],
+        )
 
     def get_input_form(self) -> dict[str, dict]:
         return {}
@@ -51,8 +56,8 @@ class ListOperations(ComponentBase,ABC):
         if not isinstance(self.inputs, list):
             raise TypeError("The input of List Operations should be an array.")
         self.set_input_value(inputs, self.inputs)
-        if self._param.operations == "topN":
-            self._topN()
+        if self._param.operations == "nth":
+            self._nth()
         elif self._param.operations == "head":
             self._head()
         elif self._param.operations == "tail":
@@ -70,35 +75,74 @@ class ListOperations(ComponentBase,ABC):
             return int(getattr(self._param, "n", 0))
         except Exception:
             return 0
-        
+
+    def _is_strict(self):
+        strict = getattr(self._param, "strict", False)
+        if isinstance(strict, str):
+            return strict.strip().lower() in {"1", "true", "yes", "on"}
+        return bool(strict)
+
     def _set_outputs(self, outputs):
         self._param.outputs["result"]["value"] = outputs
         self._param.outputs["first"]["value"] = outputs[0] if outputs else None
         self._param.outputs["last"]["value"]  = outputs[-1] if outputs else None
-        
-    def _topN(self):
+
+    def _raise_strict_range_error(self, operation, n):
+        raise ValueError(
+            f"{operation} requires n to be within the valid range in strict mode, got {n}."
+        )
+
+    def _nth(self):
         n = self._coerce_n()
-        if n < 1:
+        strict = self._is_strict()
+        if n == 0:
+            if strict:
+                self._raise_strict_range_error("nth", n)
             outputs = []
+        elif n > 0:
+            if n <= len(self.inputs):
+                outputs = [self.inputs[n - 1]]
+            elif strict:
+                self._raise_strict_range_error("nth", n)
+            else:
+                outputs = []
         else:
-            n = min(n, len(self.inputs))
-            outputs = self.inputs[:n]
+            if abs(n) <= len(self.inputs):
+                outputs = [self.inputs[n]]
+            elif strict:
+                self._raise_strict_range_error("nth", n)
+            else:
+                outputs = []
         self._set_outputs(outputs)
 
     def _head(self):
         n = self._coerce_n()
-        if 1 <= n <= len(self.inputs):
-            outputs = [self.inputs[n - 1]]
+        strict = self._is_strict()
+        if strict:
+            if 1 <= n <= len(self.inputs):
+                outputs = self.inputs[:n]
+            else:
+                self._raise_strict_range_error("head", n)
         else:
-            outputs = []
+            if n < 1:
+                outputs = []
+            else:
+                outputs = self.inputs[:n]
         self._set_outputs(outputs)
 
     def _tail(self):
         n = self._coerce_n()
-        if 1 <= n <= len(self.inputs):
-            outputs = [self.inputs[-n]]
+        strict = self._is_strict()
+        if strict:
+            if 1 <= n <= len(self.inputs):
+                outputs = self.inputs[-n:]
+            else:
+                self._raise_strict_range_error("tail", n)
         else:
-            outputs = []
+            if n < 1:
+                outputs = []
+            else:
+                outputs = self.inputs[-n:]
         self._set_outputs(outputs)
 
     def _filter(self):
@@ -107,7 +151,7 @@ class ListOperations(ComponentBase,ABC):
     def _norm(self,v):
         s = "" if v is None else str(v)
         return s
-    
+
     def _eval(self, v, operator, value):
         if operator == "=":
             return v == value
@@ -163,6 +207,6 @@ class ListOperations(ComponentBase,ABC):
         if isinstance(x, set):
             return tuple(sorted(self._hashable(v) for v in x))
         return x
-    
+
     def thoughts(self) -> str:
         return "ListOperation in progress"

agent/component/llm.py

@@ -23,9 +23,9 @@ from typing import Any, AsyncGenerator
 import json_repair
 from functools import partial
 from common.constants import LLMType
+from api.db.services.dialog_service import _stream_with_think_delta
 from api.db.services.llm_service import LLMBundle
-from api.db.services.tenant_llm_service import TenantLLMService
-from api.db.joint_services.tenant_model_service import get_model_config_by_type_and_name
+from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance, get_model_type_by_name
 from agent.component.base import ComponentBase, ComponentParamBase
 from common.connection_utils import timeout
 from rag.prompts.generator import tool_call_summary, message_fit_in, citation_prompt, structured_output_prompt
@@ -85,7 +85,9 @@ class LLM(ComponentBase):
 
     def __init__(self, canvas, component_id, param: ComponentParamBase):
         super().__init__(canvas, component_id, param)
-        chat_model_config = get_model_config_by_type_and_name(self._canvas.get_tenant_id(), TenantLLMService.llm_id2llm_type(self._param.llm_id), self._param.llm_id)
+        model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
+        model_type = "chat" if "chat" in model_types else model_types[0]
+        chat_model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
         self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), chat_model_config,
                                   max_retries=self._param.max_retries,
                                   retry_interval=self._param.delay_after_error)
@@ -247,9 +249,16 @@ class LLM(ComponentBase):
             self.set_input_value(k, args[k])
 
         self.imgs = self._uniq_images(self.imgs + extracted_imgs)
-        if self.imgs and TenantLLMService.llm_id2llm_type(self._param.llm_id) == LLMType.CHAT.value:
-            self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), LLMType.IMAGE2TEXT.value,
-                                      self._param.llm_id, max_retries=self._param.max_retries,
+        model_types = get_model_type_by_name(self._canvas.get_tenant_id(), self._param.llm_id)
+        if self.imgs and LLMType.IMAGE2TEXT.value in model_types:
+            model_type = LLMType.IMAGE2TEXT.value
+        elif LLMType.CHAT.value in model_types:
+            model_type = LLMType.CHAT.value
+        else:
+            model_type = model_types[0]
+        model_config = get_model_config_from_provider_instance(self._canvas.get_tenant_id(), model_type, self._param.llm_id)
+        if self.imgs:
+            self.chat_mdl = LLMBundle(self._canvas.get_tenant_id(), model_config, max_retries=self._param.max_retries,
                                       retry_interval=self._param.delay_after_error
                                       )
 
@@ -276,82 +285,23 @@ class LLM(ComponentBase):
         return await self.chat_mdl.async_chat(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)
 
     async def _generate_streamly(self, msg: list[dict], **kwargs) -> AsyncGenerator[str, None]:
-        async def delta_wrapper(txt_iter):
-            ans = ""
-            last_idx = 0
-            endswith_think = False
-
-            def delta(txt):
-                nonlocal ans, last_idx, endswith_think
-                delta_ans = txt[last_idx:]
-                ans = txt
-
-                if delta_ans.find("<think>") == 0:
-                    last_idx += len("<think>")
-                    return "<think>"
-                elif delta_ans.find("<think>") > 0:
-                    delta_ans = txt[last_idx:last_idx + delta_ans.find("<think>")]
-                    last_idx += delta_ans.find("<think>")
-                    return delta_ans
-                elif delta_ans.endswith("</think>"):
-                    endswith_think = True
-                elif endswith_think:
-                    endswith_think = False
-                    return "</think>"
-
-                last_idx = len(ans)
-                if ans.endswith("</think>"):
-                    last_idx -= len("</think>")
-                return re.sub(r"(<think>|</think>)", "", delta_ans)
-
-            async for t in txt_iter:
-                yield delta(t)
-
-        if not self.imgs:
-            async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **kwargs)):
-                yield t
-            return
-
-        async for t in delta_wrapper(self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), images=self.imgs, **kwargs)):
-            yield t
+        stream_kwargs = {"images": self.imgs} if self.imgs else {}
+        stream_kwargs.update(kwargs)
+        stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs)
+        async for _, value, _ in _stream_with_think_delta(stream, min_tokens=0):
+            yield value
 
     async def _stream_output_async(self, prompt, msg):
         _, msg = message_fit_in([{"role": "system", "content": prompt}, *msg], int(self.chat_mdl.max_length * 0.97))
         answer = ""
-        last_idx = 0
-        endswith_think = False
-
-        def delta(txt):
-            nonlocal answer, last_idx, endswith_think
-            delta_ans = txt[last_idx:]
-            answer = txt
-
-            if delta_ans.find("<think>") == 0:
-                last_idx += len("<think>")
-                return "<think>"
-            elif delta_ans.find("<think>") > 0:
-                delta_ans = txt[last_idx:last_idx + delta_ans.find("<think>")]
-                last_idx += delta_ans.find("<think>")
-                return delta_ans
-            elif delta_ans.endswith("</think>"):
-                endswith_think = True
-            elif endswith_think:
-                endswith_think = False
-                return "</think>"
-
-            last_idx = len(answer)
-            if answer.endswith("</think>"):
-                last_idx -= len("</think>")
-            return re.sub(r"(<think>|</think>)", "", delta_ans)
-
         stream_kwargs = {"images": self.imgs} if self.imgs else {}
-        async for ans in self.chat_mdl.async_chat_streamly(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs):
+        extra_chat_kwargs = self._get_chat_template_kwargs()
+        stream_kwargs.update(extra_chat_kwargs)
+        stream = self.chat_mdl.async_chat_streamly_delta(msg[0]["content"], msg[1:], self._param.gen_conf(), **stream_kwargs)
+        async for _, ans, _ in _stream_with_think_delta(stream, min_tokens=0):
             if self.check_if_canceled("LLM streaming"):
                 return
 
-            if isinstance(ans, int):
-                continue
-
             if ans.find("**ERROR**") >= 0:
                 if self.get_exception_default_value():
                     self.set_output("content", self.get_exception_default_value())
@@ -360,7 +310,8 @@ class LLM(ComponentBase):
                     self.set_output("_ERROR", ans)
                 return
 
-            yield delta(ans)
+            answer += ans
+            yield ans
 
         self.set_output("content", answer)
 
@@ -375,6 +326,7 @@ class LLM(ComponentBase):
             return re.sub(r"```\n*$", "", ans, flags=re.DOTALL)
 
         prompt, msg, _ = self._prepare_prompt_variables()
+        extra_chat_kwargs = self._get_chat_template_kwargs()
         error: str = ""
         output_structure = None
         try:
@@ -393,7 +345,7 @@ class LLM(ComponentBase):
                     int(self.chat_mdl.max_length * 0.97),
                 )
                 error = ""
-                ans = await self._generate_async(msg_fit)
+                ans = await self._generate_async(msg_fit, **extra_chat_kwargs)
                 msg_fit.pop(0)
                 if ans.find("**ERROR**") >= 0:
                     logging.error(f"LLM response error: {ans}")
@@ -426,7 +378,7 @@ class LLM(ComponentBase):
                 [{"role": "system", "content": prompt}, *deepcopy(msg)], int(self.chat_mdl.max_length * 0.97)
             )
             error = ""
-            ans = await self._generate_async(msg_fit)
+            ans = await self._generate_async(msg_fit, **extra_chat_kwargs)
             msg_fit.pop(0)
             if ans.find("**ERROR**") >= 0:
                 logging.error(f"LLM response error: {ans}")
@@ -445,6 +397,24 @@ class LLM(ComponentBase):
     def _invoke(self, **kwargs):
         return asyncio.run(self._invoke_async(**kwargs))
 
+    def _get_chat_template_kwargs(self) -> dict[str, Any]:
+        chat_template_kwargs = self._canvas.globals.get("sys.chat_template_kwargs")
+        if chat_template_kwargs is None:
+            return {}
+
+        # The API should pass this as a JSON object, but accept a JSON string for compatibility.
+        if isinstance(chat_template_kwargs, str):
+            try:
+                chat_template_kwargs = json_repair.loads(chat_template_kwargs)
+            except Exception:
+                logging.warning("Ignore invalid sys.chat_template_kwargs: expected JSON object or JSON string object.")
+                return {}
+
+        if not isinstance(chat_template_kwargs, dict):
+            logging.warning("Ignore invalid sys.chat_template_kwargs type: %s", type(chat_template_kwargs).__name__)
+            return {}
+        return {"chat_template_kwargs": chat_template_kwargs}
+
     async def add_memory(self, user:str, assist:str, func_name: str, params: dict, results: str, user_defined_prompt:dict={}):
         summ = await tool_call_summary(self.chat_mdl, func_name, params, results, user_defined_prompt)
         logging.info(f"[MEMORY]: {summ}")

agent/component/loop.py

@@ -56,7 +56,7 @@ class Loop(ComponentBase, ABC):
 
         for item in self._param.loop_variables:
             if any([not item.get("variable"), not item.get("input_mode"), not item.get("value"),not item.get("type")]):
-                assert "Loop Variable is not complete."
+                raise ValueError("Loop Variable is not complete.")
             if item["input_mode"]=="variable":
                 self.set_output(item["variable"],self._canvas.get_variable_value(item["value"]))
             elif item["input_mode"]=="constant":

agent/component/loopitem.py

@@ -64,6 +64,16 @@ class LoopItem(ComponentBase, ABC):
             elif operator == "not empty":
                 return var != ""
 
+        elif isinstance(var, bool):
+            if operator == "is":
+                return var is value
+            elif operator == "is not":
+                return var is not value
+            elif operator == "empty":
+                return var is None
+            elif operator == "not empty":
+                return var is not None
+
         elif isinstance(var, (int, float)):
             if operator == "=":
                 return var == value
@@ -82,16 +92,6 @@ class LoopItem(ComponentBase, ABC):
             elif operator == "not empty":
                 return var is not None
 
-        elif isinstance(var, bool):
-            if operator == "is":
-                return var is value
-            elif operator == "is not":
-                return var is not value
-            elif operator == "empty":
-                return var is None
-            elif operator == "not empty":
-                return var is not None
-
         elif isinstance(var, dict):
             if operator == "empty":
                 return len(var) == 0

agent/component/message.py

@@ -75,6 +75,22 @@ class Message(ComponentBase):
             key in value for key in ("doc_id", "filename", "mime_type")
         )
 
+    @staticmethod
+    def _download_info_includes_content(value: Any) -> bool:
+        return isinstance(value, dict) and bool(value.get("include_download_info_in_content"))
+
+    @staticmethod
+    def _normalize_download_info(value: Any) -> Any:
+        if isinstance(value, list):
+            return [Message._normalize_download_info(item) for item in value]
+
+        if not isinstance(value, dict):
+            return value
+
+        normalized = value.copy()
+        normalized.pop("include_download_info_in_content", None)
+        return normalized
+
     def _extract_downloads(self, value: Any) -> list[dict[str, Any]]:
         if isinstance(value, str):
             try:
@@ -100,7 +116,19 @@ class Message(ComponentBase):
         extracted_downloads = self._extract_downloads(value)
         if extracted_downloads:
             if downloads is not None:
-                downloads.extend(extracted_downloads)
+                downloads.extend(self._normalize_download_info(item) for item in extracted_downloads)
+            if any(self._download_info_includes_content(item) for item in extracted_downloads):
+                if isinstance(value, str):
+                    try:
+                        value = json.loads(value)
+                    except Exception:
+                        return value
+                try:
+                    return json.dumps(self._normalize_download_info(value), ensure_ascii=False)
+                except Exception:
+                    if fallback_to_str:
+                        return str(value)
+                    return ""
             return ""
 
         if value is None:
@@ -133,7 +161,7 @@ class Message(ComponentBase):
             if k in kwargs:
                 continue
             v = v["value"]
-            if not v:
+            if v is None:
                 v = ""
             ans = ""
             if isinstance(v, partial):

agent/component/string_transform.py

@@ -105,7 +105,7 @@ class StringTransform(Message, ABC):
                 pass
 
         for k,v in kwargs.items():
-            if not v:
+            if v is None:
                 v = ""
             script = re.sub(k, lambda match: v, script)
 

agent/component/switch.py

@@ -88,7 +88,7 @@ class Switch(ComponentBase, ABC):
                     self.set_output("_next", cond["to"])
                     return
 
-            if all(res):
+            if res and all(res):
                 self.set_output("next", [self._canvas.get_component_name(cpn_id) for cpn_id in cond["to"]])
                 self.set_output("_next", cond["to"])
                 return

agent/component/variable_assigner.py

@@ -48,7 +48,7 @@ class VariableAssigner(ComponentBase,ABC):
         else:
             for item in self._param.variables:
                 if any([not item.get("variable"), not item.get("operator"), not item.get("parameter")]):
-                    assert "Variable is not complete."
+                    raise ValueError("Variable is not complete.")
                 variable=item["variable"]
                 operator=item["operator"]
                 parameter=item["parameter"]
@@ -92,12 +92,12 @@ class VariableAssigner(ComponentBase,ABC):
             return ""
         elif isinstance(variable,dict):
             return {}
+        elif isinstance(variable,bool):
+            return False
         elif isinstance(variable,int):
             return 0
         elif isinstance(variable,float):
             return 0.0
-        elif isinstance(variable,bool):
-            return False
         else:
             return None
 
@@ -141,20 +141,18 @@ class VariableAssigner(ComponentBase,ABC):
             return variable + parameter
 
     def _remove_first(self,variable):
-        if len(variable)==0:
-            return variable
         if not isinstance(variable,list):
             return "ERROR:VARIABLE_NOT_LIST"
-        else:
-            return variable[1:]
+        if len(variable)==0:
+            return variable
+        return variable[1:]
 
     def _remove_last(self,variable):
-        if len(variable)==0:
-            return variable
         if not isinstance(variable,list):
             return "ERROR:VARIABLE_NOT_LIST"
-        else:
-            return variable[:-1]
+        if len(variable)==0:
+            return variable
+        return variable[:-1]
 
     def is_number(self, value):
         if isinstance(value, bool):

agent/sandbox/client.py

@@ -27,7 +27,7 @@ from typing import Dict, Any, Optional
 
 from api.db.services.system_settings_service import SystemSettingsService
 from agent.sandbox.providers import ProviderManager
-from agent.sandbox.providers.base import ExecutionResult
+from agent.sandbox.providers.base import ExecutionResult, SandboxProviderConfigError
 
 logger = logging.getLogger(__name__)
 
@@ -48,7 +48,6 @@ def get_provider_manager() -> ProviderManager:
     if _provider_manager is not None:
         return _provider_manager
 
-    # Initialize provider manager with system settings
     _provider_manager = ProviderManager()
     _load_provider_from_settings()
 
@@ -59,8 +58,8 @@ def _load_provider_from_settings() -> None:
     """
     Load sandbox provider from system settings and configure the provider manager.
 
-    This function reads the system settings to determine which provider is active
-    and initializes it with the appropriate configuration.
+    This function resolves the active provider type, then loads configuration
+    from system settings.
     """
     global _provider_manager
 
@@ -68,38 +67,24 @@ def _load_provider_from_settings() -> None:
         return
 
     try:
-        # Get active provider type
-        provider_type_settings = SystemSettingsService.get_by_name("sandbox.provider_type")
-        if not provider_type_settings:
-            raise RuntimeError(
-                "Sandbox provider type not configured. Please set 'sandbox.provider_type' in system settings."
-            )
-        provider_type = provider_type_settings[0].value
-
-        # Get provider configuration
-        provider_config_settings = SystemSettingsService.get_by_name(f"sandbox.{provider_type}")
-
-        if not provider_config_settings:
-            logger.warning(f"No configuration found for provider: {provider_type}")
-            config = {}
-        else:
-            try:
-                config = json.loads(provider_config_settings[0].value)
-            except json.JSONDecodeError as e:
-                logger.error(f"Failed to parse sandbox config for {provider_type}: {e}")
-                config = {}
+        provider_type = _resolve_provider_type()
+        config = _load_provider_config(provider_type)
 
         # Import and instantiate the provider
         from agent.sandbox.providers import (
             SelfManagedProvider,
             AliyunCodeInterpreterProvider,
             E2BProvider,
+            LocalProvider,
+            SSHProvider,
         )
 
         provider_classes = {
             "self_managed": SelfManagedProvider,
             "aliyun_codeinterpreter": AliyunCodeInterpreterProvider,
             "e2b": E2BProvider,
+            "local": LocalProvider,
+            "ssh": SSHProvider,
         }
 
         if provider_type not in provider_classes:
@@ -111,17 +96,44 @@ def _load_provider_from_settings() -> None:
 
         # Initialize the provider
         if not provider.initialize(config):
-            logger.error(f"Failed to initialize sandbox provider: {provider_type}. Config keys: {list(config.keys())}")
+            message = f"Failed to initialize sandbox provider: {provider_type}. Config keys: {list(config.keys())}"
+            if provider_type in {"local", "ssh"}:
+                raise SandboxProviderConfigError(message)
+            logger.error(message)
             return
 
         # Set the active provider
         _provider_manager.set_provider(provider_type, provider)
         logger.info(f"Sandbox provider '{provider_type}' initialized successfully")
 
+    except SandboxProviderConfigError:
+        raise
     except Exception as e:
         logger.error(f"Failed to load sandbox provider from settings: {e}")
         import traceback
         traceback.print_exc()
+def _load_provider_config_from_settings(provider_type: str) -> Dict[str, Any]:
+    provider_config_settings = SystemSettingsService.get_by_name(f"sandbox.{provider_type}")
+    if not provider_config_settings:
+        logger.warning(f"No configuration found for provider: {provider_type}")
+        return {}
+
+    try:
+        return json.loads(provider_config_settings[0].value)
+    except json.JSONDecodeError as e:
+        logger.error(f"Failed to parse sandbox config for {provider_type}: {e}")
+        return {}
+
+
+def _resolve_provider_type() -> str:
+    provider_type_settings = SystemSettingsService.get_by_name("sandbox.provider_type")
+    if not provider_type_settings:
+        return "self_managed"
+    return provider_type_settings[0].value
+
+
+def _load_provider_config(provider_type: str) -> Dict[str, Any]:
+    return _load_provider_config_from_settings(provider_type)
 
 
 def reload_provider() -> None:
@@ -166,6 +178,14 @@ def execute_code(
         )
 
     provider = provider_manager.get_provider()
+    provider_name = provider_manager.get_provider_name() or getattr(provider, "__class__", type(provider)).__name__
+
+    logger.info(
+        "CodeExec using sandbox provider '%s' (language=%s, timeout=%ss)",
+        provider_name,
+        language,
+        timeout,
+    )
 
     # Create a sandbox instance
     instance = provider.create_instance(template=language)

agent/sandbox/executor_manager/Dockerfile

@@ -1,6 +1,10 @@
 FROM python:3.11-slim-bookworm
 
-RUN grep -rl 'deb.debian.org' /etc/apt/ | xargs sed -i 's|http[s]*://deb.debian.org|https://mirrors.tuna.tsinghua.edu.cn|g' && \
+ARG NEED_MIRROR=1
+
+RUN if [ "$NEED_MIRROR" = 1 ]; then \
+        grep -rl 'deb.debian.org' /etc/apt/ | xargs sed -i 's|http[s]*://deb.debian.org|https://mirrors.tuna.tsinghua.edu.cn|g'; \
+    fi; \
     apt-get update && \
     apt-get install -y curl gcc && \
     rm -rf /var/lib/apt/lists/*
@@ -27,11 +31,11 @@ RUN set -eux; \
     ln -sf /usr/local/bin/docker /usr/bin/docker
 
 COPY --from=ghcr.io/astral-sh/uv:0.7.5 /uv /uvx /bin/
-ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
 
 WORKDIR /app
 COPY . .
 
-RUN uv pip install --system -r requirements.txt
+RUN if [ "$NEED_MIRROR" = 1 ]; then export UV_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"; else export UV_INDEX_URL="https://pypi.org/simple"; fi && \
+    uv pip install --system -r requirements.txt
 
 CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "9385"]

agent/sandbox/executor_manager/services/security.py

@@ -26,7 +26,7 @@ class SecurePythonAnalyzer(ast.NodeVisitor):
     An AST-based analyzer for detecting unsafe Python code patterns.
     """
 
-    DANGEROUS_IMPORTS = {"os", "subprocess", "sys", "shutil", "socket", "ctypes", "pickle", "threading", "multiprocessing", "asyncio", "http.client", "ftplib", "telnetlib"}
+    DANGEROUS_IMPORTS = {"os", "subprocess", "sys", "shutil", "socket", "ctypes", "pickle", "threading", "multiprocessing", "asyncio", "http.client", "ftplib", "telnetlib", "builtins"}
 
     DANGEROUS_CALLS = {
         "eval",
@@ -77,6 +77,16 @@ class SecurePythonAnalyzer(ast.NodeVisitor):
         """Check for dangerous function calls."""
         if isinstance(node.func, ast.Name) and node.func.id in self.DANGEROUS_CALLS:
             self.unsafe_items.append((f"Call: {node.func.id}", node.lineno))
+        elif isinstance(node.func, ast.Attribute) and node.func.attr in self.DANGEROUS_CALLS:
+            # Surface the attribute-style match in the analyzer log so that
+            # incident response can grep for it just like the other unsafe-item
+            # findings; the bare append is invisible to operators.
+            logger.warning(
+                "[SafeCheck] Attribute-style dangerous call detected: %s (line %s)",
+                node.func.attr,
+                node.lineno,
+            )
+            self.unsafe_items.append((f"Call: {node.func.attr}", node.lineno))
         self.generic_visit(node)
 
     def visit_Attribute(self, node: ast.Attribute):
@@ -154,9 +164,9 @@ class SecurePythonAnalyzer(ast.NodeVisitor):
 
 class SecureJavaScriptAnalyzer:
     DANGEROUS_PATTERNS = [
-        (re.compile(r"""require\s*\(\s*['"]child_process['"]\s*\)"""), "Require: child_process"),
-        (re.compile(r"""require\s*\(\s*['"]fs['"]\s*\)"""), "Require: fs"),
-        (re.compile(r"""require\s*\(\s*['"]worker_threads['"]\s*\)"""), "Require: worker_threads"),
+        (re.compile(r"""require\s*\(\s*['"`]child_process['"`]\s*\)"""), "Require: child_process"),
+        (re.compile(r"""require\s*\(\s*['"`]fs['"`]\s*\)"""), "Require: fs"),
+        (re.compile(r"""require\s*\(\s*['"`]worker_threads['"`]\s*\)"""), "Require: worker_threads"),
         (re.compile(r"""\beval\s*\("""), "Call: eval"),
         (re.compile(r"""\bFunction\s*\("""), "Call: Function"),
         (re.compile(r"""\bprocess\s*\.\s*binding\s*\("""), "Call: process.binding"),

agent/sandbox/providers/__init__.py

@@ -24,20 +24,27 @@ This package contains:
 - aliyun_codeinterpreter.py: Aliyun Code Interpreter provider implementation
   Official Documentation: https://help.aliyun.com/zh/functioncompute/fc/sandbox-sandbox-code-interepreter
 - e2b.py: E2B provider implementation
+- local.py: Local process provider implementation
+- ssh.py: Remote SSH provider implementation
 """
 
-from .base import SandboxProvider, SandboxInstance, ExecutionResult
+from .base import SandboxProvider, SandboxInstance, ExecutionResult, SandboxProviderConfigError
 from .manager import ProviderManager
 from .self_managed import SelfManagedProvider
 from .aliyun_codeinterpreter import AliyunCodeInterpreterProvider
 from .e2b import E2BProvider
+from .local import LocalProvider
+from .ssh import SSHProvider
 
 __all__ = [
     "SandboxProvider",
     "SandboxInstance",
     "ExecutionResult",
+    "SandboxProviderConfigError",
     "ProviderManager",
     "SelfManagedProvider",
     "AliyunCodeInterpreterProvider",
     "E2BProvider",
+    "LocalProvider",
+    "SSHProvider",
 ]

agent/sandbox/providers/aliyun_codeinterpreter.py

@@ -30,7 +30,6 @@ https://api.aliyun.com/api/AgentRun/2025-09-10/CreateSandbox?lang=PYTHON
 import logging
 import os
 import time
-import base64
 import json
 from typing import Dict, Any, List, Optional
 from datetime import datetime, timezone
@@ -39,10 +38,10 @@ from agentrun.sandbox import TemplateType, CodeLanguage, Template, TemplateInput
 from agentrun.utils.config import Config
 from agentrun.utils.exception import ServerError
 
+from agent.sandbox.result_protocol import build_javascript_wrapper, build_python_wrapper, extract_structured_result
 from .base import SandboxProvider, SandboxInstance, ExecutionResult
 
 logger = logging.getLogger(__name__)
-RESULT_MARKER_PREFIX = "__RAGFLOW_RESULT__:"
 
 
 class AliyunCodeInterpreterProvider(SandboxProvider):
@@ -234,9 +233,9 @@ class AliyunCodeInterpreterProvider(SandboxProvider):
             # Matches self_managed provider behavior: call main(**arguments)
             args_json = json.dumps(arguments or {})
             wrapped_code = (
-                self._build_python_wrapper(code, args_json)
+                build_python_wrapper(code, args_json)
                 if normalized_lang == "python"
-                else self._build_javascript_wrapper(code, args_json)
+                else build_javascript_wrapper(code, args_json)
             )
             logger.debug(f"Aliyun Code Interpreter: Wrapped code (first 200 chars): {wrapped_code[:200]}")
 
@@ -284,7 +283,7 @@ class AliyunCodeInterpreterProvider(SandboxProvider):
 
             stdout = "\n".join(stdout_parts)
             stderr = "\n".join(stderr_parts)
-            stdout, structured_result = self._extract_structured_result(stdout)
+            stdout, structured_result = extract_structured_result(stdout)
 
             logger.info(f"Aliyun Code Interpreter: stdout length={len(stdout)}, stderr length={len(stderr)}, exit_code={exit_code}")
             if stdout:
@@ -364,71 +363,6 @@ class AliyunCodeInterpreterProvider(SandboxProvider):
             # If we get any response (even an error), the service is reachable
             return "connection" not in str(e).lower()
 
-    @staticmethod
-    def _build_python_wrapper(code: str, args_json: str) -> str:
-        marker = RESULT_MARKER_PREFIX
-        return f'''{code}
-
-if __name__ == "__main__":
-    import base64
-    import json
-
-    result = main(**{args_json})
-    payload = json.dumps({{"present": True, "value": result, "type": "json"}}, ensure_ascii=False, separators=(",", ":"))
-    print("{marker}" + base64.b64encode(payload.encode("utf-8")).decode("ascii"))
-'''
-
-    @staticmethod
-    def _build_javascript_wrapper(code: str, args_json: str) -> str:
-        marker = RESULT_MARKER_PREFIX
-        return f'''{code}
-
-const __ragflowArgs = {args_json};
-
-(async () => {{
-  try {{
-    const output = await Promise.resolve(main(__ragflowArgs));
-    if (typeof output === 'undefined') {{
-      throw new Error('main() must return a value. Use null for an empty result.');
-    }}
-    const payload = JSON.stringify({{ present: true, value: output, type: 'json' }});
-    if (typeof payload === 'undefined') {{
-      throw new Error('main() returned a non-JSON-serializable value.');
-    }}
-    console.log('{marker}' + Buffer.from(payload, 'utf8').toString('base64'));
-  }} catch (err) {{
-    console.error(err instanceof Error ? err.stack || err.message : String(err));
-  }}
-}})();
-'''
-
-    @staticmethod
-    def _extract_structured_result(stdout: str) -> tuple[str, Dict[str, Any]]:
-        if not stdout:
-            return "", {}
-
-        cleaned_lines: list[str] = []
-        structured_result: Dict[str, Any] = {}
-
-        for line in str(stdout).splitlines():
-            if line.startswith(RESULT_MARKER_PREFIX):
-                payload_b64 = line[len(RESULT_MARKER_PREFIX) :].strip()
-                if not payload_b64:
-                    continue
-                try:
-                    payload = base64.b64decode(payload_b64).decode("utf-8")
-                    structured_result = json.loads(payload)
-                except Exception as exc:
-                    logger.warning(f"Aliyun Code Interpreter: failed to decode structured result marker: {exc}")
-                    cleaned_lines.append(line)
-                continue
-            cleaned_lines.append(line)
-
-        cleaned_stdout = "\n".join(cleaned_lines)
-        if stdout.endswith("\n") and cleaned_stdout and not cleaned_stdout.endswith("\n"):
-            cleaned_stdout += "\n"
-        return cleaned_stdout, structured_result
-
     def get_supported_languages(self) -> List[str]:
         """
         Get list of supported programming languages.

agent/sandbox/providers/base.py

@@ -26,6 +26,10 @@ from dataclasses import dataclass
 from typing import Dict, Any, Optional, List
 
 
+class SandboxProviderConfigError(Exception):
+    """Raised when the selected provider is explicitly configured but unusable."""
+
+
 @dataclass
 class SandboxInstance:
     """Represents a sandbox execution instance"""
@@ -209,4 +213,4 @@ class SandboxProvider(ABC):
             >>>     return True, None
         """
         # Default implementation: no custom validation
-        return True, None
+        return True, None

agent/sandbox/providers/local.py

@@ -0,0 +1,352 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import base64
+import json
+import mimetypes
+import os
+import shutil
+import signal
+import subprocess
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from agent.sandbox.result_protocol import build_javascript_wrapper, build_python_wrapper, extract_structured_result
+from .base import ExecutionResult, SandboxInstance, SandboxProvider, SandboxProviderConfigError
+
+
+ALLOWED_ARTIFACT_EXTENSIONS = {
+    ".csv",
+    ".html",
+    ".jpeg",
+    ".jpg",
+    ".json",
+    ".pdf",
+    ".png",
+    ".svg",
+}
+
+LOCAL_PYTHON_THREAD_ENV_VARS = (
+    "OPENBLAS_NUM_THREADS",
+    "OMP_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "BLIS_NUM_THREADS",
+    "VECLIB_MAXIMUM_THREADS",
+)
+class LocalProvider(SandboxProvider):
+    """
+    Execute code as a local child process.
+
+    This provider is intentionally gated by SANDBOX_LOCAL_ENABLED because it is
+    not a sandbox boundary. Use a low-privilege runtime account.
+    """
+
+    def __init__(self):
+        self.python_bin = "python3"
+        self.node_bin = "node"
+        self.work_dir = Path("/tmp/ragflow-codeexec")
+        self.timeout = 30
+        self.max_memory_mb = 512
+        self.max_output_bytes = 1024 * 1024
+        self.max_artifacts = 20
+        self.max_artifact_bytes = 10 * 1024 * 1024
+        self._initialized = False
+        self._instances: dict[str, Path] = {}
+
+    def initialize(self, config: Dict[str, Any]) -> bool:
+        self.python_bin = str(config.get("python_bin", "python3"))
+        self.node_bin = str(config.get("node_bin", "node"))
+        self.work_dir = Path(str(config.get("work_dir", "/tmp/ragflow-codeexec"))).resolve()
+        self.timeout = int(config.get("timeout", 30))
+        self.max_memory_mb = int(config.get("max_memory_mb", 512))
+        self.max_output_bytes = int(config.get("max_output_bytes", 1024 * 1024))
+        self.max_artifacts = int(config.get("max_artifacts", 20))
+        self.max_artifact_bytes = int(config.get("max_artifact_bytes", 10 * 1024 * 1024))
+
+        self._validate_limits()
+        self.work_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
+        self._initialized = True
+        return True
+
+    def create_instance(self, template: str = "python") -> SandboxInstance:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+
+        language = self._normalize_language(template)
+        instance_id = str(uuid.uuid4())
+        instance_dir = self.work_dir / instance_id
+        instance_dir.mkdir(mode=0o700)
+        (instance_dir / "artifacts").mkdir(mode=0o700)
+        self._instances[instance_id] = instance_dir
+
+        return SandboxInstance(
+            instance_id=instance_id,
+            provider="local",
+            status="running",
+            metadata={"language": language, "work_dir": str(instance_dir)},
+        )
+
+    def execute_code(
+        self,
+        instance_id: str,
+        code: str,
+        language: str,
+        timeout: int = 10,
+        arguments: Optional[Dict[str, Any]] = None,
+    ) -> ExecutionResult:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+
+        normalized_lang = self._normalize_language(language)
+        instance_dir = self._instances[instance_id]
+        args_json = json.dumps(arguments or {}, ensure_ascii=False)
+        command, script_path = self._prepare_script(instance_dir, normalized_lang, code, args_json)
+        requested_timeout = self.timeout if timeout is None else int(timeout)
+        if requested_timeout <= 0:
+            raise RuntimeError(f"Execution timeout must be greater than 0 seconds, got {requested_timeout}.")
+        exec_timeout = min(requested_timeout, self.timeout)
+
+        start_time = time.time()
+        process = subprocess.Popen(
+            command,
+            cwd=instance_dir,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            env=self._build_child_env(instance_dir),
+            preexec_fn=self._limit_child_process if os.name == "posix" else None,
+            start_new_session=os.name == "posix",
+        )
+
+        try:
+            stdout, stderr = process.communicate(timeout=exec_timeout)
+        except subprocess.TimeoutExpired:
+            if os.name == "posix":
+                os.killpg(process.pid, signal.SIGKILL)
+            else:
+                process.kill()
+            process.communicate()
+            raise TimeoutError(f"Execution timed out after {exec_timeout} seconds")
+
+        execution_time = time.time() - start_time
+        self._validate_output_size(stdout, stderr)
+        stdout, structured_result = extract_structured_result(stdout)
+
+        return ExecutionResult(
+            stdout=stdout,
+            stderr=stderr,
+            exit_code=process.returncode,
+            execution_time=execution_time,
+            metadata={
+                "instance_id": instance_id,
+                "language": normalized_lang,
+                "script_path": str(script_path),
+                "status": "ok" if process.returncode == 0 else "error",
+                "timeout": exec_timeout,
+                "artifacts": self._collect_artifacts(instance_dir / "artifacts"),
+                "result_present": structured_result.get("present", False),
+                "result_value": structured_result.get("value"),
+                "result_type": structured_result.get("type"),
+            },
+        )
+
+    def destroy_instance(self, instance_id: str) -> bool:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+
+        instance_dir = self._instances.pop(instance_id)
+        shutil.rmtree(instance_dir)
+        return True
+
+    def health_check(self) -> bool:
+        return self._initialized and self.work_dir.exists() and os.access(self.work_dir, os.W_OK)
+
+    def get_supported_languages(self) -> List[str]:
+        return ["python", "javascript", "nodejs"]
+
+    @staticmethod
+    def get_config_schema() -> Dict[str, Dict]:
+        return {
+            "python_bin": {
+                "type": "string",
+                "required": False,
+                "default": "python3",
+                "label": "Python Binary",
+                "description": "Python executable used for local code execution.",
+            },
+            "node_bin": {
+                "type": "string",
+                "required": False,
+                "default": "node",
+                "label": "Node.js Binary",
+                "description": "Node.js executable used for local JavaScript execution.",
+            },
+            "work_dir": {
+                "type": "string",
+                "required": False,
+                "default": "/tmp/ragflow-codeexec",
+                "label": "Working Directory",
+                "description": "Directory used to store temporary scripts and artifacts on the current host.",
+            },
+            "timeout": {
+                "type": "integer",
+                "required": False,
+                "default": 30,
+                "label": "Timeout (seconds)",
+                "description": "Maximum execution time for each local run. Unit: seconds.",
+                "min": 1,
+                "max": 600,
+            },
+            "max_memory_mb": {
+                "type": "integer",
+                "required": False,
+                "default": 512,
+                "label": "Max Memory (MB)",
+                "description": "Address-space memory limit for the local child process. Unit: MB.",
+                "min": 1,
+                "max": 65536,
+            },
+            "max_output_bytes": {
+                "type": "integer",
+                "required": False,
+                "default": 1048576,
+                "label": "Max Output (bytes)",
+                "description": "Maximum combined stdout and stderr size. Unit: bytes.",
+                "min": 1024,
+                "max": 10485760,
+            },
+            "max_artifacts": {
+                "type": "integer",
+                "required": False,
+                "default": 20,
+                "label": "Max Artifacts",
+                "description": "Maximum number of files collected from the artifacts directory.",
+                "min": 0,
+                "max": 100,
+            },
+            "max_artifact_bytes": {
+                "type": "integer",
+                "required": False,
+                "default": 10485760,
+                "label": "Max Artifact Size (bytes)",
+                "description": "Maximum size of a single artifact file. Unit: bytes.",
+                "min": 1024,
+                "max": 104857600,
+            },
+        }
+
+    def _validate_limits(self) -> None:
+        if self.timeout <= 0:
+            raise SandboxProviderConfigError("SANDBOX_LOCAL_TIMEOUT must be greater than 0.")
+        if self.max_memory_mb <= 0:
+            raise SandboxProviderConfigError("SANDBOX_LOCAL_MAX_MEMORY_MB must be greater than 0.")
+        if self.max_output_bytes <= 0:
+            raise SandboxProviderConfigError("SANDBOX_LOCAL_MAX_OUTPUT_BYTES must be greater than 0.")
+        if self.max_artifacts < 0:
+            raise SandboxProviderConfigError("SANDBOX_LOCAL_MAX_ARTIFACTS must be greater than or equal to 0.")
+        if self.max_artifact_bytes <= 0:
+            raise SandboxProviderConfigError("SANDBOX_LOCAL_MAX_ARTIFACT_BYTES must be greater than 0.")
+
+    def _prepare_script(self, instance_dir: Path, language: str, code: str, args_json: str) -> tuple[list[str], Path]:
+        if language == "python":
+            script_path = instance_dir / "main.py"
+            script_path.write_text(build_python_wrapper(code, args_json), encoding="utf-8")
+            return [self.python_bin, str(script_path)], script_path
+        if language in {"javascript", "nodejs"}:
+            script_path = instance_dir / "main.js"
+            script_path.write_text(build_javascript_wrapper(code, args_json), encoding="utf-8")
+            return [self.node_bin, str(script_path)], script_path
+        raise RuntimeError(f"Unsupported language for local provider: {language}")
+
+    def _build_child_env(self, instance_dir: Path) -> dict[str, str]:
+        env = {
+            "HOME": str(instance_dir),
+            "MPLBACKEND": "Agg",
+            "PATH": os.environ.get("PATH", ""),
+            "PYTHONUNBUFFERED": "1",
+            "TMPDIR": str(instance_dir),
+        }
+        for name in LOCAL_PYTHON_THREAD_ENV_VARS:
+            value = os.environ.get(name)
+            if value is not None:
+                env[name] = value
+        return env
+
+    def _limit_child_process(self) -> None:
+        import resource
+
+        self._set_resource_limit(resource.RLIMIT_CPU, self.timeout + 1)
+        self._set_resource_limit(resource.RLIMIT_AS, self.max_memory_mb * 1024 * 1024)
+        self._set_resource_limit(resource.RLIMIT_FSIZE, self.max_artifact_bytes)
+        self._set_resource_limit(resource.RLIMIT_NOFILE, 64)
+
+    @staticmethod
+    def _set_resource_limit(kind: int, value: int) -> None:
+        import resource
+
+        _, hard = resource.getrlimit(kind)
+        limit = value if hard == resource.RLIM_INFINITY else min(value, hard)
+        resource.setrlimit(kind, (limit, limit))
+
+    def _validate_output_size(self, stdout: str, stderr: str) -> None:
+        output_size = len((stdout or "").encode("utf-8")) + len((stderr or "").encode("utf-8"))
+        if output_size > self.max_output_bytes:
+            raise RuntimeError(f"Local execution output exceeded {self.max_output_bytes} bytes.")
+
+    def _collect_artifacts(self, artifacts_dir: Path) -> list[dict[str, Any]]:
+        artifacts: list[dict[str, Any]] = []
+        for path in sorted(artifacts_dir.rglob("*")):
+            if path.is_symlink():
+                raise RuntimeError(f"Artifact symlinks are not allowed: {path.name}")
+            if path.is_dir():
+                continue
+            if not path.is_file():
+                raise RuntimeError(f"Unsupported artifact entry: {path.name}")
+
+            if len(artifacts) >= self.max_artifacts:
+                raise RuntimeError(f"Local execution produced more than {self.max_artifacts} artifacts.")
+
+            size = path.stat().st_size
+            if size > self.max_artifact_bytes:
+                raise RuntimeError(f"Artifact exceeds {self.max_artifact_bytes} bytes: {path.name}")
+
+            ext = path.suffix.lower()
+            if ext not in ALLOWED_ARTIFACT_EXTENSIONS:
+                raise RuntimeError(f"Unsupported artifact type: {path.name}")
+
+            artifacts.append(
+                {
+                    "name": path.relative_to(artifacts_dir).as_posix(),
+                    "content_b64": base64.b64encode(path.read_bytes()).decode("ascii"),
+                    "mime_type": mimetypes.guess_type(path.name)[0] or "application/octet-stream",
+                    "size": size,
+                }
+            )
+        return artifacts
+
+    @staticmethod
+    def _normalize_language(language: str) -> str:
+        lang_lower = (language or "python").lower()
+        if lang_lower in {"python", "python3"}:
+            return "python"
+        if lang_lower in {"javascript", "nodejs"}:
+            return "nodejs"
+        return lang_lower

agent/sandbox/providers/self_managed.py

@@ -22,6 +22,7 @@ a pool of Docker containers with gVisor for secure code execution.
 """
 
 import base64
+import os
 import time
 import uuid
 from typing import Dict, Any, List, Optional
@@ -40,10 +41,10 @@ class SelfManagedProvider(SandboxProvider):
     """
 
     def __init__(self):
-        self.endpoint: str = "http://localhost:9385"
+        self.endpoint: str = "http://sandbox-executor-manager:9385"
         self.timeout: int = 30
         self.max_retries: int = 3
-        self.pool_size: int = 10
+        self.pool_size: int = 3
         self._initialized: bool = False
 
     def initialize(self, config: Dict[str, Any]) -> bool:
@@ -52,7 +53,7 @@ class SelfManagedProvider(SandboxProvider):
 
         Args:
             config: Configuration dictionary with keys:
-                - endpoint: HTTP endpoint (default: "http://localhost:9385")
+                - endpoint: HTTP endpoint (default: "http://sandbox-executor-manager:9385")
                 - timeout: Request timeout in seconds (default: 30)
                 - max_retries: Maximum retry attempts (default: 3)
                 - pool_size: Container pool size for info (default: 10)
@@ -60,30 +61,13 @@ class SelfManagedProvider(SandboxProvider):
         Returns:
             True if initialization successful, False otherwise
         """
-        self.endpoint = config.get("endpoint", "http://localhost:9385")
+        self.endpoint = config.get("endpoint", "http://sandbox-executor-manager:9385")
         self.timeout = config.get("timeout", 30)
         self.max_retries = config.get("max_retries", 3)
-        self.pool_size = config.get("pool_size", 10)
+        self.pool_size = config.get("executor_manager_pool_size", config.get("pool_size", 3))
 
         # Validate endpoint is accessible
         if not self.health_check():
-            # Try to fall back to SANDBOX_HOST from settings if we are using localhost
-            if "localhost" in self.endpoint or "127.0.0.1" in self.endpoint:
-                try:
-                    from common import settings
-                    if settings.SANDBOX_HOST and settings.SANDBOX_HOST not in self.endpoint:
-                        original_endpoint = self.endpoint
-                        self.endpoint = f"http://{settings.SANDBOX_HOST}:9385"
-                        if self.health_check():
-                            import logging
-                            logging.warning(f"Sandbox self_managed: Connected using settings.SANDBOX_HOST fallback: {self.endpoint} (original: {original_endpoint})")
-                            self._initialized = True
-                            return True
-                        else:
-                            self.endpoint = original_endpoint # Restore if fallback also fails
-                except ImportError:
-                    pass
-
             return False
 
         self._initialized = True
@@ -270,9 +254,11 @@ class SelfManagedProvider(SandboxProvider):
                 "type": "string",
                 "required": True,
                 "label": "Executor Manager Endpoint",
-                "placeholder": "http://localhost:9385",
-                "default": "http://localhost:9385",
-                "description": "HTTP endpoint of the executor_manager service"
+                "placeholder": "http://sandbox-executor-manager:9385",
+                "default": "http://sandbox-executor-manager:9385",
+                "description": "HTTP endpoint used by RAGFlow to call sandbox-executor-manager.",
+                "scope": "runtime",
+                "readonly": False,
             },
             "timeout": {
                 "type": "integer",
@@ -281,26 +267,86 @@ class SelfManagedProvider(SandboxProvider):
                 "default": 30,
                 "min": 5,
                 "max": 300,
-                "description": "HTTP request timeout for code execution"
+                "description": "Maximum request time for a single code execution call. Unit: seconds.",
+                "scope": "runtime",
+                "readonly": False,
             },
-            "max_retries": {
-                "type": "integer",
+            "executor_manager_image": {
+                "type": "string",
                 "required": False,
-                "label": "Max Retries",
-                "default": 3,
-                "min": 0,
-                "max": 10,
-                "description": "Maximum number of retry attempts for failed requests"
+                "label": "Executor Manager Image",
+                "default": os.getenv("SANDBOX_EXECUTOR_MANAGER_IMAGE", "infiniflow/sandbox-executor-manager:latest"),
+                "description": "Docker image used by sandbox-executor-manager.",
+                "scope": "deployment",
+                "readonly": True,
             },
-            "pool_size": {
+            "executor_manager_pool_size": {
                 "type": "integer",
                 "required": False,
                 "label": "Container Pool Size",
-                "default": 10,
+                "default": int(os.getenv("SANDBOX_EXECUTOR_MANAGER_POOL_SIZE", "3")),
                 "min": 1,
                 "max": 100,
-                "description": "Size of the container pool (configured in executor_manager)"
-            }
+                "description": "Container pool size used by sandbox-executor-manager.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "base_python_image": {
+                "type": "string",
+                "required": False,
+                "label": "Base Python Image",
+                "default": os.getenv("SANDBOX_BASE_PYTHON_IMAGE", "infiniflow/sandbox-base-python:latest"),
+                "description": "Python runtime image used by executor-managed containers.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "base_nodejs_image": {
+                "type": "string",
+                "required": False,
+                "label": "Base Node.js Image",
+                "default": os.getenv("SANDBOX_BASE_NODEJS_IMAGE", "infiniflow/sandbox-base-nodejs:latest"),
+                "description": "Node.js runtime image used by executor-managed containers.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "executor_manager_port": {
+                "type": "integer",
+                "required": False,
+                "label": "Executor Manager Port",
+                "default": int(os.getenv("SANDBOX_EXECUTOR_MANAGER_PORT", "9385")),
+                "min": 1,
+                "max": 65535,
+                "description": "Host port exposed by sandbox-executor-manager.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "enable_seccomp": {
+                "type": "boolean",
+                "required": False,
+                "label": "Enable Seccomp",
+                "default": os.getenv("SANDBOX_ENABLE_SECCOMP", "false").lower() == "true",
+                "description": "Whether sandbox-executor-manager starts containers with seccomp enabled.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "max_memory": {
+                "type": "string",
+                "required": False,
+                "label": "Max Memory",
+                "default": os.getenv("SANDBOX_MAX_MEMORY", "256m"),
+                "description": "Memory limit applied to each sandbox container. Common format: 256m or 1g.",
+                "scope": "deployment",
+                "readonly": True,
+            },
+            "sandbox_timeout": {
+                "type": "string",
+                "required": False,
+                "label": "Sandbox Timeout",
+                "default": os.getenv("SANDBOX_TIMEOUT", "10s"),
+                "description": "Executor-manager container timeout for each sandbox run. Common format: 10s or 1m.",
+                "scope": "deployment",
+                "readonly": True,
+            },
         }
 
     def _normalize_language(self, language: str) -> str:
@@ -347,7 +393,7 @@ class SelfManagedProvider(SandboxProvider):
                 return False, f"Invalid endpoint format: {endpoint}. Must start with http:// or https://"
 
         # Validate pool_size is positive
-        pool_size = config.get("pool_size", 10)
+        pool_size = config.get("executor_manager_pool_size", config.get("pool_size", 3))
         if isinstance(pool_size, int) and pool_size <= 0:
             return False, "Pool size must be greater than 0"
 

agent/sandbox/providers/ssh.py

@@ -0,0 +1,664 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import mimetypes
+import os
+import posixpath
+import shlex
+import stat
+import time
+import uuid
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from agent.sandbox.result_protocol import (
+    build_javascript_wrapper,
+    build_python_wrapper,
+    extract_structured_result,
+)
+from .base import (
+    ExecutionResult,
+    SandboxInstance,
+    SandboxProvider,
+    SandboxProviderConfigError,
+)
+
+if TYPE_CHECKING:
+    import paramiko
+
+
+ALLOWED_ARTIFACT_EXTENSIONS = {
+    ".csv",
+    ".html",
+    ".jpeg",
+    ".jpg",
+    ".json",
+    ".pdf",
+    ".png",
+    ".svg",
+}
+
+
+class SSHProvider(SandboxProvider):
+    """Execute code on a remote host through SSH."""
+
+    def __init__(self):
+        self.host = ""
+        self.port = 22
+        self.username = ""
+        self.password = ""
+        self.private_key = ""
+        self.passphrase = ""
+        self.python_bin = "python3"
+        self.node_bin = "node"
+        self.work_dir = "/tmp"
+        self.timeout = 30
+        self.max_output_bytes = 1024 * 1024
+        self.max_artifacts = 20
+        self.max_artifact_bytes = 10 * 1024 * 1024
+        self._initialized = False
+        self._instances: dict[str, dict[str, Any]] = {}
+
+    def initialize(self, config: Dict[str, Any]) -> bool:
+        self.host = str(config.get("host", "")).strip()
+        self.port = int(config.get("port", 22) or 22)
+        self.username = str(config.get("username", "")).strip()
+        self.password = str(config.get("password", "") or "")
+        self.private_key = str(config.get("private_key", "") or "")
+        self.passphrase = str(config.get("passphrase", "") or "")
+        self.python_bin = str(config.get("python_bin", "python3") or "python3").strip() or "python3"
+        self.node_bin = str(config.get("node_bin", "node") or "node").strip() or "node"
+        self.work_dir = str(config.get("work_dir", "/tmp") or "/tmp").strip() or "/tmp"
+        self.timeout = int(config.get("timeout", 30) or 30)
+        self.max_output_bytes = int(config.get("max_output_bytes", 1024 * 1024) or 1024 * 1024)
+        self.max_artifacts = int(config.get("max_artifacts", 20) or 20)
+        self.max_artifact_bytes = int(config.get("max_artifact_bytes", 10 * 1024 * 1024) or 10 * 1024 * 1024)
+
+        is_valid, error_message = self.validate_config(
+            {
+                "host": self.host,
+                "port": self.port,
+                "username": self.username,
+                "password": self.password,
+                "private_key": self.private_key,
+                "passphrase": self.passphrase,
+                "python_bin": self.python_bin,
+                "node_bin": self.node_bin,
+                "work_dir": self.work_dir,
+                "timeout": self.timeout,
+                "max_output_bytes": self.max_output_bytes,
+                "max_artifacts": self.max_artifacts,
+                "max_artifact_bytes": self.max_artifact_bytes,
+            }
+        )
+        if not is_valid:
+            raise SandboxProviderConfigError(error_message or "Invalid SSH provider configuration.")
+
+        self._assert_connectivity()
+
+        self._initialized = True
+        return True
+
+    def create_instance(self, template: str = "python") -> SandboxInstance:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+
+        language = self._normalize_language(template)
+        client = self._create_ssh_client()
+        sftp = client.open_sftp()
+
+        try:
+            remote_work_dir = self._create_remote_workspace(client)
+            stdout, stderr, exit_code = self._run_remote_command(
+                client,
+                f"mkdir -p {shlex.quote(posixpath.join(remote_work_dir, 'artifacts'))}",
+                timeout=min(self.timeout, 10),
+            )
+            if exit_code != 0:
+                raise RuntimeError(
+                    f"Failed to create remote artifacts directory: {stderr or stdout or 'unknown error'}"
+                )
+        except Exception:
+            sftp.close()
+            client.close()
+            raise
+
+        instance_id = str(uuid.uuid4())
+        self._instances[instance_id] = {
+            "client": client,
+            "sftp": sftp,
+            "remote_work_dir": remote_work_dir,
+            "language": language,
+        }
+
+        return SandboxInstance(
+            instance_id=instance_id,
+            provider="ssh",
+            status="running",
+            metadata={"language": language, "remote_work_dir": remote_work_dir},
+        )
+
+    def execute_code(
+        self,
+        instance_id: str,
+        code: str,
+        language: str,
+        timeout: int = 10,
+        arguments: Optional[Dict[str, Any]] = None,
+    ) -> ExecutionResult:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+        if instance_id not in self._instances:
+            raise RuntimeError(f"Unknown SSH sandbox instance: {instance_id}")
+
+        normalized_lang = self._normalize_language(language)
+        instance = self._instances[instance_id]
+        client: paramiko.SSHClient = instance["client"]
+        sftp: paramiko.SFTPClient = instance["sftp"]
+        remote_work_dir: str = instance["remote_work_dir"]
+
+        args_json = json.dumps(arguments or {}, ensure_ascii=False)
+        remote_script_path, command = self._upload_script(
+            sftp=sftp,
+            remote_work_dir=remote_work_dir,
+            language=normalized_lang,
+            code=code,
+            args_json=args_json,
+        )
+
+        requested_timeout = self.timeout if timeout is None else int(timeout)
+        if requested_timeout <= 0:
+            raise RuntimeError(f"Execution timeout must be greater than 0 seconds, got {requested_timeout}.")
+        exec_timeout = min(requested_timeout, self.timeout)
+
+        start_time = time.time()
+        stdout, stderr, exit_code = self._run_remote_command(client, command, timeout=exec_timeout)
+        execution_time = time.time() - start_time
+
+        self._validate_output_size(stdout, stderr)
+        stdout, structured_result = extract_structured_result(stdout)
+
+        return ExecutionResult(
+            stdout=stdout,
+            stderr=stderr,
+            exit_code=exit_code,
+            execution_time=execution_time,
+            metadata={
+                "instance_id": instance_id,
+                "language": normalized_lang,
+                "script_path": remote_script_path,
+                "remote_work_dir": remote_work_dir,
+                "status": "ok" if exit_code == 0 else "error",
+                "timeout": exec_timeout,
+                "command": command,
+                "artifacts": self._collect_artifacts(
+                    sftp, posixpath.join(remote_work_dir, "artifacts")
+                ),
+                "result_present": structured_result.get("present", False),
+                "result_value": structured_result.get("value"),
+                "result_type": structured_result.get("type"),
+            },
+        )
+
+    def destroy_instance(self, instance_id: str) -> bool:
+        if not self._initialized:
+            raise RuntimeError("Provider not initialized. Call initialize() first.")
+        if instance_id not in self._instances:
+            return True
+
+        instance = self._instances.pop(instance_id)
+        client: paramiko.SSHClient = instance["client"]
+        sftp: paramiko.SFTPClient = instance["sftp"]
+        remote_work_dir: str = instance["remote_work_dir"]
+
+        cleanup_error: Optional[Exception] = None
+        try:
+            stdout, stderr, exit_code = self._run_remote_command(
+                client,
+                f"rm -rf {shlex.quote(remote_work_dir)}",
+                timeout=min(self.timeout, 10),
+            )
+            if exit_code != 0:
+                raise RuntimeError(stderr or stdout or "unknown error")
+        except Exception as exc:
+            cleanup_error = exc
+        finally:
+            try:
+                sftp.close()
+            finally:
+                client.close()
+
+        if cleanup_error is not None:
+            raise RuntimeError(f"Failed to clean remote workspace {remote_work_dir}: {cleanup_error}")
+        return True
+
+    def health_check(self) -> bool:
+        try:
+            self._assert_connectivity()
+            return True
+        except Exception:
+            return False
+
+    def _assert_connectivity(self) -> None:
+        try:
+            client = self._create_ssh_client()
+            try:
+                _, stderr, exit_code = self._run_remote_command(
+                    client,
+                    "true",
+                    timeout=min(self.timeout, 10),
+                )
+                if exit_code != 0:
+                    raise SandboxProviderConfigError(
+                        f"SSH connectivity check failed on {self.username}@{self.host}:{self.port}: "
+                        f"{stderr or 'remote command returned non-zero exit status'}"
+                    )
+            finally:
+                client.close()
+        except SandboxProviderConfigError:
+            raise
+        except Exception as exc:
+            raise SandboxProviderConfigError(
+                f"Failed to connect to SSH host {self.username}@{self.host}:{self.port}: {exc}"
+            ) from exc
+
+    def get_supported_languages(self) -> List[str]:
+        return ["python", "javascript", "nodejs"]
+
+    @staticmethod
+    def get_config_schema() -> Dict[str, Dict]:
+        return {
+            "host": {
+                "type": "string",
+                "required": True,
+                "label": "SSH Host",
+                "placeholder": "192.168.1.10",
+                "description": "Remote host that will execute generated code.",
+            },
+            "port": {
+                "type": "integer",
+                "required": True,
+                "label": "SSH Port",
+                "default": 22,
+                "min": 1,
+                "max": 65535,
+                "description": "SSH port on the remote host.",
+            },
+            "username": {
+                "type": "string",
+                "required": True,
+                "label": "SSH Username",
+                "placeholder": "ragflow",
+                "description": "Username used to connect to the remote host.",
+            },
+            "password": {
+                "type": "string",
+                "required": False,
+                "label": "SSH Password",
+                "secret": True,
+                "placeholder": "Optional when using a private key",
+                "description": "Password-based SSH authentication.",
+            },
+            "private_key": {
+                "type": "string",
+                "required": False,
+                "label": "SSH Private Key",
+                "secret": True,
+                "multiline": True,
+                "placeholder": "Paste PEM content or enter a local file path",
+                "description": "Private key PEM content or a readable private key path on the RAGFlow host.",
+            },
+            "passphrase": {
+                "type": "string",
+                "required": False,
+                "label": "Private Key Passphrase",
+                "secret": True,
+                "placeholder": "Optional",
+                "description": "Passphrase for the private key if it is encrypted.",
+            },
+            "python_bin": {
+                "type": "string",
+                "required": False,
+                "default": "python3",
+                "label": "Python Binary",
+                "description": "Python executable used for remote code execution.",
+            },
+            "node_bin": {
+                "type": "string",
+                "required": False,
+                "default": "node",
+                "label": "Node.js Binary",
+                "description": "Node.js executable used for remote JavaScript execution.",
+            },
+            "work_dir": {
+                "type": "string",
+                "required": False,
+                "label": "Remote Workspace Root",
+                "default": "/tmp",
+                "placeholder": "/tmp",
+                "description": "Writable remote directory used to create a temporary workspace.",
+            },
+            "timeout": {
+                "type": "integer",
+                "required": False,
+                "label": "Timeout (seconds)",
+                "default": 30,
+                "min": 1,
+                "max": 600,
+                "description": "Maximum SSH execution time for a single run.",
+            },
+            "max_output_bytes": {
+                "type": "integer",
+                "required": False,
+                "label": "Max Output Bytes",
+                "default": 1048576,
+                "min": 1024,
+                "max": 10485760,
+                "description": "Maximum combined stdout and stderr size.",
+            },
+            "max_artifacts": {
+                "type": "integer",
+                "required": False,
+                "label": "Max Artifacts",
+                "default": 20,
+                "min": 0,
+                "max": 100,
+                "description": "Maximum number of files collected from the remote artifacts directory.",
+            },
+            "max_artifact_bytes": {
+                "type": "integer",
+                "required": False,
+                "label": "Max Artifact Bytes",
+                "default": 10485760,
+                "min": 1024,
+                "max": 104857600,
+                "description": "Maximum size of a single artifact file in bytes.",
+            },
+        }
+
+    def validate_config(self, config: Dict[str, Any]) -> tuple[bool, Optional[str]]:
+        host = str(config.get("host", "") or "").strip()
+        username = str(config.get("username", "") or "").strip()
+        password = str(config.get("password", "") or "")
+        private_key = str(config.get("private_key", "") or "")
+        python_bin = str(config.get("python_bin", "python3") or "python3").strip()
+        node_bin = str(config.get("node_bin", "node") or "node").strip()
+
+        if not host:
+            return False, "SSH host is required"
+        if not username:
+            return False, "SSH username is required"
+        if not password and not private_key:
+            return False, "Either password or private_key must be provided"
+        if not python_bin:
+            return False, "Python binary is required"
+        if not node_bin:
+            return False, "Node.js binary is required"
+
+        try:
+            port = int(config.get("port", 22) or 22)
+        except (TypeError, ValueError):
+            return False, "SSH port must be an integer"
+        if port <= 0 or port > 65535:
+            return False, "SSH port must be between 1 and 65535"
+
+        for key in ("timeout", "max_output_bytes", "max_artifacts", "max_artifact_bytes"):
+            try:
+                value = int(config.get(key, 0) or 0)
+            except (TypeError, ValueError):
+                return False, f"{key} must be an integer"
+            if key == "max_artifacts":
+                if value < 0:
+                    return False, "max_artifacts must be greater than or equal to 0"
+            elif value <= 0:
+                return False, f"{key} must be greater than 0"
+
+        return True, None
+
+    def _create_ssh_client(self) -> paramiko.SSHClient:
+        paramiko = _get_paramiko_module()
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        connect_kwargs: dict[str, Any] = {
+            "hostname": self.host,
+            "port": self.port,
+            "username": self.username,
+            "timeout": self.timeout,
+            "banner_timeout": self.timeout,
+            "auth_timeout": self.timeout,
+            "look_for_keys": False,
+            "allow_agent": False,
+        }
+        if self.private_key:
+            connect_kwargs["pkey"] = self._load_private_key()
+        if self.password:
+            connect_kwargs["password"] = self.password
+
+        client.connect(**connect_kwargs)
+        return client
+
+    def _load_private_key(self) -> paramiko.PKey:
+        paramiko = _get_paramiko_module()
+        loaders = (
+            paramiko.RSAKey,
+            paramiko.Ed25519Key,
+            paramiko.ECDSAKey,
+            paramiko.DSSKey,
+        )
+        errors: list[str] = []
+        private_key_value = self.private_key.strip()
+        passphrase = self.passphrase or None
+
+        if os.path.exists(private_key_value):
+            for key_cls in loaders:
+                try:
+                    return key_cls.from_private_key_file(private_key_value, password=passphrase)
+                except Exception as exc:
+                    errors.append(str(exc))
+        else:
+            for key_cls in loaders:
+                try:
+                    return key_cls.from_private_key(io.StringIO(private_key_value), password=passphrase)
+                except Exception as exc:
+                    errors.append(str(exc))
+
+        raise SandboxProviderConfigError(
+            "Failed to load SSH private key. " + "; ".join(error for error in errors if error)
+        )
+
+    def _create_remote_workspace(self, client: paramiko.SSHClient) -> str:
+        base_dir = self.work_dir.rstrip("/") or "/tmp"
+        template = posixpath.join(base_dir, "ragflow-codeexec.XXXXXX")
+        stdout, stderr, exit_code = self._run_remote_command(
+            client,
+            f"mkdir -p {shlex.quote(base_dir)} && mktemp -d {shlex.quote(template)}",
+            timeout=min(self.timeout, 10),
+        )
+        if exit_code != 0:
+            raise RuntimeError(
+                f"Failed to create remote workspace on {self.host}: {stderr or stdout or 'unknown error'}"
+            )
+
+        remote_work_dir = stdout.strip().splitlines()[-1] if stdout.strip() else ""
+        if not remote_work_dir:
+            raise RuntimeError("Remote workspace creation did not return a path.")
+        return remote_work_dir
+
+    def _upload_script(
+        self,
+        sftp: paramiko.SFTPClient,
+        remote_work_dir: str,
+        language: str,
+        code: str,
+        args_json: str,
+    ) -> tuple[str, str]:
+        if language == "python":
+            script_name = "main.py"
+            script_content = build_python_wrapper(code, args_json)
+        elif language in {"javascript", "nodejs"}:
+            script_name = "main.js"
+            script_content = build_javascript_wrapper(code, args_json)
+        else:
+            raise RuntimeError(f"Unsupported language for SSH provider: {language}")
+
+        remote_script_path = posixpath.join(remote_work_dir, script_name)
+        with sftp.file(remote_script_path, "w") as remote_file:
+            remote_file.write(script_content)
+
+        command = self._build_execution_command(remote_work_dir, remote_script_path, language)
+        return remote_script_path, command
+
+    def _build_execution_command(self, remote_work_dir: str, remote_script_path: str, language: str) -> str:
+        normalized_lang = self._normalize_language(language)
+        if normalized_lang == "python":
+            executable = self.python_bin
+        elif normalized_lang == "nodejs":
+            executable = self.node_bin
+        else:
+            raise RuntimeError(f"Unsupported language for SSH provider: {language}")
+
+        return (
+            f"cd {shlex.quote(remote_work_dir)} && "
+            f"{shlex.quote(executable)} {shlex.quote(remote_script_path)}"
+        )
+
+    def _run_remote_command(
+        self,
+        client: paramiko.SSHClient,
+        command: str,
+        timeout: int,
+    ) -> tuple[str, str, int]:
+        stdin, stdout_stream, stderr_stream = client.exec_command(command, timeout=timeout)
+        stdin.close()
+        channel = stdout_stream.channel
+
+        stdout_chunks: list[bytes] = []
+        stderr_chunks: list[bytes] = []
+        deadline = time.time() + timeout
+
+        while True:
+            while channel.recv_ready():
+                stdout_chunks.append(channel.recv(65536))
+            while channel.recv_stderr_ready():
+                stderr_chunks.append(channel.recv_stderr(65536))
+
+            if channel.exit_status_ready():
+                break
+            if time.time() > deadline:
+                channel.close()
+                raise TimeoutError(f"Execution timed out after {timeout} seconds")
+            time.sleep(0.1)
+
+        while channel.recv_ready():
+            stdout_chunks.append(channel.recv(65536))
+        while channel.recv_stderr_ready():
+            stderr_chunks.append(channel.recv_stderr(65536))
+
+        exit_code = channel.recv_exit_status()
+        stdout = b"".join(stdout_chunks).decode("utf-8", errors="replace")
+        stderr = b"".join(stderr_chunks).decode("utf-8", errors="replace")
+        return stdout, stderr, exit_code
+
+    def _validate_output_size(self, stdout: str, stderr: str) -> None:
+        output_size = len((stdout or "").encode("utf-8")) + len((stderr or "").encode("utf-8"))
+        if output_size > self.max_output_bytes:
+            raise RuntimeError(f"SSH execution output exceeded {self.max_output_bytes} bytes.")
+
+    def _collect_artifacts(
+        self,
+        sftp: paramiko.SFTPClient,
+        artifacts_dir: str,
+    ) -> list[dict[str, Any]]:
+        artifacts: list[dict[str, Any]] = []
+        self._collect_artifacts_recursive(sftp, artifacts_dir, "", artifacts)
+        return artifacts
+
+    def _collect_artifacts_recursive(
+        self,
+        sftp: paramiko.SFTPClient,
+        current_dir: str,
+        relative_dir: str,
+        artifacts: list[dict[str, Any]],
+    ) -> None:
+        try:
+            entries = sftp.listdir_attr(current_dir)
+        except FileNotFoundError:
+            return
+
+        for entry in sorted(entries, key=lambda item: item.filename):
+            name = entry.filename
+            remote_path = posixpath.join(current_dir, name)
+            relative_path = posixpath.join(relative_dir, name) if relative_dir else name
+            mode = entry.st_mode
+            if mode is None:
+                mode = sftp.lstat(remote_path).st_mode
+            if mode is None:
+                raise RuntimeError(f"Unable to determine artifact entry type: {relative_path}")
+
+            if stat.S_ISLNK(mode):
+                raise RuntimeError(f"Artifact symlinks are not allowed: {relative_path}")
+            if stat.S_ISDIR(mode):
+                self._collect_artifacts_recursive(sftp, remote_path, relative_path, artifacts)
+                continue
+            if not stat.S_ISREG(mode):
+                raise RuntimeError(f"Unsupported artifact entry: {relative_path}")
+
+            if len(artifacts) >= self.max_artifacts:
+                raise RuntimeError(f"SSH execution produced more than {self.max_artifacts} artifacts.")
+
+            size = int(entry.st_size or 0)
+            if size > self.max_artifact_bytes:
+                raise RuntimeError(f"Artifact exceeds {self.max_artifact_bytes} bytes: {relative_path}")
+
+            ext = os.path.splitext(name)[1].lower()
+            if ext not in ALLOWED_ARTIFACT_EXTENSIONS:
+                raise RuntimeError(f"Unsupported artifact type: {relative_path}")
+
+            with sftp.file(remote_path, "rb") as artifact_file:
+                content = artifact_file.read()
+
+            artifacts.append(
+                {
+                    "name": relative_path,
+                    "content_b64": base64.b64encode(content).decode("ascii"),
+                    "mime_type": mimetypes.guess_type(name)[0] or "application/octet-stream",
+                    "size": size,
+                }
+            )
+
+    @staticmethod
+    def _normalize_language(language: str) -> str:
+        lang_lower = (language or "python").lower()
+        if lang_lower in {"python", "python3"}:
+            return "python"
+        if lang_lower in {"javascript", "nodejs"}:
+            return "nodejs"
+        return lang_lower
+
+
+def _get_paramiko_module():
+    try:
+        import paramiko
+    except ImportError as exc:
+        raise SandboxProviderConfigError(
+            "paramiko is required for the SSH sandbox provider. Install the project dependencies to enable it."
+        ) from exc
+    return paramiko

agent/sandbox/pyproject.toml

@@ -3,7 +3,7 @@ name = "gvisor-sandbox"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
-requires-python = ">=3.12,<3.15"
+requires-python = ">=3.13,<3.14"
 dependencies = [
     "fastapi>=0.115.12",
     "httpx>=0.28.1",

agent/sandbox/result_protocol.py

@@ -0,0 +1,85 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import base64
+import json
+from typing import Any
+
+
+RESULT_MARKER_PREFIX = "__RAGFLOW_RESULT__:"
+
+
+def build_python_wrapper(code: str, args_json: str) -> str:
+    return f'''{code}
+
+if __name__ == "__main__":
+    import base64
+    import json
+
+    result = main(**{args_json})
+    payload = json.dumps({{"present": True, "value": result, "type": "json"}}, ensure_ascii=False, separators=(",", ":"))
+    print("{RESULT_MARKER_PREFIX}" + base64.b64encode(payload.encode("utf-8")).decode("ascii"))
+'''
+
+
+def build_javascript_wrapper(code: str, args_json: str) -> str:
+    return f'''{code}
+
+const __ragflowArgs = {args_json};
+
+(async () => {{
+  const __ragflowMain = typeof main !== 'undefined' ? main : module.exports && module.exports.main;
+  if (typeof __ragflowMain !== 'function') {{
+    throw new Error('main() must be defined or exported.');
+  }}
+  const output = await Promise.resolve(__ragflowMain(__ragflowArgs));
+  if (typeof output === 'undefined') {{
+    throw new Error('main() must return a value. Use null for an empty result.');
+  }}
+  const payload = JSON.stringify({{ present: true, value: output, type: 'json' }});
+  if (typeof payload === 'undefined') {{
+    throw new Error('main() returned a non-JSON-serializable value.');
+  }}
+  console.log('{RESULT_MARKER_PREFIX}' + Buffer.from(payload, 'utf8').toString('base64'));
+}})();
+'''
+
+
+def extract_structured_result(stdout: str) -> tuple[str, dict[str, Any]]:
+    if not stdout:
+        return "", {}
+
+    cleaned_lines: list[str] = []
+    structured_result: dict[str, Any] = {}
+
+    for line in str(stdout).splitlines():
+        if line.startswith(RESULT_MARKER_PREFIX):
+            payload_b64 = line[len(RESULT_MARKER_PREFIX) :].strip()
+            if not payload_b64:
+                cleaned_lines.append(line)
+                continue
+            try:
+                payload = base64.b64decode(payload_b64, validate=True).decode("utf-8")
+                structured_result = json.loads(payload)
+            except Exception:
+                cleaned_lines.append(line)
+            continue
+        cleaned_lines.append(line)
+
+    cleaned_stdout = "\n".join(cleaned_lines)
+    if stdout.endswith("\n") and cleaned_stdout and not cleaned_stdout.endswith("\n"):
+        cleaned_stdout += "\n"
+    return cleaned_stdout, structured_result

agent/sandbox/sandbox_base_image/nodejs/Dockerfile

@@ -1,6 +1,12 @@
 FROM node:24.13-bookworm-slim
 
-RUN npm config set registry https://registry.npmmirror.com
+ARG NEED_MIRROR=1
+
+RUN if [ "$NEED_MIRROR" = 1 ]; then \
+        npm config set registry https://registry.npmmirror.com; \
+    else \
+        npm config set registry https://registry.npmjs.org; \
+    fi
 
 # RUN grep -rl 'deb.debian.org' /etc/apt/ | xargs sed -i 's|http[s]*://deb.debian.org|https://mirrors.ustc.edu.cn|g' && \
 #     apt-get update && \

agent/sandbox/sandbox_base_image/python/Dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.11-slim-bookworm
 
+ARG NEED_MIRROR=1
+
 COPY --from=ghcr.io/astral-sh/uv:0.7.5 /uv /uvx /bin/
-ENV UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
 ENV MPLBACKEND=Agg
 ENV MPLCONFIGDIR=/tmp/matplotlib
 ENV MATPLOTLIBRC=/usr/local/etc/matplotlibrc
@@ -9,12 +10,18 @@ ENV MATPLOTLIBRC=/usr/local/etc/matplotlibrc
 COPY requirements.txt .
 COPY matplotlibrc /usr/local/etc/matplotlibrc
 
-RUN grep -rl 'deb.debian.org' /etc/apt/ | xargs sed -i 's|http[s]*://deb.debian.org|https://mirrors.tuna.tsinghua.edu.cn|g' && \
+RUN if [ "$NEED_MIRROR" = 1 ]; then \
+        grep -rl 'deb.debian.org' /etc/apt/ | xargs sed -i 's|http[s]*://deb.debian.org|https://mirrors.tuna.tsinghua.edu.cn|g'; \
+        export UV_INDEX_URL="https://pypi.tuna.tsinghua.edu.cn/simple"; \
+    else \
+        export UV_INDEX_URL="https://pypi.org/simple"; \
+    fi; \
     apt-get update && \
-    apt-get install -y curl gcc && \
+    apt-get install -y --no-install-recommends curl gcc && \
     mkdir -p /tmp/matplotlib && \
-    uv pip install --system -r requirements.txt
+    uv pip install --system -r requirements.txt && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /workspace
 
-CMD ["sleep", "infinity"]
+CMD ["sleep", "infinity"]

agent/sandbox/tests/test_security.py

@@ -45,6 +45,60 @@ def test_javascript_eval_is_rejected():
     assert any("eval" in issue.lower() for issue, _ in issues)
 
 
+def test_javascript_child_process_template_literal_is_rejected():
+    """Template literal backticks bypass single/double-quote regex patterns."""
+    is_safe, issues = analyze_code_security(
+        "const cp = require(`child_process`); async function main() { return 'ok'; }",
+        SupportLanguage.NODEJS,
+    )
+
+    assert is_safe is False
+    assert any("child_process" in issue for issue, _ in issues)
+
+
+def test_javascript_fs_template_literal_is_rejected():
+    is_safe, issues = analyze_code_security(
+        "const fs = require(`fs`); async function main() { return fs.readFileSync('/etc/passwd', 'utf8'); }",
+        SupportLanguage.NODEJS,
+    )
+
+    assert is_safe is False
+    assert any("fs" in issue for issue, _ in issues)
+
+
+def test_python_builtins_import_is_rejected():
+    """builtins module gives access to eval/exec and must be blocked."""
+    is_safe, issues = analyze_code_security(
+        "import builtins\ndef main():\n    builtins.eval('1+1')",
+        SupportLanguage.PYTHON,
+    )
+
+    assert is_safe is False
+    # Pin the specific reason: rejection must come from the new ``builtins``
+    # entry in ``DANGEROUS_IMPORTS``, not from some unrelated parse error.
+    assert any("builtins" in issue for issue, _ in issues), (
+        f"expected an issue mentioning 'builtins', got {issues!r}"
+    )
+
+
+def test_python_attribute_eval_call_is_rejected():
+    """Attribute-style dangerous calls (builtins.eval) must be caught."""
+    is_safe, issues = analyze_code_security(
+        "import builtins\ndef main():\n    builtins.exec('import os')",
+        SupportLanguage.PYTHON,
+    )
+
+    assert is_safe is False
+    # Pin the specific reason: rejection must come from the new
+    # ``ast.Attribute`` branch in ``visit_Call`` flagging the ``exec`` call,
+    # not from the ``import builtins`` line above. We assert ``exec`` is in at
+    # least one finding so the test fails if visit_Call's attribute branch is
+    # ever reverted.
+    assert any("exec" in issue for issue, _ in issues), (
+        f"expected an issue mentioning 'exec', got {issues!r}"
+    )
+
+
 def test_javascript_safe_code_still_passes():
     is_safe, issues = analyze_code_security(
         "async function main(args) { return { answer: args.value ?? null }; }",

agent/sandbox/uv.lock

@@ -1,6 +1,6 @@
 version = 1
 revision = 3
-requires-python = ">=3.12, <3.15"
+requires-python = "==3.13.*"
 
 [[package]]
 name = "annotated-doc"
@@ -27,7 +27,6 @@ source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
 dependencies = [
     { name = "idna" },
     { name = "sniffio" },
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949, upload-time = "2025-03-17T00:02:54.77Z" }
 wheels = [
@@ -61,19 +60,6 @@ version = "3.4.2"
 source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" }
 wheels = [
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" },
@@ -278,20 +264,6 @@ dependencies = [
 ]
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" }
 wheels = [
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/18/8a/2b41c97f554ec8c71f2a8a5f85cb56a8b0956addfe8b0efb5b3d77e8bdc3/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc", size = 2009000, upload-time = "2025-04-23T18:31:25.863Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a1/02/6224312aacb3c8ecbaa959897af57181fb6cf3a3d7917fd44d0f2917e6f2/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7", size = 1847996, upload-time = "2025-04-23T18:31:27.341Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/d6/46/6dcdf084a523dbe0a0be59d054734b86a981726f221f4562aed313dbcb49/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e61206137cbc65e6d5256e1166f88331d3b6238e082d9f74613b9b765fb9025", size = 1880957, upload-time = "2025-04-23T18:31:28.956Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ec/6b/1ec2c03837ac00886ba8160ce041ce4e325b41d06a034adbef11339ae422/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb8c529b2819c37140eb51b914153063d27ed88e3bdc31b71198a198e921e011", size = 1964199, upload-time = "2025-04-23T18:31:31.025Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2d/1d/6bf34d6adb9debd9136bd197ca72642203ce9aaaa85cfcbfcf20f9696e83/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c52b02ad8b4e2cf14ca7b3d918f3eb0ee91e63b3167c32591e57c4317e134f8f", size = 2120296, upload-time = "2025-04-23T18:31:32.514Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/e0/94/2bd0aaf5a591e974b32a9f7123f16637776c304471a0ab33cf263cf5591a/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:96081f1605125ba0855dfda83f6f3df5ec90c61195421ba72223de35ccfb2f88", size = 2676109, upload-time = "2025-04-23T18:31:33.958Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/f9/41/4b043778cf9c4285d59742281a769eac371b9e47e35f98ad321349cc5d61/pydantic_core-2.33.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f57a69461af2a5fa6e6bbd7a5f60d3b7e6cebb687f55106933188e79ad155c1", size = 2002028, upload-time = "2025-04-23T18:31:39.095Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/cb/d5/7bb781bf2748ce3d03af04d5c969fa1308880e1dca35a9bd94e1a96a922e/pydantic_core-2.33.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:572c7e6c8bb4774d2ac88929e3d1f12bc45714ae5ee6d9a788a9fb35e60bb04b", size = 2100044, upload-time = "2025-04-23T18:31:41.034Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/fe/36/def5e53e1eb0ad896785702a5bbfd25eed546cdcf4087ad285021a90ed53/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:db4b41f9bd95fbe5acd76d89920336ba96f03e149097365afe1cb092fceb89a1", size = 2058881, upload-time = "2025-04-23T18:31:42.757Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/01/6c/57f8d70b2ee57fc3dc8b9610315949837fa8c11d86927b9bb044f8705419/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:fa854f5cf7e33842a892e5c73f45327760bc7bc516339fda888c75ae60edaeb6", size = 2227034, upload-time = "2025-04-23T18:31:44.304Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/27/b9/9c17f0396a82b3d5cbea4c24d742083422639e7bb1d5bf600e12cb176a13/pydantic_core-2.33.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5f483cfb75ff703095c59e365360cb73e00185e01aaea067cd19acffd2ab20ea", size = 2234187, upload-time = "2025-04-23T18:31:45.891Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/b0/6a/adf5734ffd52bf86d865093ad70b2ce543415e0e356f6cacabbc0d9ad910/pydantic_core-2.33.2-cp312-cp312-win32.whl", hash = "sha256:9cb1da0f5a471435a7bc7e439b8a728e8b61e59784b2af70d7c169f8dd8ae290", size = 1892628, upload-time = "2025-04-23T18:31:47.819Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/43/e4/5479fecb3606c1368d496a825d8411e126133c41224c1e7238be58b87d7e/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl", hash = "sha256:f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2", size = 1955866, upload-time = "2025-04-23T18:31:49.635Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/0d/24/8b11e8b3e2be9dd82df4b11408a67c61bb4dc4f8e11b5b0fc888b38118b5/pydantic_core-2.33.2-cp312-cp312-win_arm64.whl", hash = "sha256:cca3868ddfaccfbc4bfb1d608e2ccaaebe0ae628e1416aeb9c4d88c001bb45ab", size = 1888894, upload-time = "2025-04-23T18:31:51.609Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" },
@@ -353,7 +325,6 @@ version = "0.49.1"
 source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
 dependencies = [
     { name = "anyio" },
-    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
 ]
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/1b/3f/507c21db33b66fb027a332f2cb3abbbe924cc3a79ced12f01ed8645955c9/starlette-0.49.1.tar.gz", hash = "sha256:481a43b71e24ed8c43b11ea02f5353d77840e01480881b8cb5a26b8cae64a8cb", size = 2654703, upload-time = "2025-10-28T17:34:10.928Z" }
 wheels = [
@@ -383,11 +354,11 @@ wheels = [
 
 [[package]]
 name = "urllib3"
-version = "2.6.3"
+version = "2.7.0"
 source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
-sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" }
+sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
 wheels = [
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" },
+    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
 ]
 
 [[package]]
@@ -409,17 +380,6 @@ version = "1.17.2"
 source = { registry = "https://pypi.tuna.tsinghua.edu.cn/simple" }
 sdist = { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c3/fc/e91cc220803d7bc4db93fb02facd8461c37364151b8494762cc88b0fbcef/wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", size = 55531, upload-time = "2025-01-14T10:35:45.465Z" }
 wheels = [
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/a1/bd/ab55f849fd1f9a58ed7ea47f5559ff09741b25f00c191231f9f059c83949/wrapt-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925", size = 53799, upload-time = "2025-01-14T10:33:57.4Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/53/18/75ddc64c3f63988f5a1d7e10fb204ffe5762bc663f8023f18ecaf31a332e/wrapt-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392", size = 38821, upload-time = "2025-01-14T10:33:59.334Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/48/2a/97928387d6ed1c1ebbfd4efc4133a0633546bec8481a2dd5ec961313a1c7/wrapt-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40", size = 38919, upload-time = "2025-01-14T10:34:04.093Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/73/54/3bfe5a1febbbccb7a2f77de47b989c0b85ed3a6a41614b104204a788c20e/wrapt-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d", size = 88721, upload-time = "2025-01-14T10:34:07.163Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/25/cb/7262bc1b0300b4b64af50c2720ef958c2c1917525238d661c3e9a2b71b7b/wrapt-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b", size = 80899, upload-time = "2025-01-14T10:34:09.82Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/2a/5a/04cde32b07a7431d4ed0553a76fdb7a61270e78c5fd5a603e190ac389f14/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98", size = 89222, upload-time = "2025-01-14T10:34:11.258Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/09/28/2e45a4f4771fcfb109e244d5dbe54259e970362a311b67a965555ba65026/wrapt-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82", size = 86707, upload-time = "2025-01-14T10:34:12.49Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c6/d2/dcb56bf5f32fcd4bd9aacc77b50a539abdd5b6536872413fd3f428b21bed/wrapt-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae", size = 79685, upload-time = "2025-01-14T10:34:15.043Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/80/4e/eb8b353e36711347893f502ce91c770b0b0929f8f0bed2670a6856e667a9/wrapt-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9", size = 87567, upload-time = "2025-01-14T10:34:16.563Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/17/27/4fe749a54e7fae6e7146f1c7d914d28ef599dacd4416566c055564080fe2/wrapt-1.17.2-cp312-cp312-win32.whl", hash = "sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9", size = 36672, upload-time = "2025-01-14T10:34:17.727Z" },
-    { url = "https://pypi.tuna.tsinghua.edu.cn/packages/15/06/1dbf478ea45c03e78a6a8c4be4fdc3c3bddea5c8de8a93bc971415e47f0f/wrapt-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991", size = 38865, upload-time = "2025-01-14T10:34:19.577Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/ce/b9/0ffd557a92f3b11d4c5d5e0c5e4ad057bd9eb8586615cdaf901409920b14/wrapt-1.17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125", size = 53800, upload-time = "2025-01-14T10:34:21.571Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/c0/ef/8be90a0b7e73c32e550c73cfb2fa09db62234227ece47b0e80a05073b375/wrapt-1.17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998", size = 38824, upload-time = "2025-01-14T10:34:22.999Z" },
     { url = "https://pypi.tuna.tsinghua.edu.cn/packages/36/89/0aae34c10fe524cce30fe5fc433210376bce94cf74d05b0d68344c8ba46e/wrapt-1.17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5", size = 38920, upload-time = "2025-01-14T10:34:25.386Z" },

agent/templates/ingestion_pipeline_resume.json

@@ -242,13 +242,14 @@
                             "include_heading_content": false,
                             "levels": [
                                 [
-                                    "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:\uff1a]?\\s*$"                                
+                                    "^\\s*(?i:(?:\\d+[\\.\\)]\\s*)?(?:EDUCATION|ACADEMIC\\s*BACKGROUND|ACADEMIC\\s*HISTORY|EDUCATIONAL\\s*BACKGROUND|RELEVANT\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\s*EXPERIENCE|PROFESSIONAL\\s*EXPERIENCE|RELEVANT\\s*EXPERIENCE|EMPLOYMENT\\s*HISTORY|CAREER\\s*HISTORY|INTERNSHIP\\s*EXPERIENCE|PROJECTS|PROJECT\\s*EXPERIENCE|ACADEMIC\\s*PROJECTS|PROFESSIONAL\\s*PROJECTS|SKILLS|TECHNICAL\\s*SKILLS|CORE\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\s*OF\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\s*ACTIVITIES|ACTIVITIES\\s*(?:&|AND)\\s*SKILLS|INVOLVEMENT|CAMPUS\\s*INVOLVEMENT|VOLUNTEER\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\s*PROFILE|SUMMARY|PROFESSIONAL\\s*SUMMARY|CAREER\\s*SUMMARY|OBJECTIVE|CAREER\\s*OBJECTIVE|PERSONAL\\s*INFORMATION|CONTACT\\s*INFORMATION|ADDITIONAL\\s*INFORMATION|TRAINING))\\s*[:\uff1a]?\\s*$"
                                 ],
                                 [
                                     "^\\s*(?:\\d+[\\.\u3001\\)]\\s*)?(?:\u6559\u80b2\u80cc\u666f|\u6559\u80b2\u7ecf\u5386|\u5b66\u5386\u80cc\u666f|\u5b66\u672f\u80cc\u666f|\u6280\u672f\u80cc\u666f|\u5de5\u4f5c\u7ecf\u5386|\u5de5\u4f5c\u7ecf\u9a8c|\u5b9e\u4e60\u7ecf\u5386|\u9879\u76ee\u7ecf\u5386|\u9879\u76ee\u7ecf\u9a8c|\u79d1\u7814\u7ecf\u5386|\u7814\u7a76\u7ecf\u5386|\u6821\u56ed\u7ecf\u5386|\u5b9e\u8df5\u7ecf\u5386|\u4e13\u4e1a\u7ecf\u5386|\u804c\u4e1a\u7ecf\u5386|\u6280\u80fd|\u4e13\u4e1a\u6280\u80fd|\u6280\u80fd\u7279\u957f|\u6838\u5fc3\u6280\u80fd|\u6280\u672f\u6808|\u4e2a\u4eba\u6280\u80fd|\u5de5\u4f5c\u6280\u80fd|\u804c\u4e1a\u6280\u80fd|\u6280\u80fd\u4e0e\u8bc4\u4ef7|\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u5de5\u4f5c\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u804c\u4e1a\u6280\u80fd\u4e0e\u81ea\u6211\u8bc4\u4ef7|\u8bc1\u4e66|\u8d44\u683c\u8bc1\u4e66|\u804c\u4e1a\u8d44\u683c|\u8d44\u8d28\u8bc1\u4e66|\u83b7\u5956\u60c5\u51b5|\u83b7\u5956\u7ecf\u5386|\u8363\u8a89|\u8363\u8a89\u5956\u9879|\u5956\u9879|\u79d1\u7814\u6210\u679c|\u8bba\u6587\u53d1\u8868|\u53d1\u8868\u8bba\u6587|\u9886\u5bfc\u7ecf\u5386|\u5b66\u751f\u5de5\u4f5c|\u6821\u56ed\u6d3b\u52a8|\u793e\u56e2\u7ecf\u5386|\u6d3b\u52a8\u7ecf\u5386|\u5fd7\u613f\u7ecf\u5386|\u5fd7\u613f\u670d\u52a1|\u793e\u4f1a\u5b9e\u8df5|\u8bed\u8a00\u80fd\u529b|\u8bed\u8a00|\u81ea\u6211\u8bc4\u4ef7|\u4e2a\u4eba\u8bc4\u4ef7|\u81ea\u6211\u603b\u7ed3|\u4e2a\u4eba\u603b\u7ed3|\u4e2a\u4eba\u4f18\u52bf|\u4e2a\u4eba\u7b80\u4ecb|\u4e2a\u4eba\u4fe1\u606f|\u57fa\u672c\u4fe1\u606f|\u8054\u7cfb\u65b9\u5f0f|\u6c42\u804c\u610f\u5411|\u5e94\u8058\u610f\u5411|\u804c\u4e1a\u76ee\u6807|\u6c42\u804c\u76ee\u6807|\u5174\u8da3\u7231\u597d|\u5174\u8da3\u7279\u957f|\u57f9\u8bad\u7ecf\u5386|\u5176\u4ed6\u4fe1\u606f|\u9644\u52a0\u4fe1\u606f)\\s*[:\uff1a]?\\s*$"
                                 ]
                             ],
-                            "method": "hierarchy"
+                            "method": "hierarchy",
+                            "root_chunk_as_heading": true
                         }
                     },
                     "upstream": [
@@ -299,16 +300,6 @@
                         "target": "TitleChunker:FlatMiceFix",
                         "targetHandle": "end"
                     },
-                    {
-                        "data": {
-                            "isHovered": false
-                        },
-                        "id": "xy-edge__TitleChunker:FlatMiceFixstart-Extractor:ThreeDrinksActend",
-                        "source": "TitleChunker:FlatMiceFix",
-                        "sourceHandle": "start",
-                        "target": "Extractor:ThreeDrinksAct",
-                        "targetHandle": "end"
-                    },
                     {
                         "data": {
                             "isHovered": false
@@ -321,6 +312,19 @@
                         "targetHandle": "end",
                         "type": "buttonEdge",
                         "zIndex": 1001
+                    },
+                    {
+                        "data": {
+                            "isHovered": false
+                        },
+                        "id": "xy-edge__TitleChunker:FlatMiceFixstart-Extractor:ThreeDrinksActend",
+                        "markerEnd": "logo",
+                        "source": "TitleChunker:FlatMiceFix",
+                        "sourceHandle": "start",
+                        "target": "Extractor:ThreeDrinksAct",
+                        "targetHandle": "end",
+                        "type": "buttonEdge",
+                        "zIndex": 1001
                     }
                 ],
                 "nodes": [
@@ -331,7 +335,7 @@
                         },
                         "id": "File",
                         "measured": {
-                            "height": 50,
+                            "height": 49,
                             "width": 200
                         },
                         "position": {
@@ -460,7 +464,7 @@
                         "dragging": false,
                         "id": "Parser:HipSignsRhyme",
                         "measured": {
-                            "height": 198,
+                            "height": 197,
                             "width": 200
                         },
                         "position": {
@@ -489,12 +493,12 @@
                         "dragging": false,
                         "id": "Tokenizer:KindHandsWin",
                         "measured": {
-                            "height": 114,
+                            "height": 113,
                             "width": 200
                         },
                         "position": {
-                            "x": 876.4654525205967,
-                            "y": 189.1906747329592
+                            "x": 883.0243372012395,
+                            "y": 156.39625132974524
                         },
                         "selected": false,
                         "sourcePosition": "right",
@@ -514,6 +518,7 @@
                                     }
                                 },
                                 "promote_first_heading_to_root": false,
+                                "root_chunk_as_heading": true,
                                 "rules": [
                                     {
                                         "levels": [
@@ -537,14 +542,14 @@
                         "dragging": false,
                         "id": "TitleChunker:FlatMiceFix",
                         "measured": {
-                            "height": 74,
+                            "height": 73,
                             "width": 200
                         },
                         "position": {
                             "x": 572.7908769627791,
                             "y": 141.55515313482098
                         },
-                        "selected": false,
+                        "selected": true,
                         "sourcePosition": "right",
                         "targetPosition": "left",
                         "type": "chunkerNode"
@@ -580,12 +585,12 @@
                         "dragging": false,
                         "id": "Extractor:ThreeDrinksAct",
                         "measured": {
-                            "height": 90,
+                            "height": 89,
                             "width": 200
                         },
                         "position": {
-                            "x": 583.3659219536569,
-                            "y": 274.7600100230409
+                            "x": 623.8123774842874,
+                            "y": 236.49984938595793
                         },
                         "selected": false,
                         "sourcePosition": "right",

agent/templates/stock_market_research_assistant.json

@@ -57,7 +57,7 @@
                                     "component_name": "TavilySearch",
                                     "name": "TavilySearch",
                                     "params": {
-                                        "api_key": "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",
+                                        "api_key": "",
                                         "days": 7,
                                         "exclude_domains": [],
                                         "include_answer": false,
@@ -651,7 +651,7 @@
                                         "component_name": "TavilySearch",
                                         "name": "TavilySearch",
                                         "params": {
-                                            "api_key": "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",
+                                            "api_key": "",
                                             "days": 7,
                                             "exclude_domains": [],
                                             "include_answer": false,

agent/tools/base.py

@@ -19,11 +19,12 @@ import time
 from copy import deepcopy
 import asyncio
 from functools import partial
+from collections.abc import Mapping
 from typing import TypedDict, List, Any
 from agent.component.base import ComponentParamBase, ComponentBase
 from common.misc_utils import hash_str2int
 from rag.prompts.generator import kb_prompt
-from common.mcp_tool_call_conn import MCPToolCallSession, ToolCallSession
+from common.mcp_tool_call_conn import MCPToolBinding, MCPToolCallSession, ToolCallSession
 from timeit import default_timer as timer
 
 
@@ -52,21 +53,38 @@ class LLMToolPluginCallSession(ToolCallSession):
         self.tools_map = tools_map
         self.callback = callback
 
-    def tool_call(self, name: str, arguments: dict[str, Any]) -> Any:
-        return asyncio.run(self.tool_call_async(name, arguments))
+    def tool_call(self, name: str, arguments: dict[str, Any], timeout: float | int = 10) -> Any:
+        return asyncio.run(self.tool_call_async(name, arguments, request_timeout=timeout))
 
-    async def tool_call_async(self, name: str, arguments: dict[str, Any]) -> Any:
+    async def tool_call_async(self, name: str, arguments: dict[str, Any], request_timeout: float | int = 10) -> Any:
         assert name in self.tools_map, f"LLM tool {name} does not exist"
         logging.info(f"[ToolCall] invoke name={name} arguments={str(arguments)[:200]}")
+        if not isinstance(arguments, Mapping):
+            raise TypeError(f"Tool arguments for {name} must be an object, got {type(arguments).__name__}")
         st = timer()
         tool_obj = self.tools_map[name]
-        if isinstance(tool_obj, MCPToolCallSession):
-            resp = await thread_pool_exec(tool_obj.tool_call, name, arguments, 60)
+        if isinstance(tool_obj, MCPToolBinding):
+            resp = await thread_pool_exec(tool_obj.session.tool_call, tool_obj.original_name, arguments, request_timeout)
+        elif isinstance(tool_obj, MCPToolCallSession):
+            resp = await thread_pool_exec(tool_obj.tool_call, name, arguments, request_timeout)
         elif hasattr(tool_obj, "invoke_async") and asyncio.iscoroutinefunction(tool_obj.invoke_async):
             resp = await tool_obj.invoke_async(**arguments)
         else:
             resp = await thread_pool_exec(tool_obj.invoke, **arguments)
 
+        if resp is None and hasattr(tool_obj, "output") and callable(tool_obj.output):
+            try:
+                fallback_output = tool_obj.output()
+                if isinstance(fallback_output, dict) and fallback_output.get("content") not in (None, ""):
+                    resp = fallback_output["content"]
+                elif fallback_output not in (None, ""):
+                    resp = fallback_output
+                else:
+                    resp = fallback_output
+                logging.warning(f"[ToolCall] resp is None, fallback to output name={name} output_keys={list(fallback_output.keys()) if isinstance(fallback_output, dict) else type(fallback_output).__name__}")
+            except Exception as e:
+                logging.warning(f"[ToolCall] resp is None and output fallback failed name={name} err={e}")
+
         elapsed = timer() - st
         logging.info(f"[ToolCall] done name={name} elapsed={elapsed:.2f}s result={str(resp)[:200]}")
         self.callback(name, arguments, resp, elapsed_time=elapsed)

agent/tools/code_exec.py

@@ -24,7 +24,7 @@ from collections.abc import Mapping
 from typing import Optional
 
 from pydantic import BaseModel, Field, field_validator
-from strenum import StrEnum
+from enum import StrEnum
 
 from agent.tools.base import ToolBase, ToolMeta, ToolParamBase
 from api.db.services.file_service import FileService
@@ -37,6 +37,7 @@ SYSTEM_OUTPUT_KEYS = frozenset(
     {
         "content",
         "actual_type",
+        "attachments",
         "_ERROR",
         "_ARTIFACTS",
         "_ATTACHMENT_CONTENT",
@@ -312,7 +313,10 @@ module.exports = { main };
         self.lang = Language.PYTHON.value
         self.script = 'def main(arg1: str, arg2: str) -> dict: return {"result": arg1 + arg2}'
         self.arguments = {}
-        self.outputs = {"result": {"value": "", "type": "object"}}
+        self.outputs = {
+            "result": {"value": "", "type": "object"},
+            "attachments": {"value": [], "type": "Array<String>"},
+        }
 
     def check(self):
         self.check_valid_value(self.lang, "Support languages", ["python", "python3", "nodejs", "javascript"])
@@ -357,10 +361,21 @@ class CodeExec(ToolBase, ABC):
             # Try using the new sandbox provider system first
             try:
                 from agent.sandbox.client import execute_code as sandbox_execute_code
+                from agent.sandbox.client import get_provider_info
+                from agent.sandbox.client import reload_provider
+                from agent.sandbox.providers.base import SandboxProviderConfigError
 
                 if self.check_if_canceled("CodeExec execution"):
                     return
 
+                reload_provider()
+                provider_info = get_provider_info()
+                provider_type = provider_info.get("provider_type") or "unknown"
+                logging.info(
+                    f"[CodeExec]: dispatching execution to sandbox provider '{provider_type}' "
+                    f"(language={language}, timeout={timeout_seconds}s)"
+                )
+
                 # Execute code using the provider system
                 result = sandbox_execute_code(code=code, language=language, timeout=timeout_seconds, arguments=arguments)
 
@@ -371,14 +386,20 @@ class CodeExec(ToolBase, ABC):
                 return self._process_execution_result(
                     result.stdout,
                     result.stderr,
-                    "Provider system",
+                    f"Provider system ({provider_type})",
                     artifacts,
                     execution_metadata=result.metadata,
                 )
 
-            except (ImportError, RuntimeError) as provider_error:
-                # Provider system not available or not configured, fall back to HTTP
+            except SandboxProviderConfigError as provider_error:
+                self.set_output("_ERROR", str(provider_error))
+                return self.output()
+            except ImportError as provider_error:
+                # Provider modules are unavailable, fall back to legacy HTTP sandbox.
                 logging.info(f"[CodeExec]: Provider system not available, using HTTP fallback: {provider_error}")
+            except RuntimeError as provider_error:
+                self.set_output("_ERROR", f"Provider system execution failed: {provider_error}")
+                return self.output()
 
             # Fallback to direct HTTP request
             code_b64 = self._encode_code(code)
@@ -459,11 +480,13 @@ class CodeExec(ToolBase, ABC):
             self.set_output("_ARTIFACTS", artifact_urls or None)
             attachment_text = self._build_attachment_content(artifacts, artifact_urls)
             self.set_output("_ATTACHMENT_CONTENT", attachment_text)
+            self.set_output("attachments", self._build_attachment_markdown_list(artifact_urls))
             if attachment_text:
                 content_parts.append(attachment_text)
         else:
             self.set_output("_ARTIFACTS", None)
             self.set_output("_ATTACHMENT_CONTENT", "")
+            self.set_output("attachments", [])
 
         self.set_output("content", "\n\n".join([part for part in content_parts if part]).strip())
 
@@ -533,7 +556,7 @@ class CodeExec(ToolBase, ABC):
 
                 settings.STORAGE_IMPL.put(SANDBOX_ARTIFACT_BUCKET, storage_name, binary)
 
-                url = f"/v1/document/artifact/{storage_name}"
+                url = f"/api/v1/documents/artifact/{storage_name}"
                 uploaded.append(
                     {
                         "name": name,
@@ -623,6 +646,23 @@ class CodeExec(ToolBase, ABC):
             return f"attachment_count: {len(sections)}\n\n" + "\n\n".join(sections)
         return "attachment_count: 0"
 
+    def _build_attachment_markdown_list(self, artifact_urls: list[dict]) -> list[str]:
+        markdown_items = []
+        for art in artifact_urls:
+            name = _art_field(art, "name")
+            url = _art_field(art, "url")
+            mime_type = str(_art_field(art, "mime_type") or "").strip().lower()
+            if not name:
+                continue
+
+            if mime_type.startswith("image/") and url:
+                markdown_items.append(f"![{name}]({url})")
+            elif url:
+                markdown_items.append(f"[Download {name}]({url})")
+            else:
+                markdown_items.append(name)
+        return markdown_items
+
     def _normalize_attachment_type(self, name: str, mime_type: str) -> str:
         mime_type = str(mime_type or "").strip().lower()
         if mime_type.startswith("image/"):

agent/tools/crawler.py

@@ -19,7 +19,6 @@ from crawl4ai import AsyncWebCrawler
 from agent.tools.base import ToolParamBase, ToolBase
 
 
-
 class CrawlerParam(ToolParamBase):
     """
     Define the Crawler component parameters.
@@ -31,20 +30,26 @@ class CrawlerParam(ToolParamBase):
         self.extract_type = "markdown"
 
     def check(self):
-        self.check_valid_value(self.extract_type, "Type of content from the crawler", ['html', 'markdown', 'content'])
+        self.check_valid_value(self.extract_type, "Type of content from the crawler", ["html", "markdown", "content"])
 
 
 class Crawler(ToolBase, ABC):
     component_name = "Crawler"
 
     def _run(self, history, **kwargs):
-        from api.utils.web_utils import is_valid_url
+        from common.ssrf_guard import assert_url_is_safe, pin_dns_global
+
         ans = self.get_input()
         ans = " - ".join(ans["content"]) if "content" in ans else ""
-        if not is_valid_url(ans):
+        try:
+            _ssrf_hostname, _ssrf_ip = assert_url_is_safe(ans)
+        except ValueError:
             return Crawler.be_output("URL not valid")
         try:
-            result = asyncio.run(self.get_web(ans))
+            # pin_dns_global is used (not thread-local) because crawl4ai resolves
+            # DNS in asyncio executor threads that don't share thread-local state.
+            with pin_dns_global(_ssrf_hostname, _ssrf_ip):
+                result = asyncio.run(self.get_web(ans))
 
             return Crawler.be_output(result)
 
@@ -57,18 +62,15 @@ class Crawler(ToolBase, ABC):
 
         proxy = self._param.proxy if self._param.proxy else None
         async with AsyncWebCrawler(verbose=True, proxy=proxy) as crawler:
-            result = await crawler.arun(
-                url=url,
-                bypass_cache=True
-            )
+            result = await crawler.arun(url=url, bypass_cache=True)
 
             if self.check_if_canceled("Crawler async operation"):
                 return
 
-            if self._param.extract_type == 'html':
+            if self._param.extract_type == "html":
                 return result.cleaned_html
-            elif self._param.extract_type == 'markdown':
+            elif self._param.extract_type == "markdown":
                 return result.markdown
-            elif self._param.extract_type == 'content':
+            elif self._param.extract_type == "content":
                 return result.extracted_content
             return result.markdown

agent/tools/exesql.py

@@ -64,9 +64,9 @@ class ExeSQLParam(ToolParamBase):
         self.check_positive_integer(self.max_records, "Maximum number of records")
         if self.database == "rag_flow":
             if self.host == "ragflow-mysql":
-                raise ValueError("For the security reason, it dose not support database named rag_flow.")
+                raise ValueError("For the security reason, it does not support database named rag_flow.")
             if self.password == "infini_rag_flow":
-                raise ValueError("For the security reason, it dose not support database named rag_flow.")
+                raise ValueError("For the security reason, it does not support database named rag_flow.")
 
     def get_input_form(self) -> dict[str, dict]:
         return {
@@ -208,28 +208,37 @@ class ExeSQL(ToolBase, ABC):
                         continue
                     single_sql = re.sub(r"\[ID:[0-9]+\]", "", single_sql)
 
-                    stmt = ibm_db.exec_immediate(conn, single_sql)
-                    rows = []
-                    row = ibm_db.fetch_assoc(stmt)
-                    while row and len(rows) < self._param.max_records:
-                        if self.check_if_canceled("ExeSQL processing"):
-                            return
-                        rows.append(row)
+                    try:
+                        stmt = ibm_db.exec_immediate(conn, single_sql)
+                        rows = []
                         row = ibm_db.fetch_assoc(stmt)
+                        while row and len(rows) < self._param.max_records:
+                            if self.check_if_canceled("ExeSQL processing"):
+                                return
+                            rows.append(row)
+                            row = ibm_db.fetch_assoc(stmt)
 
-                    if not rows:
-                        sql_res.append({"content": "No record in the database!"})
+                        if not rows:
+                            sql_res.append({"content": "No record in the database!"})
+                            continue
+
+                        df = pd.DataFrame(rows)
+                        for col in df.columns:
+                            if pd.api.types.is_datetime64_any_dtype(df[col]):
+                                df[col] = df[col].dt.strftime("%Y-%m-%d")
+
+                        df = df.where(pd.notnull(df), None)
+
+                        sql_res.append(convert_decimals(df.to_dict(orient="records")))
+                        formalized_content.append(df.to_markdown(index=False, floatfmt=".6f"))
+                    except Exception as e:
+                        # Keep the node alive on a bad statement: report and continue.
+                        with contextlib.suppress(Exception):
+                            ibm_db.rollback(conn)
+                        msg = f"SQL Execution Failed: {single_sql}\n{str(e)}"
+                        sql_res.append({"content": msg})
+                        formalized_content.append(msg)
                         continue
-
-                    df = pd.DataFrame(rows)
-                    for col in df.columns:
-                        if pd.api.types.is_datetime64_any_dtype(df[col]):
-                            df[col] = df[col].dt.strftime("%Y-%m-%d")
-
-                    df = df.where(pd.notnull(df), None)
-
-                    sql_res.append(convert_decimals(df.to_dict(orient="records")))
-                    formalized_content.append(df.to_markdown(index=False, floatfmt=".6f"))
             finally:
                 with contextlib.suppress(Exception):
                     ibm_db.close(conn)
@@ -259,25 +268,37 @@ class ExeSQL(ToolBase, ABC):
                     sql_res.append({"content": "For security reasons, INSERT, UPDATE, and DELETE statements are not supported."})
                     formalized_content.append("For security reasons, INSERT, UPDATE, and DELETE statements are not supported.")
                     continue
-                cursor.execute(single_sql)
-                if cursor.rowcount == 0:
-                    sql_res.append({"content": "No record in the database!"})
-                    break
-                if self._param.db_type == 'mssql':
-                    single_res = pd.DataFrame.from_records(cursor.fetchmany(self._param.max_records),
-                                                           columns=[desc[0] for desc in cursor.description])
-                else:
-                    single_res = pd.DataFrame([i for i in cursor.fetchmany(self._param.max_records)])
-                    single_res.columns = [i[0] for i in cursor.description]
+                try:
+                    cursor.execute(single_sql)
+                    if cursor.rowcount == 0:
+                        sql_res.append({"content": "No record in the database!"})
+                        break
+                    if self._param.db_type == 'mssql':
+                        single_res = pd.DataFrame.from_records(cursor.fetchmany(self._param.max_records),
+                                                               columns=[desc[0] for desc in cursor.description])
+                    else:
+                        single_res = pd.DataFrame([i for i in cursor.fetchmany(self._param.max_records)])
+                        single_res.columns = [i[0] for i in cursor.description]
 
-                for col in single_res.columns:
-                    if pd.api.types.is_datetime64_any_dtype(single_res[col]):
-                        single_res[col] = single_res[col].dt.strftime('%Y-%m-%d')
+                    for col in single_res.columns:
+                        if pd.api.types.is_datetime64_any_dtype(single_res[col]):
+                            single_res[col] = single_res[col].dt.strftime('%Y-%m-%d')
 
-                single_res = single_res.where(pd.notnull(single_res), None)
+                    single_res = single_res.where(pd.notnull(single_res), None)
 
-                sql_res.append(convert_decimals(single_res.to_dict(orient='records')))
-                formalized_content.append(single_res.to_markdown(index=False, floatfmt=".6f"))
+                    sql_res.append(convert_decimals(single_res.to_dict(orient='records')))
+                    formalized_content.append(single_res.to_markdown(index=False, floatfmt=".6f"))
+                except Exception as e:
+                    # A failing statement must not abort the node: report it and keep
+                    # going so earlier results survive and later statements still run.
+                    # The rollback clears PostgreSQL's aborted-transaction state, which
+                    # would otherwise make every subsequent statement fail too.
+                    with contextlib.suppress(Exception):
+                        db.rollback()
+                    msg = f"SQL Execution Failed: {single_sql}\n{str(e)}"
+                    sql_res.append({"content": msg})
+                    formalized_content.append(msg)
+                    continue
         finally:
             with contextlib.suppress(Exception):
                 cursor.close()

agent/tools/retrieval.py

@@ -27,7 +27,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.memory_service import MemoryService
 from api.db.joint_services import memory_message_service
-from api.db.joint_services.tenant_model_service import get_model_config_by_type_and_name, get_tenant_default_model_by_type
+from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, get_model_config_from_provider_instance
 from common import settings
 from common.connection_utils import timeout
 from rag.app.tag import label_question
@@ -121,12 +121,12 @@ class Retrieval(ToolBase, ABC):
         embd_mdl = None
         if embd_nms:
             tenant_id = self._canvas.get_tenant_id()
-            embd_model_config = get_model_config_by_type_and_name(tenant_id, LLMType.EMBEDDING, embd_nms[0])
+            embd_model_config = get_model_config_from_provider_instance(tenant_id, LLMType.EMBEDDING, embd_nms[0])
             embd_mdl = LLMBundle(tenant_id, embd_model_config)
 
         rerank_mdl = None
         if self._param.rerank_id:
-            rerank_model_config = get_model_config_by_type_and_name(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
+            rerank_model_config = get_model_config_from_provider_instance(kbs[0].tenant_id, LLMType.RERANK, self._param.rerank_id)
             rerank_mdl = LLMBundle(kbs[0].tenant_id, rerank_model_config)
 
         vars = self.get_input_elements_from_text(query_text)
@@ -135,9 +135,18 @@ class Retrieval(ToolBase, ABC):
 
         doc_ids = []
         if self._param.meta_data_filter != {}:
-            metas = DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
+            # Defer the (potentially expensive) metadata table load — manual
+            # filters served by ES push-down never need it. The loader is
+            # invoked at most once per request by ``apply_meta_data_filter``.
+            def _load_metas() -> dict:
+                return DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
 
             def _resolve_manual_filter(flt: dict) -> dict:
+                # Return a new dict instead of mutating `flt` in place. The
+                # caller passes filters straight out of self._param.meta_data_filter,
+                # so mutating them would replace the variable reference with its
+                # resolved value and every subsequent invocation (e.g. inside an
+                # Iteration component) would reuse that stale value.
                 pat = re.compile(self.variable_ref_patt)
                 s = flt.get("value", "")
                 out_parts = []
@@ -163,8 +172,9 @@ class Retrieval(ToolBase, ABC):
                     last = m.end()
 
                 out_parts.append(s[last:])
-                flt["value"] = "".join(out_parts)
-                return flt
+                resolved = dict(flt)
+                resolved["value"] = "".join(out_parts)
+                return resolved
 
             chat_mdl = None
             if self._param.meta_data_filter.get("method") in ["auto", "semi_auto"]:
@@ -174,11 +184,13 @@ class Retrieval(ToolBase, ABC):
 
             doc_ids = await apply_meta_data_filter(
                 self._param.meta_data_filter,
-                metas,
+                None,
                 query,
                 chat_mdl,
                 doc_ids,
                 _resolve_manual_filter if self._param.meta_data_filter.get("method") == "manual" else None,
+                kb_ids=kb_ids,
+                metas_loader=_load_metas,
             )
 
         if self._param.cross_languages:
@@ -195,6 +207,7 @@ class Retrieval(ToolBase, ABC):
                 self._param.top_n,
                 self._param.similarity_threshold,
                 1 - self._param.keywords_similarity_weight,
+                top=self._param.top_k,
                 doc_ids=doc_ids,
                 aggs=True,
                 rerank_mdl=rerank_mdl,

agent/tools/searxng.py

@@ -20,6 +20,7 @@ from abc import ABC
 import requests
 from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
 from common.connection_utils import timeout
+from common.ssrf_guard import assert_url_is_safe, pin_dns
 
 
 class SearXNGParam(ToolParamBase):
@@ -36,15 +37,15 @@ class SearXNGParam(ToolParamBase):
                     "type": "string",
                     "description": "The search keywords to execute with SearXNG. The keywords should be the most important words/terms(includes synonyms) from the original request.",
                     "default": "{sys.query}",
-                    "required": True
+                    "required": True,
                 },
                 "searxng_url": {
                     "type": "string",
                     "description": "The base URL of your SearXNG instance (e.g., http://localhost:4000). This is required to connect to your SearXNG server.",
                     "required": False,
-                    "default": ""
-                }
-            }
+                    "default": "",
+                },
+            },
         }
         super().__init__()
         self.top_n = 10
@@ -61,17 +62,7 @@ class SearXNGParam(ToolParamBase):
         self.check_positive_integer(self.top_n, "Top N")
 
     def get_input_form(self) -> dict[str, dict]:
-        return {
-            "query": {
-                "name": "Query",
-                "type": "line"
-            },
-            "searxng_url": {
-                "name": "SearXNG URL",
-                "type": "line",
-                "placeholder": "http://localhost:4000"
-            }
-        }
+        return {"query": {"name": "Query", "type": "line"}, "searxng_url": {"name": "SearXNG URL", "type": "line", "placeholder": "http://localhost:4000"}}
 
 
 class SearXNG(ToolBase, ABC):
@@ -94,26 +85,22 @@ class SearXNG(ToolBase, ABC):
             self.set_output("formalized_content", "")
             return ""
 
+        try:
+            _ssrf_hostname, _ssrf_ip = assert_url_is_safe(searxng_url)
+        except ValueError as e:
+            self.set_output("_ERROR", str(e))
+            return f"SearXNG error: SSRF guard blocked {searxng_url!r}: {e}"
+
         last_e = ""
-        for _ in range(self._param.max_retries+1):
+        for _ in range(self._param.max_retries + 1):
             if self.check_if_canceled("SearXNG processing"):
                 return
 
             try:
-                search_params = {
-                    'q': query,
-                    'format': 'json',
-                    'categories': 'general',
-                    'language': 'auto',
-                    'safesearch': 1,
-                    'pageno': 1
-                }
+                search_params = {"q": query, "format": "json", "categories": "general", "language": "auto", "safesearch": 1, "pageno": 1}
 
-                response = requests.get(
-                    f"{searxng_url}/search",
-                    params=search_params,
-                    timeout=10
-                )
+                with pin_dns(_ssrf_hostname, _ssrf_ip):
+                    response = requests.get(f"{searxng_url}/search", params=search_params, timeout=10)
                 response.raise_for_status()
 
                 if self.check_if_canceled("SearXNG processing"):
@@ -128,15 +115,12 @@ class SearXNG(ToolBase, ABC):
                 if not isinstance(results, list):
                     raise ValueError("Invalid results format from SearXNG")
 
-                results = results[:self._param.top_n]
+                results = results[: self._param.top_n]
 
                 if self.check_if_canceled("SearXNG processing"):
                     return
 
-                self._retrieve_chunks(results,
-                                      get_title=lambda r: r.get("title", ""),
-                                      get_url=lambda r: r.get("url", ""),
-                                      get_content=lambda r: r.get("content", ""))
+                self._retrieve_chunks(results, get_title=lambda r: r.get("title", ""), get_url=lambda r: r.get("url", ""), get_content=lambda r: r.get("content", ""))
 
                 self.set_output("json", results)
                 return self.output("formalized_content")

api/apps/__init__.py

@@ -34,6 +34,7 @@ from quart_schema import QuartSchema
 from common import settings
 from api.utils.api_utils import server_error_response, get_json_result
 from api.constants import API_VERSION
+from common.exceptions import ModelException
 from common.misc_utils import get_uuid
 
 settings.init_settings()
@@ -56,6 +57,7 @@ def _unauthorized_message(error):
     except Exception:
         return UNAUTHORIZED_MESSAGE
 
+
 app = Quart(__name__)
 app = cors(app, allow_origin="*")
 
@@ -79,83 +81,160 @@ app.config["SESSION_REDIS"] = settings.decrypt_database_config(name="redis")
 app.config["MAX_CONTENT_LENGTH"] = int(
     os.environ.get("MAX_CONTENT_LENGTH", 1024 * 1024 * 1024)
 )
-app.config['SECRET_KEY'] = settings.SECRET_KEY
-app.secret_key = settings.SECRET_KEY
+app.config['SECRET_KEY'] = settings.get_secret_key()
+app.secret_key = settings.get_secret_key()
 commands.register_commands(app)
 
 from functools import wraps
 from typing import ParamSpec, TypeVar
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Iterable
 from werkzeug.local import LocalProxy
 
 T = TypeVar("T")
 P = ParamSpec("P")
 
+AUTH_JWT = "JWT"
+AUTH_API = "API"
+AUTH_BETA = "BETA"
+DEFAULT_AUTH_TYPES = (AUTH_JWT, AUTH_API)
 
-def _load_user():
-    jwt = Serializer(secret_key=settings.SECRET_KEY)
-    authorization = request.headers.get("Authorization")
-    g.user = None
-    if not authorization:
+
+def _normalize_auth_types(auth_types=None):
+    if auth_types is None:
+        return set(DEFAULT_AUTH_TYPES)
+    if isinstance(auth_types, str):
+        return {auth_types.upper()}
+    if isinstance(auth_types, Iterable):
+        return {str(auth_type).upper() for auth_type in auth_types}
+    return {str(auth_types).upper()}
+
+
+def _load_user_from_session():
+    """Resolve the current user from the session cookie set by ``login_user()``.
+
+    OAuth/OIDC callbacks call ``login_user(user)`` which writes ``_user_id``
+    into the session. The frontend's response interceptor wipes the
+    Authorization header from localStorage on the first 401, so post-redirect
+    requests can arrive with no header at all — we still want to honour the
+    server-side session in that window.
+
+    The same access-token validity rules used by the JWT path are applied
+    here so that tokens revoked by ``logout`` (which rewrites the column to
+    ``INVALID_<hex>``) or shortened by data corruption can't keep a stale
+    session authenticated.
+    """
+    user_id = session.get("_user_id")
+    if not user_id:
         return None
+    try:
+        users = UserService.query(id=user_id, status=StatusEnum.VALID.value)
+    except Exception:
+        logging.exception("load_user from session failed")
+        return None
+    if not users:
+        return None
+    user = users[0]
+    access_token = str(user.access_token or "").strip()
+    if not access_token or len(access_token) < 32 or access_token.startswith("INVALID_"):
+        return None
+    logging.debug("Authenticated request via session fallback for user_id=%s", user_id)
+    g.auth_type = AUTH_JWT
+    g.user = user
+    return user
+
+
+def _load_user(auth_types=None):
+    explicit_auth_types = auth_types is not None
+    auth_types = _normalize_auth_types(auth_types)
+    if getattr(g, "user", None) and (not explicit_auth_types or getattr(g, "auth_type", None) in auth_types):
+        return g.user
+    
+    # No Authorization header, try to load user from session cookie if JWT auth is allowed
+    authorization = request.headers.get("Authorization")
+    if not authorization:
+        return _load_user_from_session() if AUTH_JWT in auth_types else None
 
     # Extract auth_token based on whether Authorization starts with "bearer" (case-insensitive)
-    if authorization.lower().startswith("bearer "):
+    if authorization[:7].lower() == "bearer ":
         parts = authorization.split(maxsplit=1)
         if len(parts) < 2:
             logging.warning("Authorization header has invalid bearer format")
-            return None
+            return _load_user_from_session() if AUTH_JWT in auth_types else None
         auth_token = parts[1]
     else:
         auth_token = authorization
 
+    g.user = None
+    g.auth_type = None
+    g.auth_error_message = None
+
+    # Try Beta token
+    if AUTH_BETA in auth_types:
+        try:
+            objs = APIToken.query(beta=auth_token)
+            if objs:
+                user = UserService.query(id=objs[0].tenant_id, status=StatusEnum.VALID.value)
+                if user:
+                    g.auth_type = AUTH_BETA
+                    g.user = user[0]
+                    return user[0]
+            g.auth_error_message = 'Authentication error: API key is invalid! '
+        except Exception as e_beta:
+            logging.warning(f"load_user from beta token got exception {e_beta}")
+            g.auth_error_message = 'Authentication error: API key is invalid!'
+
     # Try JWT decoding
-    try:
-        access_token = str(jwt.loads(auth_token))
+    if AUTH_JWT in auth_types:
+        try:
+            jwt = Serializer(secret_key=settings.get_secret_key())
+            access_token = str(jwt.loads(auth_token))
 
-        if not access_token or not access_token.strip():
-            logging.warning("Authentication attempt with empty access token")
-            return None
+            if not access_token or not access_token.strip():
+                logging.warning("Authentication attempt with empty access token")
+                return _load_user_from_session()
 
-        if len(access_token.strip()) < 32:
-            logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
-            return None
+            if len(access_token.strip()) < 32:
+                logging.warning(f"Authentication attempt with invalid token format: {len(access_token)} chars")
+                return _load_user_from_session()
 
-        user = UserService.query(access_token=access_token, status=StatusEnum.VALID.value)
-        if user:
-            if not user[0].access_token or not user[0].access_token.strip():
-                logging.warning(f"User {user[0].email} has empty access_token in database")
-                return None
-            g.user = user[0]
-            return user[0]
-        return None
-    except Exception as e_jwt:
-        logging.warning(f"load_user from jwt got exception {e_jwt}")
-
-    # JWT decode failed, try as api_token
-    try:
-        objs = APIToken.query(token=auth_token)
-        if objs:
-            user = UserService.query(id=objs[0].tenant_id, status=StatusEnum.VALID.value)
+            user = UserService.query(access_token=access_token, status=StatusEnum.VALID.value)
             if user:
                 if not user[0].access_token or not user[0].access_token.strip():
                     logging.warning(f"User {user[0].email} has empty access_token in database")
-                    return None
+                    return _load_user_from_session()
+                g.auth_type = AUTH_JWT
                 g.user = user[0]
                 return user[0]
-            logging.warning(f"load_user: No user found for tenant_id={objs[0].tenant_id} from APIToken")
-        else:
-            logging.warning(f"load_user: No APIToken found for token={auth_token[:10]}...")
-    except Exception as e_api_token:
-        logging.warning(f"load_user from api token got exception {e_api_token}")
+            return _load_user_from_session()
+        except Exception as e_jwt:
+            logging.warning(f"load_user from jwt got exception {e_jwt}")
 
-    return None
+    # JWT decode failed, try as api_token
+    if AUTH_API in auth_types:
+        try:
+            objs = APIToken.query(token=auth_token)
+            if objs:
+                user = UserService.query(id=objs[0].tenant_id, status=StatusEnum.VALID.value)
+                if user:
+                    if not user[0].access_token or not user[0].access_token.strip():
+                        logging.warning(f"User {user[0].email} has empty access_token in database")
+                        return _load_user_from_session() if AUTH_JWT in auth_types else None
+                    g.auth_type = AUTH_API
+                    g.user = user[0]
+                    return user[0]
+                logging.warning(f"load_user: No user found for tenant_id={objs[0].tenant_id} from APIToken")
+            else:
+                logging.warning(f"load_user: No APIToken found for token={auth_token[:10]}...")
+        except Exception as e_api_token:
+            logging.warning(f"load_user from api token got exception {e_api_token}")
+
+    return _load_user_from_session() if AUTH_JWT in auth_types else None
 
 
 current_user = LocalProxy(_load_user)
 
 
-def login_required(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
+def login_required(func: Callable[P, Awaitable[T]] = None, auth_types=None) -> Callable[P, Awaitable[T]]:
     """A decorator to restrict route access to authenticated users.
 
     This should be used to wrap a route handler (or view function) to
@@ -175,22 +254,32 @@ def login_required(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]
 
     """
 
-    @wraps(func)
-    async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
-        timing_enabled = os.getenv("RAGFLOW_API_TIMING")
-        t_start = time.perf_counter() if timing_enabled else None
-        user = current_user
-        if timing_enabled:
-            logging.info(
-                "api_timing login_required auth_ms=%.2f path=%s",
-                (time.perf_counter() - t_start) * 1000,
-                request.path,
-            )
-        if not user:  # or not session.get("_user_id"):
-            raise QuartAuthUnauthorized()
-        return await current_app.ensure_async(func)(*args, **kwargs)
+    def decorator(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
+        @wraps(func)
+        async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+            timing_enabled = os.getenv("RAGFLOW_API_TIMING")
+            t_start = time.perf_counter() if timing_enabled else None
+            user = _load_user(auth_types)
+            if timing_enabled:
+                logging.info(
+                    "api_timing login_required auth_ms=%.2f path=%s",
+                    (time.perf_counter() - t_start) * 1000,
+                    request.path,
+                )
+            if not user:  # or not session.get("_user_id"):
+                if _normalize_auth_types(auth_types) == {AUTH_BETA}:
+                    return get_json_result(
+                        code=RetCode.DATA_ERROR,
+                        message=getattr(g, "auth_error_message", None) or "Authorization is not valid!",
+                    )
+                raise QuartAuthUnauthorized()
+            return await current_app.ensure_async(func)(*args, **kwargs)
 
-    return wrapper
+        return wrapper
+
+    if func is None:
+        return decorator
+    return decorator(func)
 
 
 def login_user(user, remember=False, duration=None, force=False, fresh=True):
@@ -251,16 +340,10 @@ def logout_user():
 
 
 def search_pages_path(page_path):
-    app_path_list = [
-        path for path in page_path.glob("*_app.py") if not path.name.startswith(".")
-    ]
-    api_path_list = [
-        path for path in page_path.glob("*sdk/*.py") if not path.name.startswith(".")
-    ]
+    app_path_list = [path for path in page_path.glob("*_app.py") if not path.name.startswith(".")]
+    api_path_list = [path for path in page_path.glob("*sdk/*.py") if not path.name.startswith(".")]
     app_path_list.extend(api_path_list)
-    restful_api_path_list = [
-        path for path in page_path.glob("*restful_apis/*.py") if not path.name.startswith(".")
-    ]
+    restful_api_path_list = [path for path in page_path.glob("*restful_apis/*.py") if not path.name.startswith(".")]
     app_path_list.extend(restful_api_path_list)
     return app_path_list
 
@@ -269,9 +352,7 @@ def register_page(page_path):
     path = f"{page_path}"
 
     page_name = page_path.stem.removesuffix("_app")
-    module_name = ".".join(
-        page_path.parts[page_path.parts.index("api"): -1] + (page_name,)
-    )
+    module_name = ".".join(page_path.parts[page_path.parts.index("api") : -1] + (page_name,))
 
     spec = spec_from_file_location(module_name, page_path)
     page = module_from_spec(spec)
@@ -280,11 +361,8 @@ def register_page(page_path):
     sys.modules[module_name] = page
     spec.loader.exec_module(page)
     page_name = getattr(page, "page_name", page_name)
-    sdk_path = "\\sdk\\" if sys.platform.startswith("win") else "/sdk/"
     restful_api_path = "\\restful_apis\\" if sys.platform.startswith("win") else "/restful_apis/"
-    url_prefix = (
-        f"/api/{API_VERSION}" if sdk_path in path or restful_api_path in path else f"/{API_VERSION}/{page_name}"
-    )
+    url_prefix = f"/api/{API_VERSION}" if restful_api_path in path else f"/{API_VERSION}/{page_name}"
 
     app.register_blueprint(page.manager, url_prefix=url_prefix)
     return url_prefix
@@ -297,9 +375,12 @@ pages_dir = [
     Path(__file__).parent.parent / "api" / "apps" / "sdk",
 ]
 
-client_urls_prefix = [
-    register_page(path) for directory in pages_dir for path in search_pages_path(directory)
-]
+client_urls_prefix = [register_page(path) for directory in pages_dir for path in search_pages_path(directory)]
+
+# Register backward compatibility routes for deprecated APIs
+from api.apps.backward_compat import register_backward_compat_routes
+
+register_backward_compat_routes(app)
 
 
 @app.errorhandler(404)
@@ -332,6 +413,13 @@ async def unauthorized_werkzeug(error):
     logging.warning("Unauthorized request (werkzeug)")
     return get_json_result(code=error.code, message=error.description), RetCode.UNAUTHORIZED
 
+
+@app.errorhandler(ModelException)
+async def handle_model_exception(error):
+    logging.warning("Forbidden request")
+    return get_json_result(code=RetCode.BAD_REQUEST, message=repr(error)), 200
+
+
 @app.teardown_request
 def _db_close(exception):
     if exception:

api/apps/auth/README.md

@@ -20,7 +20,7 @@ oauth_config = {
     "authorization_url": "https://your-oauth-provider.com/oauth/authorize",
     "token_url": "https://your-oauth-provider.com/oauth/token",
     "userinfo_url": "https://your-oauth-provider.com/oauth/userinfo",
-    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+    "redirect_uri": "https://your-app.com/api/v1/auth/oauth/<channel>/callback"
 }
 
 # OIDC configuration
@@ -29,7 +29,7 @@ oidc_config = {
     "issuer": "https://your-oauth-provider.com/oidc",
     "client_id": "your_client_id",
     "client_secret": "your_client_secret",
-    "redirect_uri": "https://your-app.com/v1/user/oauth/callback/<channel>"
+    "redirect_uri": "https://your-app.com/api/v1/auth/oauth/<channel>/callback"
 }
 
 # Github OAuth configuration

api/apps/auth/oidc.py

@@ -19,6 +19,45 @@ from common.http_client import sync_request
 from .oauth import OAuthClient
 
 
+# Asymmetric signing algorithms safe to accept for OIDC ID tokens.
+# Symmetric HMAC algorithms (HS*) are intentionally excluded — when the
+# verification key is the asymmetric public key fetched from the provider's
+# JWKS (as it is for every OIDC ID token), accepting HS256 lets an attacker
+# forge tokens by HMAC-signing them with the public key bytes
+# (RSA/HMAC algorithm-confusion attack, CWE-347). "none" is excluded for the
+# obvious reason that it disables signature verification entirely.
+_ALLOWED_OIDC_SIGNING_ALGS = frozenset({
+    "RS256", "RS384", "RS512",
+    "ES256", "ES384", "ES512",
+    "PS256", "PS384", "PS512",
+    "EdDSA",
+})
+
+# OIDC Core 1.0 § 2 makes RS256 the spec-default ``id_token_signing_alg``,
+# so this is the safe fallback when a provider's discovery document does not
+# advertise ``id_token_signing_alg_values_supported`` (or advertises only
+# algorithms outside the safe allowlist).
+_DEFAULT_OIDC_SIGNING_ALGS = ("RS256",)
+
+
+def _resolve_id_token_signing_algs(metadata):
+    """Return the algorithms to pass to ``jwt.decode(..., algorithms=...)``.
+
+    Intersects the provider-advertised
+    ``id_token_signing_alg_values_supported`` with
+    :data:`_ALLOWED_OIDC_SIGNING_ALGS`. Falls back to
+    :data:`_DEFAULT_OIDC_SIGNING_ALGS` when the provider does not advertise
+    the field or advertises only algorithms outside the safe allowlist —
+    crucially, the fallback is to RS256, **never** to whatever the JWT
+    header claims at verification time.
+    """
+    advertised = metadata.get("id_token_signing_alg_values_supported") or []
+    if not isinstance(advertised, (list, tuple)):
+        advertised = []
+    safe = [a for a in advertised if isinstance(a, str) and a in _ALLOWED_OIDC_SIGNING_ALGS]
+    return safe or list(_DEFAULT_OIDC_SIGNING_ALGS)
+
+
 class OIDCClient(OAuthClient):
     def __init__(self, config):
         """
@@ -32,7 +71,7 @@ class OIDCClient(OAuthClient):
         oidc_metadata = self._load_oidc_metadata(self.issuer)
         config.update({
             'issuer': oidc_metadata['issuer'],
-            'jwks_uri': oidc_metadata['jwks_uri'], 
+            'jwks_uri': oidc_metadata['jwks_uri'],
             'authorization_url': oidc_metadata['authorization_endpoint'],
             'token_url': oidc_metadata['token_endpoint'],
             'userinfo_url': oidc_metadata['userinfo_endpoint']
@@ -41,6 +80,11 @@ class OIDCClient(OAuthClient):
         super().__init__(config)
         self.issuer = config['issuer']
         self.jwks_uri = config['jwks_uri']
+        # Pin the accepted ID-token signing algorithms at construction time
+        # from a trusted source (provider metadata + safe allowlist) so the
+        # JWT verification step in :meth:`parse_id_token` cannot be tricked
+        # by attacker-controlled JWT headers (CWE-345 / CWE-347).
+        self.id_token_signing_algs = _resolve_id_token_signing_algs(oidc_metadata)
 
 
     @staticmethod
@@ -60,23 +104,29 @@ class OIDCClient(OAuthClient):
     def parse_id_token(self, id_token):
         """
         Parse and validate OIDC ID Token (JWT format) with signature verification.
+
+        The accepted signing algorithms come from ``self.id_token_signing_algs``
+        (pinned at construction time from the provider's discovery metadata,
+        intersected with :data:`_ALLOWED_OIDC_SIGNING_ALGS`). We deliberately
+        do **not** read the algorithm from the unverified JWT header — doing
+        so would let an attacker bypass signature verification by setting
+        ``"alg": "none"`` or pull off the classic RSA / HMAC algorithm
+        confusion by setting ``"alg": "HS256"`` and signing with the public
+        key fetched from the provider's JWKS (CWE-345 / CWE-347).
         """
         try:
-            # Decode JWT header without verifying signature
-            headers = jwt.get_unverified_header(id_token)
-            
-            # OIDC usually uses `RS256` for signing
-            alg = headers.get("alg", "RS256")
-
-            # Use PyJWT's PyJWKClient to fetch JWKS and find signing key
+            # Use PyJWT's PyJWKClient to fetch JWKS and find signing key.
+            # The client reads the ``kid`` from the JWT header internally to
+            # look up the key — that's fine: ``kid`` is not a security
+            # decision, the signature still proves which key was used.
             jwks_cli = jwt.PyJWKClient(self.jwks_uri)
             signing_key = jwks_cli.get_signing_key_from_jwt(id_token).key
 
-            # Decode and verify signature
+            # Decode and verify signature against the pinned allowlist.
             decoded_token = jwt.decode(
                 id_token,
                 key=signing_key,
-                algorithms=[alg],  
+                algorithms=list(self.id_token_signing_algs),
                 audience=str(self.client_id),
                 issuer=self.issuer,
             )

api/apps/backward_compat.py

@@ -0,0 +1,665 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+"""
+Backward compatibility layer for deprecated API endpoints.
+
+This module adds support for old API routes that were deprecated during the
+RESTful API migration. Each deprecated route forwards to the corresponding
+new API implementation.
+
+Deprecated APIs and their replacements:
+- POST /api/v1/agents/{agent_id}/completions -> POST /api/v1/agents/chat/completion
+- POST /api/v1/agents_openai/{agent_id}/chat/completions -> POST /api/v1/agents/chat/completions
+- POST /api/v1/chats/{chat_id}/completions -> POST /api/v1/chat/completions
+- POST /api/v1/chats_openai/{chat_id}/chat/completions -> POST /api/v1/openai/{chat_id}/chat/completions
+- GET /api/v1/datasets/{dataset_id}/knowledge_graph -> GET /api/v1/datasets/{dataset_id}/graph
+- DELETE /api/v1/datasets/{dataset_id}/knowledge_graph -> DELETE /api/v1/datasets/{dataset_id}/graph
+- POST /api/v1/datasets/{dataset_id}/run_graphrag -> POST /api/v1/datasets/{dataset_id}/index?type=graph
+- GET /api/v1/datasets/{dataset_id}/trace_graphrag -> GET /api/v1/datasets/{dataset_id}/index?type=graph
+- POST /api/v1/datasets/{dataset_id}/run_raptor -> POST /api/v1/datasets/{dataset_id}/index?type=raptor
+- GET /api/v1/datasets/{dataset_id}/trace_raptor -> GET /api/v1/datasets/{dataset_id}/index?type=raptor
+- PUT /api/v1/chats/{chat_id}/sessions/{session_id} -> PATCH /api/v1/chats/{chat_id}/sessions/{session_id}
+- DELETE /api/v1/chats -> DELETE /api/v1/chats/{chat_id} (with body)
+- POST /api/v1/file/convert -> POST /api/v1/files/link-to-datasets
+- GET /api/v1/file/* -> GET /api/v1/files*
+- POST /api/v1/file/* -> POST /api/v1/files*
+- GET /api/v1/document/get/{doc_id} -> GET /api/v1/documents/{doc_id}/preview
+- GET /api/v1/document/download/{doc_id} -> GET /api/v1/agents/attachments/{doc_id}/download
+- GET /v1/document/download/{attachment_id} -> GET /api/v1/agents/attachments/{attachment_id}/download
+- GET /v1/system/healthz -> GET /api/v1/system/healthz
+- POST /api/v1/sessions/related_questions -> POST /api/v1/chat/recommandation
+- PUT (chunk update) -> PATCH (chunk update)
+"""
+import logging
+
+from quart import Blueprint, jsonify, request
+
+from api.apps import login_required
+from api.apps.restful_apis import agent_api, chat_api, chunk_api, dataset_api, document_api, file2document_api, file_api, openai_api
+from api.apps.restful_apis.system_api import run_health_checks
+from api.apps.services import dataset_api_service, file_api_service
+from api.utils.api_utils import add_tenant_id_to_kwargs, get_data_error_result, get_json_result, get_request_json
+
+manager = Blueprint("backward_compat", __name__)
+legacy_v1_manager = Blueprint("backward_compat_legacy_v1", __name__)
+
+
+def _index_result(success, result):
+    if success:
+        return get_json_result(data=result)
+    return get_data_error_result(message=result)
+
+
+# =============================================================================
+# System APIs
+# =============================================================================
+
+@legacy_v1_manager.route("/system/healthz", methods=["GET"])
+async def deprecated_system_healthz():
+    """
+    Deprecated: Use GET /api/v1/system/healthz instead.
+
+    Old path: GET /v1/system/healthz
+    New path: GET /api/v1/system/healthz
+    """
+    logging.warning(
+        "API endpoint /v1/system/healthz is deprecated. "
+        "Please use /api/v1/system/healthz instead."
+    )
+    result, all_ok = run_health_checks()
+    return jsonify(result), (200 if all_ok else 500)
+
+# =============================================================================
+# Chat Completion APIs
+# =============================================================================
+
+@manager.route("/chats/<chat_id>/completions", methods=["POST"])
+@login_required
+async def deprecated_chat_completions(chat_id):
+    """
+    Deprecated: Use POST /api/v1/chat/completions instead.
+
+    Old path: POST /api/v1/chats/{chat_id}/completions
+    New path: POST /api/v1/chat/completions
+    """
+    logging.warning(
+        "API endpoint /api/v1/chats/%s/completions is deprecated. "
+        "Please use /api/v1/chat/completions instead.",
+        chat_id,
+    )
+    # Forward to the new API implementation
+    return await chat_api.session_completion(chat_id)
+
+
+@manager.route("/chats_openai/<chat_id>/chat/completions", methods=["POST"])
+@login_required
+async def deprecated_openai_chat_completions(chat_id):
+    """
+    Deprecated: Use POST /api/v1/openai/{chat_id}/chat/completions instead.
+
+    Old path: POST /api/v1/chats_openai/{chat_id}/chat/completions
+    New path: POST /api/v1/openai/{chat_id}/chat/completions
+    """
+    logging.warning(
+        "API endpoint /api/v1/chats_openai/%s/chat/completions is deprecated. "
+        "Please use /api/v1/openai/%s/chat/completions instead.",
+        chat_id, chat_id,
+    )
+    # Forward to the new API implementation
+    return await openai_api.openai_chat_completions(chat_id)
+
+
+@manager.route("/agents_openai/<agent_id>/chat/completions", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_agents_openai_chat_completions(agent_id, tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/agents/chat/completions with openai-compatible=true instead.
+
+    Old path: POST /api/v1/agents_openai/{agent_id}/chat/completions
+    New path: POST /api/v1/agents/chat/completions
+    """
+    logging.warning(
+        "API endpoint /api/v1/agents_openai/%s/chat/completions is deprecated. "
+        "Please use /api/v1/agents/chat/completions with `openai-compatible` instead.",
+        agent_id,
+    )
+    req = dict(await get_request_json())
+    req["openai-compatible"] = True
+    request._cached_payload = req
+    return await agent_api.agent_chat_completion(tenant_id=tenant_id, agent_id=agent_id)
+
+
+# =============================================================================
+# Dataset Graph and Index APIs
+# =============================================================================
+
+@manager.route("/datasets/<dataset_id>/knowledge_graph", methods=["GET"])
+@login_required
+async def deprecated_get_knowledge_graph(dataset_id):
+    """
+    Deprecated: Use GET /api/v1/datasets/{dataset_id}/graph instead.
+
+    Old path: GET /api/v1/datasets/{dataset_id}/knowledge_graph
+    New path: GET /api/v1/datasets/{dataset_id}/graph
+    """
+    logging.warning(
+        "API endpoint /api/v1/datasets/%s/knowledge_graph is deprecated. "
+        "Please use /api/v1/datasets/%s/graph instead.",
+        dataset_id, dataset_id,
+    )
+    return await dataset_api.get_knowledge_graph(dataset_id=dataset_id)
+
+
+@manager.route("/datasets/<dataset_id>/knowledge_graph", methods=["DELETE"])
+@login_required
+async def deprecated_delete_knowledge_graph(dataset_id):
+    """
+    Deprecated: Use DELETE /api/v1/datasets/{dataset_id}/graph instead.
+
+    Old path: DELETE /api/v1/datasets/{dataset_id}/knowledge_graph
+    New path: DELETE /api/v1/datasets/{dataset_id}/graph
+    """
+    logging.warning(
+        "API endpoint DELETE /api/v1/datasets/%s/knowledge_graph is deprecated. "
+        "Please use DELETE /api/v1/datasets/%s/graph instead.",
+        dataset_id, dataset_id,
+    )
+    return await dataset_api.delete_knowledge_graph(dataset_id=dataset_id)
+
+
+@manager.route("/datasets/<dataset_id>/run_graphrag", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_run_graphrag(dataset_id, tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/datasets/{dataset_id}/index?type=graph instead.
+
+    Old path: POST /api/v1/datasets/{dataset_id}/run_graphrag
+    New path: POST /api/v1/datasets/{dataset_id}/index?type=graph
+    """
+    logging.warning(
+        "API endpoint /api/v1/datasets/%s/run_graphrag is deprecated. "
+        "Please use /api/v1/datasets/%s/index?type=graph instead.",
+        dataset_id, dataset_id,
+    )
+    return _index_result(*dataset_api_service.run_index(dataset_id, tenant_id, "graph"))
+
+
+@manager.route("/datasets/<dataset_id>/trace_graphrag", methods=["GET"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_trace_graphrag(dataset_id, tenant_id=None):
+    """
+    Deprecated: Use GET /api/v1/datasets/{dataset_id}/index?type=graph instead.
+
+    Old path: GET /api/v1/datasets/{dataset_id}/trace_graphrag
+    New path: GET /api/v1/datasets/{dataset_id}/index?type=graph
+    """
+    logging.warning(
+        "API endpoint /api/v1/datasets/%s/trace_graphrag is deprecated. "
+        "Please use /api/v1/datasets/%s/index?type=graph instead.",
+        dataset_id, dataset_id,
+    )
+    return _index_result(*dataset_api_service.trace_index(dataset_id, tenant_id, "graph"))
+
+
+@manager.route("/datasets/<dataset_id>/run_raptor", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_run_raptor(dataset_id, tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/datasets/{dataset_id}/index?type=raptor instead.
+
+    Old path: POST /api/v1/datasets/{dataset_id}/run_raptor
+    New path: POST /api/v1/datasets/{dataset_id}/index?type=raptor
+    """
+    logging.warning(
+        "API endpoint /api/v1/datasets/%s/run_raptor is deprecated. "
+        "Please use /api/v1/datasets/%s/index?type=raptor instead.",
+        dataset_id, dataset_id,
+    )
+    return _index_result(*dataset_api_service.run_index(dataset_id, tenant_id, "raptor"))
+
+
+@manager.route("/datasets/<dataset_id>/trace_raptor", methods=["GET"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_trace_raptor(dataset_id, tenant_id=None):
+    """
+    Deprecated: Use GET /api/v1/datasets/{dataset_id}/index?type=raptor instead.
+
+    Old path: GET /api/v1/datasets/{dataset_id}/trace_raptor
+    New path: GET /api/v1/datasets/{dataset_id}/index?type=raptor
+    """
+    logging.warning(
+        "API endpoint /api/v1/datasets/%s/trace_raptor is deprecated. "
+        "Please use /api/v1/datasets/%s/index?type=raptor instead.",
+        dataset_id, dataset_id,
+    )
+    return _index_result(*dataset_api_service.trace_index(dataset_id, tenant_id, "raptor"))
+
+
+# =============================================================================
+# Chat Session APIs
+# =============================================================================
+
+@manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PUT"])
+@login_required
+async def deprecated_update_session(chat_id, session_id):
+    """
+    Deprecated: Use PATCH /api/v1/chats/{chat_id}/sessions/{session_id} instead.
+
+    Old path: PUT /api/v1/chats/{chat_id}/sessions/{session_id}
+    New path: PATCH /api/v1/chats/{chat_id}/sessions/{session_id}
+    """
+    logging.warning(
+        "API endpoint PUT /api/v1/chats/%s/sessions/%s is deprecated. "
+        "Please use PATCH /api/v1/chats/%s/sessions/%s instead.",
+        chat_id, session_id, chat_id, session_id,
+    )
+    # Forward to the new API implementation
+    return await chat_api.update_session(chat_id, session_id)
+
+
+# =============================================================================
+# File APIs (Old /api/v1/file/* -> New /api/v1/files*)
+# =============================================================================
+
+@manager.route("/file/get/<file_id>", methods=["GET"])
+@login_required
+async def deprecated_file_get(file_id):
+    """
+    Deprecated: Use GET /api/v1/files/{file_id} instead.
+
+    Old path: GET /api/v1/file/get/{file_id}
+    New path: GET /api/v1/files/{file_id}
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/get/%s is deprecated. "
+        "Please use /api/v1/files/%s instead.",
+        file_id, file_id,
+    )
+    # Forward to the new API implementation (download)
+    return await file_api.download(file_id=file_id)
+
+
+@manager.route("/file/list", methods=["GET"])
+@login_required
+async def deprecated_file_list():
+    """
+    Deprecated: Use GET /api/v1/files instead.
+
+    Old path: GET /api/v1/file/list?...
+    New path: GET /api/v1/files?...
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/list is deprecated. "
+        "Please use /api/v1/files instead."
+    )
+    # Forward to the new API implementation
+    return await file_api.list_files()
+
+
+@manager.route("/file/all_parent_folder", methods=["GET"])
+@login_required
+async def deprecated_file_all_parent_folder():
+    """
+    Deprecated: Use GET /api/v1/files/{file_id}/ancestors instead.
+
+    Old path: GET /api/v1/file/all_parent_folder?file_id=...
+    New path: GET /api/v1/files/{file_id}/ancestors
+    """
+    file_id = request.args.get("file_id")
+    if not file_id:
+        return get_data_error_result(message="`file_id` query parameter is required")
+    logging.warning(
+        "API endpoint /api/v1/file/all_parent_folder is deprecated. "
+        "Please use /api/v1/files/%s/ancestors instead.",
+        file_id,
+    )
+    # Forward to the new API implementation
+    return await file_api.ancestors(file_id=file_id)
+
+
+@manager.route("/file/parent_folder", methods=["GET"])
+@login_required
+async def deprecated_file_parent_folder():
+    """
+    Deprecated: Use GET /api/v1/files/{file_id}/parent instead.
+
+    Old path: GET /api/v1/file/parent_folder?file_id=...
+    New path: GET /api/v1/files/{file_id}/parent
+    """
+    file_id = request.args.get("file_id")
+    if not file_id:
+        return get_data_error_result(message="`file_id` query parameter is required")
+    logging.warning(
+        "API endpoint /api/v1/file/parent_folder is deprecated. "
+        "Please use /api/v1/files/%s/parent instead.",
+        file_id,
+    )
+    # Forward to the new API implementation
+    return await file_api.parent_folder(file_id=file_id)
+
+
+@manager.route("/file/root_folder", methods=["GET"])
+@login_required
+async def deprecated_file_root_folder():
+    """
+    Deprecated: Root folder is now accessible via GET /api/v1/files with parent_id=...
+
+    Old path: GET /api/v1/file/root_folder
+    New path: GET /api/v1/files?parent_id=<root_id>
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/root_folder is deprecated. "
+        "Please use /api/v1/files with appropriate parent_id instead."
+    )
+    # Forward to the new API implementation with empty parent_id to get root
+    return await file_api.list_files()
+
+
+@manager.route("/file/create", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_file_create(tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/files instead.
+
+    Old path: POST /api/v1/file/create
+    New path: POST /api/v1/files
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/create is deprecated. "
+        "Please use POST /api/v1/files instead."
+    )
+    # Forward to the new API implementation
+    return await file_api.create_or_upload(tenant_id=tenant_id)
+
+
+@manager.route("/file/upload", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_file_upload(tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/files (with multipart/form-data) instead.
+
+    Old path: POST /api/v1/file/upload
+    New path: POST /api/v1/files
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/upload is deprecated. "
+        "Please use POST /api/v1/files with multipart/form-data instead."
+    )
+    # Forward to the new API implementation
+    return await file_api.create_or_upload(tenant_id=tenant_id)
+
+
+@manager.route("/file/convert", methods=["POST"])
+@login_required
+async def deprecated_file_convert():
+    """
+    Deprecated: Use POST /api/v1/files/link-to-datasets instead.
+
+    Old path: POST /api/v1/file/convert
+    New path: POST /api/v1/files/link-to-datasets
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/convert is deprecated. "
+        "Please use POST /api/v1/files/link-to-datasets instead."
+    )
+    return await file2document_api.convert()
+
+
+@manager.route("/file/mv", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_file_mv(tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/files/move instead.
+
+    Old path: POST /api/v1/file/mv
+    New path: POST /api/v1/files/move
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/mv is deprecated. "
+        "Please use POST /api/v1/files/move instead."
+    )
+    # Forward to the new API implementation
+    return await file_api.move(tenant_id=tenant_id)
+
+
+@manager.route("/file/rename", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_file_rename(tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/files/move with new_name instead.
+
+    Old path: POST /api/v1/file/rename
+    New path: POST /api/v1/files/move
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/rename is deprecated. "
+        "Please use POST /api/v1/files/move with `new_name` instead."
+    )
+    # Transform the old API format to new format
+    req = await request.get_json()
+    # Old API used `file_id` and `name`, new API uses `src_file_ids` and `new_name`
+    src_file_ids = [req.get("file_id")]
+    new_name = req.get("name")
+    # Call the underlying service directly with transformed data
+    try:
+        success, result = await file_api_service.move_files(
+            tenant_id, src_file_ids, None, new_name
+        )
+        if success:
+            return get_json_result(data=result)
+        else:
+            return get_data_error_result(message=result)
+    except Exception as e:
+        logging.exception(e)
+        return get_data_error_result(message="Internal server error")
+
+
+@manager.route("/file/rm", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_file_rm(tenant_id=None):
+    """
+    Deprecated: Use DELETE /api/v1/files instead.
+
+    Old path: POST /api/v1/file/rm
+    New path: DELETE /api/v1/files
+    """
+    logging.warning(
+        "API endpoint /api/v1/file/rm is deprecated. "
+        "Please use DELETE /api/v1/files instead."
+    )
+    # Transform POST with body to DELETE behavior
+    # The new API expects a JSON body with `ids`
+    return await file_api.delete(tenant_id=tenant_id)
+
+
+# =============================================================================
+# Related Questions API
+# =============================================================================
+
+@manager.route("/sessions/related_questions", methods=["POST"])
+@login_required
+async def deprecated_related_questions():
+    """
+    Deprecated: Use POST /api/v1/chat/recommendation instead.
+
+    Old path: POST /api/v1/sessions/related_questions
+    New path: POST /api/v1/chat/recommendation
+    """
+    logging.warning(
+        "API endpoint /api/v1/sessions/related_questions is deprecated. "
+        "Please use /api/v1/chat/recommendation instead."
+    )
+    # Forward to the new API implementation
+    return await chat_api.recommendation()
+
+
+# =============================================================================
+# Chunk Update API (PUT -> PATCH)
+# =============================================================================
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["PUT"])
+@login_required
+async def deprecated_update_chunk(dataset_id, document_id, chunk_id):
+    """
+    Deprecated: Use PATCH /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks/{chunk_id} instead.
+
+    Old path: PUT /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks/{chunk_id}
+    New path: PATCH /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks/{chunk_id}
+    """
+    logging.warning(
+        "API endpoint PUT /api/v1/datasets/%s/documents/%s/chunks/%s is deprecated. "
+        "Please use PATCH instead.",
+        dataset_id, document_id, chunk_id,
+    )
+    # Forward to the new API implementation
+    return await chunk_api.update_chunk(dataset_id=dataset_id, document_id=document_id, chunk_id=chunk_id)
+
+
+# =============================================================================
+# File Upload Info API
+# =============================================================================
+
+@manager.route("/file/upload_info", methods=["POST"])
+@login_required
+async def deprecated_file_upload_info():
+    """
+    Deprecated: Use POST /api/v1/documents/upload instead.
+
+    Old path: POST /api/v1/file/upload_info
+    New path: POST /api/v1/documents/upload
+    """
+    from api.apps import current_user
+
+    logging.warning(
+        "API endpoint /api/v1/file/upload_info is deprecated. "
+        "Please use POST /api/v1/documents/upload instead."
+    )
+    # Forward to the new API implementation
+    # Need to pass tenant_id explicitly since we're calling the function directly
+    tenant_id = current_user.id
+    return await document_api.upload_info(tenant_id=tenant_id)
+
+
+# =============================================================================
+# Document APIs
+# =============================================================================
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>", methods=["PUT"])
+@login_required
+async def deprecated_update_document(dataset_id, document_id):
+    """
+    Deprecated: Use PATCH /api/v1/datasets/{dataset_id}/documents/{document_id} instead.
+
+    Old path: PUT /api/v1/datasets/{dataset_id}/documents/{document_id}
+    New path: PATCH /api/v1/datasets/{dataset_id}/documents/{document_id}
+    """
+    logging.warning(
+        "API endpoint PUT /api/v1/datasets/%s/documents/%s is deprecated. "
+        "Please use PATCH instead.",
+        dataset_id, document_id,
+    )
+    # Forward to the new API implementation
+    return await document_api.update_document(dataset_id=dataset_id, document_id=document_id)
+
+
+@manager.route("/document/get/<doc_id>", methods=["GET"])
+@login_required
+async def deprecated_document_get(doc_id):
+    """
+    Deprecated: Use GET /api/v1/documents/{doc_id}/preview instead.
+
+    Old path: GET /api/v1/document/get/{doc_id}
+    New path: GET /api/v1/documents/{doc_id}/preview
+    """
+    logging.warning(
+        "API endpoint /api/v1/document/get/%s is deprecated. "
+        "Please use /api/v1/documents/%s/preview instead.",
+        doc_id, doc_id,
+    )
+    return await document_api.get(doc_id)
+
+
+@manager.route("/document/download/<doc_id>", methods=["GET"])
+@login_required
+async def deprecated_document_download(doc_id):
+    """
+    Deprecated: Use GET /api/v1/agents/attachments/{attachment_id}/download instead.
+
+    Old path: GET /api/v1/document/download/{doc_id}
+    New path: GET /api/v1/agents/attachments/{doc_id}/download
+    """
+    logging.warning(
+        "API endpoint /api/v1/document/download/%s is deprecated. "
+        "Please use /api/v1/agents/attachments/%s/download instead.",
+        doc_id, doc_id,
+    )
+    return await agent_api.download_attachment(attachment_id=doc_id)
+
+
+@legacy_v1_manager.route("/document/download/<attachment_id>", methods=["GET"])
+@login_required
+async def document_download_v1(attachment_id):
+    """
+    Compatibility alias for document download under /v1.
+
+    Old path: GET /v1/document/download/{attachment_id}
+    New path: GET /api/v1/agents/attachments/{attachment_id}/download
+    """
+    logging.warning(
+        "API endpoint /v1/document/download/%s is deprecated. "
+        "Please use /api/v1/agents/attachments/%s/download instead.",
+        attachment_id, attachment_id,
+    )
+    return await agent_api.download_attachment(attachment_id=attachment_id)
+
+# =============================================================================
+# Agent Chat API
+# =============================================================================
+
+@manager.route("/agents/<agent_id>/completions", methods=["POST"])
+@login_required
+@add_tenant_id_to_kwargs
+async def deprecated_agent_completions(agent_id, tenant_id=None):
+    """
+    Deprecated: Use POST /api/v1/agents/chat/completions instead.
+
+    Old path: POST /api/v1/agents/{agent_id}/completions
+    New path: POST /api/v1/agents/chat/completions
+    """
+    logging.warning(
+        "API endpoint /api/v1/agents/%s/completions is deprecated. "
+        "Please use /api/v1/agents/chat/completions instead.",
+        agent_id,
+    )
+    return await agent_api.agent_chat_completion(tenant_id=tenant_id, agent_id=agent_id)
+
+def register_backward_compat_routes(app_instance):
+    """
+    Register all backward compatibility routes with the app.
+    """
+    app_instance.register_blueprint(manager, url_prefix="/api/v1")
+    app_instance.register_blueprint(legacy_v1_manager, url_prefix="/v1")
+    logging.info("Backward compatibility routes registered successfully.")

api/apps/canvas_app.py

@@ -1,755 +0,0 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import copy
-import inspect
-import json
-import logging
-from functools import partial
-from quart import request, Response, make_response
-from agent.component import LLM
-from api.db import CanvasCategory
-from api.db.services.canvas_service import CanvasTemplateService, UserCanvasService, API4ConversationService
-from api.db.services.document_service import DocumentService
-from api.db.services.file_service import FileService
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.pipeline_operation_log_service import PipelineOperationLogService
-from api.db.services.task_service import queue_dataflow, CANVAS_DEBUG_DOC_ID, TaskService
-from api.db.services.user_service import TenantService
-from api.db.services.user_canvas_version import UserCanvasVersionService
-from common.constants import RetCode
-from common.misc_utils import get_uuid, thread_pool_exec
-from api.utils.api_utils import (
-    get_json_result,
-    server_error_response,
-    validate_request,
-    get_data_error_result,
-    get_request_json,
-)
-from agent.canvas import Canvas
-from agent.dsl_migration import normalize_chunker_dsl
-from peewee import MySQLDatabase, PostgresqlDatabase
-from api.db.db_models import APIToken, Task
-
-from rag.flow.pipeline import Pipeline
-from rag.nlp import search
-from rag.utils.redis_conn import REDIS_CONN
-from common import settings
-from api.apps import login_required, current_user
-from api.apps.services.canvas_replica_service import CanvasReplicaService
-from api.db.services.canvas_service import completion as agent_completion
-
-
-@manager.route('/templates', methods=['GET'])  # noqa: F821
-@login_required
-def templates():
-    return get_json_result(data=[c.to_dict() for c in CanvasTemplateService.get_all()])
-
-
-@manager.route('/rm', methods=['POST'])  # noqa: F821
-@validate_request("canvas_ids")
-@login_required
-async def rm():
-    req = await get_request_json()
-    for i in req["canvas_ids"]:
-        if not UserCanvasService.accessible(i, current_user.id):
-            return get_json_result(
-                data=False, message='Only owner of canvas authorized for this operation.',
-                code=RetCode.OPERATING_ERROR)
-        UserCanvasService.delete_by_id(i)
-    return get_json_result(data=True)
-
-
-@manager.route('/set', methods=['POST'])  # noqa: F821
-@validate_request("dsl", "title")
-@login_required
-async def save():
-    req = await get_request_json()
-    req['release'] = bool(req.get("release", ""))
-    try:
-        req["dsl"] = CanvasReplicaService.normalize_dsl(req["dsl"])
-    except ValueError as e:
-        return get_data_error_result(message=str(e))
-    cate = req.get("canvas_category", CanvasCategory.Agent)
-    if "id" not in req:
-        req["user_id"] = current_user.id
-        if UserCanvasService.query(user_id=current_user.id, title=req["title"].strip(), canvas_category=cate):
-            return get_data_error_result(message=f"{req['title'].strip()} already exists.")
-        req["id"] = get_uuid()
-        if not UserCanvasService.save(**req):
-            return get_data_error_result(message="Fail to save canvas.")
-    else:
-        if not UserCanvasService.accessible(req["id"], current_user.id):
-            return get_json_result(
-                data=False, message='Only owner of canvas authorized for this operation.',
-                code=RetCode.OPERATING_ERROR)
-        UserCanvasService.update_by_id(req["id"], req)
-    # save version
-    UserCanvasVersionService.save_or_replace_latest(
-        user_canvas_id=req["id"],
-        dsl=req["dsl"],
-        title=UserCanvasVersionService.build_version_title(getattr(current_user, "nickname", current_user.id), req.get("title")),
-        release=req.get("release"),
-    )
-    replica_ok = CanvasReplicaService.replace_for_set(
-        canvas_id=req["id"],
-        tenant_id=str(current_user.id),
-        runtime_user_id=str(current_user.id),
-        dsl=req["dsl"],
-        canvas_category=req.get("canvas_category", cate),
-        title=req.get("title", ""),
-    )
-    if not replica_ok:
-        return get_data_error_result(message="canvas saved, but replica sync failed.")
-    return get_json_result(data=req)
-
-
-@manager.route('/get/<canvas_id>', methods=['GET'])  # noqa: F821
-@login_required
-def get(canvas_id):
-    if not UserCanvasService.accessible(canvas_id, current_user.id):
-        return get_data_error_result(message="canvas not found.")
-    e, c = UserCanvasService.get_by_canvas_id(canvas_id)
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-    try:
-        # DELETE
-        CanvasReplicaService.bootstrap(
-            canvas_id=canvas_id,
-            tenant_id=str(current_user.id),
-            runtime_user_id=str(current_user.id),
-            dsl=c.get("dsl"),
-            canvas_category=c.get("canvas_category", CanvasCategory.Agent),
-            title=c.get("title", ""),
-        )
-    except ValueError as e:
-        return get_data_error_result(message=str(e))
-
-    # Get the last publication time (latest released version's update_time)
-    last_publish_time = None
-    versions = UserCanvasVersionService.list_by_canvas_id(canvas_id)
-    if versions:
-        released_versions = [v for v in versions if v.release]
-        if released_versions:
-            # Sort by update_time descending and get the latest
-            released_versions.sort(key=lambda x: x.update_time, reverse=True)
-            last_publish_time = released_versions[0].update_time
-
-    # Add last_publish_time to response data
-    if isinstance(c, dict):
-        c["dsl"] = normalize_chunker_dsl(c.get("dsl", {}))
-        c["last_publish_time"] = last_publish_time
-    else:
-        # If c is a model object, convert to dict first
-        c = c.to_dict()
-        c["dsl"] = normalize_chunker_dsl(c.get("dsl", {}))
-        c["last_publish_time"] = last_publish_time
-
-    # For pipeline type, get associated datasets
-    if c.get("canvas_category") == CanvasCategory.DataFlow:
-        datasets = list(KnowledgebaseService.query(pipeline_id=canvas_id))
-        c["datasets"] = [{"id": d.id, "name": d.name, "avatar": d.avatar} for d in datasets]
-
-    return get_json_result(data=c)
-
-
-@manager.route('/getsse/<canvas_id>', methods=['GET'])  # type: ignore # noqa: F821
-def getsse(canvas_id):
-    token = request.headers.get('Authorization').split()
-    if len(token) != 2:
-        return get_data_error_result(message='Authorization is not valid!')
-    token = token[1]
-    objs = APIToken.query(beta=token)
-    if not objs:
-        return get_data_error_result(message='Authentication error: API key is invalid!"')
-    tenant_id = objs[0].tenant_id
-    if not UserCanvasService.query(user_id=tenant_id, id=canvas_id):
-        return get_json_result(
-            data=False,
-            message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR
-        )
-    e, c = UserCanvasService.get_by_id(canvas_id)
-    if not e or c.user_id != tenant_id:
-        return get_data_error_result(message="canvas not found.")
-    return get_json_result(data=c.to_dict())
-
-
-@manager.route('/completion', methods=['POST'])  # noqa: F821
-@validate_request("id")
-@login_required
-async def run():
-    req = await get_request_json()
-    query = req.get("query", "")
-    files = req.get("files", [])
-    inputs = req.get("inputs", {})
-    tenant_id = str(current_user.id)
-    runtime_user_id = req.get("user_id") or tenant_id
-    user_id = str(runtime_user_id)
-    if not await thread_pool_exec(UserCanvasService.accessible, req["id"], tenant_id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-
-    replica_payload = CanvasReplicaService.load_for_run(
-        canvas_id=req["id"],
-        tenant_id=tenant_id,
-        runtime_user_id=user_id,
-    )
-
-    if not replica_payload:
-        return get_data_error_result(message="canvas replica not found, please call /get/<canvas_id> first.")
-
-    replica_dsl = replica_payload.get("dsl", {})
-    canvas_title = replica_payload.get("title", "")
-    canvas_category = replica_payload.get("canvas_category", CanvasCategory.Agent)
-    dsl_str = json.dumps(replica_dsl, ensure_ascii=False)
-
-    _, cvs = await thread_pool_exec(UserCanvasService.get_by_id, req["id"])
-    if cvs.canvas_category == CanvasCategory.DataFlow:
-        task_id = get_uuid()
-        Pipeline(dsl_str, tenant_id=tenant_id, doc_id=CANVAS_DEBUG_DOC_ID, task_id=task_id, flow_id=req["id"])
-        ok, error_message = await thread_pool_exec(queue_dataflow, user_id, req["id"], task_id, CANVAS_DEBUG_DOC_ID, files[0], 0)
-        if not ok:
-            return get_data_error_result(message=error_message)
-        return get_json_result(data={"message_id": task_id})
-
-    try:
-        canvas = Canvas(dsl_str, tenant_id, canvas_id=req["id"])
-    except Exception as e:
-        return server_error_response(e)
-
-    async def sse():
-        nonlocal canvas, user_id
-        try:
-            async for ans in canvas.run(query=query, files=files, user_id=user_id, inputs=inputs):
-                yield "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
-
-            commit_ok = CanvasReplicaService.commit_after_run(
-                canvas_id=req["id"],
-                tenant_id=tenant_id,
-                runtime_user_id=user_id,
-                dsl=json.loads(str(canvas)),
-                canvas_category=canvas_category,
-                title=canvas_title,
-            )
-            if not commit_ok:
-                logging.error(
-                    "Canvas runtime replica commit failed: canvas_id=%s tenant_id=%s runtime_user_id=%s",
-                    req["id"],
-                    tenant_id,
-                    user_id,
-                )
-
-        except Exception as e:
-            logging.exception(e)
-            canvas.cancel_task()
-            yield "data:" + json.dumps({"code": 500, "message": str(e), "data": False}, ensure_ascii=False) + "\n\n"
-
-    resp = Response(sse(), mimetype="text/event-stream")
-    resp.headers.add_header("Cache-control", "no-cache")
-    resp.headers.add_header("Connection", "keep-alive")
-    resp.headers.add_header("X-Accel-Buffering", "no")
-    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
-    #resp.call_on_close(lambda: canvas.cancel_task())
-    return resp
-
-
-@manager.route("/<canvas_id>/completion", methods=["POST"])  # noqa: F821
-@login_required
-async def exp_agent_completion(canvas_id):
-    tenant_id = current_user.id
-    req = await get_request_json()
-    return_trace = bool(req.get("return_trace", False))
-    async def generate():
-        trace_items = []
-        async for answer in agent_completion(tenant_id=tenant_id, agent_id=canvas_id, **req):
-            if isinstance(answer, str):
-                try:
-                    ans = json.loads(answer[5:])  # remove "data:"
-                except Exception:
-                    continue
-
-            event = ans.get("event")
-            if event == "node_finished":
-                if return_trace:
-                    data = ans.get("data", {})
-                    trace_items.append(
-                        {
-                            "component_id": data.get("component_id"),
-                            "trace": [copy.deepcopy(data)],
-                        }
-                    )
-                    ans.setdefault("data", {})["trace"] = trace_items
-                    answer = "data:" + json.dumps(ans, ensure_ascii=False) + "\n\n"
-                yield answer
-
-            if event not in ["message", "message_end"]:
-                continue
-
-            yield answer
-
-        yield "data:[DONE]\n\n"
-
-    resp = Response(generate(), mimetype="text/event-stream")
-    resp.headers.add_header("Cache-control", "no-cache")
-    resp.headers.add_header("Connection", "keep-alive")
-    resp.headers.add_header("X-Accel-Buffering", "no")
-    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
-    return resp
-    
-
-@manager.route('/rerun', methods=['POST'])  # noqa: F821
-@validate_request("id", "dsl", "component_id")
-@login_required
-async def rerun():
-    req = await get_request_json()
-    doc = PipelineOperationLogService.get_documents_info(req["id"])
-    if not doc:
-        return get_data_error_result(message="Document not found.")
-    doc = doc[0]
-    if 0 < doc["progress"] < 1:
-        return get_data_error_result(message=f"`{doc['name']}` is processing...")
-
-    if settings.docStoreConn.index_exist(search.index_name(current_user.id), doc["kb_id"]):
-        settings.docStoreConn.delete({"doc_id": doc["id"]}, search.index_name(current_user.id), doc["kb_id"])
-    doc["progress_msg"] = ""
-    doc["chunk_num"] = 0
-    doc["token_num"] = 0
-    DocumentService.clear_chunk_num_when_rerun(doc["id"])
-    DocumentService.update_by_id(id, doc)
-    TaskService.filter_delete([Task.doc_id == id])
-
-    dsl = req["dsl"]
-    dsl["path"] = [req["component_id"]]
-    PipelineOperationLogService.update_by_id(req["id"], {"dsl": dsl})
-    queue_dataflow(tenant_id=current_user.id, flow_id=req["id"], task_id=get_uuid(), doc_id=doc["id"], priority=0, rerun=True)
-    return get_json_result(data=True)
-
-
-@manager.route('/cancel/<task_id>', methods=['PUT'])  # noqa: F821
-@login_required
-def cancel(task_id):
-    try:
-        REDIS_CONN.set(f"{task_id}-cancel", "x")
-    except Exception as e:
-        logging.exception(e)
-    return get_json_result(data=True)
-
-
-@manager.route('/reset', methods=['POST'])  # noqa: F821
-@validate_request("id")
-@login_required
-async def reset():
-    req = await get_request_json()
-    if not UserCanvasService.accessible(req["id"], current_user.id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(req["id"])
-        if not e:
-            return get_data_error_result(message="canvas not found.")
-
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
-        canvas.reset()
-        req["dsl"] = json.loads(str(canvas))
-        UserCanvasService.update_by_id(req["id"], {"dsl": req["dsl"]})
-        return get_json_result(data=req["dsl"])
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/upload/<canvas_id>", methods=["POST"])  # noqa: F821
-async def upload(canvas_id):
-    e, cvs = UserCanvasService.get_by_canvas_id(canvas_id)
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-
-    user_id = cvs["user_id"]
-    files = await request.files
-    file_objs = files.getlist("file") if files and files.get("file") else []
-    try:
-        if len(file_objs) == 1:
-            return get_json_result(data=FileService.upload_info(user_id, file_objs[0], request.args.get("url")))
-        results = [FileService.upload_info(user_id, f) for f in file_objs]
-        return get_json_result(data=results)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/input_form', methods=['GET'])  # noqa: F821
-@login_required
-def input_form():
-    cvs_id = request.args.get("id")
-    cpn_id = request.args.get("component_id")
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(cvs_id)
-        if not e:
-            return get_data_error_result(message="canvas not found.")
-        if not UserCanvasService.query(user_id=current_user.id, id=cvs_id):
-            return get_json_result(
-                data=False, message='Only owner of canvas authorized for this operation.',
-                code=RetCode.OPERATING_ERROR)
-
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
-        return get_json_result(data=canvas.get_component_input_form(cpn_id))
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/debug', methods=['POST'])  # noqa: F821
-@validate_request("id", "component_id", "params")
-@login_required
-async def debug():
-    req = await get_request_json()
-    if not UserCanvasService.accessible(req["id"], current_user.id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-    try:
-        e, user_canvas = UserCanvasService.get_by_id(req["id"])
-        canvas = Canvas(json.dumps(user_canvas.dsl), current_user.id, canvas_id=user_canvas.id)
-        canvas.reset()
-        canvas.message_id = get_uuid()
-        component = canvas.get_component(req["component_id"])["obj"]
-        component.reset()
-
-        if isinstance(component, LLM):
-            component.set_debug_inputs(req["params"])
-        component.invoke(**{k: o["value"] for k,o in req["params"].items()})
-        outputs = component.output()
-        for k in outputs.keys():
-            if isinstance(outputs[k], partial):
-                txt = ""
-                iter_obj = outputs[k]()
-                if inspect.isasyncgen(iter_obj):
-                    async for c in iter_obj:
-                        txt += c
-                else:
-                    for c in iter_obj:
-                        txt += c
-                outputs[k] = txt
-        return get_json_result(data=outputs)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/test_db_connect', methods=['POST'])  # noqa: F821
-@validate_request("db_type", "database", "username", "host", "port", "password")
-@login_required
-async def test_db_connect():
-    req = await get_request_json()
-    try:
-        if req["db_type"] in ["mysql", "mariadb"]:
-            db = MySQLDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
-                               password=req["password"])
-        elif req["db_type"] == "oceanbase":
-            db = MySQLDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
-                               password=req["password"], charset="utf8mb4")
-        elif req["db_type"] == 'postgres':
-            db = PostgresqlDatabase(req["database"], user=req["username"], host=req["host"], port=req["port"],
-                                    password=req["password"])
-        elif req["db_type"] == 'mssql':
-            import pyodbc
-            connection_string = (
-                f"DRIVER={{ODBC Driver 17 for SQL Server}};"
-                f"SERVER={req['host']},{req['port']};"
-                f"DATABASE={req['database']};"
-                f"UID={req['username']};"
-                f"PWD={req['password']};"
-            )
-            db = pyodbc.connect(connection_string)
-            cursor = db.cursor()
-            cursor.execute("SELECT 1")
-            cursor.close()
-        elif req["db_type"] == 'IBM DB2':
-            import ibm_db
-            conn_str = (
-                f"DATABASE={req['database']};"
-                f"HOSTNAME={req['host']};"
-                f"PORT={req['port']};"
-                f"PROTOCOL=TCPIP;"
-                f"UID={req['username']};"
-                f"PWD={req['password']};"
-            )
-            redacted_conn_str = (
-                f"DATABASE={req['database']};"
-                f"HOSTNAME={req['host']};"
-                f"PORT={req['port']};"
-                f"PROTOCOL=TCPIP;"
-                f"UID={req['username']};"
-                f"PWD=****;"
-            )
-            logging.info(redacted_conn_str)
-            conn = ibm_db.connect(conn_str, "", "")
-            stmt = ibm_db.exec_immediate(conn, "SELECT 1 FROM sysibm.sysdummy1")
-            ibm_db.fetch_assoc(stmt)
-            ibm_db.close(conn)
-            return get_json_result(data="Database Connection Successful!")
-        elif req["db_type"] == 'trino':
-            def _parse_catalog_schema(db_name: str):
-                if not db_name:
-                    return None, None
-                if "." in db_name:
-                    catalog_name, schema_name = db_name.split(".", 1)
-                elif "/" in db_name:
-                    catalog_name, schema_name = db_name.split("/", 1)
-                else:
-                    catalog_name, schema_name = db_name, "default"
-                return catalog_name, schema_name
-            try:
-                import trino
-                import os
-            except Exception as e:
-                return server_error_response(f"Missing dependency 'trino'. Please install: pip install trino, detail: {e}")
-
-            catalog, schema = _parse_catalog_schema(req["database"])
-            if not catalog:
-                return server_error_response("For Trino, 'database' must be 'catalog.schema' or at least 'catalog'.")
-
-            http_scheme = "https" if os.environ.get("TRINO_USE_TLS", "0") == "1" else "http"
-
-            auth = None
-            if http_scheme == "https" and req.get("password"):
-                auth = trino.BasicAuthentication(req.get("username") or "ragflow", req["password"])
-
-            conn = trino.dbapi.connect(
-                host=req["host"],
-                port=int(req["port"] or 8080),
-                user=req["username"] or "ragflow",
-                catalog=catalog,
-                schema=schema or "default",
-                http_scheme=http_scheme,
-                auth=auth
-            )
-            cur = conn.cursor()
-            cur.execute("SELECT 1")
-            cur.fetchall()
-            cur.close()
-            conn.close()
-            return get_json_result(data="Database Connection Successful!")
-        else:
-            return server_error_response("Unsupported database type.")
-        if req["db_type"] != 'mssql':
-            db.connect()
-        db.close()
-
-        return get_json_result(data="Database Connection Successful!")
-    except Exception as e:
-        return server_error_response(e)
-
-
-#api get list version dsl of canvas
-@manager.route('/getlistversion/<canvas_id>', methods=['GET'])  # noqa: F821
-@login_required
-def getlistversion(canvas_id):
-    try:
-        versions =sorted([c.to_dict() for c in UserCanvasVersionService.list_by_canvas_id(canvas_id)], key=lambda x: x["update_time"]*-1)
-        return get_json_result(data=versions)
-    except Exception as e:
-        return get_data_error_result(message=f"Error getting history files: {e}")
-
-
-#api get version dsl of canvas
-@manager.route('/getversion/<version_id>', methods=['GET'])  # noqa: F821
-@login_required
-def getversion( version_id):
-    try:
-        e, version = UserCanvasVersionService.get_by_id(version_id)
-        if version:
-            return get_json_result(data=version.to_dict())
-    except Exception as e:
-        return get_json_result(data=f"Error getting history file: {e}")
-
-
-@manager.route('/list', methods=['GET'])  # noqa: F821
-@login_required
-def list_canvas():
-    keywords = request.args.get("keywords", "")
-    page_number = int(request.args.get("page", 0))
-    items_per_page = int(request.args.get("page_size", 0))
-    orderby = request.args.get("orderby", "create_time")
-    canvas_category = request.args.get("canvas_category")
-    if request.args.get("desc", "true").lower() == "false":
-        desc = False
-    else:
-        desc = True
-    owner_ids = [id for id in request.args.get("owner_ids", "").strip().split(",") if id]
-    if not owner_ids:
-        tenants = TenantService.get_joined_tenants_by_user_id(current_user.id)
-        tenants = [m["tenant_id"] for m in tenants]
-        tenants.append(current_user.id)
-        canvas, total = UserCanvasService.get_by_tenant_ids(
-            tenants, current_user.id, page_number,
-            items_per_page, orderby, desc, keywords, canvas_category)
-    else:
-        tenants = owner_ids
-        canvas, total = UserCanvasService.get_by_tenant_ids(
-            tenants, current_user.id, 0,
-            0, orderby, desc, keywords, canvas_category)
-    return get_json_result(data={"canvas": canvas, "total": total})
-
-
-@manager.route('/setting', methods=['POST'])  # noqa: F821
-@validate_request("id", "title", "permission")
-@login_required
-async def setting():
-    req = await get_request_json()
-    req["user_id"] = current_user.id
-
-    if not UserCanvasService.accessible(req["id"], current_user.id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-
-    e,flow = UserCanvasService.get_by_id(req["id"])
-    if not e:
-        return get_data_error_result(message="canvas not found.")
-    flow = flow.to_dict()
-    flow["title"] = req["title"]
-
-    for key in ["description", "permission", "avatar"]:
-        if value := req.get(key):
-            flow[key] = value
-
-    num= UserCanvasService.update_by_id(req["id"], flow)
-    return get_json_result(data=num)
-
-
-@manager.route('/trace', methods=['GET'])  # noqa: F821
-def trace():
-    cvs_id = request.args.get("canvas_id")
-    msg_id = request.args.get("message_id")
-    try:
-        binary = REDIS_CONN.get(f"{cvs_id}-{msg_id}-logs")
-        if not binary:
-            return get_json_result(data={})
-
-        return get_json_result(data=json.loads(binary.encode("utf-8")))
-    except Exception as e:
-        logging.exception(e)
-
-
-@manager.route('/<canvas_id>/sessions', methods=['GET'])  # noqa: F821
-@login_required
-def sessions(canvas_id):
-    tenant_id = current_user.id
-    if not UserCanvasService.accessible(canvas_id, tenant_id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-
-    user_id = request.args.get("user_id")
-    page_number = int(request.args.get("page", 1))
-    items_per_page = int(request.args.get("page_size", 30))
-    keywords = request.args.get("keywords")
-    from_date = request.args.get("from_date")
-    to_date = request.args.get("to_date")
-    orderby = request.args.get("orderby", "update_time")
-    exp_user_id = request.args.get("exp_user_id")
-    if request.args.get("desc") == "False" or request.args.get("desc") == "false":
-        desc = False
-    else:
-        desc = True
-
-    if exp_user_id:
-        sess = API4ConversationService.get_names(canvas_id, exp_user_id)
-        return get_json_result(data={"total": len(sess), "sessions": sess})
-    
-    # dsl defaults to True in all cases except for False and false
-    include_dsl = request.args.get("dsl") != "False" and request.args.get("dsl") != "false"
-    total, sess = API4ConversationService.get_list(canvas_id, tenant_id, page_number, items_per_page, orderby, desc,
-                                             None, user_id, include_dsl, keywords, from_date, to_date, exp_user_id=exp_user_id)
-    try:
-        return get_json_result(data={"total": total, "sessions": sess})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/<canvas_id>/sessions', methods=['PUT'])  # noqa: F821
-@login_required
-async def set_session(canvas_id):
-    req = await get_request_json()
-    tenant_id = current_user.id
-    e, cvs = UserCanvasService.get_by_id(canvas_id)
-    assert e, "Agent not found."
-    if not isinstance(cvs.dsl, str):
-        cvs.dsl = json.dumps(cvs.dsl, ensure_ascii=False)
-    session_id=get_uuid()
-    canvas = Canvas(cvs.dsl, tenant_id, canvas_id, canvas_id=cvs.id)
-    canvas.reset()
-    # Get the version title for this canvas (using latest, not necessarily released)
-    version_title = UserCanvasVersionService.get_latest_version_title(cvs.id, release_mode=False)
-    conv = {
-        "id": session_id,
-        "name": req.get("name", ""),
-        "dialog_id": cvs.id,
-        "user_id": tenant_id,
-        "exp_user_id": tenant_id,
-        "message": [],
-        "source": "agent",
-        "dsl": cvs.dsl,
-        "reference": [],
-        "version_title": version_title
-    }
-    API4ConversationService.save(**conv)
-    return get_json_result(data=conv)
-
-
-@manager.route('/<canvas_id>/sessions/<session_id>', methods=['GET'])  # noqa: F821
-@login_required
-def get_session(canvas_id, session_id):
-    tenant_id = current_user.id
-    if not UserCanvasService.accessible(canvas_id, tenant_id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-    _, conv = API4ConversationService.get_by_id(session_id)
-    return get_json_result(data=conv.to_dict())
-
-
-@manager.route('/<canvas_id>/sessions/<session_id>', methods=['DELETE'])  # noqa: F821
-@login_required
-def del_session(canvas_id, session_id):
-    tenant_id = current_user.id
-    if not UserCanvasService.accessible(canvas_id, tenant_id):
-        return get_json_result(
-            data=False, message='Only owner of canvas authorized for this operation.',
-            code=RetCode.OPERATING_ERROR)
-    return get_json_result(data=API4ConversationService.delete_by_id(session_id))
-
-
-@manager.route('/prompts', methods=['GET'])  # noqa: F821
-@login_required
-def prompts():
-    from rag.prompts.generator import ANALYZE_TASK_SYSTEM, ANALYZE_TASK_USER, NEXT_STEP, REFLECT, CITATION_PROMPT_TEMPLATE
-
-    return get_json_result(data={
-        "task_analysis": ANALYZE_TASK_SYSTEM +"\n\n"+ ANALYZE_TASK_USER,
-        "plan_generation": NEXT_STEP,
-        "reflection": REFLECT,
-        #"context_summary": SUMMARY4MEMORY,
-        #"context_ranking": RANK_MEMORY,
-        "citation_guidelines": CITATION_PROMPT_TEMPLATE
-    })
-
-
-@manager.route('/download', methods=['GET'])  # noqa: F821
-async def download():
-    id = request.args.get("id")
-    created_by = request.args.get("created_by")
-    blob = FileService.get_blob(created_by, id)
-    return await make_response(blob)

api/apps/chunk_app.py

@@ -1,580 +0,0 @@
-#
-#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-import base64
-import datetime
-import json
-import logging
-import re
-import xxhash
-from quart import request
-
-from api.db.services.document_service import DocumentService
-from api.db.services.doc_metadata_service import DocMetadataService
-from api.utils.image_utils import store_chunk_image
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.llm_service import LLMBundle
-from common.metadata_utils import apply_meta_data_filter
-from api.db.services.search_service import SearchService
-from api.db.services.user_service import UserTenantService
-from api.db.joint_services.tenant_model_service import get_model_config_by_id, get_tenant_default_model_by_type, get_model_config_by_type_and_name
-from api.utils.api_utils import (
-    get_data_error_result,
-    get_json_result,
-    server_error_response,
-    validate_request,
-    get_request_json,
-)
-from common.misc_utils import thread_pool_exec
-from common.tag_feature_utils import validate_tag_features
-from rag.app.qa import beAdoc, rmPrefix
-from rag.app.tag import label_question
-from rag.nlp import rag_tokenizer, search
-from rag.prompts.generator import cross_languages, keyword_extraction
-from common.string_utils import is_content_empty, remove_redundant_spaces
-from common.constants import RetCode, LLMType, ParserType, PAGERANK_FLD
-from common import settings
-from api.apps import login_required, current_user
-
-@manager.route('/list', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("doc_id")
-async def list_chunk():
-    req = await get_request_json()
-    doc_id = req["doc_id"]
-    page = int(req.get("page", 1))
-    size = int(req.get("size", 30))
-    question = req.get("keywords", "")
-    try:
-        tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-        if not tenant_id:
-            return get_data_error_result(message="Tenant not found!")
-        e, doc = DocumentService.get_by_id(doc_id)
-        if not e:
-            return get_data_error_result(message="Document not found!")
-        kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
-        query = {
-            "doc_ids": [doc_id], "page": page, "size": size, "question": question, "sort": True
-        }
-        if "available_int" in req:
-            query["available_int"] = int(req["available_int"])
-        sres = await settings.retriever.search(query, search.index_name(tenant_id), kb_ids, highlight=["content_ltks"])
-        res = {"total": sres.total, "chunks": [], "doc": doc.to_dict()}
-        for id in sres.ids:
-            d = {
-                "chunk_id": id,
-                "content_with_weight": remove_redundant_spaces(sres.highlight[id]) if question and id in sres.highlight else sres.field[
-                    id].get(
-                    "content_with_weight", ""),
-                "doc_id": sres.field[id]["doc_id"],
-                "docnm_kwd": sres.field[id]["docnm_kwd"],
-                "important_kwd": sres.field[id].get("important_kwd", []),
-                "question_kwd": sres.field[id].get("question_kwd", []),
-                "image_id": sres.field[id].get("img_id", ""),
-                "available_int": int(sres.field[id].get("available_int", 1)),
-                "positions": sres.field[id].get("position_int", []),
-                "doc_type_kwd": sres.field[id].get("doc_type_kwd")
-            }
-            assert isinstance(d["positions"], list)
-            assert len(d["positions"]) == 0 or (isinstance(d["positions"][0], list) and len(d["positions"][0]) == 5)
-            res["chunks"].append(d)
-        return get_json_result(data=res)
-    except Exception as e:
-        if str(e).find("not_found") > 0:
-            return get_json_result(data=False, message='No chunk found!',
-                                   code=RetCode.DATA_ERROR)
-        return server_error_response(e)
-
-
-@manager.route('/get', methods=['GET'])  # noqa: F821
-@login_required
-def get():
-    chunk_id = request.args["chunk_id"]
-    try:
-        chunk = None
-        tenants = UserTenantService.query(user_id=current_user.id)
-        if not tenants:
-            return get_data_error_result(message="Tenant not found!")
-        for tenant in tenants:
-            kb_ids = KnowledgebaseService.get_kb_ids(tenant.tenant_id)
-            chunk = settings.docStoreConn.get(chunk_id, search.index_name(tenant.tenant_id), kb_ids)
-            if chunk:
-                break
-        if chunk is None:
-            return server_error_response(Exception("Chunk not found"))
-
-        k = []
-        for n in chunk.keys():
-            if re.search(r"(_vec$|_sm_|_tks|_ltks)", n):
-                k.append(n)
-        for n in k:
-            del chunk[n]
-
-        return get_json_result(data=chunk)
-    except Exception as e:
-        if str(e).find("NotFoundError") >= 0:
-            return get_json_result(data=False, message='Chunk not found!',
-                                   code=RetCode.DATA_ERROR)
-        return server_error_response(e)
-
-
-@manager.route('/set', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("doc_id", "chunk_id", "content_with_weight")
-async def set():
-    req = await get_request_json()
-    content_with_weight = req["content_with_weight"]
-    if not isinstance(content_with_weight, (str, bytes)):
-        raise TypeError("expected string or bytes-like object")
-    if isinstance(content_with_weight, bytes):
-        content_with_weight = content_with_weight.decode("utf-8", errors="ignore")
-    if is_content_empty(content_with_weight):
-        return get_data_error_result(message="`content_with_weight` is required")
-    d = {
-        "id": req["chunk_id"],
-        "content_with_weight": content_with_weight}
-    d["content_ltks"] = rag_tokenizer.tokenize(content_with_weight)
-    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
-    if "important_kwd" in req:
-        if not isinstance(req["important_kwd"], list):
-            return get_data_error_result(message="`important_kwd` should be a list")
-        d["important_kwd"] = req["important_kwd"]
-        d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_kwd"]))
-    if "question_kwd" in req:
-        if not isinstance(req["question_kwd"], list):
-            return get_data_error_result(message="`question_kwd` should be a list")
-        d["question_kwd"] = req["question_kwd"]
-        d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["question_kwd"]))
-    if "tag_kwd" in req:
-        if not isinstance(req["tag_kwd"], list):
-            return get_data_error_result(message="`tag_kwd` should be a list")
-        if not all(isinstance(t, str) for t in req["tag_kwd"]):
-            return get_data_error_result(message="`tag_kwd` must be a list of strings")
-        d["tag_kwd"] = req["tag_kwd"]
-    if "tag_feas" in req:
-        try:
-            d["tag_feas"] = validate_tag_features(req["tag_feas"])
-        except ValueError as exc:
-            return get_data_error_result(message=f"`tag_feas` {exc}")
-    if "available_int" in req:
-        d["available_int"] = req["available_int"]
-
-    try:
-        def _set_sync():
-            tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-            if not tenant_id:
-                return get_data_error_result(message="Tenant not found!")
-
-            e, doc = DocumentService.get_by_id(req["doc_id"])
-            if not e:
-                return get_data_error_result(message="Document not found!")
-
-            tenant_embd_id = DocumentService.get_tenant_embd_id(req["doc_id"])
-            if tenant_embd_id:
-                embd_model_config = get_model_config_by_id(tenant_embd_id)
-            else:
-                embd_id = DocumentService.get_embd_id(req["doc_id"])
-                if embd_id:
-                    embd_model_config = get_model_config_by_type_and_name(tenant_id, LLMType.EMBEDDING, embd_id)
-                else:
-                    embd_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.EMBEDDING)
-            embd_mdl = LLMBundle(tenant_id, embd_model_config)
-
-            _d = d
-            if doc.parser_id == ParserType.QA:
-                arr = [
-                    t for t in re.split(
-                        r"[\n\t]",
-                        req["content_with_weight"]) if len(t) > 1]
-                q, a = rmPrefix(arr[0]), rmPrefix("\n".join(arr[1:]))
-                _d = beAdoc(d, q, a, not any(
-                    [rag_tokenizer.is_chinese(t) for t in q + a]))
-
-            v, c = embd_mdl.encode([doc.name, content_with_weight if not _d.get("question_kwd") else "\n".join(_d["question_kwd"])])
-            v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
-            _d["q_%d_vec" % len(v)] = v.tolist()
-            settings.docStoreConn.update({"id": req["chunk_id"]}, _d, search.index_name(tenant_id), doc.kb_id)
-
-            # update image
-            image_base64 = req.get("image_base64", None)
-            img_id = req.get("img_id", "")
-            if image_base64 and img_id and "-" in img_id:
-                bkt, name = img_id.split("-", 1)
-                image_binary = base64.b64decode(image_base64)
-                settings.STORAGE_IMPL.put(bkt, name, image_binary)
-            return get_json_result(data=True)
-
-        return await thread_pool_exec(_set_sync)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/switch', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("chunk_ids", "available_int", "doc_id")
-async def switch():
-    req = await get_request_json()
-    try:
-        def _switch_sync():
-            e, doc = DocumentService.get_by_id(req["doc_id"])
-            if not e:
-                return get_data_error_result(message="Document not found!")
-            for cid in req["chunk_ids"]:
-                if not settings.docStoreConn.update({"id": cid},
-                                                    {"available_int": int(req["available_int"])},
-                                                    search.index_name(DocumentService.get_tenant_id(req["doc_id"])),
-                                                    doc.kb_id):
-                    return get_data_error_result(message="Index updating failure")
-            return get_json_result(data=True)
-
-        return await thread_pool_exec(_switch_sync)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/rm', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("doc_id")
-async def rm():
-    req = await get_request_json()
-    try:
-        def _rm_sync():
-            deleted_chunk_ids = req.get("chunk_ids")
-            if isinstance(deleted_chunk_ids, list):
-                unique_chunk_ids = list(dict.fromkeys(deleted_chunk_ids))
-                has_ids = len(unique_chunk_ids) > 0
-            elif deleted_chunk_ids is not None:
-                unique_chunk_ids = [deleted_chunk_ids]
-                has_ids = deleted_chunk_ids not in (None, "")
-            else:
-                unique_chunk_ids = []
-                has_ids = False
-            if not has_ids:
-                if req.get("delete_all") is True:
-                    e, doc = DocumentService.get_by_id(req["doc_id"])
-                    if not e:
-                        return get_data_error_result(message="Document not found!")
-                    tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-                    # Clean up storage assets while index rows still exist for discovery
-                    DocumentService.delete_chunk_images(doc, tenant_id)
-                    condition = {"doc_id": req["doc_id"]}
-                    try:
-                        deleted_count = settings.docStoreConn.delete(condition, search.index_name(tenant_id), doc.kb_id)
-                    except Exception:
-                        return get_data_error_result(message="Chunk deleting failure")
-                    if deleted_count > 0:
-                        DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, deleted_count, 0)
-                    return get_json_result(data=True)
-                return get_json_result(data=True)
-
-            e, doc = DocumentService.get_by_id(req["doc_id"])
-            if not e:
-                return get_data_error_result(message="Document not found!")
-            condition = {"id": req["chunk_ids"], "doc_id": req["doc_id"]}
-            try:
-                deleted_count = settings.docStoreConn.delete(condition,
-                                                             search.index_name(DocumentService.get_tenant_id(req["doc_id"])),
-                                                             doc.kb_id)
-            except Exception:
-                return get_data_error_result(message="Chunk deleting failure")
-            if has_ids and deleted_count == 0:
-                return get_data_error_result(message="Index updating failure")
-            if deleted_count > 0 and deleted_count < len(unique_chunk_ids):
-                deleted_count += settings.docStoreConn.delete({"doc_id": req["doc_id"]},
-                                                              search.index_name(DocumentService.get_tenant_id(req["doc_id"])),
-                                                              doc.kb_id)
-            chunk_number = deleted_count
-            DocumentService.decrement_chunk_num(doc.id, doc.kb_id, 1, chunk_number, 0)
-            for cid in deleted_chunk_ids:
-                if settings.STORAGE_IMPL.obj_exist(doc.kb_id, cid):
-                    settings.STORAGE_IMPL.rm(doc.kb_id, cid)
-            return get_json_result(data=True)
-
-        return await thread_pool_exec(_rm_sync)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/create', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("doc_id", "content_with_weight")
-async def create():
-    req = await get_request_json()
-    req_id = request.headers.get("X-Request-ID")
-    chunck_id = xxhash.xxh64((req["content_with_weight"] + req["doc_id"]).encode("utf-8")).hexdigest()
-    d = {"id": chunck_id, "content_ltks": rag_tokenizer.tokenize(req["content_with_weight"]),
-         "content_with_weight": req["content_with_weight"]}
-    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
-    d["important_kwd"] = req.get("important_kwd", [])
-    if not isinstance(d["important_kwd"], list):
-        return get_data_error_result(message="`important_kwd` is required to be a list")
-    d["important_tks"] = rag_tokenizer.tokenize(" ".join(d["important_kwd"]))
-    d["question_kwd"] = req.get("question_kwd", [])
-    if not isinstance(d["question_kwd"], list):
-        return get_data_error_result(message="`question_kwd` is required to be a list")
-    d["question_tks"] = rag_tokenizer.tokenize("\n".join(d["question_kwd"]))
-    d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
-    d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
-    if "tag_kwd" in req:
-        if not isinstance(req["tag_kwd"], list):
-            return get_data_error_result(message="`tag_kwd` is required to be a list")
-        if not all(isinstance(t, str) for t in req["tag_kwd"]):
-            return get_data_error_result(message="`tag_kwd` must be a list of strings")
-        d["tag_kwd"] = req["tag_kwd"]
-    if "tag_feas" in req:
-        try:
-            d["tag_feas"] = validate_tag_features(req["tag_feas"])
-        except ValueError as exc:
-            return get_data_error_result(message=f"`tag_feas` {exc}")
-    image_base64 = req.get("image_base64", None)
-
-    try:
-        def _log_response(resp, code, message):
-            logging.info(
-                "chunk_create response req_id=%s status=%s code=%s message=%s",
-                req_id,
-                getattr(resp, "status_code", None),
-                code,
-                message,
-            )
-
-        def _create_sync():
-            e, doc = DocumentService.get_by_id(req["doc_id"])
-            if not e:
-                resp = get_data_error_result(message="Document not found!")
-                _log_response(resp, RetCode.DATA_ERROR, "Document not found!")
-                return resp
-            d["kb_id"] = [doc.kb_id]
-            d["docnm_kwd"] = doc.name
-            d["title_tks"] = rag_tokenizer.tokenize(doc.name)
-            d["doc_id"] = doc.id
-
-            tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-            if not tenant_id:
-                resp = get_data_error_result(message="Tenant not found!")
-                _log_response(resp, RetCode.DATA_ERROR, "Tenant not found!")
-                return resp
-
-            e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
-            if not e:
-                resp = get_data_error_result(message="Knowledgebase not found!")
-                _log_response(resp, RetCode.DATA_ERROR, "Knowledgebase not found!")
-                return resp
-            if kb.pagerank:
-                d[PAGERANK_FLD] = kb.pagerank
-
-            tenant_embd_id = DocumentService.get_tenant_embd_id(req["doc_id"])
-            if tenant_embd_id:
-                embd_model_config = get_model_config_by_id(tenant_embd_id)
-            else:
-                embd_id = DocumentService.get_embd_id(req["doc_id"])
-                if embd_id:
-                    embd_model_config = get_model_config_by_type_and_name(tenant_id, LLMType.EMBEDDING, embd_id)
-                else:
-                    embd_model_config = get_tenant_default_model_by_type(tenant_id, LLMType.EMBEDDING)
-            embd_mdl = LLMBundle(tenant_id, embd_model_config)
-
-            if image_base64:
-                d["img_id"] = "{}-{}".format(doc.kb_id, chunck_id)
-                d["doc_type_kwd"] = "image"
-
-            v, c = embd_mdl.encode([doc.name, req["content_with_weight"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
-            v = 0.1 * v[0] + 0.9 * v[1]
-            d["q_%d_vec" % len(v)] = v.tolist()
-            settings.docStoreConn.insert([d], search.index_name(tenant_id), doc.kb_id)
-
-            if image_base64:
-                store_chunk_image(doc.kb_id, chunck_id, base64.b64decode(image_base64))
-
-            DocumentService.increment_chunk_num(
-                doc.id, doc.kb_id, c, 1, 0)
-            resp = get_json_result(data={"chunk_id": chunck_id, "image_id": d.get("img_id", "")})
-            _log_response(resp, RetCode.SUCCESS, "success")
-            return resp
-
-        return await thread_pool_exec(_create_sync)
-    except Exception as e:
-        logging.info("chunk_create exception req_id=%s error=%r", req_id, e)
-        return server_error_response(e)
-
-
-@manager.route('/retrieval_test', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("kb_id", "question")
-async def retrieval_test():
-    req = await get_request_json()
-    page = int(req.get("page", 1))
-    size = int(req.get("size", 30))
-    question = req["question"]
-    kb_ids = req["kb_id"]
-    if isinstance(kb_ids, str):
-        kb_ids = [kb_ids]
-    if not kb_ids:
-        return get_json_result(data=False, message='Please specify dataset firstly.',
-                               code=RetCode.DATA_ERROR)
-
-    doc_ids = req.get("doc_ids", [])
-    use_kg = req.get("use_kg", False)
-    top = int(req.get("top_k", 1024))
-    langs = req.get("cross_languages", [])
-    user_id = current_user.id
-
-    async def _retrieval():
-        local_doc_ids = list(doc_ids) if doc_ids else []
-        tenant_ids = []
-
-        meta_data_filter = {}
-        chat_mdl = None
-        if req.get("search_id", ""):
-            search_config = SearchService.get_detail(req.get("search_id", "")).get("search_config", {})
-            meta_data_filter = search_config.get("meta_data_filter", {})
-            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
-                chat_id = search_config.get("chat_id", "")
-                if chat_id:
-                    chat_model_config = get_model_config_by_type_and_name(user_id, LLMType.CHAT, search_config["chat_id"])
-                else:
-                    chat_model_config = get_tenant_default_model_by_type(user_id, LLMType.CHAT)
-                chat_mdl = LLMBundle(user_id, chat_model_config)
-        else:
-            meta_data_filter = req.get("meta_data_filter") or {}
-            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
-                chat_model_config = get_tenant_default_model_by_type(user_id, LLMType.CHAT)
-                chat_mdl = LLMBundle(user_id, chat_model_config)
-
-        if meta_data_filter:
-            metas = DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
-            local_doc_ids = await apply_meta_data_filter(meta_data_filter, metas, question, chat_mdl, local_doc_ids)
-
-        tenants = UserTenantService.query(user_id=user_id)
-        for kb_id in kb_ids:
-            for tenant in tenants:
-                if KnowledgebaseService.query(
-                        tenant_id=tenant.tenant_id, id=kb_id):
-                    tenant_ids.append(tenant.tenant_id)
-                    break
-            else:
-                return get_json_result(
-                    data=False, message='Only owner of dataset authorized for this operation.',
-                    code=RetCode.OPERATING_ERROR)
-
-        e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
-        if not e:
-            return get_data_error_result(message="Knowledgebase not found!")
-
-        _question = question
-        if langs:
-            _question = await cross_languages(kb.tenant_id, None, _question, langs)
-        if kb.tenant_embd_id:
-            embd_model_config = get_model_config_by_id(kb.tenant_embd_id)
-        elif kb.embd_id:
-            embd_model_config = get_model_config_by_type_and_name(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
-        else:
-            embd_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.EMBEDDING)
-        embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
-
-        rerank_mdl = None
-        if req.get("tenant_rerank_id"):
-            rerank_model_config = get_model_config_by_id(req["tenant_rerank_id"])
-            rerank_mdl = LLMBundle(kb.tenant_id, rerank_model_config)
-        elif req.get("rerank_id"):
-            rerank_model_config = get_model_config_by_type_and_name(kb.tenant_id, LLMType.RERANK.value, req["rerank_id"])
-            rerank_mdl = LLMBundle(kb.tenant_id, rerank_model_config)
-
-        if req.get("keyword", False):
-            default_chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
-            chat_mdl = LLMBundle(kb.tenant_id, default_chat_model_config)
-            _question += await keyword_extraction(chat_mdl, _question)
-
-        labels = label_question(_question, [kb])
-        ranks = await settings.retriever.retrieval(
-                        _question,
-                        embd_mdl,
-                        tenant_ids,
-                        kb_ids,
-                        page,
-                        size,
-                        float(req.get("similarity_threshold", 0.0)),
-                        float(req.get("vector_similarity_weight", 0.3)),
-                        doc_ids=local_doc_ids,
-                        top=top,
-                        rerank_mdl=rerank_mdl,
-                        rank_feature=labels
-                    )
-
-        if use_kg:
-            default_chat_model_config = get_tenant_default_model_by_type(user_id, LLMType.CHAT)
-            ck = await settings.kg_retriever.retrieval(_question,
-                                                   tenant_ids,
-                                                   kb_ids,
-                                                   embd_mdl,
-                                                   LLMBundle(kb.tenant_id, default_chat_model_config))
-            if ck["content_with_weight"]:
-                ranks["chunks"].insert(0, ck)
-        ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], tenant_ids)
-
-        for c in ranks["chunks"]:
-            c.pop("vector", None)
-        ranks["labels"] = labels
-
-        return get_json_result(data=ranks)
-
-    try:
-        return await _retrieval()
-    except Exception as e:
-        if str(e).find("not_found") > 0:
-            return get_json_result(data=False, message='No chunk found! Check the chunk status please!',
-                                   code=RetCode.DATA_ERROR)
-        return server_error_response(e)
-
-
-@manager.route('/knowledge_graph', methods=['GET'])  # noqa: F821
-@login_required
-async def knowledge_graph():
-    doc_id = request.args["doc_id"]
-    tenant_id = DocumentService.get_tenant_id(doc_id)
-    kb_ids = KnowledgebaseService.get_kb_ids(tenant_id)
-    req = {
-        "doc_ids": [doc_id],
-        "knowledge_graph_kwd": ["graph", "mind_map"]
-    }
-    sres = await settings.retriever.search(req, search.index_name(tenant_id), kb_ids)
-    obj = {"graph": {}, "mind_map": {}}
-    for id in sres.ids[:2]:
-        ty = sres.field[id]["knowledge_graph_kwd"]
-        try:
-            content_json = json.loads(sres.field[id]["content_with_weight"])
-        except Exception:
-            continue
-
-        if ty == 'mind_map':
-            node_dict = {}
-
-            def repeat_deal(content_json, node_dict):
-                if 'id' in content_json:
-                    if content_json['id'] in node_dict:
-                        node_name = content_json['id']
-                        content_json['id'] += f"({node_dict[content_json['id']]})"
-                        node_dict[node_name] += 1
-                    else:
-                        node_dict[content_json['id']] = 1
-                if 'children' in content_json and content_json['children']:
-                    for item in content_json['children']:
-                        repeat_deal(item, node_dict)
-
-            repeat_deal(content_json, node_dict)
-
-        obj[ty] = content_json
-
-    return get_json_result(data=obj)

api/apps/document_app.py

@@ -1,716 +0,0 @@
-#
-#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License
-#
-import os.path
-import re
-from pathlib import Path, PurePosixPath, PureWindowsPath
-
-from quart import make_response, request
-
-from api.apps import current_user, login_required
-from api.common.check_team_permission import check_kb_team_permission
-from api.constants import FILE_NAME_LEN_LIMIT, IMG_BASE64_PREFIX
-from api.db import VALID_FILE_TYPES, FileType
-from api.db.db_models import Task
-from api.db.services import duplicate_name
-from api.db.services.doc_metadata_service import DocMetadataService
-from api.db.services.document_service import DocumentService, doc_upload_and_parse
-from api.db.services.file2document_service import File2DocumentService
-from api.db.services.file_service import FileService
-from api.db.services.knowledgebase_service import KnowledgebaseService
-from api.db.services.task_service import TaskService, cancel_all_task_of
-from api.db.services.user_service import UserTenantService
-from api.utils.api_utils import (
-    get_data_error_result,
-    get_json_result,
-    get_request_json,
-    server_error_response,
-    validate_request,
-)
-from api.utils.file_utils import filename_type, thumbnail
-from api.utils.web_utils import CONTENT_TYPE_MAP, apply_safe_file_response_headers, html2pdf, is_valid_url
-from common import settings
-from common.constants import SANDBOX_ARTIFACT_BUCKET, VALID_TASK_STATUS, ParserType, RetCode, TaskStatus
-from common.file_utils import get_project_base_directory
-from common.misc_utils import get_uuid, thread_pool_exec
-from deepdoc.parser.html_parser import RAGFlowHtmlParser
-from rag.nlp import search
-
-
-def _is_safe_download_filename(name: str) -> bool:
-    if not name or name in {".", ".."}:
-        return False
-    if "\x00" in name or len(name) > 255:
-        return False
-    if name != PurePosixPath(name).name:
-        return False
-    if name != PureWindowsPath(name).name:
-        return False
-    return True
-
-
-@manager.route("/web_crawl", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("kb_id", "name", "url")
-async def web_crawl():
-    form = await request.form
-    kb_id = form.get("kb_id")
-    if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
-    name = form.get("name")
-    url = form.get("url")
-    if not is_valid_url(url):
-        return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
-    e, kb = KnowledgebaseService.get_by_id(kb_id)
-    if not e:
-        raise LookupError("Can't find this dataset!")
-    if not check_kb_team_permission(kb, current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-
-    blob = html2pdf(url)
-    if not blob:
-        return server_error_response(ValueError("Download failure."))
-
-    root_folder = FileService.get_root_folder(current_user.id)
-    pf_id = root_folder["id"]
-    FileService.init_knowledgebase_docs(pf_id, current_user.id)
-    kb_root_folder = FileService.get_kb_folder(current_user.id)
-    kb_folder = FileService.new_a_file_from_kb(kb.tenant_id, kb.name, kb_root_folder["id"])
-
-    try:
-        filename = duplicate_name(DocumentService.query, name=name + ".pdf", kb_id=kb.id)
-        filetype = filename_type(filename)
-        if filetype == FileType.OTHER.value:
-            raise RuntimeError("This type of file has not been supported yet!")
-
-        location = filename
-        while settings.STORAGE_IMPL.obj_exist(kb_id, location):
-            location += "_"
-        settings.STORAGE_IMPL.put(kb_id, location, blob)
-        doc = {
-            "id": get_uuid(),
-            "kb_id": kb.id,
-            "parser_id": kb.parser_id,
-            "parser_config": kb.parser_config,
-            "created_by": current_user.id,
-            "type": filetype,
-            "name": filename,
-            "location": location,
-            "size": len(blob),
-            "thumbnail": thumbnail(filename, blob),
-            "suffix": Path(filename).suffix.lstrip("."),
-        }
-        if doc["type"] == FileType.VISUAL:
-            doc["parser_id"] = ParserType.PICTURE.value
-        if doc["type"] == FileType.AURAL:
-            doc["parser_id"] = ParserType.AUDIO.value
-        if re.search(r"\.(ppt|pptx|pages)$", filename):
-            doc["parser_id"] = ParserType.PRESENTATION.value
-        if re.search(r"\.(eml)$", filename):
-            doc["parser_id"] = ParserType.EMAIL.value
-        DocumentService.insert(doc)
-        FileService.add_file_from_kb(doc, kb_folder["id"], kb.tenant_id)
-    except Exception as e:
-        return server_error_response(e)
-    return get_json_result(data=True)
-
-
-@manager.route("/create", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("name", "kb_id")
-async def create():
-    req = await get_request_json()
-    kb_id = req["kb_id"]
-    if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
-    if len(req["name"].encode("utf-8")) > FILE_NAME_LEN_LIMIT:
-        return get_json_result(data=False, message=f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", code=RetCode.ARGUMENT_ERROR)
-
-    if req["name"].strip() == "":
-        return get_json_result(data=False, message="File name can't be empty.", code=RetCode.ARGUMENT_ERROR)
-    req["name"] = req["name"].strip()
-
-    try:
-        e, kb = KnowledgebaseService.get_by_id(kb_id)
-        if not e:
-            return get_data_error_result(message="Can't find this dataset!")
-
-        if DocumentService.query(name=req["name"], kb_id=kb_id):
-            return get_data_error_result(message="Duplicated document name in the same dataset.")
-
-        kb_root_folder = FileService.get_kb_folder(kb.tenant_id)
-        if not kb_root_folder:
-            return get_data_error_result(message="Cannot find the root folder.")
-        kb_folder = FileService.new_a_file_from_kb(
-            kb.tenant_id,
-            kb.name,
-            kb_root_folder["id"],
-        )
-        if not kb_folder:
-            return get_data_error_result(message="Cannot find the kb folder for this file.")
-
-        doc = DocumentService.insert(
-            {
-                "id": get_uuid(),
-                "kb_id": kb.id,
-                "parser_id": kb.parser_id,
-                "pipeline_id": kb.pipeline_id,
-                "parser_config": kb.parser_config,
-                "created_by": current_user.id,
-                "type": FileType.VIRTUAL,
-                "name": req["name"],
-                "suffix": Path(req["name"]).suffix.lstrip("."),
-                "location": "",
-                "size": 0,
-            }
-        )
-
-        FileService.add_file_from_kb(doc.to_dict(), kb_folder["id"], kb.tenant_id)
-
-        return get_json_result(data=doc.to_json())
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/filter", methods=["POST"])  # noqa: F821
-@login_required
-async def get_filter():
-    req = await get_request_json()
-
-    kb_id = req.get("kb_id")
-    if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
-    tenants = UserTenantService.query(user_id=current_user.id)
-    for tenant in tenants:
-        if KnowledgebaseService.query(tenant_id=tenant.tenant_id, id=kb_id):
-            break
-    else:
-        return get_json_result(data=False, message="Only owner of dataset authorized for this operation.", code=RetCode.OPERATING_ERROR)
-
-    keywords = req.get("keywords", "")
-
-    suffix = req.get("suffix", [])
-
-    run_status = req.get("run_status", [])
-    if run_status:
-        invalid_status = {s for s in run_status if s not in VALID_TASK_STATUS}
-        if invalid_status:
-            return get_data_error_result(message=f"Invalid filter run status conditions: {', '.join(invalid_status)}")
-
-    types = req.get("types", [])
-    if types:
-        invalid_types = {t for t in types if t not in VALID_FILE_TYPES}
-        if invalid_types:
-            return get_data_error_result(message=f"Invalid filter conditions: {', '.join(invalid_types)} type{'s' if len(invalid_types) > 1 else ''}")
-
-    try:
-        filter, total = DocumentService.get_filter_by_kb_id(kb_id, keywords, run_status, types, suffix)
-        return get_json_result(data={"total": total, "filter": filter})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/infos", methods=["POST"])  # noqa: F821
-@login_required
-async def doc_infos():
-    req = await get_request_json()
-    doc_ids = req["doc_ids"]
-    for doc_id in doc_ids:
-        if not DocumentService.accessible(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-    docs = DocumentService.get_by_ids(doc_ids)
-    docs_list = list(docs.dicts())
-    # Add meta_fields for each document
-    for doc in docs_list:
-        doc["meta_fields"] = DocMetadataService.get_document_metadata(doc["id"])
-    return get_json_result(data=docs_list)
-
-
-@manager.route("/metadata/update", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_ids")
-async def metadata_update():
-    req = await get_request_json()
-    kb_id = req.get("kb_id")
-    document_ids = req.get("doc_ids")
-    updates = req.get("updates", []) or []
-    deletes = req.get("deletes", []) or []
-
-    if not kb_id:
-        return get_json_result(data=False, message='Lack of "KB ID"', code=RetCode.ARGUMENT_ERROR)
-
-    if not isinstance(updates, list) or not isinstance(deletes, list):
-        return get_json_result(data=False, message="updates and deletes must be lists.", code=RetCode.ARGUMENT_ERROR)
-
-    for upd in updates:
-        if not isinstance(upd, dict) or not upd.get("key") or "value" not in upd:
-            return get_json_result(data=False, message="Each update requires key and value.", code=RetCode.ARGUMENT_ERROR)
-    for d in deletes:
-        if not isinstance(d, dict) or not d.get("key"):
-            return get_json_result(data=False, message="Each delete requires key.", code=RetCode.ARGUMENT_ERROR)
-
-    updated = DocMetadataService.batch_update_metadata(kb_id, document_ids, updates, deletes)
-    return get_json_result(data={"updated": updated, "matched_docs": len(document_ids)})
-
-
-@manager.route("/update_metadata_setting", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_id", "metadata")
-async def update_metadata_setting():
-    req = await get_request_json()
-    if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-
-    e, doc = DocumentService.get_by_id(req["doc_id"])
-    if not e:
-        return get_data_error_result(message="Document not found!")
-
-    DocumentService.update_parser_config(doc.id, {"metadata": req["metadata"]})
-    e, doc = DocumentService.get_by_id(doc.id)
-    if not e:
-        return get_data_error_result(message="Document not found!")
-
-    return get_json_result(data=doc.to_dict())
-
-
-@manager.route("/thumbnails", methods=["GET"])  # noqa: F821
-# @login_required
-def thumbnails():
-    doc_ids = request.args.getlist("doc_ids")
-    if not doc_ids:
-        return get_json_result(data=False, message='Lack of "Document ID"', code=RetCode.ARGUMENT_ERROR)
-
-    try:
-        docs = DocumentService.get_thumbnails(doc_ids)
-
-        for doc_item in docs:
-            if doc_item["thumbnail"] and not doc_item["thumbnail"].startswith(IMG_BASE64_PREFIX):
-                doc_item["thumbnail"] = f"/v1/document/image/{doc_item['kb_id']}-{doc_item['thumbnail']}"
-
-        return get_json_result(data={d["id"]: d["thumbnail"] for d in docs})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/change_status", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_ids", "status")
-async def change_status():
-    req = await get_request_json()
-    doc_ids = req.get("doc_ids", [])
-    status = str(req.get("status", ""))
-
-    if status not in ["0", "1"]:
-        return get_json_result(data=False, message='"Status" must be either 0 or 1!', code=RetCode.ARGUMENT_ERROR)
-
-    result = {}
-    has_error = False
-    for doc_id in doc_ids:
-        if not DocumentService.accessible(doc_id, current_user.id):
-            result[doc_id] = {"error": "No authorization."}
-            has_error = True
-            continue
-
-        try:
-            e, doc = DocumentService.get_by_id(doc_id)
-            if not e:
-                result[doc_id] = {"error": "No authorization."}
-                has_error = True
-                continue
-            e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
-            if not e:
-                result[doc_id] = {"error": "Can't find this dataset!"}
-                has_error = True
-                continue
-            current_status = str(doc.status)
-            if current_status == status:
-                result[doc_id] = {"status": status}
-                continue
-            if not DocumentService.update_by_id(doc_id, {"status": str(status)}):
-                result[doc_id] = {"error": "Database error (Document update)!"}
-                has_error = True
-                continue
-
-            status_int = int(status)
-            if getattr(doc, "chunk_num", 0) > 0:
-                try:
-                    ok = settings.docStoreConn.update(
-                        {"doc_id": doc_id},
-                        {"available_int": status_int},
-                        search.index_name(kb.tenant_id),
-                        doc.kb_id,
-                    )
-                except Exception as exc:
-                    msg = str(exc)
-                    if "3022" in msg:
-                        result[doc_id] = {"error": "Document store table missing."}
-                    else:
-                        result[doc_id] = {"error": f"Document store update failed: {msg}"}
-                    has_error = True
-                    continue
-                if not ok:
-                    result[doc_id] = {"error": "Database error (docStore update)!"}
-                    has_error = True
-                    continue
-            result[doc_id] = {"status": status}
-        except Exception as e:
-            result[doc_id] = {"error": f"Internal server error: {str(e)}"}
-            has_error = True
-
-    if has_error:
-        return get_json_result(data=result, message="Partial failure", code=RetCode.SERVER_ERROR)
-    return get_json_result(data=result)
-
-
-@manager.route("/rm", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_id")
-async def rm():
-    req = await get_request_json()
-    doc_ids = req["doc_id"]
-    if isinstance(doc_ids, str):
-        doc_ids = [doc_ids]
-
-    for doc_id in doc_ids:
-        if not DocumentService.accessible4deletion(doc_id, current_user.id):
-            return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-
-    errors = await thread_pool_exec(FileService.delete_docs, doc_ids, current_user.id)
-
-    if errors:
-        return get_json_result(data=False, message=errors, code=RetCode.SERVER_ERROR)
-
-    return get_json_result(data=True)
-
-
-@manager.route("/run", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_ids", "run")
-async def run():
-    req = await get_request_json()
-    uid = current_user.id
-    try:
-
-        def _run_sync():
-            for doc_id in req["doc_ids"]:
-                if not DocumentService.accessible(doc_id, uid):
-                    return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-
-            kb_table_num_map = {}
-            for id in req["doc_ids"]:
-                info = {"run": str(req["run"]), "progress": 0}
-                if str(req["run"]) == TaskStatus.RUNNING.value and req.get("delete", False):
-                    info["progress_msg"] = ""
-                    info["chunk_num"] = 0
-                    info["token_num"] = 0
-
-                tenant_id = DocumentService.get_tenant_id(id)
-                if not tenant_id:
-                    return get_data_error_result(message="Tenant not found!")
-                e, doc = DocumentService.get_by_id(id)
-                if not e:
-                    return get_data_error_result(message="Document not found!")
-
-                if str(req["run"]) == TaskStatus.CANCEL.value:
-                    tasks = list(TaskService.query(doc_id=id))
-                    has_unfinished_task = any((task.progress or 0) < 1 for task in tasks)
-                    if str(doc.run) in [TaskStatus.RUNNING.value, TaskStatus.CANCEL.value] or has_unfinished_task:
-                        cancel_all_task_of(id)
-                    else:
-                        return get_data_error_result(message="Cannot cancel a task that is not in RUNNING status")
-                if all([("delete" not in req or req["delete"]), str(req["run"]) == TaskStatus.RUNNING.value, str(doc.run) == TaskStatus.DONE.value]):
-                    DocumentService.clear_chunk_num_when_rerun(doc.id)
-
-                DocumentService.update_by_id(id, info)
-                if req.get("delete", False):
-                    TaskService.filter_delete([Task.doc_id == id])
-                    if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
-                        settings.docStoreConn.delete({"doc_id": id}, search.index_name(tenant_id), doc.kb_id)
-
-                if str(req["run"]) == TaskStatus.RUNNING.value:
-                    if req.get("apply_kb"):
-                        e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
-                        if not e:
-                            raise LookupError("Can't find this dataset!")
-                        doc.parser_config["llm_id"] = kb.parser_config.get("llm_id")
-                        doc.parser_config["enable_metadata"] = kb.parser_config.get("enable_metadata", False)
-                        doc.parser_config["metadata"] = kb.parser_config.get("metadata", {})
-                        DocumentService.update_parser_config(doc.id, doc.parser_config)
-                    doc_dict = doc.to_dict()
-                    DocumentService.run(tenant_id, doc_dict, kb_table_num_map)
-
-            return get_json_result(data=True)
-
-        return await thread_pool_exec(_run_sync)
-    except Exception as e:
-        return server_error_response(e)
-
-@manager.route("/get/<doc_id>", methods=["GET"])  # noqa: F821
-@login_required
-async def get(doc_id):
-    try:
-        e, doc = DocumentService.get_by_id(doc_id)
-        if not e:
-            return get_data_error_result(message="Document not found!")
-
-        b, n = File2DocumentService.get_storage_address(doc_id=doc_id)
-        data = await thread_pool_exec(settings.STORAGE_IMPL.get, b, n)
-        response = await make_response(data)
-
-        ext = re.search(r"\.([^.]+)$", doc.name.lower())
-        ext = ext.group(1) if ext else None
-        content_type = None
-        if ext:
-            fallback_prefix = "image" if doc.type == FileType.VISUAL.value else "application"
-            content_type = CONTENT_TYPE_MAP.get(ext, f"{fallback_prefix}/{ext}")
-        apply_safe_file_response_headers(response, content_type, ext)
-        return response
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/download/<attachment_id>", methods=["GET"])  # noqa: F821
-@login_required
-async def download_attachment(attachment_id):
-    try:
-        ext = request.args.get("ext", "markdown")
-        data = await thread_pool_exec(settings.STORAGE_IMPL.get, current_user.id, attachment_id)
-        response = await make_response(data)
-        content_type = CONTENT_TYPE_MAP.get(ext, f"application/{ext}")
-        apply_safe_file_response_headers(response, content_type, ext)
-
-        return response
-
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/change_parser", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("doc_id")
-async def change_parser():
-    req = await get_request_json()
-    if not DocumentService.accessible(req["doc_id"], current_user.id):
-        return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-
-    e, doc = DocumentService.get_by_id(req["doc_id"])
-    if not e:
-        return get_data_error_result(message="Document not found!")
-
-    def reset_doc():
-        nonlocal doc
-        e = DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"], "parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
-        if not e:
-            return get_data_error_result(message="Document not found!")
-        if doc.token_num > 0:
-            e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duration * -1)
-            if not e:
-                return get_data_error_result(message="Document not found!")
-            tenant_id = DocumentService.get_tenant_id(req["doc_id"])
-            if not tenant_id:
-                return get_data_error_result(message="Tenant not found!")
-            DocumentService.delete_chunk_images(doc, tenant_id)
-            if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
-                settings.docStoreConn.delete({"doc_id": doc.id}, search.index_name(tenant_id), doc.kb_id)
-        return None
-
-    try:
-        if "pipeline_id" in req and req["pipeline_id"] != "":
-            if doc.pipeline_id == req["pipeline_id"]:
-                return get_json_result(data=True)
-            DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"]})
-            reset_doc()
-            return get_json_result(data=True)
-
-        if doc.parser_id.lower() == req["parser_id"].lower():
-            if "parser_config" in req:
-                if req["parser_config"] == doc.parser_config:
-                    return get_json_result(data=True)
-            else:
-                return get_json_result(data=True)
-
-        if (doc.type == FileType.VISUAL and req["parser_id"] != "picture") or (re.search(r"\.(ppt|pptx|pages)$", doc.name) and req["parser_id"] != "presentation"):
-            return get_data_error_result(message="Not supported yet!")
-        if "parser_config" in req:
-            DocumentService.update_parser_config(doc.id, req["parser_config"])
-        reset_doc()
-        return get_json_result(data=True)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/image/<image_id>", methods=["GET"])  # noqa: F821
-# @login_required
-async def get_image(image_id):
-    try:
-        arr = image_id.split("-")
-        if len(arr) != 2:
-            return get_data_error_result(message="Image not found.")
-        bkt, nm = image_id.split("-")
-        data = await thread_pool_exec(settings.STORAGE_IMPL.get, bkt, nm)
-        response = await make_response(data)
-        response.headers.set("Content-Type", "image/JPEG")
-        return response
-    except Exception as e:
-        return server_error_response(e)
-
-
-ARTIFACT_CONTENT_TYPES = {
-    ".png": "image/png",
-    ".jpg": "image/jpeg",
-    ".jpeg": "image/jpeg",
-    ".svg": "image/svg+xml",
-    ".pdf": "application/pdf",
-    ".csv": "text/csv",
-    ".json": "application/json",
-    ".html": "text/html",
-}
-
-
-@manager.route("/artifact/<filename>", methods=["GET"])  # noqa: F821
-@login_required
-async def get_artifact(filename):
-    try:
-        bucket = SANDBOX_ARTIFACT_BUCKET
-        # Validate filename: must be uuid hex + allowed extension, nothing else
-        basename = os.path.basename(filename)
-        if basename != filename or "/" in filename or "\\" in filename:
-            return get_data_error_result(message="Invalid filename.")
-        ext = os.path.splitext(basename)[1].lower()
-        if ext not in ARTIFACT_CONTENT_TYPES:
-            return get_data_error_result(message="Invalid file type.")
-        data = await thread_pool_exec(settings.STORAGE_IMPL.get, bucket, basename)
-        if not data:
-            return get_data_error_result(message="Artifact not found.")
-        content_type = ARTIFACT_CONTENT_TYPES.get(ext, "application/octet-stream")
-        response = await make_response(data)
-        safe_filename = re.sub(r"[^\w.\-]", "_", basename)
-        apply_safe_file_response_headers(response, content_type, ext)
-        if not response.headers.get("Content-Disposition"):
-            response.headers.set("Content-Disposition", f'inline; filename="{safe_filename}"')
-        return response
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route("/upload_and_parse", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("conversation_id")
-async def upload_and_parse():
-    files = await request.files
-    if "file" not in files:
-        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
-
-    file_objs = files.getlist("file")
-    for file_obj in file_objs:
-        if file_obj.filename == "":
-            return get_json_result(data=False, message="No file selected!", code=RetCode.ARGUMENT_ERROR)
-
-    form = await request.form
-    doc_ids = doc_upload_and_parse(form.get("conversation_id"), file_objs, current_user.id)
-    return get_json_result(data=doc_ids)
-
-
-@manager.route("/parse", methods=["POST"])  # noqa: F821
-@login_required
-async def parse():
-    req = await get_request_json()
-    url = req.get("url", "")
-    if url:
-        if not is_valid_url(url):
-            return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
-        download_path = os.path.join(get_project_base_directory(), "logs/downloads")
-        os.makedirs(download_path, exist_ok=True)
-        from seleniumwire.webdriver import Chrome, ChromeOptions
-
-        options = ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("prefs", {"download.default_directory": download_path, "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True})
-        driver = Chrome(options=options)
-        driver.get(url)
-        res_headers = [r.response.headers for r in driver.requests if r and r.response]
-        if len(res_headers) > 1:
-            sections = RAGFlowHtmlParser().parser_txt(driver.page_source)
-            driver.quit()
-            return get_json_result(data="\n".join(sections))
-
-        class File:
-            filename: str
-            filepath: str
-
-            def __init__(self, filename, filepath):
-                self.filename = filename
-                self.filepath = filepath
-
-            def read(self):
-                with open(self.filepath, "rb") as f:
-                    return f.read()
-
-        r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
-        if not r or not r.group(1):
-            return get_json_result(data=False, message="Can't not identify downloaded file", code=RetCode.ARGUMENT_ERROR)
-        filename = r.group(1).strip()
-        if not _is_safe_download_filename(filename):
-            return get_json_result(data=False, message="Invalid downloaded filename", code=RetCode.ARGUMENT_ERROR)
-        filepath = os.path.join(download_path, filename)
-        f = File(filename, filepath)
-        txt = FileService.parse_docs([f], current_user.id)
-        return get_json_result(data=txt)
-
-    files = await request.files
-    if "file" not in files:
-        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
-
-    file_objs = files.getlist("file")
-    txt = FileService.parse_docs(file_objs, current_user.id)
-
-    return get_json_result(data=txt)
-
-
-@manager.route("/upload_info", methods=["POST"])  # noqa: F821
-@login_required
-async def upload_info():
-    files = await request.files
-    file_objs = files.getlist("file") if files and files.get("file") else []
-    url = request.args.get("url")
-
-    if file_objs and url:
-        return get_json_result(
-            data=False,
-            message="Provide either multipart file(s) or ?url=..., not both.",
-            code=RetCode.BAD_REQUEST,
-        )
-
-    if not file_objs and not url:
-        return get_json_result(
-            data=False,
-            message="Missing input: provide multipart file(s) or url",
-            code=RetCode.BAD_REQUEST,
-        )
-
-    try:
-        if url and not file_objs:
-            return get_json_result(data=FileService.upload_info(current_user.id, None, url))
-
-        if len(file_objs) == 1:
-            return get_json_result(data=FileService.upload_info(current_user.id, file_objs[0], None))
-
-        results = [FileService.upload_info(current_user.id, f, None) for f in file_objs]
-        return get_json_result(data=results)
-    except Exception as e:
-        return server_error_response(e)

api/apps/evaluation_app.py

@@ -1,479 +0,0 @@
-#
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-"""
-RAG Evaluation API Endpoints
-
-Provides REST API for RAG evaluation functionality including:
-- Dataset management
-- Test case management
-- Evaluation execution
-- Results retrieval
-- Configuration recommendations
-"""
-
-from quart import request
-from api.apps import login_required, current_user
-from api.db.services.evaluation_service import EvaluationService
-from api.utils.api_utils import (
-    get_data_error_result,
-    get_json_result,
-    get_request_json,
-    server_error_response,
-    validate_request
-)
-from common.constants import RetCode
-
-
-# ==================== Dataset Management ====================
-
-@manager.route('/dataset/create', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("name", "kb_ids")
-async def create_dataset():
-    """
-    Create a new evaluation dataset.
-    
-    Request body:
-    {
-        "name": "Dataset name",
-        "description": "Optional description",
-        "kb_ids": ["kb_id1", "kb_id2"]
-    }
-    """
-    try:
-        req = await get_request_json()
-        name = req.get("name", "").strip()
-        description = req.get("description", "")
-        kb_ids = req.get("kb_ids", [])
-        
-        if not name:
-            return get_data_error_result(message="Dataset name cannot be empty")
-        
-        if not kb_ids or not isinstance(kb_ids, list):
-            return get_data_error_result(message="kb_ids must be a non-empty list")
-        
-        success, result = EvaluationService.create_dataset(
-            name=name,
-            description=description,
-            kb_ids=kb_ids,
-            tenant_id=current_user.id,
-            user_id=current_user.id
-        )
-        
-        if not success:
-            return get_data_error_result(message=result)
-        
-        return get_json_result(data={"dataset_id": result})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/list', methods=['GET'])  # noqa: F821
-@login_required
-async def list_datasets():
-    """
-    List evaluation datasets for current tenant.
-    
-    Query params:
-    - page: Page number (default: 1)
-    - page_size: Items per page (default: 20)
-    """
-    try:
-        page = int(request.args.get("page", 1))
-        page_size = int(request.args.get("page_size", 20))
-        
-        result = EvaluationService.list_datasets(
-            tenant_id=current_user.id,
-            user_id=current_user.id,
-            page=page,
-            page_size=page_size
-        )
-        
-        return get_json_result(data=result)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/<dataset_id>', methods=['GET'])  # noqa: F821
-@login_required
-async def get_dataset(dataset_id):
-    """Get dataset details by ID"""
-    try:
-        dataset = EvaluationService.get_dataset(dataset_id)
-        if not dataset:
-            return get_data_error_result(
-                message="Dataset not found",
-                code=RetCode.DATA_ERROR
-            )
-        
-        return get_json_result(data=dataset)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/<dataset_id>', methods=['PUT'])  # noqa: F821
-@login_required
-async def update_dataset(dataset_id):
-    """
-    Update dataset.
-    
-    Request body:
-    {
-        "name": "New name",
-        "description": "New description",
-        "kb_ids": ["kb_id1", "kb_id2"]
-    }
-    """
-    try:
-        req = await get_request_json()
-        
-        # Remove fields that shouldn't be updated
-        req.pop("id", None)
-        req.pop("tenant_id", None)
-        req.pop("created_by", None)
-        req.pop("create_time", None)
-        
-        success = EvaluationService.update_dataset(dataset_id, **req)
-        
-        if not success:
-            return get_data_error_result(message="Failed to update dataset")
-        
-        return get_json_result(data={"dataset_id": dataset_id})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/<dataset_id>', methods=['DELETE'])  # noqa: F821
-@login_required
-async def delete_dataset(dataset_id):
-    """Delete dataset (soft delete)"""
-    try:
-        success = EvaluationService.delete_dataset(dataset_id)
-        
-        if not success:
-            return get_data_error_result(message="Failed to delete dataset")
-        
-        return get_json_result(data={"dataset_id": dataset_id})
-    except Exception as e:
-        return server_error_response(e)
-
-
-# ==================== Test Case Management ====================
-
-@manager.route('/dataset/<dataset_id>/case/add', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("question")
-async def add_test_case(dataset_id):
-    """
-    Add a test case to a dataset.
-    
-    Request body:
-    {
-        "question": "Test question",
-        "reference_answer": "Optional ground truth answer",
-        "relevant_doc_ids": ["doc_id1", "doc_id2"],
-        "relevant_chunk_ids": ["chunk_id1", "chunk_id2"],
-        "metadata": {"key": "value"}
-    }
-    """
-    try:
-        req = await get_request_json()
-        question = req.get("question", "").strip()
-        
-        if not question:
-            return get_data_error_result(message="Question cannot be empty")
-        
-        success, result = EvaluationService.add_test_case(
-            dataset_id=dataset_id,
-            question=question,
-            reference_answer=req.get("reference_answer"),
-            relevant_doc_ids=req.get("relevant_doc_ids"),
-            relevant_chunk_ids=req.get("relevant_chunk_ids"),
-            metadata=req.get("metadata")
-        )
-        
-        if not success:
-            return get_data_error_result(message=result)
-        
-        return get_json_result(data={"case_id": result})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/<dataset_id>/case/import', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("cases")
-async def import_test_cases(dataset_id):
-    """
-    Bulk import test cases.
-    
-    Request body:
-    {
-        "cases": [
-            {
-                "question": "Question 1",
-                "reference_answer": "Answer 1",
-                ...
-            },
-            {
-                "question": "Question 2",
-                ...
-            }
-        ]
-    }
-    """
-    try:
-        req = await get_request_json()
-        cases = req.get("cases", [])
-        
-        if not cases or not isinstance(cases, list):
-            return get_data_error_result(message="cases must be a non-empty list")
-        
-        success_count, failure_count = EvaluationService.import_test_cases(
-            dataset_id=dataset_id,
-            cases=cases
-        )
-        
-        return get_json_result(data={
-            "success_count": success_count,
-            "failure_count": failure_count,
-            "total": len(cases)
-        })
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/dataset/<dataset_id>/cases', methods=['GET'])  # noqa: F821
-@login_required
-async def get_test_cases(dataset_id):
-    """Get all test cases for a dataset"""
-    try:
-        cases = EvaluationService.get_test_cases(dataset_id)
-        return get_json_result(data={"cases": cases, "total": len(cases)})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/case/<case_id>', methods=['DELETE'])  # noqa: F821
-@login_required
-async def delete_test_case(case_id):
-    """Delete a test case"""
-    try:
-        success = EvaluationService.delete_test_case(case_id)
-        
-        if not success:
-            return get_data_error_result(message="Failed to delete test case")
-        
-        return get_json_result(data={"case_id": case_id})
-    except Exception as e:
-        return server_error_response(e)
-
-
-# ==================== Evaluation Execution ====================
-
-@manager.route('/run/start', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("dataset_id", "dialog_id")
-async def start_evaluation():
-    """
-    Start an evaluation run.
-    
-    Request body:
-    {
-        "dataset_id": "dataset_id",
-        "dialog_id": "dialog_id",
-        "name": "Optional run name"
-    }
-    """
-    try:
-        req = await get_request_json()
-        dataset_id = req.get("dataset_id")
-        dialog_id = req.get("dialog_id")
-        name = req.get("name")
-        
-        success, result = EvaluationService.start_evaluation(
-            dataset_id=dataset_id,
-            dialog_id=dialog_id,
-            user_id=current_user.id,
-            name=name
-        )
-        
-        if not success:
-            return get_data_error_result(message=result)
-        
-        return get_json_result(data={"run_id": result})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/run/<run_id>', methods=['GET'])  # noqa: F821
-@login_required
-async def get_evaluation_run(run_id):
-    """Get evaluation run details"""
-    try:
-        result = EvaluationService.get_run_results(run_id)
-        
-        if not result:
-            return get_data_error_result(
-                message="Evaluation run not found",
-                code=RetCode.DATA_ERROR
-            )
-        
-        return get_json_result(data=result)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/run/<run_id>/results', methods=['GET'])  # noqa: F821
-@login_required
-async def get_run_results(run_id):
-    """Get detailed results for an evaluation run"""
-    try:
-        result = EvaluationService.get_run_results(run_id)
-        
-        if not result:
-            return get_data_error_result(
-                message="Evaluation run not found",
-                code=RetCode.DATA_ERROR
-            )
-        
-        return get_json_result(data=result)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/run/list', methods=['GET'])  # noqa: F821
-@login_required
-async def list_evaluation_runs():
-    """
-    List evaluation runs.
-    
-    Query params:
-    - dataset_id: Filter by dataset (optional)
-    - dialog_id: Filter by dialog (optional)
-    - page: Page number (default: 1)
-    - page_size: Items per page (default: 20)
-    """
-    try:
-        # TODO: Implement list_runs in EvaluationService
-        return get_json_result(data={"runs": [], "total": 0})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/run/<run_id>', methods=['DELETE'])  # noqa: F821
-@login_required
-async def delete_evaluation_run(run_id):
-    """Delete an evaluation run"""
-    try:
-        # TODO: Implement delete_run in EvaluationService
-        return get_json_result(data={"run_id": run_id})
-    except Exception as e:
-        return server_error_response(e)
-
-
-# ==================== Analysis & Recommendations ====================
-
-@manager.route('/run/<run_id>/recommendations', methods=['GET'])  # noqa: F821
-@login_required
-async def get_recommendations(run_id):
-    """Get configuration recommendations based on evaluation results"""
-    try:
-        recommendations = EvaluationService.get_recommendations(run_id)
-        return get_json_result(data={"recommendations": recommendations})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/compare', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("run_ids")
-async def compare_runs():
-    """
-    Compare multiple evaluation runs.
-    
-    Request body:
-    {
-        "run_ids": ["run_id1", "run_id2", "run_id3"]
-    }
-    """
-    try:
-        req = await get_request_json()
-        run_ids = req.get("run_ids", [])
-        
-        if not run_ids or not isinstance(run_ids, list) or len(run_ids) < 2:
-            return get_data_error_result(
-                message="run_ids must be a list with at least 2 run IDs"
-            )
-        
-        # TODO: Implement compare_runs in EvaluationService
-        return get_json_result(data={"comparison": {}})
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/run/<run_id>/export', methods=['GET'])  # noqa: F821
-@login_required
-async def export_results(run_id):
-    """Export evaluation results as JSON/CSV"""
-    try:
-        # format_type = request.args.get("format", "json")  # TODO: Use for CSV export
-        
-        result = EvaluationService.get_run_results(run_id)
-        
-        if not result:
-            return get_data_error_result(
-                message="Evaluation run not found",
-                code=RetCode.DATA_ERROR
-            )
-        
-        # TODO: Implement CSV export
-        return get_json_result(data=result)
-    except Exception as e:
-        return server_error_response(e)
-
-
-# ==================== Real-time Evaluation ====================
-
-@manager.route('/evaluate_single', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("question", "dialog_id")
-async def evaluate_single():
-    """
-    Evaluate a single question-answer pair in real-time.
-    
-    Request body:
-    {
-        "question": "Test question",
-        "dialog_id": "dialog_id",
-        "reference_answer": "Optional ground truth",
-        "relevant_chunk_ids": ["chunk_id1", "chunk_id2"]
-    }
-    """
-    try:
-        # req = await get_request_json()  # TODO: Use for single evaluation implementation
-        
-        # TODO: Implement single evaluation
-        # This would execute the RAG pipeline and return metrics immediately
-        
-        return get_json_result(data={
-            "answer": "",
-            "metrics": {},
-            "retrieved_chunks": []
-        })
-    except Exception as e:
-        return server_error_response(e)

api/apps/file_app.py

@@ -1,464 +0,0 @@
-# #
-# #  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
-# #
-# #  Licensed under the Apache License, Version 2.0 (the "License");
-# #  you may not use this file except in compliance with the License.
-# #  You may obtain a copy of the License at
-# #
-# #      http://www.apache.org/licenses/LICENSE-2.0
-# #
-# #  Unless required by applicable law or agreed to in writing, software
-# #  distributed under the License is distributed on an "AS IS" BASIS,
-# #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# #  See the License for the specific language governing permissions and
-# #  limitations under the License
-# #
-# import logging
-# import os
-# import pathlib
-# import re
-# from quart import request, make_response
-# from api.apps import login_required, current_user
-#
-# from api.common.check_team_permission import check_file_team_permission
-# from api.db.services.document_service import DocumentService
-# from api.db.services.file2document_service import File2DocumentService
-# from api.utils.api_utils import server_error_response, get_data_error_result, validate_request
-# from common.misc_utils import get_uuid, thread_pool_exec
-# from common.constants import RetCode, FileSource
-# from api.db import FileType
-# from api.db.services import duplicate_name
-# from api.db.services.file_service import FileService
-# from api.utils.api_utils import get_json_result, get_request_json
-# from api.utils.file_utils import filename_type
-# from api.utils.web_utils import CONTENT_TYPE_MAP, apply_safe_file_response_headers
-# from common import settings
-#
-# @manager.route('/upload', methods=['POST'])  # noqa: F821
-# @login_required
-# # @validate_request("parent_id")
-# async def upload():
-#     form = await request.form
-#     pf_id = form.get("parent_id")
-#
-#     if not pf_id:
-#         root_folder = FileService.get_root_folder(current_user.id)
-#         pf_id = root_folder["id"]
-#
-#     files = await request.files
-#     if 'file' not in files:
-#         return get_json_result(
-#             data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR)
-#     file_objs = files.getlist('file')
-#
-#     for file_obj in file_objs:
-#         if file_obj.filename == '':
-#             return get_json_result(
-#                 data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR)
-#     file_res = []
-#     try:
-#         e, pf_folder = FileService.get_by_id(pf_id)
-#         if not e:
-#             return get_data_error_result( message="Can't find this folder!")
-#
-#         async def _handle_single_file(file_obj):
-#             MAX_FILE_NUM_PER_USER: int = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0))
-#             if 0 < MAX_FILE_NUM_PER_USER <= await thread_pool_exec(DocumentService.get_doc_count, current_user.id):
-#                 return get_data_error_result( message="Exceed the maximum file number of a free user!")
-#
-#             # split file name path
-#             if not file_obj.filename:
-#                 file_obj_names = [pf_folder.name, file_obj.filename]
-#             else:
-#                 full_path = '/' + file_obj.filename
-#                 file_obj_names = full_path.split('/')
-#             file_len = len(file_obj_names)
-#
-#             # get folder
-#             file_id_list = await thread_pool_exec(FileService.get_id_list_by_id, pf_id, file_obj_names, 1, [pf_id])
-#             len_id_list = len(file_id_list)
-#
-#             # create folder
-#             if file_len != len_id_list:
-#                 e, file = await thread_pool_exec(FileService.get_by_id, file_id_list[len_id_list - 1])
-#                 if not e:
-#                     return get_data_error_result(message="Folder not found!")
-#                 last_folder = await thread_pool_exec(FileService.create_folder, file, file_id_list[len_id_list - 1], file_obj_names,
-#                                                         len_id_list)
-#             else:
-#                 e, file = await thread_pool_exec(FileService.get_by_id, file_id_list[len_id_list - 2])
-#                 if not e:
-#                     return get_data_error_result(message="Folder not found!")
-#                 last_folder = await thread_pool_exec(FileService.create_folder, file, file_id_list[len_id_list - 2], file_obj_names,
-#                                                         len_id_list)
-#
-#             # file type
-#             filetype = filename_type(file_obj_names[file_len - 1])
-#             location = file_obj_names[file_len - 1]
-#             while await thread_pool_exec(settings.STORAGE_IMPL.obj_exist, last_folder.id, location):
-#                 location += "_"
-#             blob = await thread_pool_exec(file_obj.read)
-#             filename = await thread_pool_exec(
-#                 duplicate_name,
-#                 FileService.query,
-#                 name=file_obj_names[file_len - 1],
-#                 parent_id=last_folder.id)
-#             await thread_pool_exec(settings.STORAGE_IMPL.put, last_folder.id, location, blob)
-#             file_data = {
-#                 "id": get_uuid(),
-#                 "parent_id": last_folder.id,
-#                 "tenant_id": current_user.id,
-#                 "created_by": current_user.id,
-#                 "type": filetype,
-#                 "name": filename,
-#                 "location": location,
-#                 "size": len(blob),
-#             }
-#             inserted = await thread_pool_exec(FileService.insert, file_data)
-#             return inserted.to_json()
-#
-#         for file_obj in file_objs:
-#             res = await _handle_single_file(file_obj)
-#             file_res.append(res)
-#
-#         return get_json_result(data=file_res)
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/create', methods=['POST'])  # noqa: F821
-# @login_required
-# @validate_request("name")
-# async def create():
-#     req = await get_request_json()
-#     pf_id = req.get("parent_id")
-#     input_file_type = req.get("type")
-#     if not pf_id:
-#         root_folder = FileService.get_root_folder(current_user.id)
-#         pf_id = root_folder["id"]
-#
-#     try:
-#         if not FileService.is_parent_folder_exist(pf_id):
-#             return get_json_result(
-#                 data=False, message="Parent Folder Doesn't Exist!", code=RetCode.OPERATING_ERROR)
-#         if FileService.query(name=req["name"], parent_id=pf_id):
-#             return get_data_error_result(
-#                 message="Duplicated folder name in the same folder.")
-#
-#         if input_file_type == FileType.FOLDER.value:
-#             file_type = FileType.FOLDER.value
-#         else:
-#             file_type = FileType.VIRTUAL.value
-#
-#         file = FileService.insert({
-#             "id": get_uuid(),
-#             "parent_id": pf_id,
-#             "tenant_id": current_user.id,
-#             "created_by": current_user.id,
-#             "name": req["name"],
-#             "location": "",
-#             "size": 0,
-#             "type": file_type
-#         })
-#
-#         return get_json_result(data=file.to_json())
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/list', methods=['GET'])  # noqa: F821
-# @login_required
-# def list_files():
-#     pf_id = request.args.get("parent_id")
-#
-#     keywords = request.args.get("keywords", "")
-#
-#     page_number = int(request.args.get("page", 1))
-#     items_per_page = int(request.args.get("page_size", 15))
-#     orderby = request.args.get("orderby", "create_time")
-#     desc = request.args.get("desc", True)
-#     if not pf_id:
-#         root_folder = FileService.get_root_folder(current_user.id)
-#         pf_id = root_folder["id"]
-#         FileService.init_knowledgebase_docs(pf_id, current_user.id)
-#     try:
-#         e, file = FileService.get_by_id(pf_id)
-#         if not e:
-#             return get_data_error_result(message="Folder not found!")
-#
-#         files, total = FileService.get_by_pf_id(
-#             current_user.id, pf_id, page_number, items_per_page, orderby, desc, keywords)
-#
-#         parent_folder = FileService.get_parent_folder(pf_id)
-#         if not parent_folder:
-#             return get_json_result(message="File not found!")
-#
-#         return get_json_result(data={"total": total, "files": files, "parent_folder": parent_folder.to_json()})
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/root_folder', methods=['GET'])  # noqa: F821
-# @login_required
-# def get_root_folder():
-#     try:
-#         root_folder = FileService.get_root_folder(current_user.id)
-#         return get_json_result(data={"root_folder": root_folder})
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/parent_folder', methods=['GET'])  # noqa: F821
-# @login_required
-# def get_parent_folder():
-#     file_id = request.args.get("file_id")
-#     try:
-#         e, file = FileService.get_by_id(file_id)
-#         if not e:
-#             return get_data_error_result(message="Folder not found!")
-#
-#         parent_folder = FileService.get_parent_folder(file_id)
-#         return get_json_result(data={"parent_folder": parent_folder.to_json()})
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/all_parent_folder', methods=['GET'])  # noqa: F821
-# @login_required
-# def get_all_parent_folders():
-#     file_id = request.args.get("file_id")
-#     try:
-#         e, file = FileService.get_by_id(file_id)
-#         if not e:
-#             return get_data_error_result(message="Folder not found!")
-#
-#         parent_folders = FileService.get_all_parent_folders(file_id)
-#         parent_folders_res = []
-#         for parent_folder in parent_folders:
-#             parent_folders_res.append(parent_folder.to_json())
-#         return get_json_result(data={"parent_folders": parent_folders_res})
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route("/rm", methods=["POST"])  # noqa: F821
-# @login_required
-# @validate_request("file_ids")
-# async def rm():
-#     req = await get_request_json()
-#     file_ids = req["file_ids"]
-#     uid = current_user.id
-#
-#     try:
-#         def _delete_single_file(file):
-#             try:
-#                 if file.location:
-#                     settings.STORAGE_IMPL.rm(file.parent_id, file.location)
-#             except Exception as e:
-#                 logging.exception(f"Fail to remove object: {file.parent_id}/{file.location}, error: {e}")
-#
-#             informs = File2DocumentService.get_by_file_id(file.id)
-#             for inform in informs:
-#                 doc_id = inform.document_id
-#                 e, doc = DocumentService.get_by_id(doc_id)
-#                 if e and doc:
-#                     tenant_id = DocumentService.get_tenant_id(doc_id)
-#                     if tenant_id:
-#                         DocumentService.remove_document(doc, tenant_id)
-#                 File2DocumentService.delete_by_file_id(file.id)
-#
-#             FileService.delete(file)
-#
-#         def _delete_folder_recursive(folder, tenant_id):
-#             sub_files = FileService.list_all_files_by_parent_id(folder.id)
-#             for sub_file in sub_files:
-#                 if sub_file.type == FileType.FOLDER.value:
-#                     _delete_folder_recursive(sub_file, tenant_id)
-#                 else:
-#                     _delete_single_file(sub_file)
-#
-#             FileService.delete(folder)
-#
-#         def _rm_sync():
-#             for file_id in file_ids:
-#                 e, file = FileService.get_by_id(file_id)
-#                 if not e or not file:
-#                     return get_data_error_result(message="File or Folder not found!")
-#                 if not file.tenant_id:
-#                     return get_data_error_result(message="Tenant not found!")
-#                 if not check_file_team_permission(file, uid):
-#                     return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
-#
-#                 if file.source_type == FileSource.KNOWLEDGEBASE:
-#                     continue
-#
-#                 if file.type == FileType.FOLDER.value:
-#                     _delete_folder_recursive(file, uid)
-#                     continue
-#
-#                 _delete_single_file(file)
-#
-#             return get_json_result(data=True)
-#
-#         return await thread_pool_exec(_rm_sync)
-#
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/rename', methods=['POST'])  # noqa: F821
-# @login_required
-# @validate_request("file_id", "name")
-# async def rename():
-#     req = await get_request_json()
-#     try:
-#         e, file = FileService.get_by_id(req["file_id"])
-#         if not e:
-#             return get_data_error_result(message="File not found!")
-#         if not check_file_team_permission(file, current_user.id):
-#             return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
-#         if file.type != FileType.FOLDER.value \
-#             and pathlib.Path(req["name"].lower()).suffix != pathlib.Path(
-#                 file.name.lower()).suffix:
-#             return get_json_result(
-#                 data=False,
-#                 message="The extension of file can't be changed",
-#                 code=RetCode.ARGUMENT_ERROR)
-#         for file in FileService.query(name=req["name"], pf_id=file.parent_id):
-#             if file.name == req["name"]:
-#                 return get_data_error_result(
-#                     message="Duplicated file name in the same folder.")
-#
-#         if not FileService.update_by_id(
-#                 req["file_id"], {"name": req["name"]}):
-#             return get_data_error_result(
-#                 message="Database error (File rename)!")
-#
-#         informs = File2DocumentService.get_by_file_id(req["file_id"])
-#         if informs:
-#             if not DocumentService.update_by_id(
-#                     informs[0].document_id, {"name": req["name"]}):
-#                 return get_data_error_result(
-#                     message="Database error (Document rename)!")
-#
-#         return get_json_result(data=True)
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route('/get/<file_id>', methods=['GET'])  # noqa: F821
-# @login_required
-# async def get(file_id):
-#     try:
-#         e, file = FileService.get_by_id(file_id)
-#         if not e:
-#             return get_data_error_result(message="Document not found!")
-#         if not check_file_team_permission(file, current_user.id):
-#             return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR)
-#
-#         blob = await thread_pool_exec(settings.STORAGE_IMPL.get, file.parent_id, file.location)
-#         if not blob:
-#             b, n = File2DocumentService.get_storage_address(file_id=file_id)
-#             blob = await thread_pool_exec(settings.STORAGE_IMPL.get, b, n)
-#
-#         response = await make_response(blob)
-#         ext = re.search(r"\.([^.]+)$", file.name.lower())
-#         ext = ext.group(1) if ext else None
-#         content_type = None
-#         if ext:
-#             fallback_prefix = "image" if file.type == FileType.VISUAL.value else "application"
-#             content_type = CONTENT_TYPE_MAP.get(ext, f"{fallback_prefix}/{ext}")
-#         apply_safe_file_response_headers(response, content_type, ext)
-#         return response
-#     except Exception as e:
-#         return server_error_response(e)
-#
-#
-# @manager.route("/mv", methods=["POST"])  # noqa: F821
-# @login_required
-# @validate_request("src_file_ids", "dest_file_id")
-# async def move():
-#     req = await get_request_json()
-#     try:
-#         file_ids = req["src_file_ids"]
-#         dest_parent_id = req["dest_file_id"]
-#
-#         ok, dest_folder = FileService.get_by_id(dest_parent_id)
-#         if not ok or not dest_folder:
-#             return get_data_error_result(message="Parent folder not found!")
-#
-#         files = FileService.get_by_ids(file_ids)
-#         if not files:
-#             return get_data_error_result(message="Source files not found!")
-#
-#         files_dict = {f.id: f for f in files}
-#
-#         for file_id in file_ids:
-#             file = files_dict.get(file_id)
-#             if not file:
-#                 return get_data_error_result(message="File or folder not found!")
-#             if not file.tenant_id:
-#                 return get_data_error_result(message="Tenant not found!")
-#             if not check_file_team_permission(file, current_user.id):
-#                 return get_json_result(
-#                     data=False,
-#                     message="No authorization.",
-#                     code=RetCode.AUTHENTICATION_ERROR,
-#                 )
-#
-#         def _move_entry_recursive(source_file_entry, dest_folder):
-#             if source_file_entry.type == FileType.FOLDER.value:
-#                 existing_folder = FileService.query(name=source_file_entry.name, parent_id=dest_folder.id)
-#                 if existing_folder:
-#                     new_folder = existing_folder[0]
-#                 else:
-#                     new_folder = FileService.insert(
-#                         {
-#                             "id": get_uuid(),
-#                             "parent_id": dest_folder.id,
-#                             "tenant_id": source_file_entry.tenant_id,
-#                             "created_by": current_user.id,
-#                             "name": source_file_entry.name,
-#                             "location": "",
-#                             "size": 0,
-#                             "type": FileType.FOLDER.value,
-#                         }
-#                     )
-#
-#                 sub_files = FileService.list_all_files_by_parent_id(source_file_entry.id)
-#                 for sub_file in sub_files:
-#                     _move_entry_recursive(sub_file, new_folder)
-#
-#                 FileService.delete_by_id(source_file_entry.id)
-#                 return
-#
-#             old_parent_id = source_file_entry.parent_id
-#             old_location = source_file_entry.location
-#             filename = source_file_entry.name
-#
-#             new_location = filename
-#             while settings.STORAGE_IMPL.obj_exist(dest_folder.id, new_location):
-#                 new_location += "_"
-#
-#             try:
-#                 settings.STORAGE_IMPL.move(old_parent_id, old_location, dest_folder.id, new_location)
-#             except Exception as storage_err:
-#                 raise RuntimeError(f"Move file failed at storage layer: {str(storage_err)}")
-#
-#             FileService.update_by_id(
-#                 source_file_entry.id,
-#                 {
-#                     "parent_id": dest_folder.id,
-#                     "location": new_location,
-#                 },
-#             )
-#
-#         def _move_sync():
-#             for file in files:
-#                 _move_entry_recursive(file, dest_folder)
-#             return get_json_result(data=True)
-#
-#         return await thread_pool_exec(_move_sync)
-#
-#     except Exception as e:
-#         return server_error_response(e)

api/apps/llm_app.py

@@ -25,8 +25,23 @@ from api.db.services.llm_service import LLMService
 from api.utils.api_utils import get_allowed_llm_factories, get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request
 from common.constants import StatusEnum, LLMType
 from api.db.db_models import TenantLLM
-from rag.utils.base64_image import test_image
-from rag.llm import EmbeddingModel, ChatModel, RerankModel, CvModel, TTSModel, OcrModel, Seq2txtModel
+
+
+def _resolve_my_llm_is_tools(o_dict: dict) -> bool:
+    decode_api_key_config = getattr(TenantLLMService, "_decode_api_key_config", None)
+    if callable(decode_api_key_config):
+        _, is_tools, _ = decode_api_key_config(o_dict.get("api_key", ""))
+        if is_tools is not None:
+            return bool(is_tools)
+
+    try:
+        base_name, fid = TenantLLMService.split_model_name_and_factory(o_dict["llm_name"])
+        llm_cfg = LLMService.query(llm_name=base_name, fid=fid) if fid else LLMService.query(llm_name=base_name)
+        if not llm_cfg and fid:
+            llm_cfg = LLMService.query(llm_name=base_name)
+        return bool(llm_cfg[0].is_tools) if llm_cfg else False
+    except Exception:
+        return False
 
 
 @manager.route("/factories", methods=["GET"])  # noqa: F821
@@ -61,6 +76,8 @@ def factories():
 @validate_request("llm_factory", "api_key")
 async def set_api_key():
     req = await get_request_json()
+    from rag.llm import ChatModel, EmbeddingModel, RerankModel
+
     # test if api key works
     chat_passed, embd_passed, rerank_passed = False, False, False
     factory = req["llm_factory"]
@@ -112,7 +129,9 @@ async def set_api_key():
             except Exception as e:
                 msg += f"\nFail to access model({llm.fid}/{llm.llm_name}) using this api key." + str(e)
         elif not rerank_passed and llm.model_type == LLMType.RERANK.value:
-            assert factory in RerankModel, f"Re-rank model from {factory} is not supported yet."
+            if factory not in RerankModel:
+                msg += f"\nRerank model from {factory} is not supported yet."
+                continue
             mdl = RerankModel[factory](req["api_key"], llm.llm_name, base_url=base_url)
             try:
                 arr, tc = await asyncio.wait_for(
@@ -161,21 +180,68 @@ async def set_api_key():
 @validate_request("llm_factory")
 async def add_llm():
     req = await get_request_json()
+    from rag.llm import ChatModel, CvModel, EmbeddingModel, OcrModel, RerankModel, Seq2txtModel, TTSModel
+
     factory = req["llm_factory"]
-    api_key = req.get("api_key", "x")
     llm_name = req.get("llm_name")
     timeout_seconds = int(os.environ.get("LLM_TIMEOUT_SECONDS", 10))
 
     if factory not in [f.name for f in get_allowed_llm_factories()]:
         return get_data_error_result(message=f"LLM factory {factory} is not allowed")
 
+    # When editing an existing model the frontend leaves the api_key input blank
+    # and strips it from the payload, so req["api_key"] is missing. Without a
+    # fallback the validation below would run with the "x" placeholder and the
+    # upstream provider would return "Your API key is invalid" — recover the
+    # saved key from DB. Use only the *decoded* api_key (never the raw JSON
+    # payload) so factories that pack extra fields into api_key
+    # (OpenRouter, Bedrock, …) can rebuild their JSON correctly with whatever
+    # new fields the user did provide via apikey_json.
+    if req.get("api_key") is None and llm_name:
+        _LLM_NAME_SUFFIX = {
+            "LocalAI": "___LocalAI",
+            "HuggingFace": "___HuggingFace",
+            "OpenAI-API-Compatible": "___OpenAI-API",
+            "VLLM": "___VLLM",
+        }
+        saved_llm_name = llm_name + _LLM_NAME_SUFFIX.get(factory, "")
+        logging.debug(
+            "add_llm: attempting api_key recovery factory=%s llm_name=%s saved_llm_name=%s tenant_id=%s",
+            factory, llm_name, saved_llm_name, current_user.id,
+        )
+        existing_llms = TenantLLMService.query(
+            tenant_id=current_user.id,
+            llm_factory=factory,
+            llm_name=saved_llm_name,
+        )
+        logging.debug(
+            "add_llm: api_key recovery query matched=%d factory=%s saved_llm_name=%s",
+            len(existing_llms) if existing_llms else 0, factory, saved_llm_name,
+        )
+        if existing_llms:
+            existing_api_key, _, _ = TenantLLMService._decode_api_key_config(
+                existing_llms[0].api_key
+            )
+            logging.debug(
+                "add_llm: api_key recovery decoded=%s factory=%s saved_llm_name=%s",
+                "present" if existing_api_key else "absent", factory, saved_llm_name,
+            )
+            if existing_api_key:
+                req["api_key"] = existing_api_key
+                logging.info(
+                    "add_llm: recovered saved api_key from existing record factory=%s saved_llm_name=%s tenant_id=%s",
+                    factory, saved_llm_name, current_user.id,
+                )
+
+    api_key = req.get("api_key", "x")
+
     def apikey_json(keys):
         nonlocal req
         return json.dumps({k: req.get(k, "") for k in keys})
 
     if factory == "VolcEngine":
         # For VolcEngine, due to its special authentication method
-        # Assemble ark_api_key endpoint_id into api_key
+        # Assemble ark_api_key model_id into api_key; keep endpoint_id in backend payload for compatibility
         api_key = apikey_json(["ark_api_key", "endpoint_id"])
 
     elif factory == "Tencent Cloud":
@@ -185,7 +251,9 @@ async def add_llm():
     elif factory == "Bedrock":
         # For Bedrock, due to its special authentication method
         # Assemble bedrock_ak, bedrock_sk, bedrock_region
-        api_key = apikey_json(["auth_mode", "bedrock_ak", "bedrock_sk", "bedrock_region", "aws_role_arn"])
+        # Write into req["api_key"] to prevent the "existing key" override logic from replacing it
+        req["api_key"] = apikey_json(["auth_mode", "bedrock_ak", "bedrock_sk", "bedrock_region", "aws_role_arn"])
+        api_key = req["api_key"]
 
     elif factory == "LocalAI":
         llm_name += "___LocalAI"
@@ -226,6 +294,9 @@ async def add_llm():
     elif factory == "PaddleOCR":
         api_key = apikey_json(["api_key", "provider_order"])
 
+    elif factory == "OpenDataLoader":
+        api_key = apikey_json(["api_key", "provider_order"])
+
     llm = {
         "tenant_id": current_user.id,
         "llm_factory": factory,
@@ -281,21 +352,25 @@ async def add_llm():
                 msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
 
         case LLMType.RERANK.value:
-            assert factory in RerankModel, f"RE-rank model from {factory} is not supported yet."
-            try:
-                mdl = RerankModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
-                arr, tc = await asyncio.wait_for(
-                    asyncio.to_thread(mdl.similarity, "Hello~ RAGFlower!", ["Hi, there!", "Ohh, my friend!"]),
-                    timeout=timeout_seconds,
-                )
-                if len(arr) == 0:
-                    raise Exception("Not known.")
-            except KeyError:
-                msg += f"{factory} dose not support this model({factory}/{mdl_nm})"
-            except Exception as e:
-                msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
+            if factory not in RerankModel:
+                msg += f"\nRerank model from {factory} is not supported yet."
+            else:
+                try:
+                    mdl = RerankModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
+                    arr, tc = await asyncio.wait_for(
+                        asyncio.to_thread(mdl.similarity, "Hello~ RAGFlower!", ["Hi, there!", "Ohh, my friend!"]),
+                        timeout=timeout_seconds,
+                    )
+                    if len(arr) == 0:
+                        raise Exception("Not known.")
+                except KeyError:
+                    msg += f"{factory} does not support this model({factory}/{mdl_nm})"
+                except Exception as e:
+                    msg += f"\nFail to access model({factory}/{mdl_nm})." + str(e)
 
         case LLMType.IMAGE2TEXT.value:
+            from rag.utils.base64_image import test_image
+
             assert factory in CvModel, f"Image to text model from {factory} is not supported yet."
             mdl = CvModel[factory](key=model_api_key, model_name=mdl_nm, base_url=model_base_url)
             try:
@@ -350,6 +425,9 @@ async def add_llm():
     if msg:
         return get_data_error_result(message=msg)
 
+    if "is_tools" in req:
+        llm["api_key"] = TenantLLMService._encode_api_key_config(llm["api_key"], bool(req["is_tools"]))
+
     if not TenantLLMService.filter_update([TenantLLM.tenant_id == current_user.id, TenantLLM.llm_factory == factory, TenantLLM.llm_name == llm["llm_name"]], llm):
         TenantLLMService.save(**llm)
 
@@ -390,6 +468,7 @@ async def delete_factory():
 def my_llms():
     try:
         TenantLLMService.ensure_mineru_from_env(current_user.id)
+        TenantLLMService.ensure_opendataloader_from_env(current_user.id)
         include_details = request.args.get("include_details", "false").lower() == "true"
 
         if include_details:
@@ -417,6 +496,7 @@ def my_llms():
                         "api_base": o_dict["api_base"] or "",
                         "max_tokens": o_dict["max_tokens"] or 8192,
                         "status": o_dict["status"] or "1",
+                        "is_tools": _resolve_my_llm_is_tools(o_dict),
                     }
                 )
         else:

api/apps/restful_apis/_generation_params.py

@@ -0,0 +1,38 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from copy import deepcopy
+
+GENERATION_CONFIG_KEYS = ("temperature", "top_p", "frequency_penalty", "presence_penalty", "max_tokens")
+
+
+def extract_generation_config(req):
+    return {key: req[key] for key in GENERATION_CONFIG_KEYS if key in req and req[key] is not None}
+
+
+def pop_generation_config(req):
+    generation_config = extract_generation_config(req)
+    for key in GENERATION_CONFIG_KEYS:
+        req.pop(key, None)
+    return generation_config
+
+
+def merge_generation_config(dialog, generation_config):
+    if not generation_config:
+        return
+    llm_setting = deepcopy(getattr(dialog, "llm_setting", None) or {})
+    llm_setting.update(generation_config)
+    dialog.llm_setting = llm_setting

api/apps/restful_apis/bot_api.py

@@ -0,0 +1,510 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import copy
+import json
+import re
+
+import logging
+
+from quart import Response, request
+
+from agent.canvas import Canvas
+from api.apps import AUTH_BETA, login_required
+from api.db.services.api_service import API4ConversationService
+from api.db.services.canvas_service import UserCanvasService
+from api.db.services.canvas_service import completion as agent_completion
+from api.db.services.conversation_service import async_iframe_completion as iframe_completion
+from api.db.services.dialog_service import DialogService, async_ask, gen_mindmap
+from api.db.services.doc_metadata_service import DocMetadataService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import LLMBundle
+from api.db.services.user_service import TenantService
+from common.metadata_utils import apply_meta_data_filter
+from api.db.services.search_service import SearchService
+from api.db.services.user_service import UserTenantService
+from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, get_model_config_from_provider_instance
+from common.misc_utils import thread_pool_exec
+from api.utils.api_utils import get_error_data_result, get_json_result, \
+    add_tenant_id_to_kwargs, get_result, get_request_json, server_error_response, validate_request
+from rag.app.tag import label_question
+from rag.prompts.template import load_prompt
+from rag.prompts.generator import cross_languages, keyword_extraction
+from common.constants import RetCode, LLMType, StatusEnum
+from common import settings
+from api.utils.reference_metadata_utils import (
+    enrich_chunks_with_document_metadata,
+    resolve_reference_metadata_preferences,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@manager.route("/chatbots/<dialog_id>/completions", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+async def chatbot_completions(dialog_id, tenant_id=None):
+    req = await get_request_json()
+
+    exists, dialog = DialogService.get_by_id(dialog_id)
+    if (not exists
+            or getattr(dialog, "tenant_id", None) != tenant_id
+            or str(getattr(dialog, "status", "")) != StatusEnum.VALID.value):
+        logger.warning(
+            "Denied chatbot access: reason=%s tenant_id=%s dialog_id=%s user_id=%s session_id=%s",
+            "no access to this chatbot",
+            tenant_id,
+            dialog_id,
+            req.get("user_id"),
+            req.get("session_id"),
+        )
+        return get_error_data_result(message="Authentication error: no access to this chatbot!")
+
+    if "quote" not in req:
+        req["quote"] = False
+
+    def _validate_iframe_access():
+        if req.get("session_id"):
+            exists, conv = API4ConversationService.get_by_id(req.get("session_id"))
+            if not exists:
+                raise AssertionError("Session not found!")
+            if conv.dialog_id != dialog_id:
+                raise AssertionError("Session does not belong to this dialog")
+            if tenant_id and conv.user_id and conv.user_id != tenant_id:
+                raise AssertionError("Session does not belong to this tenant")
+
+    if req.get("stream", True):
+        try:
+            _validate_iframe_access()
+        except AssertionError:
+            logger.warning(
+                "Denied chatbot completion stream: reason=%s tenant_id=%s dialog_id=%s user_id=%s session_id=%s",
+                "no access to this chatbot",
+                tenant_id,
+                dialog_id,
+                req.get("user_id"),
+                req.get("session_id"),
+            )
+            return get_error_data_result(message="Authentication error: no access to this chatbot!")
+
+        resp = Response(iframe_completion(dialog_id, tenant_id=tenant_id, **req), mimetype="text/event-stream")
+        resp.headers.add_header("Cache-control", "no-cache")
+        resp.headers.add_header("Connection", "keep-alive")
+        resp.headers.add_header("X-Accel-Buffering", "no")
+        resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+        return resp
+
+    try:
+        _validate_iframe_access()
+        async for answer in iframe_completion(dialog_id, tenant_id=tenant_id, **req):
+            return get_result(data=answer)
+    except AssertionError:
+        logger.warning(
+            "Denied chatbot completion: reason=%s tenant_id=%s dialog_id=%s user_id=%s session_id=%s",
+            "no access to this chatbot",
+            tenant_id,
+            dialog_id,
+            req.get("user_id"),
+            req.get("session_id"),
+        )
+        return get_error_data_result(message="Authentication error: no access to this chatbot!")
+
+    return None
+
+@manager.route("/chatbots/<dialog_id>/info", methods=["GET"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+async def chatbots_inputs(dialog_id, tenant_id=None):
+    exists, dialog = await thread_pool_exec(DialogService.get_by_id, dialog_id)
+    if (not exists
+            or getattr(dialog, "tenant_id", None) != tenant_id
+            or str(getattr(dialog, "status", "")) != StatusEnum.VALID.value):
+        request_args = getattr(request, "args", {}) or {}
+        request_user_id = request_args.get("user_id") if hasattr(request_args, "get") else None
+        request_session_id = request_args.get("session_id") if hasattr(request_args, "get") else None
+        logger.warning(
+            "Denied chatbot access: reason=%s tenant_id=%s dialog_id=%s user_id=%s session_id=%s",
+            "no access to this chatbot",
+            tenant_id,
+            dialog_id,
+            request_user_id,
+            request_session_id,
+        )
+        return get_error_data_result(message="Authentication error: no access to this chatbot!")
+    return get_result(
+        data={
+            "title": dialog.name,
+            "avatar": dialog.icon,
+            "prologue": dialog.prompt_config.get("prologue", ""),
+            "has_tavily_key": bool(dialog.prompt_config.get("tavily_api_key", "").strip()),
+            "llm_id": dialog.llm_id or "",
+        }
+    )
+
+
+@manager.route("/agentbots/<agent_id>/completions", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+async def agent_bot_completions(agent_id, tenant_id=None):
+    req = await get_request_json()
+
+    if req.get("stream", True):
+        async def stream():
+            try:
+                async for answer in agent_completion(tenant_id, agent_id, **req):
+                    yield answer
+            except Exception as e:
+                logging.exception(e)
+                error_result = get_error_data_result(message=str(e) or "Unknown error")
+                yield "data:" + json.dumps(
+                    {
+                        "event": "message",
+                        "data": {"content": f"Error {error_result['code']}: {error_result['message']}\n\n"},
+                        **error_result,
+                    },
+                    ensure_ascii=False,
+                ) + "\n\n"
+
+        resp = Response(stream(), mimetype="text/event-stream")
+        resp.headers.add_header("Cache-control", "no-cache")
+        resp.headers.add_header("Connection", "keep-alive")
+        resp.headers.add_header("X-Accel-Buffering", "no")
+        resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+        return resp
+
+    try:
+        full_content = ""
+        reference = {}
+        structured_output = {}
+        final_ans = {}
+        async for answer in agent_completion(tenant_id, agent_id, **req):
+            # agent_completion yields SSE-formatted strings. A single yielded
+            # chunk can contain multiple "data:..." frames separated by "\n\n"
+            # plus blank or comment lines, so parse line-by-line rather than
+            # assuming one frame per chunk.
+            if not isinstance(answer, str):
+                continue
+            for line in answer.splitlines():
+                line = line.strip()
+                if not line.startswith("data:"):
+                    continue
+                payload = line[len("data:"):].strip()
+                if not payload:
+                    continue
+                try:
+                    ans = json.loads(payload)
+                except Exception as e:
+                    logging.debug("agent_bot_completions: skipping malformed SSE frame: %s", e)
+                    continue
+                event = ans.get("event")
+                if event == "message":
+                    full_content += ans.get("data", {}).get("content", "") or ""
+                if ans.get("data", {}).get("reference"):
+                    reference.update(ans["data"]["reference"])
+                if event == "node_finished":
+                    data = ans.get("data", {})
+                    node_out = data.get("outputs") or {}
+                    component_id = data.get("component_id")
+                    if component_id is not None and "structured" in node_out:
+                        structured_output[component_id] = copy.deepcopy(node_out["structured"])
+                final_ans = ans
+
+        if not final_ans:
+            return get_result(data={})
+
+        if "data" not in final_ans or not isinstance(final_ans["data"], dict):
+            final_ans["data"] = {}
+        final_ans["data"]["content"] = full_content
+        final_ans["data"]["reference"] = reference
+        if structured_output:
+            final_ans["data"]["structured"] = structured_output
+        return get_result(data=final_ans)
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message=str(e) or "Unknown error")
+
+
+@manager.route("/agentbots/<agent_id>/inputs", methods=["GET"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+async def begin_inputs(agent_id, tenant_id=None):
+    e, cvs = await thread_pool_exec(UserCanvasService.get_by_id, agent_id)
+    if not e:
+        return get_error_data_result(f"Can't find agent by ID: {agent_id}")
+
+    canvas = Canvas(json.dumps(cvs.dsl), tenant_id, canvas_id=cvs.id)
+    return get_result(
+        data={"title": cvs.title, "avatar": cvs.avatar, "inputs": canvas.get_component_input_form("begin"),
+              "prologue": canvas.get_prologue(), "mode": canvas.get_mode()})
+
+
+@manager.route("/searchbots/ask", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+@validate_request("question", "kb_ids")
+async def ask_about_embedded(tenant_id=None):
+    req = await get_request_json()
+    uid = tenant_id
+
+    search_id = req.get("search_id", "")
+    search_config = {}
+    if search_id:
+        if search_app := await thread_pool_exec(SearchService.get_detail, search_id):
+            search_config = search_app.get("search_config", {})
+
+    chat_llm_name = ""
+    if not search_config or not search_config.get("chat_id"):
+        _, tenant_info = TenantService.get_by_id(uid)
+        chat_llm_name = tenant_info.llm_id
+
+    async def stream():
+        nonlocal req, uid
+        try:
+            async for ans in async_ask(req["question"], req["kb_ids"], uid, chat_llm_name=chat_llm_name, search_config=search_config):
+                yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
+        except Exception as e:
+            yield "data:" + json.dumps(
+                {"code": 500, "message": str(e), "data": {"answer": "**ERROR**: " + str(e), "reference": []}},
+                ensure_ascii=False) + "\n\n"
+        yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
+
+    resp = Response(stream(), mimetype="text/event-stream")
+    resp.headers.add_header("Cache-control", "no-cache")
+    resp.headers.add_header("Connection", "keep-alive")
+    resp.headers.add_header("X-Accel-Buffering", "no")
+    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
+    return resp
+
+
+@manager.route("/searchbots/retrieval_test", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+@validate_request("kb_id", "question")
+async def retrieval_test_embedded(tenant_id=None):
+    req = await get_request_json()
+    page = int(req.get("page", 1))
+    size = int(req.get("size", 30))
+    question = req["question"]
+    kb_ids = req["kb_id"]
+    if isinstance(kb_ids, str):
+        kb_ids = [kb_ids]
+    if not kb_ids:
+        return get_json_result(data=False, message='Please specify dataset firstly.',
+                               code=RetCode.DATA_ERROR)
+    doc_ids = req.get("doc_ids", [])
+    similarity_threshold = float(req.get("similarity_threshold", 0.0))
+    vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
+    use_kg = req.get("use_kg", False)
+    top = int(req.get("top_k", 1024))
+    if top <= 0:
+        return get_error_data_result("`top_k` must be greater than 0")
+    langs = req.get("cross_languages", [])
+    rerank_id = req.get("rerank_id", "")
+    if not tenant_id:
+        return get_error_data_result(message="permission denined.")
+    search_config = {}
+
+    async def _retrieval():
+        nonlocal similarity_threshold, vector_similarity_weight, top, rerank_id
+        local_doc_ids = list(doc_ids) if doc_ids else []
+        tenant_ids = []
+        _question = question
+
+        meta_data_filter = {}
+        chat_mdl = None
+        if req.get("search_id", ""):
+            nonlocal search_config
+            detail = await thread_pool_exec(SearchService.get_detail, req.get("search_id", ""))
+            if detail:
+                search_config = detail.get("search_config", {})
+                meta_data_filter = search_config.get("meta_data_filter", {})
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
+                chat_id = search_config.get("chat_id", "")
+                if chat_id:
+                    chat_model_config = await thread_pool_exec(get_model_config_from_provider_instance, tenant_id, LLMType.CHAT, chat_id)
+                else:
+                    chat_model_config = await thread_pool_exec(get_tenant_default_model_by_type, tenant_id, LLMType.CHAT)
+                chat_mdl = LLMBundle(tenant_id, chat_model_config)
+            # Apply search_config settings if not explicitly provided in request
+            if not req.get("similarity_threshold"):
+                similarity_threshold = float(search_config.get("similarity_threshold", similarity_threshold))
+            if not req.get("vector_similarity_weight"):
+                vector_similarity_weight = float(search_config.get("vector_similarity_weight", vector_similarity_weight))
+            if not req.get("top_k"):
+                top = int(search_config.get("top_k", top))
+            if not req.get("rerank_id"):
+                rerank_id = search_config.get("rerank_id", "")
+        else:
+            meta_data_filter = req.get("meta_data_filter") or {}
+            if meta_data_filter.get("method") in ["auto", "semi_auto"]:
+                chat_model_config = await thread_pool_exec(get_tenant_default_model_by_type, tenant_id, LLMType.CHAT)
+                chat_mdl = LLMBundle(tenant_id, chat_model_config)
+
+        if meta_data_filter:
+            local_doc_ids = await apply_meta_data_filter(
+                meta_data_filter,
+                None,
+                _question,
+                chat_mdl,
+                local_doc_ids,
+                kb_ids=kb_ids,
+                metas_loader=lambda: DocMetadataService.get_flatted_meta_by_kbs(kb_ids),
+            )
+
+        tenants = await thread_pool_exec(UserTenantService.query, user_id=tenant_id)
+        for kb_id in kb_ids:
+            for tenant in tenants:
+                if await thread_pool_exec(KnowledgebaseService.query, tenant_id=tenant.tenant_id, id=kb_id):
+                    tenant_ids.append(tenant.tenant_id)
+                    break
+            else:
+                return get_json_result(data=False, message="Only owner of dataset authorized for this operation.",
+                                       code=RetCode.OPERATING_ERROR)
+
+        e, kb = await thread_pool_exec(KnowledgebaseService.get_by_id, kb_ids[0])
+        if not e:
+            return get_error_data_result(message="Knowledgebase not found!")
+
+        if langs:
+            _question = await cross_languages(kb.tenant_id, None, _question, langs)
+        embd_model_config = await thread_pool_exec(get_model_config_from_provider_instance, kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+        embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
+
+        rerank_mdl = None
+        if rerank_id:
+            rerank_model_config = await thread_pool_exec(get_model_config_from_provider_instance, tenant_id, LLMType.RERANK, rerank_id)
+            rerank_mdl = LLMBundle(kb.tenant_id, rerank_model_config)
+
+        if req.get("keyword", False):
+            default_chat_model = await thread_pool_exec(get_tenant_default_model_by_type, kb.tenant_id, LLMType.CHAT)
+            chat_mdl = LLMBundle(kb.tenant_id, default_chat_model)
+            _question += await keyword_extraction(chat_mdl, _question)
+
+        labels = label_question(_question, [kb])
+        ranks = await settings.retriever.retrieval(
+            _question, embd_mdl, tenant_ids, kb_ids, page, size, similarity_threshold, vector_similarity_weight, top,
+            local_doc_ids, rerank_mdl=rerank_mdl, highlight=req.get("highlight"), rank_feature=labels
+        )
+        if use_kg:
+            default_chat_model = await thread_pool_exec(get_tenant_default_model_by_type, kb.tenant_id, LLMType.CHAT)
+            ck = await settings.kg_retriever.retrieval(_question, tenant_ids, kb_ids, embd_mdl,
+                                                 LLMBundle(kb.tenant_id, default_chat_model))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
+        for c in ranks["chunks"]:
+            c.pop("vector", None)
+
+        include_metadata, metadata_fields = _resolve_reference_metadata(req, search_config)
+        if include_metadata:
+            enrich_chunks_with_document_metadata(ranks["chunks"], metadata_fields)
+
+        ranks["labels"] = labels
+
+        return get_json_result(data=ranks)
+
+    try:
+        return await _retrieval()
+    except Exception as e:
+        if "not_found" in str(e):
+            return get_json_result(data=False, message="No chunk found! Check the chunk status please!",
+                                   code=RetCode.DATA_ERROR)
+        return server_error_response(e)
+
+
+@manager.route("/searchbots/related_questions", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+@validate_request("question")
+async def related_questions_embedded(tenant_id=None):
+    req = await get_request_json()
+    if not tenant_id:
+        return get_error_data_result(message="permission denined.")
+
+    search_id = req.get("search_id", "")
+    search_config = {}
+    if search_id:
+        if search_app := await thread_pool_exec(SearchService.get_detail, search_id):
+            search_config = search_app.get("search_config", {})
+
+    question = req["question"]
+
+    chat_id = search_config.get("chat_id", "")
+    if chat_id:
+        chat_model_config = await thread_pool_exec(get_model_config_from_provider_instance, tenant_id, LLMType.CHAT, chat_id)
+    else:
+        chat_model_config = await thread_pool_exec(get_tenant_default_model_by_type, tenant_id, LLMType.CHAT)
+    chat_mdl = LLMBundle(tenant_id, chat_model_config)
+
+    gen_conf = search_config.get("llm_setting", {"temperature": 0.9})
+    prompt = load_prompt("related_question")
+    ans = await chat_mdl.async_chat(
+        prompt,
+        [
+            {
+                "role": "user",
+                "content": f"""
+Keywords: {question}
+Related search terms:
+    """,
+            }
+        ],
+        gen_conf,
+    )
+    return get_json_result(data=[re.sub(r"^[0-9]\. ", "", a) for a in ans.split("\n") if re.match(r"^[0-9]\. ", a)])
+
+
+@manager.route("/searchbots/detail", methods=["GET"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+async def detail_share_embedded(tenant_id=None):
+    search_id = request.args["search_id"]
+    if not tenant_id:
+        return get_error_data_result(message="permission denined.")
+    try:
+        tenants = await thread_pool_exec(UserTenantService.query, user_id=tenant_id)
+        for tenant in tenants:
+            if await thread_pool_exec(SearchService.query, tenant_id=tenant.tenant_id, id=search_id):
+                break
+        else:
+            return get_json_result(data=False, message="Has no permission for this operation.",
+                                   code=RetCode.OPERATING_ERROR)
+
+        search = await thread_pool_exec(SearchService.get_detail, search_id)
+        if not search:
+            return get_error_data_result(message="Can't find this Search App!")
+        return get_json_result(data=search)
+    except Exception as e:
+        return server_error_response(e)
+
+
+@manager.route("/searchbots/mindmap", methods=["POST"])  # noqa: F821
+@login_required(auth_types=AUTH_BETA)
+@add_tenant_id_to_kwargs
+@validate_request("question", "kb_ids")
+async def mindmap(tenant_id=None):
+    req = await get_request_json()
+
+    search_id = req.get("search_id", "")
+    search_app = await thread_pool_exec(SearchService.get_detail, search_id) if search_id else {}
+
+    mind_map =await gen_mindmap(req["question"], req["kb_ids"], tenant_id, search_app.get("search_config", {}))
+    if "error" in mind_map:
+        return server_error_response(Exception(mind_map["error"]))
+    return get_json_result(data=mind_map)
+
+
+def _resolve_reference_metadata(req, search_config=None):
+    return resolve_reference_metadata_preferences(req, search_config)

api/apps/restful_apis/chat_api.py

@@ -16,25 +16,26 @@
 
 import json
 import logging
+import math
 import os
 import re
 import tempfile
 from copy import deepcopy
+from types import SimpleNamespace
 
 from quart import Response, request
 
 from api.apps import current_user, login_required
+from api.apps.restful_apis._generation_params import merge_generation_config, pop_generation_config
 from api.db.joint_services.tenant_model_service import (
-    get_model_config_by_type_and_name,
-    get_tenant_default_model_by_type,
+    get_tenant_default_model_by_type, get_model_config_from_provider_instance, get_api_key, split_model_name
 )
 from api.db.services.chunk_feedback_service import ChunkFeedbackService
 from api.db.services.conversation_service import ConversationService, structure_answer
-from api.db.services.dialog_service import DialogService, async_ask, async_chat, gen_mindmap
+from api.db.services.dialog_service import DialogService, async_chat, gen_mindmap
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.search_service import SearchService
-from api.db.services.tenant_llm_service import TenantLLMService
 from api.db.services.user_service import TenantService, UserTenantService
 from api.utils.api_utils import (
     check_duplicate_ids,
@@ -44,12 +45,45 @@ from api.utils.api_utils import (
     server_error_response,
     validate_request,
 )
-from api.utils.tenant_utils import ensure_tenant_model_id_for_params
+from api.utils.pagination_utils import validate_rest_api_page_size
 from common.constants import LLMType, RetCode, StatusEnum
-from common.misc_utils import get_uuid
+from common import settings
+from common.misc_utils import get_uuid, thread_pool_exec
 from rag.prompts.generator import chunks_format
 from rag.prompts.template import load_prompt
 
+def _sanitize_json_floats(obj):
+    """Replace NaN/Infinity floats with None so the result is RFC 8259 JSON.
+
+    `json.dumps` emits the literal tokens `NaN`/`Infinity` by default
+    (allow_nan=True). Those tokens are valid Python JSON output but invalid
+    per the JSON spec, and downstream proxies / Go consumers reject the
+    response with `failed to encode response: json: unsupported value: NaN`
+    (fixes #15245). Retrieval scores (similarity, vector_similarity,
+    term_similarity) can become NaN when an aggregation runs over an empty
+    set or when a similarity denominator is zero, so the chat completions
+    stream is the realistic trigger.
+
+    `isinstance(obj, float)` alone catches Python float and numpy.float64
+    (a float subclass) but misses numpy.float32 / numpy.float16 and any
+    other duck-typed numeric. Probe via math.isnan/isinf in a try/except
+    so any object math can evaluate gets sanitized — without changing
+    upstream callers like chunks_format or rag/nlp/search.py.
+    """
+    try:
+        if math.isnan(obj) or math.isinf(obj):
+            return None
+    except TypeError:
+        pass
+    if isinstance(obj, dict):
+        return {k: _sanitize_json_floats(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_sanitize_json_floats(v) for v in obj]
+    if isinstance(obj, tuple):
+        return tuple(_sanitize_json_floats(v) for v in obj)
+    return obj
+
+
 _DEFAULT_PROMPT_CONFIG = {
     "system": (
         'You are an intelligent assistant. Please summarize the content of the dataset to answer the question. '
@@ -67,6 +101,15 @@ _DEFAULT_PROMPT_CONFIG = {
     "tts": False,
     "refine_multiturn": True,
 }
+_DEFAULT_DIRECT_CHAT_PROMPT_CONFIG = {
+    "system": "",
+    "prologue": "",
+    "parameters": [],
+    "empty_response": "",
+    "quote": False,
+    "tts": False,
+    "refine_multiturn": True,
+}
 _DEFAULT_RERANK_MODELS = {"BAAI/bge-reranker-v2-m3", "maidalun1020/bce-reranker-base_v1"}
 _READONLY_FIELDS = {"id", "tenant_id", "created_by", "create_time", "create_date", "update_time", "update_date"}
 _PERSISTED_FIELDS = set(DialogService.model._meta.fields)
@@ -118,45 +161,155 @@ def _build_session_response(conv: dict) -> dict:
     return conv
 
 
-def _ensure_owned_chat(chat_id):
-    return DialogService.query(
+async def _ensure_owned_chat(chat_id):
+    return await thread_pool_exec(
+        DialogService.query,
         tenant_id=current_user.id, id=chat_id, status=StatusEnum.VALID.value
     )
 
 
-def _validate_llm_id(llm_id, tenant_id, llm_setting=None):
+def _build_default_completion_dialog():
+    return SimpleNamespace(
+        tenant_id=current_user.id,
+        llm_id="",
+        tenant_llm_id=None,
+        llm_setting={},
+        prompt_config=deepcopy(_DEFAULT_DIRECT_CHAT_PROMPT_CONFIG),
+        kb_ids=[],
+        top_n=6,
+        top_k=1024,
+        rerank_id="",
+        similarity_threshold=0.1,
+        vector_similarity_weight=0.3,
+        meta_data_filter=None,
+    )
+
+
+async def _create_session_for_completion(chat_id, dialog, user_id):
+    conv = {
+        "id": get_uuid(),
+        "dialog_id": chat_id,
+        "name": "New session",
+        "message": [{"role": "assistant", "content": dialog.prompt_config.get("prologue", "")}],
+        "user_id": user_id,
+        "reference": [],
+    }
+    await thread_pool_exec(ConversationService.save, **conv)
+    ok, conv_obj = await thread_pool_exec(ConversationService.get_by_id, conv["id"])
+    if not ok:
+        raise LookupError("Fail to create a session!")
+    return conv_obj
+
+
+def _get_bool_request_flag(req, *names, default=False):
+    for name in names:
+        if name not in req:
+            continue
+        value = req.pop(name)
+        if isinstance(value, str):
+            return value.strip().lower() in {"1", "true", "yes", "on"}
+        return bool(value)
+    return default
+
+
+def _normalize_completion_messages(req):
+    messages = req.get("messages")
+    if messages is None:
+        question = req.get("question")
+        if question is None:
+            return None, get_data_error_result(
+                code=RetCode.ARGUMENT_ERROR,
+                message="required argument are missing: messages",
+            )
+        messages = [{"role": "user", "content": question}]
+        if req.get("files"):
+            messages[-1]["files"] = req["files"]
+
+    if not isinstance(messages, list) or not messages:
+        return None, get_data_error_result(
+            code=RetCode.ARGUMENT_ERROR,
+            message="`messages` must be a non-empty list.",
+        )
+
+    for message in messages:
+        if not isinstance(message, dict):
+            return None, get_data_error_result(
+                code=RetCode.ARGUMENT_ERROR,
+                message="Every item in `messages` must be an object.",
+            )
+        if "role" not in message or "content" not in message:
+            return None, get_data_error_result(
+                code=RetCode.ARGUMENT_ERROR,
+                message="Every item in `messages` must include `role` and `content`.",
+            )
+
+    msg = []
+    for m in messages:
+        if m["role"] == "system":
+            continue
+        if m["role"] == "assistant" and not msg:
+            continue
+        msg.append(m)
+
+    if not msg:
+        return None, get_data_error_result(
+            code=RetCode.ARGUMENT_ERROR,
+            message="`messages` must contain a user message.",
+        )
+    if msg[-1]["role"] != "user":
+        return None, get_data_error_result(
+            code=RetCode.ARGUMENT_ERROR,
+            message="The last message must be from user.",
+        )
+    if not msg[-1].get("id"):
+        msg[-1]["id"] = get_uuid()
+
+    # till now, message and msg are sharing the same copy
+    return (messages, msg), None
+
+
+async def _validate_llm_id(llm_id, tenant_id, llm_setting=None):
     if not llm_id:
         return None
 
-    llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(llm_id)
-    model_type = (llm_setting or {}).get("model_type")
-    if model_type not in {"chat", "image2text"}:
+    conf_model_type = (llm_setting or {}).get("model_type")
+    if isinstance(conf_model_type, str):
+        model_type = conf_model_type if conf_model_type in {"chat", "image2text"} else "chat"
+    elif isinstance(conf_model_type, list):
+        model_type = "image2text" if "image2text" in conf_model_type else "chat"
+    else:
         model_type = "chat"
-
-    if not TenantLLMService.query(
-        tenant_id=tenant_id,
-        llm_name=llm_name,
-        llm_factory=llm_factory,
-        model_type=model_type,
-    ):
+    try:
+        await thread_pool_exec(
+            get_model_config_from_provider_instance,
+            tenant_id=tenant_id,
+            model_name=llm_id,
+            model_type=model_type,
+        )
+    except Exception as e:
+        logging.error(f"Fail to get model config for {llm_id}: {e}")
         return f"`llm_id` {llm_id} doesn't exist"
+
     return None
 
-
-def _validate_rerank_id(rerank_id, tenant_id):
+async def _validate_rerank_id(rerank_id, tenant_id):
     if not rerank_id:
         return None
-    llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(rerank_id)
+    parts = rerank_id.split('@')
+    llm_name = parts[0]
     if llm_name in _DEFAULT_RERANK_MODELS:
         return None
-    if TenantLLMService.query(
-        tenant_id=tenant_id,
-        llm_name=llm_name,
-        llm_factory=llm_factory,
-        model_type="rerank",
-    ):
-        return None
-    return f"`rerank_id` {rerank_id} doesn't exist"
+    try:
+        await thread_pool_exec(
+            get_model_config_from_provider_instance,
+            tenant_id=tenant_id,
+            model_name=rerank_id,
+            model_type="rerank",
+        )
+    except Exception as e:
+        logging.error(f"Fail to get model config for {rerank_id}: {e}")
+        return f"`rerank_id` {rerank_id} doesn't exist"
+    return None
 
 
 # def _validate_prompt_config(prompt_config):
@@ -168,7 +321,7 @@ def _validate_rerank_id(rerank_id, tenant_id):
 #     return None
 
 
-def _validate_dataset_ids(dataset_ids, tenant_id):
+async def _validate_dataset_ids(dataset_ids, tenant_id):
     if dataset_ids is None:
         return []
     if not isinstance(dataset_ids, list):
@@ -177,9 +330,9 @@ def _validate_dataset_ids(dataset_ids, tenant_id):
     normalized_ids = [dataset_id for dataset_id in dataset_ids if dataset_id]
     kbs = []
     for dataset_id in normalized_ids:
-        if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        if not await thread_pool_exec(KnowledgebaseService.accessible, kb_id=dataset_id, user_id=tenant_id):
             return f"You don't own the dataset {dataset_id}"
-        matches = KnowledgebaseService.query(id=dataset_id)
+        matches = await thread_pool_exec(KnowledgebaseService.query, id=dataset_id)
         if not matches:
             return f"You don't own the dataset {dataset_id}"
         kb = matches[0]
@@ -187,7 +340,7 @@ def _validate_dataset_ids(dataset_ids, tenant_id):
             return f"The dataset {dataset_id} doesn't own parsed file"
         kbs.append(kb)
 
-    embd_ids = [TenantLLMService.split_model_name_and_factory(kb.embd_id)[0] for kb in kbs]
+    embd_ids = [split_model_name(kb.embd_id)[0] for kb in kbs]
     if len(set(embd_ids)) > 1:
         return f'Datasets use different embedding models: {[kb.embd_id for kb in kbs]}'
 
@@ -225,19 +378,19 @@ async def create():
         req["name"] = name
 
         if "dataset_ids" in req:
-            kb_ids = _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
+            kb_ids = await _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
             if isinstance(kb_ids, str):
                 return get_data_error_result(message=kb_ids)
             req["kb_ids"] = kb_ids
             req.pop("dataset_ids", None)
 
         if "llm_id" in req:
-            err = _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
+            err = await _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
             if err:
                 return get_data_error_result(message=err)
 
         if "rerank_id" in req:
-            err = _validate_rerank_id(req.get("rerank_id"), current_user.id)
+            err = await _validate_rerank_id(req.get("rerank_id"), current_user.id)
             if err:
                 return get_data_error_result(message=err)
 
@@ -265,7 +418,6 @@ async def create():
         # if err:
         #     return get_data_error_result(message=err)
 
-        req = ensure_tenant_model_id_for_params(current_user.id, req)
         req = {field: value for field, value in req.items() if field in _PERSISTED_FIELDS}
         for field in _READONLY_FIELDS:
             req.pop(field, None)
@@ -292,7 +444,7 @@ async def create():
 
 @manager.route("/chats", methods=["GET"])  # noqa: F821
 @login_required
-def list_chats():
+async def list_chats():
     chat_id = request.args.get("id")
     name = request.args.get("name")
     keywords = request.args.get("keywords", "")
@@ -305,11 +457,12 @@ def list_chats():
 
     try:
         page_number = int(request.args.get("page", 0))
-        items_per_page = int(request.args.get("page_size", 0))
+        items_per_page = validate_rest_api_page_size(int(request.args.get("page_size", 0)))
 
         if owner_ids:
-            chats, total = DialogService.get_by_tenant_ids(
-                owner_ids, current_user.id, 0, 0, orderby, desc, keywords, **exact_filters
+            chats, total = await thread_pool_exec(
+                DialogService.get_by_tenant_ids,
+                owner_ids, current_user.id, 0, 0, orderby, desc, keywords, **exact_filters,
             )
             chats = [chat for chat in chats if chat["tenant_id"] in owner_ids]
             total = len(chats)
@@ -317,8 +470,9 @@ def list_chats():
                 start = (page_number - 1) * items_per_page
                 chats = chats[start : start + items_per_page]
         else:
-            chats, total = DialogService.get_by_tenant_ids(
-                [], current_user.id, page_number, items_per_page, orderby, desc, keywords, **exact_filters
+            chats, total = await thread_pool_exec(
+                DialogService.get_by_tenant_ids,
+                [], current_user.id, page_number, items_per_page, orderby, desc, keywords, **exact_filters,
             )
 
         return get_json_result(
@@ -330,12 +484,13 @@ def list_chats():
 
 @manager.route("/chats/<chat_id>", methods=["GET"])  # noqa: F821
 @login_required
-def get_chat(chat_id):
+async def get_chat(chat_id):
     try:
-        tenants = UserTenantService.query(user_id=current_user.id)
+        tenants = await thread_pool_exec(UserTenantService.query, user_id=current_user.id)
         for tenant in tenants:
-            if DialogService.query(
-                tenant_id=tenant.tenant_id, id=chat_id, status=StatusEnum.VALID.value
+            if await thread_pool_exec(
+                DialogService.query,
+                tenant_id=tenant.tenant_id, id=chat_id, status=StatusEnum.VALID.value,
             ):
                 break
         else:
@@ -345,7 +500,7 @@ def get_chat(chat_id):
                 code=RetCode.AUTHENTICATION_ERROR,
             )
 
-        ok, chat = DialogService.get_by_id(chat_id)
+        ok, chat = await thread_pool_exec(DialogService.get_by_id, chat_id)
         if not ok:
             return get_data_error_result(message="Chat not found!")
         return get_json_result(data=_build_chat_response(chat))
@@ -356,7 +511,7 @@ def get_chat(chat_id):
 @manager.route("/chats/<chat_id>", methods=["PUT"])  # noqa: F821
 @login_required
 async def update_chat(chat_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(
             data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR
         )
@@ -382,19 +537,19 @@ async def update_chat(chat_id):
             req["name"] = name
 
         if "dataset_ids" in req:
-            kb_ids = _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
+            kb_ids = await _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
             if isinstance(kb_ids, str):
                 return get_data_error_result(message=kb_ids)
             req["kb_ids"] = kb_ids
             req.pop("dataset_ids", None)
 
         if "llm_id" in req:
-            err = _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
+            err = await _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
             if err:
                 return get_data_error_result(message=err)
 
         if "rerank_id" in req:
-            err = _validate_rerank_id(req.get("rerank_id"), current_user.id)
+            err = await _validate_rerank_id(req.get("rerank_id"), current_user.id)
             if err:
                 return get_data_error_result(message=err)
 
@@ -411,8 +566,6 @@ async def update_chat(chat_id):
         # kb_ids = req.get("kb_ids", current_chat.get("kb_ids", []))
         # if not kb_ids and not prompt_config.get("tavily_api_key") and _has_knowledge_placeholder(prompt_config):
         #     return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no dataset / Tavily used here.")
-
-        req = ensure_tenant_model_id_for_params(current_user.id, req)
         req = {field: value for field, value in req.items() if field in _PERSISTED_FIELDS}
         for field in _READONLY_FIELDS:
             req.pop(field, None)
@@ -442,7 +595,7 @@ async def update_chat(chat_id):
 @manager.route("/chats/<chat_id>", methods=["PATCH"])  # noqa: F821
 @login_required
 async def patch_chat(chat_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(
             data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR
         )
@@ -466,19 +619,19 @@ async def patch_chat(chat_id):
                 req["name"] = name
 
         if "dataset_ids" in req:
-            kb_ids = _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
+            kb_ids = await _validate_dataset_ids(req.get("dataset_ids"), current_user.id)
             if isinstance(kb_ids, str):
                 return get_data_error_result(message=kb_ids)
             req["kb_ids"] = kb_ids
             req.pop("dataset_ids", None)
 
         if "llm_id" in req:
-            err = _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
+            err = await _validate_llm_id(req.get("llm_id"), current_user.id, req.get("llm_setting"))
             if err:
                 return get_data_error_result(message=err)
 
         if "rerank_id" in req:
-            err = _validate_rerank_id(req.get("rerank_id"), current_user.id)
+            err = await _validate_rerank_id(req.get("rerank_id"), current_user.id)
             if err:
                 return get_data_error_result(message=err)
 
@@ -503,7 +656,6 @@ async def patch_chat(chat_id):
         #     if not kb_ids and not prompt_config.get("tavily_api_key") and _has_knowledge_placeholder(prompt_config):
         #         return get_data_error_result(message="Please remove `{knowledge}` in system prompt since no dataset / Tavily used here.")
 
-        req = ensure_tenant_model_id_for_params(current_user.id, req)
         req = {field: value for field, value in req.items() if field in _PERSISTED_FIELDS}
         for field in _READONLY_FIELDS:
             req.pop(field, None)
@@ -532,8 +684,8 @@ async def patch_chat(chat_id):
 
 @manager.route("/chats/<chat_id>", methods=["DELETE"])  # noqa: F821
 @login_required
-def delete_chat(chat_id):
-    if not _ensure_owned_chat(chat_id):
+async def delete_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(
             data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR
         )
@@ -565,6 +717,15 @@ async def bulk_delete_chats():
             if not ids:
                 return get_json_result(data={})
         else:
+            # keep backward compatibility, DELETE with chat_id in request body
+            chat_id = req.get("chat_id")
+            if chat_id:
+                try:
+                    if not DialogService.update_by_id(chat_id, {"status": StatusEnum.INVALID.value}):
+                        return get_data_error_result(message=f"Failed to delete chat {chat_id}")
+                    return get_json_result(data=True)
+                except Exception as ex:
+                    return server_error_response(ex)
             return get_json_result(data={})
 
     errors = []
@@ -572,7 +733,7 @@ async def bulk_delete_chats():
     unique_ids, duplicate_messages = check_duplicate_ids(ids, "chat")
 
     for chat_id in unique_ids:
-        if not _ensure_owned_chat(chat_id):
+        if not await _ensure_owned_chat(chat_id):
             errors.append(f"Chat({chat_id}) not found.")
             continue
         success_count += DialogService.update_by_id(chat_id, {"status": StatusEnum.INVALID.value})
@@ -592,7 +753,8 @@ async def bulk_delete_chats():
 @manager.route("/chats/<chat_id>/sessions", methods=["POST"])  # noqa: F821
 @login_required
 async def create_session(chat_id):
-    if not _ensure_owned_chat(chat_id):
+    """Create a new conversation session for the given chat, owned by the authenticated user."""
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         req = await get_request_json()
@@ -608,7 +770,7 @@ async def create_session(chat_id):
             "dialog_id": chat_id,
             "name": name,
             "message": [{"role": "assistant", "content": dia.prompt_config.get("prologue", "")}],
-            "user_id": req.get("user_id", current_user.id),
+            "user_id": current_user.id,
             "reference": [],
         }
         ConversationService.save(**conv)
@@ -622,16 +784,16 @@ async def create_session(chat_id):
 
 @manager.route("/chats/<chat_id>/sessions", methods=["GET"])  # noqa: F821
 @login_required
-def list_sessions(chat_id):
+async def list_sessions(chat_id):
     try:
-        if not _ensure_owned_chat(chat_id):
+        if not await _ensure_owned_chat(chat_id):
             return get_json_result(
                 data=False,
                 message="No authorization.",
                 code=RetCode.AUTHENTICATION_ERROR,
             )
         page_number = int(request.args.get("page", 1))
-        items_per_page = int(request.args.get("page_size", 30))
+        items_per_page = validate_rest_api_page_size(int(request.args.get("page_size", 30)))
         orderby = request.args.get("orderby", "create_time")
         desc = request.args.get("desc", "true").lower() != "false"
         session_id = request.args.get("id")
@@ -650,15 +812,15 @@ def list_sessions(chat_id):
 @manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["GET"])  # noqa: F821
 @login_required
 async def get_session(chat_id, session_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
-        ok, conv = ConversationService.get_by_id(session_id)
+        ok, conv = await thread_pool_exec(ConversationService.get_by_id, session_id)
         if not ok:
             return get_data_error_result(message="Session not found!")
         if conv.dialog_id != chat_id:
             return get_data_error_result(message="Session does not belong to this chat!")
-        dialog = _ensure_owned_chat(chat_id)
+        dialog = await _ensure_owned_chat(chat_id)
         avatar = dialog[0].icon if dialog else ""
         for ref in conv.reference:
             if isinstance(ref, list):
@@ -671,10 +833,10 @@ async def get_session(chat_id, session_id):
         return server_error_response(ex)
 
 
-@manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PUT"])  # noqa: F821
+@manager.route("/chats/<chat_id>/sessions/<session_id>", methods=["PATCH"])  # noqa: F821
 @login_required
 async def update_session(chat_id, session_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         req = await get_request_json()
@@ -703,7 +865,7 @@ async def update_session(chat_id, session_id):
 @manager.route("/chats/<chat_id>/sessions", methods=["DELETE"])  # noqa: F821
 @login_required
 async def delete_sessions(chat_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         req = await get_request_json()
@@ -725,6 +887,17 @@ async def delete_sessions(chat_id):
             if not ConversationService.query(id=sid, dialog_id=chat_id):
                 errors.append(f"The chat doesn't own the session {sid}")
                 continue
+            ok, conv = ConversationService.get_by_id(sid)
+            if ok:
+                for msg in conv.message or []:
+                    for file in msg.get("files") or []:
+                        file_id = file.get("id")
+                        if not file_id:
+                            continue
+                        try:
+                            settings.STORAGE_IMPL.rm(f"{current_user.id}-downloads", file_id)
+                        except Exception:
+                            logging.warning("Failed to delete chat upload blob %s/%s", current_user.id, file_id)
             ConversationService.delete_by_id(sid)
             success_count += 1
         all_errors = errors + duplicate_messages
@@ -743,7 +916,7 @@ async def delete_sessions(chat_id):
 @manager.route("/chats/<chat_id>/sessions/<session_id>/messages/<msg_id>", methods=["DELETE"])  # noqa: F821
 @login_required
 async def delete_session_message(chat_id, session_id, msg_id):
-    if not _ensure_owned_chat(chat_id):
+    if not await _ensure_owned_chat(chat_id):
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
         ok, conv = ConversationService.get_by_id(session_id)
@@ -767,7 +940,7 @@ async def delete_session_message(chat_id, session_id, msg_id):
 @manager.route("/chats/<chat_id>/sessions/<session_id>/messages/<msg_id>/feedback", methods=["PUT"])  # noqa: F821
 @login_required
 async def update_message_feedback(chat_id, session_id, msg_id):
-    owned = _ensure_owned_chat(chat_id)
+    owned = await _ensure_owned_chat(chat_id)
     if not owned:
         return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
     try:
@@ -805,12 +978,14 @@ async def update_message_feedback(chat_id, session_id, msg_id):
                     reference = conv_dict["reference"][ref_index]
                     if reference:
                         if isinstance(prior_thumb, bool) and prior_thumb != thumb_raw:
-                            ChunkFeedbackService.apply_feedback(
+                            await thread_pool_exec(
+                                ChunkFeedbackService.apply_feedback,
                                 tenant_id=current_user.id,
                                 reference=reference,
                                 is_positive=not prior_thumb,
                             )
-                        feedback_result = ChunkFeedbackService.apply_feedback(
+                        feedback_result = await thread_pool_exec(
+                            ChunkFeedbackService.apply_feedback,
                             tenant_id=current_user.id,
                             reference=reference,
                             is_positive=thumb_raw is True,
@@ -823,13 +998,13 @@ async def update_message_feedback(chat_id, session_id, msg_id):
             except Exception as e:
                 logging.warning("Failed to apply chunk feedback: %s", e)
 
-        ConversationService.update_by_id(conv_dict["id"], conv_dict)
+        await thread_pool_exec(ConversationService.update_by_id, conv_dict["id"], conv_dict)
         return get_json_result(data=_build_session_response(conv_dict))
     except Exception as ex:
         return server_error_response(ex)
 
 
-@manager.route("/chats/tts", methods=["POST"])  # noqa: F821
+@manager.route("/chat/audio/speech", methods=["POST"])  # noqa: F821
 @login_required
 async def tts():
     req = await get_request_json()
@@ -857,9 +1032,9 @@ async def tts():
     return resp
 
 
-@manager.route("/chats/transcriptions", methods=["POST"])  # noqa: F821
+@manager.route("/chat/audio/transcription", methods=["POST"])  # noqa: F821
 @login_required
-async def transcriptions():
+async def transcription():
     req = await request.form
     stream_mode = req.get("stream", "false").lower() == "true"
     files = await request.files
@@ -915,7 +1090,7 @@ async def transcriptions():
     return Response(event_stream(), content_type="text/event-stream")
 
 
-@manager.route("/chats/mindmap", methods=["POST"])  # noqa: F821
+@manager.route("/chat/mindmap", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question", "kb_ids")
 async def mindmap():
@@ -933,10 +1108,10 @@ async def mindmap():
     return get_json_result(data=mind_map)
 
 
-@manager.route("/chats/related_questions", methods=["POST"])  # noqa: F821
+@manager.route("/chat/recommendation", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("question")
-async def related_questions():
+async def recommendation():
     req = await get_request_json()
 
     search_id = req.get("search_id", "")
@@ -949,7 +1124,7 @@ async def related_questions():
 
     chat_id = search_config.get("chat_id", "")
     if chat_id:
-        chat_model_config = get_model_config_by_type_and_name(current_user.id, LLMType.CHAT, chat_id)
+        chat_model_config = get_model_config_from_provider_instance(current_user.id, LLMType.CHAT, chat_id)
     else:
         chat_model_config = get_tenant_default_model_by_type(current_user.id, LLMType.CHAT)
     chat_mdl = LLMBundle(current_user.id, chat_model_config)
@@ -971,61 +1146,156 @@ async def related_questions():
     return get_json_result(data=[re.sub(r"^[0-9]\. ", "", a) for a in ans.split("\n") if re.match(r"^[0-9]\. ", a)])
 
 
-@manager.route("/chats/<chat_id>/sessions/<session_id>/completions", methods=["POST"])  # noqa: F821
+@manager.route("/chat/completions", methods=["POST"])  # noqa: F821
 @login_required
-@validate_request("messages")
-async def session_completion(chat_id, session_id):
+async def session_completion(chat_id_in_arg=""):
+    """Handle chat completion requests, streaming or non-streaming, scoped to the authenticated user."""
     req = await get_request_json()
-    msg = []
-    for m in req["messages"]:
-        if m["role"] == "system":
-            continue
-        if m["role"] == "assistant" and not msg:
-            continue
-        msg.append(m)
-    message_id = msg[-1].get("id") if msg else None
+    normalized, error = _normalize_completion_messages(req)
+    if error:
+        return error
+    request_messages, request_msg = normalized
+    pass_all_history_messages = _get_bool_request_flag(req, "pass_all_history_messages", "pass_all_history", default=False)
+    msg = request_msg
+    message_id = request_msg[-1].get("id")
+    chat_id = req.pop("chat_id", "") or ""
+    chat_id = chat_id or chat_id_in_arg
+    session_id = req.pop("session_id", "") or req.pop("conversation_id", "") or ""
     chat_model_id = req.pop("llm_id", "")
 
-    chat_model_config = {}
-    for model_config in ["temperature", "top_p", "frequency_penalty", "presence_penalty", "max_tokens"]:
-        config = req.get(model_config)
-        if config:
-            chat_model_config[model_config] = config
+    chat_model_config = pop_generation_config(req)
 
     try:
-        e, conv = ConversationService.get_by_id(session_id)
-        if not e:
-            return get_data_error_result(message="Session not found!")
-        if conv.dialog_id != chat_id:
-            return get_data_error_result(message="Session does not belong to this chat!")
-        conv.message = deepcopy(req["messages"])
-        e, dia = DialogService.get_by_id(chat_id)
-        if not e:
-            return get_data_error_result(message="Chat not found!")
-        del req["messages"]
+        conv = None
+        if session_id and not chat_id:
+            return get_data_error_result(message="`chat_id` is required when `session_id` is provided.")
 
-        if not conv.reference:
-            conv.reference = []
-        conv.reference = [r for r in conv.reference if r]
-        conv.reference.append({"chunks": [], "doc_aggs": []})
+        if chat_id:
+            if not await _ensure_owned_chat(chat_id):
+                return get_json_result(
+                    data=False,
+                    message="No authorization.",
+                    code=RetCode.AUTHENTICATION_ERROR,
+                )
+            e, dia = await thread_pool_exec(DialogService.get_by_id, chat_id)
+            if not e:
+                return get_data_error_result(message="Chat not found!")
+            if session_id:
+                e, conv = await thread_pool_exec(ConversationService.get_by_id, session_id)
+                if not e:
+                    return get_data_error_result(message="Session not found!")
+                if conv.dialog_id != chat_id:
+                    return get_data_error_result(message="Session does not belong to this chat!")
+            else:
+                conv = await _create_session_for_completion(chat_id, dia, current_user.id)
+                session_id = conv.id
+
+            if pass_all_history_messages:
+                conv.message = deepcopy(request_messages)
+                msg = request_msg
+            else:
+                if not conv.message:
+                    conv.message = []
+                conv.message.append(deepcopy(request_msg[-1]))
+                msg = []
+                for m in conv.message:
+                    if m["role"] == "system":
+                        continue
+                    if m["role"] == "assistant" and not msg:
+                        continue
+                    msg.append(m)
+        else:
+            dia = _build_default_completion_dialog()
+
+        req.pop("messages", None)
+        req.pop("question", None)
+
+        if conv is not None:
+            if not conv.reference:
+                conv.reference = []
+            conv.reference = [r for r in conv.reference if r]
+            conv.reference.append({"chunks": [], "doc_aggs": []})
 
         if chat_model_id:
-            if not TenantLLMService.get_api_key(tenant_id=dia.tenant_id, model_name=chat_model_id):
+            if not await thread_pool_exec(get_api_key, tenant_id=dia.tenant_id, model_name=chat_model_id):
                 return get_data_error_result(message=f"Cannot use specified model {chat_model_id}.")
             dia.llm_id = chat_model_id
             dia.llm_setting = chat_model_config
+        elif not dia.llm_id:
+            logging.info("empty chat_model_id in req, use default chat model.")
+            _, tenant_info = TenantService.get_by_id(dia.tenant_id)
+            if not tenant_info or not tenant_info.llm_id:
+                raise LookupError("No default chat model for tenant.")
+            dia.llm_id = tenant_info.llm_id
+            merge_generation_config(dia, chat_model_config)
 
-        is_embedded = bool(chat_model_id)
+        legacy = _get_bool_request_flag(
+            req,
+            "legacy",
+            default=False,
+        )
         stream_mode = req.pop("stream", True)
 
+        def _format_answer(ans):
+            """Wrap a raw answer dict with session and chat identifiers."""
+            formatted = structure_answer(conv, ans, message_id, session_id)
+            if chat_id:
+                formatted["chat_id"] = chat_id
+            return formatted
+
         async def stream():
+            """Yield SSE-formatted chunks from the async chat generator."""
             nonlocal dia, msg, req, conv
             try:
-                async for ans in async_chat(dia, msg, True, **req):
-                    ans = structure_answer(conv, ans, message_id, conv.id)
-                    yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
-                if not is_embedded:
-                    ConversationService.update_by_id(conv.id, conv.to_dict())
+                if legacy:
+                    # v0.23.0-style streaming: emit accumulated answer text and
+                    # reconstruct raw <think>...</think> markers from the newer
+                    # start_to_think/end_to_think events.
+                    legacy_answer = ""
+                    final_answer = None
+                    async for ans in async_chat(dia, msg, True, session_id=session_id, **req):
+                        ans = _format_answer(ans)
+                        if ans.get("final"):
+                            final_answer = ans
+                            continue
+                        if ans.get("start_to_think"):
+                            legacy_answer += "<think>"
+                            legacy_chunk = {**ans, "answer": legacy_answer}
+                            legacy_chunk.pop("start_to_think", None)
+                            legacy_chunk.pop("end_to_think", None)
+                            payload = _sanitize_json_floats({"code": 0, "message": "", "data": legacy_chunk})
+                            yield "data:" + json.dumps(payload, ensure_ascii=False) + "\n\n"
+                            continue
+                        if ans.get("end_to_think"):
+                            legacy_answer += "</think>"
+                            legacy_chunk = {**ans, "answer": legacy_answer}
+                            legacy_chunk.pop("start_to_think", None)
+                            legacy_chunk.pop("end_to_think", None)
+                            payload = _sanitize_json_floats({"code": 0, "message": "", "data": legacy_chunk})
+                            yield "data:" + json.dumps(payload, ensure_ascii=False) + "\n\n"
+                            continue
+                        delta = ans.get("answer") or ""
+                        if not delta:
+                            continue
+                        legacy_answer += delta
+                        legacy_chunk = {**ans, "answer": legacy_answer}
+                        legacy_chunk.pop("start_to_think", None)
+                        legacy_chunk.pop("end_to_think", None)
+                        payload = _sanitize_json_floats({"code": 0, "message": "", "data": legacy_chunk})
+                        yield "data:" + json.dumps(payload, ensure_ascii=False) + "\n\n"
+                    if final_answer is not None:
+                        final_chunk = {**final_answer, "answer": final_answer.get("answer") or legacy_answer}
+                        final_chunk.pop("start_to_think", None)
+                        final_chunk.pop("end_to_think", None)
+                        payload = _sanitize_json_floats({"code": 0, "message": "", "data": final_chunk})
+                        yield "data:" + json.dumps(payload, ensure_ascii=False) + "\n\n"
+                else:
+                    async for ans in async_chat(dia, msg, True, session_id=session_id, **req):
+                        ans = _format_answer(ans)
+                        payload = _sanitize_json_floats({"code": 0, "message": "", "data": ans})
+                        yield "data:" + json.dumps(payload, ensure_ascii=False) + "\n\n"
+                if conv is not None:
+                    await thread_pool_exec(ConversationService.update_by_id, conv.id, conv.to_dict())
             except Exception as ex:
                 logging.exception(ex)
                 yield "data:" + json.dumps({"code": 500, "message": str(ex), "data": {"answer": "**ERROR**: " + str(ex), "reference": []}}, ensure_ascii=False) + "\n\n"
@@ -1040,41 +1310,11 @@ async def session_completion(chat_id, session_id):
             return resp
 
         answer = None
-        async for ans in async_chat(dia, msg, **req):
-            answer = structure_answer(conv, ans, message_id, conv.id)
-            if not is_embedded:
-                ConversationService.update_by_id(conv.id, conv.to_dict())
+        async for ans in async_chat(dia, msg, False, session_id=session_id, **req):
+            answer = _format_answer(ans)
+            if conv is not None:
+                await thread_pool_exec(ConversationService.update_by_id, conv.id, conv.to_dict())
             break
-        return get_json_result(data=answer)
+        return get_json_result(data=_sanitize_json_floats(answer))
     except Exception as ex:
         return server_error_response(ex)
-
-
-@manager.route("/chats/ask", methods=["POST"])  # noqa: F821
-@login_required
-@validate_request("question", "kb_ids")
-async def ask():
-    req = await get_request_json()
-    uid = current_user.id
-
-    search_id = req.get("search_id", "")
-    search_config = {}
-    if search_id:
-        if search_app := SearchService.get_detail(search_id):
-            search_config = search_app.get("search_config", {})
-
-    async def stream():
-        nonlocal req, uid
-        try:
-            async for ans in async_ask(req["question"], req["kb_ids"], uid, search_config=search_config):
-                yield "data:" + json.dumps({"code": 0, "message": "", "data": ans}, ensure_ascii=False) + "\n\n"
-        except Exception as ex:
-            yield "data:" + json.dumps({"code": 500, "message": str(ex), "data": {"answer": "**ERROR**: " + str(ex), "reference": []}}, ensure_ascii=False) + "\n\n"
-        yield "data:" + json.dumps({"code": 0, "message": "", "data": True}, ensure_ascii=False) + "\n\n"
-
-    resp = Response(stream(), mimetype="text/event-stream")
-    resp.headers.add_header("Cache-control", "no-cache")
-    resp.headers.add_header("Connection", "keep-alive")
-    resp.headers.add_header("X-Accel-Buffering", "no")
-    resp.headers.add_header("Content-Type", "text/event-stream; charset=utf-8")
-    return resp

api/apps/restful_apis/chat_channel_api.py

@@ -0,0 +1,117 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+
+from api.apps import current_user, login_required
+from api.db.services.chat_channel_service import ChatChannelService
+from api.db.services.dialog_service import DialogService
+from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, validate_request
+from common.constants import RetCode
+from common.misc_utils import get_uuid
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _chat_channel_auth_error(channel_id: str, user_id: str):
+    """Return the chat channel authorization failure response and log the denial."""
+    LOGGER.warning("chat channel access denied: channel_id=%s user_id=%s", channel_id, user_id)
+    return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+
+
+@manager.route("/chat-channels", methods=["POST"])  # noqa: F821
+@login_required
+@validate_request("name", "channel", "config")
+async def create_chat_channel():
+    """Create a chat channel bot owned by the current tenant."""
+    req = await get_request_json()
+    channel = {
+        "id": get_uuid(),
+        "tenant_id": current_user.id,
+        "name": req["name"],
+        "channel": req["channel"],
+        "config": req["config"],
+        "chat_id": req.get("chat_id") or None
+    }
+    ChatChannelService.insert(**channel)
+
+    e, conn = ChatChannelService.get_by_id(channel["id"])
+    if not e:
+        return get_data_error_result(message="Failed to create chat channel!")
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/chat-channels", methods=["GET"])  # noqa: F821
+@login_required
+def list_chat_channel():
+    """List chat channel bots owned by the current tenant."""
+    return get_json_result(data=ChatChannelService.list(current_user.id))
+
+
+@manager.route("/chat-channels/<channel_id>", methods=["GET"])  # noqa: F821
+@login_required
+def get_chat_channel(channel_id):
+    """Return a chat channel bot's details when the current user can access it."""
+    if not ChatChannelService.accessible(channel_id, current_user.id):
+        return _chat_channel_auth_error(channel_id, current_user.id)
+
+    e, conn = ChatChannelService.get_by_id(channel_id)
+    if not e:
+        return get_data_error_result(message="Can't find this chat channel!")
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/chat-channels/<channel_id>", methods=["PATCH"])  # noqa: F821
+@login_required
+async def update_chat_channel(channel_id):
+    """Update an accessible chat channel bot's name/config/status."""
+    if not ChatChannelService.accessible(channel_id, current_user.id):
+        return _chat_channel_auth_error(channel_id, current_user.id)
+
+    e, conn = ChatChannelService.get_by_id(channel_id)
+    if not e:
+        return get_data_error_result(message="Can't find this chat channel!")
+
+    req = await get_request_json()
+    if isinstance(req, dict) and isinstance(req.get("data"), dict):
+        req = req["data"]
+
+    # Validate the connected dialog (if provided) belongs to the channel's tenant.
+    if req.get("chat_id"):
+        e, dia = DialogService.get_by_id(req["chat_id"])
+        if not e:
+            return get_data_error_result(message="Can't find this chat assistant!")
+        if dia.tenant_id != conn.tenant_id:
+            return _chat_channel_auth_error(channel_id, current_user.id)
+
+    update_fields = {fld: req[fld] for fld in ["name", "config", "chat_id"] if fld in req}
+    if update_fields:
+        ChatChannelService.update_by_id(channel_id, update_fields)
+
+    e, conn = ChatChannelService.get_by_id(channel_id)
+    if not e:
+        return get_data_error_result(message="Can't find this chat channel!")
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/chat-channels/<channel_id>", methods=["DELETE"])  # noqa: F821
+@login_required
+def rm_chat_channel(channel_id):
+    """Delete an accessible chat channel bot."""
+    if not ChatChannelService.accessible(channel_id, current_user.id):
+        return _chat_channel_auth_error(channel_id, current_user.id)
+
+    ChatChannelService.delete_by_id(channel_id)
+    return get_json_result(data=True)

api/apps/restful_apis/chunk_api.py

@@ -0,0 +1,768 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import base64
+import binascii
+import datetime
+import logging
+import re
+
+import xxhash
+from pydantic import BaseModel, Field, validator
+from quart import request
+
+from api.apps import login_required
+from api.db.joint_services.tenant_model_service import (
+    split_model_name,
+    get_model_config_from_provider_instance,
+    get_tenant_default_model_by_type,
+)
+from api.db.db_models import Document, Task
+from api.db.services.doc_metadata_service import DocMetadataService
+from api.db.services.document_service import DocumentService
+from api.db.services.file2document_service import File2DocumentService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import LLMBundle
+from api.db.services.task_service import TaskService, cancel_all_task_of, queue_tasks
+from api.db.services.tenant_llm_service import TenantLLMService
+from api.utils.api_utils import (
+    add_tenant_id_to_kwargs,
+    check_duplicate_ids,
+    construct_json_result,
+    get_error_data_result,
+    get_request_json,
+    get_result,
+    server_error_response,
+)
+from api.utils.pagination_utils import validate_rest_api_page_size
+from api.utils.image_utils import store_chunk_image
+from api.utils.reference_metadata_utils import (
+    enrich_chunks_with_document_metadata,
+    resolve_reference_metadata_preferences,
+)
+from common import settings
+from common.constants import LLMType, ParserType, RetCode, TaskStatus
+from common.metadata_utils import convert_conditions, meta_filter
+from common.misc_utils import thread_pool_exec
+from common.string_utils import is_content_empty, remove_redundant_spaces
+from common.tag_feature_utils import validate_tag_features
+from rag.app.tag import label_question
+from rag.nlp import search
+from rag.prompts.generator import cross_languages, keyword_extraction
+
+
+DOC_STOP_PARSING_INVALID_STATE_MESSAGE = "Can't stop parsing document that has not started or already completed"
+DOC_STOP_PARSING_INVALID_STATE_ERROR_CODE = "DOC_STOP_PARSING_INVALID_STATE"
+
+
+def _decode_chunk_image_base64(image_base64):
+    if not isinstance(image_base64, str) or not image_base64.strip():
+        return None, "`image_base64` must be a non-empty string"
+    try:
+        image_binary = base64.b64decode(image_base64, validate=True)
+    except (binascii.Error, ValueError):
+        return None, "Invalid `image_base64`"
+    if not image_binary:
+        return None, "`image_base64` is empty"
+    return image_binary, None
+
+
+def _store_chunk_image_or_error(dataset_id, chunk_id, image_binary):
+    try:
+        store_chunk_image(dataset_id, chunk_id, image_binary)
+    except Exception:
+        logging.exception(
+            "Failed to store chunk image. dataset_id=%s chunk_id=%s",
+            dataset_id,
+            chunk_id,
+        )
+        return "Failed to store chunk image"
+    return None
+
+
+class Chunk(BaseModel):
+    id: str = ""
+    content: str = ""
+    document_id: str = ""
+    docnm_kwd: str = ""
+    important_keywords: list = Field(default_factory=list)
+    tag_kwd: list = Field(default_factory=list)
+    questions: list = Field(default_factory=list)
+    question_tks: str = ""
+    image_id: str = ""
+    available: bool = True
+    positions: list[list[int]] = Field(default_factory=list)
+
+    @validator("positions")
+    def validate_positions(cls, value):
+        for sublist in value:
+            if len(sublist) != 5:
+                raise ValueError("Each sublist in positions must have a length of 5")
+        return value
+
+
+def _map_doc(doc):
+    key_mapping = {
+        "chunk_num": "chunk_count",
+        "kb_id": "dataset_id",
+        "token_num": "token_count",
+        "parser_id": "chunk_method",
+    }
+    run_mapping = {
+        "0": "UNSTART",
+        "1": "RUNNING",
+        "2": "CANCEL",
+        "3": "DONE",
+        "4": "FAIL",
+    }
+    renamed_doc = {}
+    for key, value in doc.to_dict().items():
+        renamed_doc[key_mapping.get(key, key)] = value
+        if key == "run":
+            renamed_doc["run"] = run_mapping.get(str(value))
+    return renamed_doc
+
+
+def _strip_chunk_runtime_fields(chunk):
+    for name in [name for name in chunk.keys() if re.search(r"(_vec$|_sm_|_tks|_ltks)", name)]:
+        del chunk[name]
+    return chunk
+
+
+def _get_dataset_tenant_id(dataset_id):
+    ok, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if not ok:
+        return None
+    return kb.tenant_id
+
+
+def _resolve_reference_metadata(req: dict, search_config: dict | None = None):
+    return resolve_reference_metadata_preferences(req, search_config)
+
+
+def _enrich_chunks_with_document_metadata(chunks: list[dict], metadata_fields=None) -> None:
+    enrich_chunks_with_document_metadata(chunks, metadata_fields)
+
+
+@manager.route("/datasets/<dataset_id>/chunks", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def parse(tenant_id, dataset_id):
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    req = await get_request_json()
+    if not req.get("document_ids"):
+        return get_error_data_result("`document_ids` is required")
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    not_found = []
+    success_count = 0
+    for id in doc_list:
+        doc = DocumentService.query(id=id, kb_id=dataset_id)
+        if not doc:
+            not_found.append(id)
+            continue
+        if not doc:
+            return get_error_data_result(message=f"You don't own the document {id}.")
+        info = {"run": "1", "progress": 0, "progress_msg": "", "chunk_num": 0, "token_num": 0}
+        if (
+            DocumentService.filter_update(
+                [
+                    Document.id == id,
+                    ((Document.run.is_null(True)) | (Document.run != TaskStatus.RUNNING.value)),
+                ],
+                info,
+            )
+            == 0
+        ):
+            return get_error_data_result("Can't parse document that is currently being processed")
+        index_name = search.index_name(tenant_id)
+        if settings.docStoreConn.index_exist(index_name, dataset_id):
+            settings.docStoreConn.delete({"doc_id": id}, index_name, dataset_id)
+        else:
+            logging.info(
+                "Skipping chunk delete during parse for doc %s: index %s/%s does not exist",
+                id,
+                index_name,
+                dataset_id,
+            )
+        TaskService.filter_delete([Task.doc_id == id])
+        e, doc = DocumentService.get_by_id(id)
+        doc = doc.to_dict()
+        doc["tenant_id"] = tenant_id
+        bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
+        queue_tasks(doc, bucket, name, 0)
+        success_count += 1
+    if not_found:
+        return get_result(message=f"Documents not found: {not_found}", code=RetCode.DATA_ERROR)
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially parsed {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+
+    return get_result()
+
+
+@manager.route("/datasets/<dataset_id>/chunks", methods=["DELETE"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def stop_parsing(tenant_id, dataset_id):
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    req = await get_request_json()
+
+    if not req.get("document_ids"):
+        return get_error_data_result("`document_ids` is required")
+    doc_list = req.get("document_ids")
+    unique_doc_ids, duplicate_messages = check_duplicate_ids(doc_list, "document")
+    doc_list = unique_doc_ids
+
+    success_count = 0
+    for id in doc_list:
+        doc = DocumentService.query(id=id, kb_id=dataset_id)
+        if not doc:
+            return get_error_data_result(message=f"You don't own the document {id}.")
+        if doc[0].run != TaskStatus.RUNNING.value:
+            return construct_json_result(
+                code=RetCode.DATA_ERROR,
+                message=DOC_STOP_PARSING_INVALID_STATE_MESSAGE,
+                data={"error_code": DOC_STOP_PARSING_INVALID_STATE_ERROR_CODE},
+            )
+        cancel_all_task_of(id)
+        info = {"run": "2", "progress": 0, "chunk_num": 0}
+        DocumentService.update_by_id(id, info)
+        index_name = search.index_name(tenant_id)
+        if settings.docStoreConn.index_exist(index_name, dataset_id):
+            settings.docStoreConn.delete({"doc_id": doc[0].id}, index_name, dataset_id)
+        else:
+            logging.info(
+                "Skipping chunk delete during stop_parsing for doc %s: index %s/%s does not exist",
+                doc[0].id,
+                index_name,
+                dataset_id,
+            )
+        success_count += 1
+    if duplicate_messages:
+        if success_count > 0:
+            return get_result(
+                message=f"Partially stopped {success_count} documents with {len(duplicate_messages)} errors",
+                data={"success_count": success_count, "errors": duplicate_messages},
+            )
+        else:
+            return get_error_data_result(message=";".join(duplicate_messages))
+    return get_result()
+
+
+@manager.route("/retrieval", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def retrieval_test(tenant_id):
+    req = await get_request_json()
+    if not req.get("dataset_ids"):
+        return get_error_data_result("`dataset_ids` is required.")
+    kb_ids = req["dataset_ids"]
+    if not isinstance(kb_ids, list):
+        return get_error_data_result("`dataset_ids` should be a list")
+    for id in kb_ids:
+        if not KnowledgebaseService.accessible(kb_id=id, user_id=tenant_id):
+            return get_error_data_result(f"You don't own the dataset {id}.")
+    kbs = KnowledgebaseService.get_by_ids(kb_ids)
+    embd_nms = list(set([split_model_name(kb.embd_id)[0] for kb in kbs]))
+    if len(embd_nms) != 1:
+        return get_result(message="Datasets use different embedding models.", code=RetCode.DATA_ERROR)
+    if "question" not in req:
+        return get_error_data_result("`question` is required.")
+    page = int(req.get("page", 1))
+    size = validate_rest_api_page_size(int(req.get("page_size", 30)))
+    question = req["question"].strip() if isinstance(req["question"], str) else req["question"]
+    if not question:
+        return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})
+    doc_ids = req.get("document_ids", [])
+    use_kg = req.get("use_kg", False)
+    toc_enhance = req.get("toc_enhance", False)
+    langs = req.get("cross_languages", [])
+    if not isinstance(doc_ids, list):
+        return get_error_data_result("`documents` should be a list")
+    if doc_ids:
+        doc_ids_list = KnowledgebaseService.list_documents_by_ids(kb_ids)
+        for doc_id in doc_ids:
+            if doc_id not in doc_ids_list:
+                return get_error_data_result(f"The datasets don't own the document {doc_id}")
+    if not doc_ids:
+        metadata_condition = req.get("metadata_condition")
+        if metadata_condition:
+            metas = DocMetadataService.get_flatted_meta_by_kbs(kb_ids)
+            doc_ids = meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and"))
+            if not doc_ids and metadata_condition.get("conditions"):
+                return get_result(data={"total": 0, "chunks": [], "doc_aggs": {}})
+            if metadata_condition and not doc_ids:
+                doc_ids = ["-999"]
+        else:
+            doc_ids = None
+    similarity_threshold = float(req.get("similarity_threshold", 0.2))
+    vector_similarity_weight = float(req.get("vector_similarity_weight", 0.3))
+    top = int(req.get("top_k", 1024))
+    if top <= 0:
+        return get_error_data_result("`top_k` must be greater than 0")
+    highlight_val = req.get("highlight", None)
+    if highlight_val is None:
+        highlight = False
+    elif isinstance(highlight_val, bool):
+        highlight = highlight_val
+    elif isinstance(highlight_val, str) and highlight_val.lower() in ["true", "false"]:
+        highlight = highlight_val.lower() == "true"
+    else:
+        return get_error_data_result("`highlight` should be a boolean")
+    include_metadata, metadata_fields = _resolve_reference_metadata(req)
+    try:
+        tenant_ids = list(set([kb.tenant_id for kb in kbs]))
+        e, kb = KnowledgebaseService.get_by_id(kb_ids[0])
+        if not e:
+            return get_error_data_result(message="Dataset not found!")
+        embd_model_config = get_model_config_from_provider_instance(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+        embd_mdl = LLMBundle(kb.tenant_id, embd_model_config)
+
+        rerank_mdl = None
+        if req.get("rerank_id"):
+            rerank_model_config = get_model_config_from_provider_instance(kb.tenant_id, LLMType.RERANK, req["rerank_id"])
+            rerank_mdl = LLMBundle(kb.tenant_id, rerank_model_config)
+
+        if langs:
+            question = await cross_languages(kb.tenant_id, None, question, langs)
+        if req.get("keyword", False):
+            chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
+            question += await keyword_extraction(LLMBundle(kb.tenant_id, chat_model_config), question)
+
+        ranks = await settings.retriever.retrieval(
+            question, embd_mdl, tenant_ids, kb_ids, page, size, similarity_threshold,
+            vector_similarity_weight, top, doc_ids, rerank_mdl=rerank_mdl,
+            highlight=highlight, rank_feature=label_question(question, kbs),
+        )
+        if toc_enhance:
+            chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
+            cks = await settings.retriever.retrieval_by_toc(question, ranks["chunks"], tenant_ids, LLMBundle(kb.tenant_id, chat_model_config), size)
+            if cks:
+                ranks["chunks"] = cks
+        ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], tenant_ids)
+        if use_kg:
+            chat_model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
+            ck = await settings.kg_retriever.retrieval(question, [k.tenant_id for k in kbs], kb_ids, embd_mdl, LLMBundle(kb.tenant_id, chat_model_config))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
+        for c in ranks["chunks"]:
+            c.pop("vector", None)
+        if include_metadata:
+            logging.info("sdk.retrieval reference_metadata enabled dataset_ids=%s fields=%s chunks=%s", kb_ids, sorted(metadata_fields) if metadata_fields else None, len(ranks["chunks"]))
+            enrich_chunks_with_document_metadata(ranks["chunks"], metadata_fields)
+
+        key_mapping = {
+            "chunk_id": "id",
+            "content_with_weight": "content",
+            "doc_id": "document_id",
+            "important_kwd": "important_keywords",
+            "question_kwd": "questions",
+            "docnm_kwd": "document_keyword",
+            "kb_id": "dataset_id",
+        }
+        ranks["chunks"] = [{key_mapping.get(key, key): value for key, value in chunk.items()} for chunk in ranks["chunks"]]
+        return get_result(data=ranks)
+    except Exception as e:
+        if "not_found" in str(e):
+            return get_result(message="No chunk found! Check the chunk status please!", code=RetCode.DATA_ERROR)
+        return server_error_response(e)
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def list_chunks(tenant_id, dataset_id, document_id):
+    from rag.nlp import search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    doc = DocumentService.query(id=document_id, kb_id=dataset_id)
+    if not doc:
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
+    doc = doc[0]
+    req = request.args
+    page = int(req.get("page", 1))
+    size = validate_rest_api_page_size(int(req.get("page_size", 30)))
+    question = req.get("keywords", "")
+    query = {
+        "doc_ids": [document_id],
+        "page": page,
+        "size": size,
+        "question": question,
+        "sort": True,
+    }
+    if "available" in req:
+        query["available_int"] = 1 if req["available"] == "true" else 0
+
+    res = {"total": 0, "chunks": [], "doc": _map_doc(doc)}
+    if req.get("id"):
+        chunk = settings.docStoreConn.get(req.get("id"), search.index_name(dataset_tenant_id), [dataset_id])
+        if not chunk:
+            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=RetCode.DATA_ERROR)
+        if str(chunk.get("doc_id", chunk.get("document_id"))) != str(document_id):
+            return get_result(message=f"Chunk not found: {dataset_id}/{req.get('id')}", code=RetCode.DATA_ERROR)
+        _strip_chunk_runtime_fields(chunk)
+        res["total"] = 1
+        final_chunk = {
+            "id": chunk.get("id", chunk.get("chunk_id")),
+            "content": chunk["content_with_weight"],
+            "document_id": chunk.get("doc_id", chunk.get("document_id")),
+            "docnm_kwd": chunk["docnm_kwd"],
+            "important_keywords": chunk.get("important_kwd", []),
+            "questions": chunk.get("question_kwd", []),
+            "dataset_id": chunk.get("kb_id", chunk.get("dataset_id")),
+            "image_id": chunk.get("img_id", ""),
+            "available": bool(chunk.get("available_int", 1)),
+            "positions": chunk.get("position_int", []),
+            "tag_kwd": chunk.get("tag_kwd", []),
+            "tag_feas": chunk.get("tag_feas", {}),
+        }
+        res["chunks"].append(final_chunk)
+        _ = Chunk(**final_chunk)
+    elif settings.docStoreConn.index_exist(search.index_name(dataset_tenant_id), dataset_id):
+        sres = await settings.retriever.search(
+            query,
+            search.index_name(dataset_tenant_id),
+            [dataset_id],
+            emb_mdl=None,
+            highlight=True,
+        )
+        res["total"] = sres.total
+        for chunk_id in sres.ids:
+            d = {
+                "id": chunk_id,
+                "content": (
+                    remove_redundant_spaces(sres.highlight[chunk_id])
+                    if question and chunk_id in sres.highlight
+                    else sres.field[chunk_id].get("content_with_weight", "")
+                ),
+                "document_id": sres.field[chunk_id]["doc_id"],
+                "docnm_kwd": sres.field[chunk_id]["docnm_kwd"],
+                "important_keywords": sres.field[chunk_id].get("important_kwd", []),
+                "tag_kwd": sres.field[chunk_id].get("tag_kwd", []),
+                "questions": sres.field[chunk_id].get("question_kwd", []),
+                "dataset_id": sres.field[chunk_id].get("kb_id", sres.field[chunk_id].get("dataset_id")),
+                "image_id": sres.field[chunk_id].get("img_id", ""),
+                "available": bool(int(sres.field[chunk_id].get("available_int", "1"))),
+                "positions": sres.field[chunk_id].get("position_int", []),
+            }
+            res["chunks"].append(d)
+            _ = Chunk(**d)
+    return get_result(data=res)
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def get_chunk(tenant_id, dataset_id, document_id, chunk_id):
+    from rag.nlp import search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    doc = DocumentService.query(id=document_id, kb_id=dataset_id)
+    if not doc:
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
+    try:
+        chunk = settings.docStoreConn.get(chunk_id, search.index_name(dataset_tenant_id), [dataset_id])
+        if chunk is None or str(chunk.get("doc_id", chunk.get("document_id"))) != str(document_id):
+            return get_result(data=False, message="Chunk not found!", code=RetCode.DATA_ERROR)
+        return get_result(data=_strip_chunk_runtime_fields(chunk))
+    except Exception as e:
+        if str(e).find("NotFoundError") >= 0:
+            return get_result(data=False, message="Chunk not found!", code=RetCode.DATA_ERROR)
+        return server_error_response(e)
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def add_chunk(tenant_id, dataset_id, document_id):
+    from rag.nlp import rag_tokenizer, search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    doc = DocumentService.query(id=document_id, kb_id=dataset_id)
+    if not doc:
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
+    doc = doc[0]
+    req = await get_request_json()
+    if is_content_empty(req.get("content")):
+        return get_error_data_result(message="`content` is required")
+    if "important_keywords" in req and not isinstance(req["important_keywords"], list):
+        return get_error_data_result("`important_keywords` is required to be a list")
+    if "questions" in req and not isinstance(req["questions"], list):
+        return get_error_data_result("`questions` is required to be a list")
+
+    chunk_id = xxhash.xxh64((req["content"] + document_id).encode("utf-8")).hexdigest()
+    d = {
+        "id": chunk_id,
+        "content_ltks": rag_tokenizer.tokenize(req["content"]),
+        "content_with_weight": req["content"],
+    }
+    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
+    d["important_kwd"] = req.get("important_keywords", [])
+    d["important_tks"] = rag_tokenizer.tokenize(" ".join(req.get("important_keywords", [])))
+    d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
+    d["question_tks"] = rag_tokenizer.tokenize("\n".join(req.get("questions", [])))
+    d["create_time"] = str(datetime.datetime.now()).replace("T", " ")[:19]
+    d["create_timestamp_flt"] = datetime.datetime.now().timestamp()
+    d["kb_id"] = dataset_id
+    d["docnm_kwd"] = doc.name
+    d["doc_id"] = document_id
+
+    if "tag_kwd" in req:
+        if not isinstance(req["tag_kwd"], list):
+            return get_error_data_result("`tag_kwd` is required to be a list")
+        if not all(isinstance(t, str) for t in req["tag_kwd"]):
+            return get_error_data_result("`tag_kwd` must be a list of strings")
+        d["tag_kwd"] = req["tag_kwd"]
+    if "tag_feas" in req:
+        try:
+            d["tag_feas"] = validate_tag_features(req["tag_feas"])
+        except ValueError as exc:
+            return get_error_data_result(f"`tag_feas` {exc}")
+
+    if "image_base64" in req:
+        image_binary, image_err = _decode_chunk_image_base64(req.get("image_base64"))
+        if image_err:
+            return get_error_data_result(message=image_err)
+        store_err = _store_chunk_image_or_error(dataset_id, chunk_id, image_binary)
+        if store_err:
+            return get_error_data_result(message=store_err)
+        d["img_id"] = f"{dataset_id}-{chunk_id}"
+        d["doc_type_kwd"] = "image"
+
+    embd_id = DocumentService.get_embd_id(document_id)
+    model_config = get_model_config_from_provider_instance(dataset_tenant_id, LLMType.EMBEDDING.value, embd_id)
+    embd_mdl = TenantLLMService.model_instance(model_config)
+    v, c = embd_mdl.encode([doc.name, req["content"] if not d["question_kwd"] else "\n".join(d["question_kwd"])])
+    v = 0.1 * v[0] + 0.9 * v[1]
+    d[f"q_{len(v)}_vec"] = v.tolist()
+    settings.docStoreConn.insert([d], search.index_name(dataset_tenant_id), dataset_id)
+
+    DocumentService.increment_chunk_num(doc.id, doc.kb_id, c, 1, 0)
+    key_mapping = {
+        "id": "id",
+        "content_with_weight": "content",
+        "doc_id": "document_id",
+        "important_kwd": "important_keywords",
+        "tag_kwd": "tag_kwd",
+        "question_kwd": "questions",
+        "kb_id": "dataset_id",
+        "create_timestamp_flt": "create_timestamp",
+        "create_time": "create_time",
+        "document_keyword": "document",
+        "img_id": "image_id",
+    }
+    renamed_chunk = {new_key: d[key] for key, new_key in key_mapping.items() if key in d}
+    _ = Chunk(**renamed_chunk)
+    return get_result(data={"chunk": renamed_chunk})
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["DELETE"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def rm_chunk(tenant_id, dataset_id, document_id):
+    from rag.nlp import search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    docs = DocumentService.query(id=document_id, kb_id=dataset_id)
+    if not docs:
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
+    req = await get_request_json()
+    if not req:
+        return get_result()
+
+    chunk_ids = req.get("chunk_ids")
+    if not chunk_ids:
+        if req.get("delete_all") is True:
+            doc = docs[0]
+            DocumentService.delete_chunk_images(doc, dataset_tenant_id)
+            chunk_number = settings.docStoreConn.delete({"doc_id": document_id}, search.index_name(dataset_tenant_id), dataset_id)
+            if chunk_number != 0:
+                DocumentService.decrement_chunk_num(document_id, dataset_id, 1, chunk_number, 0)
+            return get_result(message=f"deleted {chunk_number} chunks")
+        return get_result()
+
+    unique_chunk_ids, duplicate_messages = check_duplicate_ids(chunk_ids, "chunk")
+    chunk_number = settings.docStoreConn.delete(
+        {"doc_id": document_id, "id": unique_chunk_ids},
+        search.index_name(dataset_tenant_id),
+        dataset_id,
+    )
+    if chunk_number != 0:
+        DocumentService.decrement_chunk_num(document_id, dataset_id, 1, chunk_number, 0)
+    if chunk_number != len(unique_chunk_ids):
+        if len(unique_chunk_ids) == 0:
+            return get_result(message=f"deleted {chunk_number} chunks")
+        return get_error_data_result(message=f"rm_chunk deleted chunks {chunk_number}, expect {len(unique_chunk_ids)}")
+    if duplicate_messages:
+        return get_result(
+            message=f"Partially deleted {chunk_number} chunks with {len(duplicate_messages)} errors",
+            data={"success_count": chunk_number, "errors": duplicate_messages},
+        )
+    return get_result(message=f"deleted {chunk_number} chunks")
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks/<chunk_id>", methods=["PATCH"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def update_chunk(tenant_id, dataset_id, document_id, chunk_id):
+    from rag.app.qa import beAdoc, rmPrefix
+    from rag.nlp import rag_tokenizer, search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    doc = DocumentService.query(id=document_id, kb_id=dataset_id)
+    if not doc:
+        return get_error_data_result(message=f"You don't own the document {document_id}.")
+    doc = doc[0]
+    chunk = settings.docStoreConn.get(chunk_id, search.index_name(dataset_tenant_id), [dataset_id])
+    if chunk is None or str(chunk.get("doc_id", chunk.get("document_id"))) != str(document_id):
+        return get_error_data_result(f"Can't find this chunk {chunk_id}")
+    req = await get_request_json()
+    content = req.get("content")
+    if content is not None:
+        if is_content_empty(content):
+            return get_error_data_result(message="`content` is required")
+    else:
+        content = chunk.get("content_with_weight", "")
+    d = {"id": chunk_id, "content_with_weight": content}
+    d["content_ltks"] = rag_tokenizer.tokenize(d["content_with_weight"])
+    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])
+    if "important_keywords" in req:
+        if not isinstance(req["important_keywords"], list):
+            return get_error_data_result("`important_keywords` should be a list")
+        d["important_kwd"] = req.get("important_keywords", [])
+        d["important_tks"] = rag_tokenizer.tokenize(" ".join(req["important_keywords"]))
+    if "questions" in req:
+        if not isinstance(req["questions"], list):
+            return get_error_data_result("`questions` should be a list")
+        d["question_kwd"] = [str(q).strip() for q in req.get("questions", []) if str(q).strip()]
+        d["question_tks"] = rag_tokenizer.tokenize("\n".join(req["questions"]))
+    if "available" in req:
+        d["available_int"] = int(req["available"])
+    if "positions" in req:
+        if not isinstance(req["positions"], list):
+            return get_error_data_result("`positions` should be a list")
+        d["position_int"] = req["positions"]
+    if "tag_kwd" in req:
+        if not isinstance(req["tag_kwd"], list):
+            return get_error_data_result("`tag_kwd` should be a list")
+        if not all(isinstance(t, str) for t in req["tag_kwd"]):
+            return get_error_data_result("`tag_kwd` must be a list of strings")
+        d["tag_kwd"] = req["tag_kwd"]
+    if "tag_feas" in req:
+        try:
+            d["tag_feas"] = validate_tag_features(req["tag_feas"])
+        except ValueError as exc:
+            return get_error_data_result(f"`tag_feas` {exc}")
+    if "image_base64" in req:
+        image_binary, image_err = _decode_chunk_image_base64(req.get("image_base64"))
+        if image_err:
+            return get_error_data_result(message=image_err)
+        store_err = _store_chunk_image_or_error(dataset_id, chunk_id, image_binary)
+        if store_err:
+            return get_error_data_result(message=store_err)
+        d["img_id"] = f"{dataset_id}-{chunk_id}"
+        d["doc_type_kwd"] = "image"
+
+    embd_id = DocumentService.get_embd_id(document_id)
+    model_config = get_model_config_from_provider_instance(dataset_tenant_id, LLMType.EMBEDDING.value, embd_id)
+    embd_mdl = TenantLLMService.model_instance(model_config)
+    if doc.parser_id == ParserType.QA:
+        arr = [t for t in re.split(r"[\n\t]", d["content_with_weight"]) if len(t) > 1]
+        if len(arr) != 2:
+            return get_error_data_result(message="Q&A must be separated by TAB/ENTER key.")
+        q, a = rmPrefix(arr[0]), rmPrefix(arr[1])
+        d = beAdoc(d, arr[0], arr[1], not any([rag_tokenizer.is_chinese(t) for t in q + a]))
+
+    v, _ = embd_mdl.encode(
+        [
+            doc.name,
+            d["content_with_weight"] if not d.get("question_kwd") else "\n".join(d["question_kwd"]),
+        ]
+    )
+    v = 0.1 * v[0] + 0.9 * v[1] if doc.parser_id != ParserType.QA else v[1]
+    d[f"q_{len(v)}_vec"] = v.tolist()
+    settings.docStoreConn.update({"id": chunk_id}, d, search.index_name(dataset_tenant_id), dataset_id)
+    return get_result()
+
+
+@manager.route("/datasets/<dataset_id>/documents/<document_id>/chunks", methods=["PATCH"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def switch_chunks(tenant_id, dataset_id, document_id):
+    from rag.nlp import search
+
+    if not KnowledgebaseService.accessible(kb_id=dataset_id, user_id=tenant_id):
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    dataset_tenant_id = _get_dataset_tenant_id(dataset_id)
+    if not dataset_tenant_id:
+        return get_error_data_result(message=f"You don't own the dataset {dataset_id}.")
+    req = await get_request_json()
+    if not req.get("chunk_ids"):
+        return get_error_data_result(message="`chunk_ids` is required.")
+    if "available_int" not in req and "available" not in req:
+        return get_error_data_result(message="`available_int` or `available` is required.")
+    available_int = int(req["available_int"]) if "available_int" in req else (1 if req.get("available") else 0)
+
+    try:
+        def _switch_sync():
+            e, doc = DocumentService.get_by_id(document_id)
+            if not e:
+                return get_error_data_result(message="Document not found!")
+            if not doc or str(doc.kb_id) != str(dataset_id):
+                return get_error_data_result(message="Document not found!")
+            for cid in req["chunk_ids"]:
+                if not settings.docStoreConn.update(
+                    {"id": cid},
+                    {"available_int": available_int},
+                    search.index_name(dataset_tenant_id),
+                    doc.kb_id,
+                ):
+                    return get_error_data_result(message="Index updating failure")
+            return get_result(data=True)
+
+        return await thread_pool_exec(_switch_sync)
+    except Exception as e:
+        return server_error_response(e)

api/apps/restful_apis/connector_api.py

@@ -27,6 +27,7 @@ from google_auth_oauthlib.flow import Flow
 from api.db import InputType
 from api.db.services.connector_service import ConnectorService, SyncLogsService
 from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, validate_request
+from api.utils.pagination_utils import validate_rest_api_page_size
 from common.constants import RetCode, TaskStatus
 from common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, GMAIL_WEB_OAUTH_REDIRECT_URI, BOX_WEB_OAUTH_REDIRECT_URI, DocumentSource
 from common.data_source.google_util.constant import WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPES
@@ -36,14 +37,61 @@ from api.apps import login_required, current_user
 from box_sdk_gen import BoxOAuth, OAuthConfig, GetAuthorizeUrlOptions
 
 
-@manager.route("/set", methods=["POST"])  # noqa: F821
+LOGGER = logging.getLogger(__name__)
+
+
+def _connector_auth_error(connector_id: str, user_id: str):
+    """Return the connector authorization failure response and log the denial."""
+    LOGGER.warning("connector access denied: connector_id=%s user_id=%s", connector_id, user_id)
+    return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+
+
+@manager.route("/connectors/<connector_id>", methods=["PATCH"])  # noqa: F821
 @login_required
-async def set_connector():
+async def update_connector(connector_id):
+    """Update an accessible connector's polling configuration."""
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
     req = await get_request_json()
-    if req.get("id"):
-        conn = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
-        ConnectorService.update_by_id(req["id"], conn)
-    else:
+    if isinstance(req, dict) and isinstance(req.get("data"), dict):
+        req = req["data"]
+
+    e, conn = ConnectorService.get_by_id(connector_id)
+    if not e:
+        return get_data_error_result(message="Can't find this Connector!")
+
+    should_sleep = False
+    if req:
+        update_fields = {fld: req[fld] for fld in ["prune_freq", "refresh_freq", "config", "timeout_secs"] if fld in req}
+        if update_fields:
+            update_fields["id"] = connector_id
+            ConnectorService.update_by_id(connector_id, update_fields)
+            should_sleep = True
+
+        if req.get("reschedule"):
+            ConnectorService.cancel_tasks(connector_id)
+            ConnectorService.schedule_tasks(connector_id)
+        elif req.get("status") in [TaskStatus.CANCEL, "CANCEL"]:
+            ConnectorService.cancel_tasks(connector_id)
+        elif req.get("status") in [TaskStatus.SCHEDULE, "SCHEDULE"]:
+            ConnectorService.schedule_tasks(connector_id)
+
+    if should_sleep:
+        await asyncio.sleep(1)
+    e, conn = ConnectorService.get_by_id(connector_id)
+    if not e:
+        return get_data_error_result(message="Can't find this Connector!")
+
+    return get_json_result(data=conn.to_dict())
+
+
+@manager.route("/connectors", methods=["POST"])  # noqa: F821
+@login_required
+async def create_connector():
+    """Create a connector owned by the current tenant."""
+    req = await get_request_json()
+    if req:
         req["id"] = get_uuid()
         conn = {
             "id": req["id"],
@@ -53,9 +101,9 @@ async def set_connector():
             "input_type": InputType.POLL,
             "config": req["config"],
             "refresh_freq": int(req.get("refresh_freq", 5)),
-            "prune_freq": int(req.get("prune_freq", 720)),
+            "prune_freq": int(req.get("prune_freq", 5)),
             "timeout_secs": int(req.get("timeout_secs", 60 * 29)),
-            "status": TaskStatus.SCHEDULE,
+            "status": TaskStatus.UNSTART,
         }
         ConnectorService.save(**conn)
 
@@ -65,59 +113,122 @@ async def set_connector():
     return get_json_result(data=conn.to_dict())
 
 
-@manager.route("/list", methods=["GET"])  # noqa: F821
+@manager.route("/connectors", methods=["GET"])  # noqa: F821
 @login_required
 def list_connector():
+    """List connectors owned by the current tenant."""
     return get_json_result(data=ConnectorService.list(current_user.id))
 
 
-@manager.route("/<connector_id>", methods=["GET"])  # noqa: F821
+@manager.route("/connectors/<connector_id>", methods=["GET"])  # noqa: F821
 @login_required
 def get_connector(connector_id):
+    """Return connector details when the current user can access it."""
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
     e, conn = ConnectorService.get_by_id(connector_id)
     if not e:
         return get_data_error_result(message="Can't find this Connector!")
     return get_json_result(data=conn.to_dict())
 
 
-@manager.route("/<connector_id>/logs", methods=["GET"])  # noqa: F821
+@manager.route("/connectors/<connector_id>/logs", methods=["GET"])  # noqa: F821
 @login_required
 def list_logs(connector_id):
+    """List sync logs for a connector the current user can access."""
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
     req = request.args.to_dict(flat=True)
-    arr, total = SyncLogsService.list_sync_tasks(connector_id, int(req.get("page", 1)), int(req.get("page_size", 15)))
+    arr, total = SyncLogsService.list_sync_tasks(
+        connector_id,
+        int(req.get("page", 1)),
+        validate_rest_api_page_size(int(req.get("page_size", 15))),
+    )
     return get_json_result(data={"total": total, "logs": arr})
 
 
-@manager.route("/<connector_id>/resume", methods=["PUT"])  # noqa: F821
+@manager.route("/connectors/<connector_id>/rebuild", methods=["POST"])  # noqa: F821
 @login_required
-async def resume(connector_id):
-    req = await get_request_json()
-    if req.get("resume"):
-        ConnectorService.resume(connector_id, TaskStatus.SCHEDULE)
-    else:
-        ConnectorService.resume(connector_id, TaskStatus.CANCEL)
-    return get_json_result(data=True)
-
-
-@manager.route("/<connector_id>/rebuild", methods=["PUT"])  # noqa: F821
-@login_required
-@validate_request("kb_id")
 async def rebuild(connector_id):
+    """Schedule a rebuild for an accessible connector and knowledge base."""
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
     req = await get_request_json()
+    if "kb_id" not in req:
+        return get_json_result(code=RetCode.ARGUMENT_ERROR, message="required argument is missing: kb_id")
+
     err = ConnectorService.rebuild(req["kb_id"], connector_id, current_user.id)
     if err:
         return get_json_result(data=False, message=err, code=RetCode.SERVER_ERROR)
     return get_json_result(data=True)
 
 
-@manager.route("/<connector_id>/rm", methods=["POST"])  # noqa: F821
+@manager.route("/connectors/<connector_id>", methods=["DELETE"])  # noqa: F821
 @login_required
 def rm_connector(connector_id):
-    ConnectorService.resume(connector_id, TaskStatus.CANCEL)
+    """Delete an accessible connector after canceling its sync tasks."""
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
+    ConnectorService.cancel_tasks(connector_id)
     ConnectorService.delete_by_id(connector_id)
     return get_json_result(data=True)
 
 
+@manager.route("/connectors/<connector_id>/test", methods=["POST"])  # noqa: F821
+@login_required
+async def test_connector(connector_id):
+    """Validate connector configuration without persisting changes or triggering sync.
+
+    For the REST API connector, this uses `RestAPIConnector.validate_config`
+    against the existing saved configuration.
+    """
+    if not ConnectorService.accessible(connector_id, current_user.id):
+        return _connector_auth_error(connector_id, current_user.id)
+
+    from common.data_source.rest_api_connector import RestAPIConnector
+    from common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationError
+
+    ok, conn = ConnectorService.get_by_id(connector_id)
+    if not ok:
+        return get_data_error_result(message="Can't find this Connector!")
+
+    if conn.source != DocumentSource.REST_API:
+        return get_json_result(
+            code=RetCode.ARGUMENT_ERROR,
+            message="Test endpoint currently supports only REST API connectors.",
+            data=False,
+        )
+
+    config = conn.config or {}
+    credentials = config.get("credentials") or {}
+
+    try:
+        await asyncio.to_thread(
+            RestAPIConnector.validate_config,
+            config=config,
+            credentials=credentials,
+        )
+    except (ConnectorValidationError, ConnectorMissingCredentialError) as exc:
+        return get_json_result(
+            code=RetCode.DATA_ERROR,
+            message=str(exc),
+            data=False,
+        )
+    except Exception as exc:
+        logging.exception("REST API connector validation failed: %s", exc)
+        return get_json_result(
+            code=RetCode.SERVER_ERROR,
+            message="REST API connector validation failed, please check logs.",
+            data=False,
+        )
+
+    return get_json_result(data=True)
+
+
 WEB_FLOW_TTL_SECS = 15 * 60
 
 
@@ -157,6 +268,22 @@ def _get_web_client_config(credentials: dict[str, Any]) -> dict[str, Any]:
     return {"web": web_section}
 
 
+def _exchange_google_web_oauth_code(
+    client_config: dict[str, Any],
+    scopes: list[str],
+    redirect_uri: str,
+    code: str,
+    code_verifier: str | None,
+) -> Flow:
+    flow = Flow.from_client_config(client_config, scopes=scopes)
+    flow.redirect_uri = redirect_uri
+    fetch_token_kwargs: dict[str, Any] = {"code": code}
+    if code_verifier:
+        fetch_token_kwargs["code_verifier"] = code_verifier
+    flow.fetch_token(**fetch_token_kwargs)
+    return flow
+
+
 async def _render_web_oauth_popup(flow_id: str, success: bool, message: str, source="drive"):
     status = "success" if success else "error"
     auto_close = "window.close();" if success else ""
@@ -185,7 +312,7 @@ async def _render_web_oauth_popup(flow_id: str, success: bool, message: str, sou
     return response
 
 
-@manager.route("/google/oauth/web/start", methods=["POST"])  # noqa: F821
+@manager.route("/connectors/google/oauth/web/start", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("credentials")
 async def start_google_web_oauth():
@@ -252,6 +379,7 @@ async def start_google_web_oauth():
         "user_id": current_user.id,
         "client_config": client_config,
         "redirect_uri": redirect_uri,
+        "code_verifier": flow.code_verifier,
         "created_at": int(time.time()),
     }
     REDIS_CONN.set_obj(_web_state_cache_key(flow_id, source), cache_payload, WEB_FLOW_TTL_SECS)
@@ -265,7 +393,7 @@ async def start_google_web_oauth():
     )
 
 
-@manager.route("/gmail/oauth/web/callback", methods=["GET"])  # noqa: F821
+@manager.route("/connectors/gmail/oauth/web/callback", methods=["GET"])  # noqa: F821
 async def google_gmail_web_oauth_callback():
     state_id = request.args.get("state")
     error = request.args.get("error")
@@ -283,6 +411,7 @@ async def google_gmail_web_oauth_callback():
     state_obj = json.loads(state_cache)
     client_config = state_obj.get("client_config")
     redirect_uri = state_obj.get("redirect_uri", GMAIL_WEB_OAUTH_REDIRECT_URI)
+    code_verifier = state_obj.get("code_verifier")
     if not client_config:
         REDIS_CONN.delete(_web_state_cache_key(state_id, source))
         return await _render_web_oauth_popup(state_id, False, "Authorization session was invalid. Please retry.", source)
@@ -296,10 +425,13 @@ async def google_gmail_web_oauth_callback():
         return await _render_web_oauth_popup(state_id, False, "Missing authorization code from Google.", source)
 
     try:
-        # TODO(google-oauth): branch scopes/redirect_uri based on source_type (drive vs gmail)
-        flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GMAIL])
-        flow.redirect_uri = redirect_uri
-        flow.fetch_token(code=code)
+        flow = _exchange_google_web_oauth_code(
+            client_config=client_config,
+            scopes=GOOGLE_SCOPES[DocumentSource.GMAIL],
+            redirect_uri=redirect_uri,
+            code=code,
+            code_verifier=code_verifier,
+        )
     except Exception as exc:  # pragma: no cover - defensive
         logging.exception("Failed to exchange Google OAuth code: %s", exc)
         REDIS_CONN.delete(_web_state_cache_key(state_id, source))
@@ -316,7 +448,7 @@ async def google_gmail_web_oauth_callback():
     return await _render_web_oauth_popup(state_id, True, "Authorization completed successfully.", source)
 
 
-@manager.route("/google-drive/oauth/web/callback", methods=["GET"])  # noqa: F821
+@manager.route("/connectors/google-drive/oauth/web/callback", methods=["GET"])  # noqa: F821
 async def google_drive_web_oauth_callback():
     state_id = request.args.get("state")
     error = request.args.get("error")
@@ -334,6 +466,7 @@ async def google_drive_web_oauth_callback():
     state_obj = json.loads(state_cache)
     client_config = state_obj.get("client_config")
     redirect_uri = state_obj.get("redirect_uri", GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI)
+    code_verifier = state_obj.get("code_verifier")
     if not client_config:
         REDIS_CONN.delete(_web_state_cache_key(state_id, source))
         return await _render_web_oauth_popup(state_id, False, "Authorization session was invalid. Please retry.", source)
@@ -347,10 +480,13 @@ async def google_drive_web_oauth_callback():
         return await _render_web_oauth_popup(state_id, False, "Missing authorization code from Google.", source)
 
     try:
-        # TODO(google-oauth): branch scopes/redirect_uri based on source_type (drive vs gmail)
-        flow = Flow.from_client_config(client_config, scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE])
-        flow.redirect_uri = redirect_uri
-        flow.fetch_token(code=code)
+        flow = _exchange_google_web_oauth_code(
+            client_config=client_config,
+            scopes=GOOGLE_SCOPES[DocumentSource.GOOGLE_DRIVE],
+            redirect_uri=redirect_uri,
+            code=code,
+            code_verifier=code_verifier,
+        )
     except Exception as exc:  # pragma: no cover - defensive
         logging.exception("Failed to exchange Google OAuth code: %s", exc)
         REDIS_CONN.delete(_web_state_cache_key(state_id, source))
@@ -366,7 +502,7 @@ async def google_drive_web_oauth_callback():
 
     return await _render_web_oauth_popup(state_id, True, "Authorization completed successfully.", source)
 
-@manager.route("/google/oauth/web/result", methods=["POST"])  # noqa: F821
+@manager.route("/connectors/google/oauth/web/result", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("flow_id")
 async def poll_google_web_result():
@@ -386,7 +522,7 @@ async def poll_google_web_result():
     REDIS_CONN.delete(_web_result_cache_key(flow_id, source))
     return get_json_result(data={"credentials": result.get("credentials")})
 
-@manager.route("/box/oauth/web/start", methods=["POST"])  # noqa: F821
+@manager.route("/connectors/box/oauth/web/start", methods=["POST"])  # noqa: F821
 @login_required
 async def start_box_web_oauth():
     req = await get_request_json()
@@ -429,7 +565,7 @@ async def start_box_web_oauth():
             "expires_in": WEB_FLOW_TTL_SECS,}
     )
 
-@manager.route("/box/oauth/web/callback", methods=["GET"])  # noqa: F821
+@manager.route("/connectors/box/oauth/web/callback", methods=["GET"])  # noqa: F821
 async def box_web_oauth_callback():
     flow_id = request.args.get("state")
     if not flow_id:
@@ -471,7 +607,7 @@ async def box_web_oauth_callback():
 
     return await _render_web_oauth_popup(flow_id, True, "Authorization completed successfully.", "box")
 
-@manager.route("/box/oauth/web/result", methods=["POST"])  # noqa: F821
+@manager.route("/connectors/box/oauth/web/result", methods=["POST"])  # noqa: F821
 @login_required
 @validate_request("flow_id")
 async def poll_box_web_result():

api/apps/restful_apis/dataset_api.py

@@ -19,11 +19,14 @@ from peewee import OperationalError
 from quart import request
 from common.constants import RetCode
 from api.apps import login_required, current_user
-from api.utils.api_utils import get_error_argument_result, get_error_data_result, get_result, add_tenant_id_to_kwargs
+from api.utils.api_utils import get_error_argument_result, get_error_data_result, get_json_result, get_result, add_tenant_id_to_kwargs
+from api.utils.pagination_utils import validate_rest_api_page_size
 from api.utils.validation_utils import (
     CreateDatasetReq,
     DeleteDatasetReq,
     ListDatasetReq,
+    SearchDatasetReq,
+    SearchDatasetsReq,
     UpdateDatasetReq,
     validate_and_parse_json_request,
     validate_and_parse_request_args,
@@ -31,10 +34,54 @@ from api.utils.validation_utils import (
 from api.apps.services import dataset_api_service
 
 
+@manager.route("/datasets/tags/aggregation", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def aggregate_tags(tenant_id):
+    dataset_ids = request.args.get("dataset_ids", "").split(",")
+    dataset_ids = [d for d in dataset_ids if d]
+    if not dataset_ids:
+        return get_error_data_result(message="Lack of dataset_ids in query parameters")
+
+    try:
+        success, result = dataset_api_service.aggregate_tags(dataset_ids, tenant_id)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/metadata/flattened", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def get_flattened_metadata(tenant_id):
+    dataset_ids = request.args.get("dataset_ids", "").split(",")
+    dataset_ids = [d for d in dataset_ids if d]
+    if not dataset_ids:
+        return get_error_data_result(message="Lack of dataset_ids in query parameters")
+
+    try:
+        success, result = dataset_api_service.get_flattened_metadata(dataset_ids, tenant_id)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
 @manager.route("/datasets", methods=["POST"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-async def create(tenant_id: str=None):
+async def create(tenant_id: str = None):
     """
     Create a new dataset.
     ---
@@ -102,6 +149,10 @@ async def create(tenant_id: str=None):
             return get_result(data=result)
         else:
             return get_error_data_result(message=result)
+    except LookupError as e:
+        return get_error_argument_result(str(e))
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
@@ -330,50 +381,252 @@ def list_datasets(tenant_id):
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route('/datasets/<dataset_id>/knowledge_graph', methods=['GET'])  # noqa: F821
+@manager.route("/datasets/<dataset_id>", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-async def knowledge_graph(tenant_id, dataset_id):
+def get_dataset(tenant_id, dataset_id):
+    try:
+        success, result = dataset_api_service.get_dataset(dataset_id, tenant_id)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/ingestions/summary", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def get_ingestion_summary(tenant_id, dataset_id):
+    try:
+        success, result = dataset_api_service.get_ingestion_summary(dataset_id, tenant_id)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/tags", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def list_tags(tenant_id, dataset_id):
+    try:
+        success, result = dataset_api_service.list_tags(dataset_id, tenant_id)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/tags", methods=["DELETE"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def delete_tags(tenant_id, dataset_id):
+    req = await request.get_json()
+    if not req or "tags" not in req:
+        return get_error_data_result(message="Lack of tags in request body")
+    if not isinstance(req["tags"], list) or not all(isinstance(t, str) for t in req["tags"]):
+        return get_error_argument_result("tags must be a list of strings")
+
+    try:
+        success, result = dataset_api_service.delete_tags(dataset_id, tenant_id, req["tags"])
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/tags", methods=["PUT"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def rename_tag(tenant_id, dataset_id):
+    req = await request.get_json()
+    if not req or "from_tag" not in req or "to_tag" not in req:
+        return get_error_data_result(message="Lack of from_tag or to_tag in request body")
+    if not isinstance(req["from_tag"], str) or not isinstance(req["to_tag"], str):
+        return get_error_argument_result("from_tag and to_tag must be strings")
+
+    if not req["from_tag"].strip() or not req["to_tag"].strip():
+        return get_error_argument_result("from_tag and to_tag must not be empty")
+
+    try:
+        success, result = dataset_api_service.rename_tag(dataset_id, tenant_id, req["from_tag"], req["to_tag"])
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/search", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def search_datasets(tenant_id):
+    """Search (retrieval test) across multiple datasets.
+
+    POST /api/v1/datasets/search
+    JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,
+               "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
+               "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}
+    Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
+    Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
+    """
+    req, err = await validate_and_parse_json_request(request, SearchDatasetsReq)
+    if err is not None:
+        return get_error_argument_result(err)
+    success, result = await dataset_api_service.search_datasets(tenant_id, req)
+    if success:
+        return get_result(data=result)
+    else:
+        return get_error_data_result(message=result)
+
+
+@manager.route("/datasets/<dataset_id>/search", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def search(tenant_id, dataset_id):
+    """Search (retrieval test) within a dataset.
+
+    POST /api/v1/datasets/<dataset_id>/search
+    JSON body: {"question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,
+               "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
+               "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}
+    Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
+    Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
+    """
+    req, err = await validate_and_parse_json_request(request, SearchDatasetReq)
+    if err is not None:
+        return get_error_argument_result(err)
+    req['dataset_ids'] = [dataset_id]
+    try:
+        success, result = await dataset_api_service.search_datasets(tenant_id, req)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except Exception as e:
+        logging.exception(e)
+        if "not_found" in str(e):
+            return get_error_data_result(message="No chunk found! Check the chunk status please!")
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/graph", methods=["GET"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def get_knowledge_graph(tenant_id, dataset_id):
+    """Get the knowledge graph of a dataset.
+
+    GET /api/v1/datasets/<dataset_id>/graph
+    Query params: optional filter params.
+    Success: {"code": 0, "data": {...}}
+    Errors: AUTHENTICATION_ERROR for access denied; DATA_ERROR for internal errors.
+    """
     try:
         success, result = await dataset_api_service.get_knowledge_graph(dataset_id, tenant_id)
         if success:
             return get_result(data=result)
         else:
-            return get_result(
-                data=False,
-                message=result,
-                code=RetCode.AUTHENTICATION_ERROR
-            )
+            return get_result(data=False, message=result, code=RetCode.AUTHENTICATION_ERROR)
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route('/datasets/<dataset_id>/knowledge_graph', methods=['DELETE'])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/index", methods=["POST"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def delete_knowledge_graph(tenant_id, dataset_id):
+async def run_index(tenant_id, dataset_id):
+    index_type = request.args.get("type", "")
+    index_type = index_type.lower()
     try:
-        success, result = dataset_api_service.delete_knowledge_graph(dataset_id, tenant_id)
+        success, result = dataset_api_service.run_index(dataset_id, tenant_id, index_type)
         if success:
             return get_result(data=result)
         else:
-            return get_result(
-                data=False,
-                message=result,
-                code=RetCode.AUTHENTICATION_ERROR
-            )
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/run_graphrag", methods=["POST"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/index", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-async def run_graphrag(tenant_id, dataset_id):
+def trace_index(tenant_id, dataset_id):
+    index_type = request.args.get("type", "")
+    index_type = index_type.lower()
     try:
-        success, result = dataset_api_service.run_graphrag(dataset_id, tenant_id)
+        success, result = dataset_api_service.trace_index(dataset_id, tenant_id, index_type)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/<index_type>", methods=["DELETE"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/index", methods=["DELETE"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+def delete_index(tenant_id, dataset_id, index_type=None):
+    index_type = (index_type or request.args.get("type", "")).lower()
+    if index_type not in dataset_api_service._VALID_INDEX_TYPES:
+        return get_error_argument_result(f"Invalid index type '{index_type}'")
+    # `wipe` controls whether the persisted index artefacts (graph rows /
+    # raptor summaries) are removed. Default true preserves historical
+    # behaviour; pass wipe=false to cancel the running task while keeping
+    # prior progress so it can be resumed later.
+    wipe_arg = (request.args.get("wipe", "true") or "true").strip().lower()
+    wipe = wipe_arg not in ("false", "0", "no", "off")
+    try:
+        success, result = dataset_api_service.delete_index(dataset_id, tenant_id, index_type, wipe=wipe)
+        if success:
+            return get_result(data=result)
+        else:
+            return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
+    except Exception as e:
+        logging.exception(e)
+        return get_error_data_result(message="Internal server error")
+
+
+@manager.route("/datasets/<dataset_id>/embedding", methods=["POST"])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def run_embedding(tenant_id, dataset_id):
+    try:
+        success, result = dataset_api_service.run_embedding(dataset_id, tenant_id)
         if success:
             return get_result(data=result)
         else:
@@ -383,14 +636,19 @@ async def run_graphrag(tenant_id, dataset_id):
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/trace_graphrag", methods=["GET"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/embedding/check", methods=["POST"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def trace_graphrag(tenant_id, dataset_id):
+async def check_embedding(tenant_id, dataset_id):
     try:
-        success, result = dataset_api_service.trace_graphrag(dataset_id, tenant_id)
-        if success:
+        req = await request.get_json()
+        if not req or not req.get("embd_id"):
+            return get_error_data_result(message="`embd_id` is required.")
+        status, result = dataset_api_service.check_embedding(dataset_id, tenant_id, req)
+        if status is True:
             return get_result(data=result)
+        elif status == "not_effective":
+            return get_json_result(code=result["code"], message=result["message"], data=result["data"])
         else:
             return get_error_data_result(message=result)
     except Exception as e:
@@ -398,37 +656,50 @@ def trace_graphrag(tenant_id, dataset_id):
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/run_raptor", methods=["POST"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/ingestions", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-async def run_raptor(tenant_id, dataset_id):
+def list_ingestion_logs(tenant_id, dataset_id):
     try:
-        success, result = dataset_api_service.run_raptor(dataset_id, tenant_id)
+        page = int(request.args.get("page", 0))
+        page_size = validate_rest_api_page_size(int(request.args.get("page_size", 0)))
+        orderby = request.args.get("orderby", "create_time")
+        desc = request.args.get("desc", "true").lower() != "false"
+        operation_status = request.args.getlist("operation_status")
+        create_date_from = request.args.get("create_date_from", None)
+        create_date_to = request.args.get("create_date_to", None)
+        log_type = request.args.get("log_type", "dataset")
+        keywords = request.args.get("keywords", None)
+        success, result = dataset_api_service.list_ingestion_logs(dataset_id, tenant_id, page, page_size, orderby, desc, operation_status, create_date_from, create_date_to, log_type, keywords)
         if success:
             return get_result(data=result)
         else:
             return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/trace_raptor", methods=["GET"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/ingestions/<log_id>", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def trace_raptor(tenant_id, dataset_id):
+def get_ingestion_log(tenant_id, dataset_id, log_id):
     try:
-        success, result = dataset_api_service.trace_raptor(dataset_id, tenant_id)
+        success, result = dataset_api_service.get_ingestion_log(dataset_id, tenant_id, log_id)
         if success:
             return get_result(data=result)
         else:
             return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/auto_metadata", methods=["GET"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/metadata/config", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
 def get_auto_metadata(tenant_id, dataset_id):
@@ -462,12 +733,14 @@ def get_auto_metadata(tenant_id, dataset_id):
             return get_result(data=result)
         else:
             return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
 
 
-@manager.route("/datasets/<dataset_id>/auto_metadata", methods=["PUT"])  # noqa: F821
+@manager.route("/datasets/<dataset_id>/metadata/config", methods=["PUT"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
 async def update_auto_metadata(tenant_id, dataset_id):
@@ -502,6 +775,7 @@ async def update_auto_metadata(tenant_id, dataset_id):
           type: object
     """
     from api.utils.validation_utils import AutoMetadataConfig
+
     cfg, err = await validate_and_parse_json_request(request, AutoMetadataConfig)
     if err is not None:
         return get_error_argument_result(err)
@@ -512,6 +786,8 @@ async def update_auto_metadata(tenant_id, dataset_id):
             return get_result(data=result)
         else:
             return get_error_data_result(message=result)
+    except ValueError as e:
+        return get_error_argument_result(str(e))
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")

api/apps/restful_apis/dify_retrieval_api.py

@@ -0,0 +1,332 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+
+from quart import jsonify, request
+from werkzeug.exceptions import BadRequest as WerkzeugBadRequest
+
+try:
+    from quart.exceptions import BadRequest as QuartBadRequest
+except ImportError:  # pragma: no cover - optional dependency
+    QuartBadRequest = None
+
+from api.db.services.document_service import DocumentService
+from api.db.services.doc_metadata_service import DocMetadataService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.llm_service import LLMBundle
+from api.db.joint_services.tenant_model_service import get_tenant_default_model_by_type, get_model_config_from_provider_instance
+from common.metadata_utils import meta_filter, convert_conditions
+from api.apps import login_required
+from api.utils.api_utils import add_tenant_id_to_kwargs, build_error_result, get_request_json, get_json_result
+from rag.app.tag import label_question
+from common.constants import RetCode, LLMType
+from common import settings
+
+logger = logging.getLogger(__name__)
+
+
+async def _read_retrieval_request():
+    try:
+        method = request.method
+    except RuntimeError:
+        # Unit tests may call the handler directly without a request context.
+        method = "POST"
+    if method == "GET":
+        query_args = request.args
+        retrieval_setting = {}
+        knowledge_id = query_args.get("knowledge_id")
+        query = query_args.get("query")
+        use_kg = str(query_args.get("use_kg", "")).lower() in {"1", "true", "yes", "on"}
+        top_k = query_args.get("top_k")
+        score_threshold = query_args.get("score_threshold")
+        try:
+            if top_k not in (None, ""):
+                retrieval_setting["top_k"] = int(top_k)
+            if score_threshold not in (None, ""):
+                retrieval_setting["score_threshold"] = float(score_threshold)
+        except (TypeError, ValueError):
+            raise ValueError("top_k must be integer and score_threshold must be numeric")
+        safe_query = f"len={len(query)}" if isinstance(query, str) else "len=0"
+        logger.debug(
+            "Dify retrieval GET normalization: knowledge_id=%s query=%s use_kg=%s top_k=%s score_threshold=%s",
+            knowledge_id,
+            safe_query,
+            use_kg,
+            retrieval_setting.get("top_k"),
+            retrieval_setting.get("score_threshold"),
+        )
+
+        req = {
+            "knowledge_id": knowledge_id,
+            "query": query,
+            "use_kg": use_kg,
+            "retrieval_setting": retrieval_setting,
+        }
+        return req
+    req = await get_request_json()
+    knowledge_id = req.get("knowledge_id") if isinstance(req, dict) else None
+    query = req.get("query") if isinstance(req, dict) else None
+    use_kg = req.get("use_kg", False) if isinstance(req, dict) else False
+    retrieval_setting = req.get("retrieval_setting", {}) if isinstance(req, dict) else {}
+    if not isinstance(retrieval_setting, dict):
+        retrieval_setting = {}
+    safe_query = f"len={len(query)}" if isinstance(query, str) else "len=0"
+    logger.debug(
+        "Dify retrieval GET normalization: knowledge_id=%s query=%s use_kg=%s top_k=%s score_threshold=%s",
+        knowledge_id,
+        safe_query,
+        use_kg,
+        retrieval_setting.get("top_k"),
+        retrieval_setting.get("score_threshold"),
+    )
+    return req
+
+
+def _parse_retrieval_options(retrieval_setting):
+    if retrieval_setting is None:
+        retrieval_setting = {}
+    if not isinstance(retrieval_setting, dict):
+        raise ValueError("retrieval_setting must be an object")
+    try:
+        similarity_threshold = float(retrieval_setting.get("score_threshold", 0.0))
+        top = int(retrieval_setting.get("top_k", 1024))
+    except (TypeError, ValueError):
+        raise ValueError("top_k must be integer and score_threshold must be numeric")
+    return retrieval_setting, similarity_threshold, top
+
+
+@manager.route('/dify/retrieval', methods=['POST', 'GET'])  # noqa: F821
+@login_required
+@add_tenant_id_to_kwargs
+async def retrieval(tenant_id):
+    """
+    Dify-compatible retrieval API
+    ---
+    tags:
+      - SDK
+    security:
+      - ApiKeyAuth: []
+    parameters:
+      - in: query
+        name: knowledge_id
+        required: false
+        type: string
+        description: Knowledge base ID (for GET requests)
+      - in: query
+        name: query
+        required: false
+        type: string
+        description: Query text (for GET requests)
+      - in: query
+        name: use_kg
+        required: false
+        type: boolean
+        description: Whether to use knowledge graph (for GET requests)
+      - in: query
+        name: top_k
+        required: false
+        type: integer
+        description: Number of results to return (for GET requests)
+      - in: query
+        name: score_threshold
+        required: false
+        type: number
+        description: Similarity threshold (for GET requests)
+      - in: body
+        name: body
+        required: false
+        schema:
+          type: object
+          required:
+            - knowledge_id
+            - query
+          properties:
+            knowledge_id:
+              type: string
+              description: Knowledge base ID
+            query:
+              type: string
+              description: Query text
+            use_kg:
+              type: boolean
+              description: Whether to use knowledge graph
+              default: false
+            retrieval_setting:
+              type: object
+              description: Retrieval configuration
+              properties:
+                score_threshold:
+                  type: number
+                  description: Similarity threshold
+                  default: 0.0
+                top_k:
+                  type: integer
+                  description: Number of results to return
+                  default: 1024
+            metadata_condition:
+              type: object
+              description: Metadata filter condition
+              properties:
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      name:
+                        type: string
+                        description: Field name
+                      comparison_operator:
+                        type: string
+                        description: Comparison operator
+                      value:
+                        type: string
+                        description: Field value
+    responses:
+      200:
+        description: Retrieval succeeded
+        schema:
+          type: object
+          properties:
+            records:
+              type: array
+              items:
+                type: object
+                properties:
+                  content:
+                    type: string
+                    description: Content text
+                  score:
+                    type: number
+                    description: Similarity score
+                  title:
+                    type: string
+                    description: Document title
+                  metadata:
+                    type: object
+                    description: Metadata info
+      404:
+        description: Knowledge base or document not found
+    """
+    parse_exception_types = (AttributeError, TypeError, ValueError, WerkzeugBadRequest)
+    if QuartBadRequest is not None:
+        parse_exception_types = parse_exception_types + (QuartBadRequest,)
+    try:
+        req = await _read_retrieval_request()
+    except parse_exception_types as e:
+        return build_error_result(
+            message=f"invalid or malformed arguments: {str(e)}; ",
+            code=RetCode.ARGUMENT_ERROR,
+        )
+    missing = [field for field in ("knowledge_id", "query") if not req.get(field)]
+    if missing:
+        return build_error_result(
+            message=f"required arguments are missing: {','.join(missing)}; ",
+            code=RetCode.ARGUMENT_ERROR,
+        )
+    question = req["query"]
+    kb_id = req["knowledge_id"]
+    use_kg = req.get("use_kg", False)
+    try:
+        _, similarity_threshold, top = _parse_retrieval_options(req.get("retrieval_setting", {}))
+    except ValueError as e:
+        return build_error_result(
+            message=f"invalid or malformed arguments: {str(e)}; ",
+            code=RetCode.ARGUMENT_ERROR,
+        )
+    metadata_condition = req.get("metadata_condition", {}) or {}
+    metas = DocMetadataService.get_flatted_meta_by_kbs([kb_id])
+
+    doc_ids = []
+    try:
+
+        e, kb = KnowledgebaseService.get_by_id(kb_id)
+        if not e:
+            return build_error_result(message="Knowledgebase not found!", code=RetCode.NOT_FOUND)
+        if not KnowledgebaseService.accessible(kb_id, tenant_id):
+            logger.warning(
+                "Rejected /dify/retrieval cross-tenant access: caller_tenant=%s knowledge_id=%s",
+                tenant_id,
+                kb_id,
+            )
+            return build_error_result(message="No authorization.", code=RetCode.AUTHENTICATION_ERROR)
+        model_config = get_model_config_from_provider_instance(kb.tenant_id, LLMType.EMBEDDING, kb.embd_id)
+        embd_mdl = LLMBundle(kb.tenant_id, model_config)
+        if metadata_condition:
+            doc_ids.extend(meta_filter(metas, convert_conditions(metadata_condition), metadata_condition.get("logic", "and")))
+        if not doc_ids and metadata_condition:
+            doc_ids = ["-999"]
+        ranks = await settings.retriever.retrieval(
+            question,
+            embd_mdl,
+            kb.tenant_id,
+            [kb_id],
+            page=1,
+            page_size=top,
+            similarity_threshold=similarity_threshold,
+            vector_similarity_weight=0.3,
+            top=top,
+            doc_ids=doc_ids,
+            rank_feature=label_question(question, [kb])
+        )
+        ranks["chunks"] = settings.retriever.retrieval_by_children(ranks["chunks"], [tenant_id])
+
+        if use_kg:
+            model_config = get_tenant_default_model_by_type(kb.tenant_id, LLMType.CHAT)
+            ck = await settings.kg_retriever.retrieval(question,
+                                                 [tenant_id],
+                                                 [kb_id],
+                                                 embd_mdl,
+                                                 LLMBundle(kb.tenant_id, model_config))
+            if ck["content_with_weight"]:
+                ranks["chunks"].insert(0, ck)
+
+        doc_ids = list(set([c["doc_id"] for c in ranks["chunks"]]))
+        docs = DocumentService.get_by_ids(doc_ids)
+        doc_map = {doc.id: doc for doc in docs}
+
+        records = []
+        for c in ranks["chunks"]:
+            doc = doc_map.get(c["doc_id"])
+            if not doc:
+                continue
+            c.pop("vector", None)
+            meta = getattr(doc, 'meta_fields', {})
+            meta["doc_id"] = c["doc_id"]
+            # Dify expects metadata.document_id for external retrieval sources.
+            meta["document_id"] = c["doc_id"]
+            records.append({
+                "content": c["content_with_weight"],
+                "score": c["similarity"],
+                "title": c["docnm_kwd"],
+                "metadata": meta
+            })
+
+        return jsonify({"records": records})
+    except Exception as e:
+        if "not_found" in str(e):
+            return build_error_result(
+                message='No chunk found! Check the chunk status please!',
+                code=RetCode.NOT_FOUND
+            )
+        logging.exception(e)
+        return build_error_result(message=str(e), code=RetCode.SERVER_ERROR)
+   
+  
+@manager.route('/dify/retrieval/health', methods=['GET'])  # noqa: F821
+async def retrieval_health_check():
+    """Health check endpoint for Dify external knowledge base connectivity verification."""
+    return get_json_result(data=True)
+  

api/apps/restful_apis/file2document_api.py

@@ -18,6 +18,7 @@ import asyncio
 import logging
 from pathlib import Path
 
+from api.common.check_team_permission import check_file_team_permission, check_kb_team_permission
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 
@@ -25,10 +26,11 @@ from api.apps import login_required, current_user
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request
 from common.misc_utils import get_uuid
-from common.constants import RetCode
 from api.db import FileType
 from api.db.services.document_service import DocumentService
 
+logger = logging.getLogger(__name__)
+
 
 def _convert_files(file_ids, kb_ids, user_id):
     """Synchronous worker: delete old docs and insert new ones for the given file/kb pairs."""
@@ -74,7 +76,7 @@ def _convert_files(file_ids, kb_ids, user_id):
             })
 
 
-@manager.route('/convert', methods=['POST'])  # noqa: F821
+@manager.route('/files/link-to-datasets', methods=['POST'])  # noqa: F821
 @login_required
 @validate_request("file_ids", "kb_ids")
 async def convert():
@@ -89,13 +91,29 @@ async def convert():
         # Validate all files exist before starting any work
         for file_id in file_ids:
             if not files_set.get(file_id):
+                logger.warning(
+                    "user_id=%s resource_type=file resource_id=%s action=validate_file_lookup result=not_found file_ids=%s kb_ids=%s",
+                    current_user.id,
+                    file_id,
+                    file_ids,
+                    kb_ids,
+                )
                 return get_data_error_result(message="File not found!")
 
         # Validate all kb_ids exist before scheduling background work
+        kb_map = {}
         for kb_id in kb_ids:
-            e, _ = KnowledgebaseService.get_by_id(kb_id)
+            e, kb = KnowledgebaseService.get_by_id(kb_id)
             if not e:
+                logger.warning(
+                    "user_id=%s resource_type=dataset resource_id=%s action=validate_dataset_lookup result=not_found file_ids=%s kb_ids=%s",
+                    current_user.id,
+                    kb_id,
+                    file_ids,
+                    kb_ids,
+                )
                 return get_data_error_result(message="Can't find this dataset!")
+            kb_map[kb_id] = kb
 
         # Expand folders to their innermost file IDs
         all_file_ids = []
@@ -107,6 +125,38 @@ async def convert():
                 all_file_ids.append(file_id)
 
         user_id = current_user.id
+        for file_id in all_file_ids:
+            e, file = FileService.get_by_id(file_id)
+            if not e or not file:
+                logger.warning(
+                    "user_id=%s resource_type=file resource_id=%s action=validate_expanded_file_lookup result=not_found file_ids=%s kb_ids=%s",
+                    user_id,
+                    file_id,
+                    file_ids,
+                    kb_ids,
+                )
+                return get_data_error_result(message="File not found!")
+            if not check_file_team_permission(file, user_id):
+                logger.warning(
+                    "user_id=%s resource_type=file resource_id=%s action=authorize_file result=denied file_ids=%s kb_ids=%s",
+                    user_id,
+                    file_id,
+                    file_ids,
+                    kb_ids,
+                )
+                return get_data_error_result(message="No authorization.")
+
+        for kb_id, kb in kb_map.items():
+            if not check_kb_team_permission(kb, user_id):
+                logger.warning(
+                    "user_id=%s resource_type=dataset resource_id=%s action=authorize_dataset result=denied file_ids=%s kb_ids=%s",
+                    user_id,
+                    kb_id,
+                    file_ids,
+                    kb_ids,
+                )
+                return get_data_error_result(message="No authorization.")
+
         # Run the blocking DB work in a thread so the event loop is not blocked.
         # For large folders this prevents 504 Gateway Timeout by returning as
         # soon as the background task is scheduled.
@@ -115,39 +165,12 @@ async def convert():
         future.add_done_callback(
             lambda f: logging.error("_convert_files failed: %s", f.exception()) if f.exception() else None
         )
-        return get_json_result(data=True)
-    except Exception as e:
-        return server_error_response(e)
-
-
-@manager.route('/rm', methods=['POST'])  # noqa: F821
-@login_required
-@validate_request("file_ids")
-async def rm():
-    req = await get_request_json()
-    file_ids = req["file_ids"]
-    if not file_ids:
-        return get_json_result(
-            data=False, message='Lack of "Files ID"', code=RetCode.ARGUMENT_ERROR)
-    try:
-        for file_id in file_ids:
-            informs = File2DocumentService.get_by_file_id(file_id)
-            if not informs:
-                return get_data_error_result(message="Inform not found!")
-            for inform in informs:
-                if not inform:
-                    return get_data_error_result(message="Inform not found!")
-                File2DocumentService.delete_by_file_id(file_id)
-                doc_id = inform.document_id
-                e, doc = DocumentService.get_by_id(doc_id)
-                if not e:
-                    return get_data_error_result(message="Document not found!")
-                tenant_id = DocumentService.get_tenant_id(doc_id)
-                if not tenant_id:
-                    return get_data_error_result(message="Tenant not found!")
-                if not DocumentService.remove_document(doc, tenant_id):
-                    return get_data_error_result(
-                        message="Database error (Document removal)!")
+        logger.info(
+            "user_id=%s resource_type=file_to_dataset_link resource_id=batch action=schedule_convert result=scheduled file_ids=%s kb_ids=%s",
+            user_id,
+            all_file_ids,
+            kb_ids,
+        )
         return get_json_result(data=True)
     except Exception as e:
         return server_error_response(e)

api/apps/restful_apis/file_api.py

@@ -24,8 +24,10 @@ from api.utils.api_utils import (
     add_tenant_id_to_kwargs,
     get_error_argument_result,
     get_error_data_result,
+    get_json_result,
     get_result,
 )
+from common.constants import RetCode
 from api.utils.validation_utils import (
     CreateFolderReq,
     DeleteFileReq,
@@ -99,7 +101,7 @@ async def create_or_upload(tenant_id: str = None):
 @manager.route("/files", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def list_files(tenant_id: str = None):
+async def list_files(tenant_id: str = None):
     """
     List files under a folder.
     ---
@@ -185,10 +187,22 @@ async def delete(tenant_id: str = None):
         return get_error_argument_result(err)
 
     try:
-        success, result = await file_api_service.delete_files(tenant_id, req["ids"])
+        # Get Authorization header to pass to Go backend
+        auth_header = request.headers.get("Authorization", "")
+        success, result = await file_api_service.delete_files(tenant_id, req["ids"], auth_header)
         if success:
             return get_result(data=result)
         else:
+            if isinstance(result, dict):
+                success_count = result.get("success_count", 0)
+                errors = result.get("errors", [])
+                return get_json_result(
+                    code=RetCode.DATA_ERROR,
+                    message=f"Partially deleted {success_count} files with {len(errors)} errors"
+                    if success_count > 0
+                    else f"Deleted files failed with {len(errors)} errors",
+                    data=result,
+                )
             return get_error_data_result(message=result)
     except Exception as e:
         logging.exception(e)
@@ -285,6 +299,13 @@ async def download(tenant_id: str = None, file_id: str = None):
         if not blob:
             b, n = File2DocumentService.get_storage_address(file_id=file_id)
             blob = await thread_pool_exec(settings.STORAGE_IMPL.get, b, n)
+        if not blob:
+            logging.warning(
+                "Download failed: empty blob after primary+fallback lookup (tenant_id=%s, file_id=%s)",
+                tenant_id,
+                file_id,
+            )
+            return get_error_data_result(message="This file is empty.")
 
         response = await make_response(blob)
         ext = re.search(r"\.([^.]+)$", file.name.lower())
@@ -303,7 +324,7 @@ async def download(tenant_id: str = None, file_id: str = None):
 @manager.route("/files/<file_id>/parent", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def parent_folder(tenant_id: str = None, file_id: str = None):
+async def parent_folder(tenant_id: str = None, file_id: str = None):
     """
     Get parent folder of a file.
     ---
@@ -321,7 +342,7 @@ def parent_folder(tenant_id: str = None, file_id: str = None):
         description: Parent folder information.
     """
     try:
-        success, result = file_api_service.get_parent_folder(file_id)
+        success, result = file_api_service.get_parent_folder(file_id, user_id=tenant_id)
         if success:
             return get_result(data=result)
         else:
@@ -334,7 +355,7 @@ def parent_folder(tenant_id: str = None, file_id: str = None):
 @manager.route("/files/<file_id>/ancestors", methods=["GET"])  # noqa: F821
 @login_required
 @add_tenant_id_to_kwargs
-def ancestors(tenant_id: str = None, file_id: str = None):
+async def ancestors(tenant_id: str = None, file_id: str = None):
     """
     Get all ancestor folders of a file.
     ---
@@ -352,7 +373,7 @@ def ancestors(tenant_id: str = None, file_id: str = None):
         description: List of ancestor folders.
     """
     try:
-        success, result = file_api_service.get_all_parent_folders(file_id)
+        success, result = file_api_service.get_all_parent_folders(file_id, user_id=tenant_id)
         if success:
             return get_result(data=result)
         else:
@@ -360,5 +381,3 @@ def ancestors(tenant_id: str = None, file_id: str = None):
     except Exception as e:
         logging.exception(e)
         return get_error_data_result(message="Internal server error")
-
-

api/apps/restful_apis/file_commit_api.py

@@ -0,0 +1,314 @@
+#
+#  Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import logging
+from functools import wraps
+
+from quart import request
+
+from api.apps import login_required, current_user
+from api.utils.api_utils import get_json_result, get_data_error_result, get_request_json, server_error_response, validate_request
+
+# manager is injected dynamically by api.apps.register_page() before this
+# module is exec'd. DO NOT assign manager = None here — it would overwrite
+# the Blueprint that register_page set on the module.
+from api.db.services.file_commit_service import FileCommitService
+from api.db.services.knowledgebase_service import KnowledgebaseService
+from api.db.services.file_service import FileService
+from common.constants import FileSource
+
+logger = logging.getLogger(__name__)
+
+_ENTITY_RESOLVERS = {}
+
+# Counter to give each generated route function a unique name,
+# preventing Quart Blueprint endpoint name collisions.
+_route_suffix = [0]
+
+
+def _register_resolver(entity_type):
+    """Decorator that registers a folder_id resolver for an entity type.
+
+    The decorated function receives (entity_id) and must return a folder_id
+    or None if the entity has no corresponding folder.
+    """
+    def decorator(func):
+        _ENTITY_RESOLVERS[entity_type] = func
+        @wraps(func)
+        def wrapper(entity_id):
+            return func(entity_id)
+        return wrapper
+    return decorator
+
+
+def _resolve_folder_id(entity_type, entity_id):
+    """Resolve an entity (dataset/memory/skill) to its folder_id."""
+    resolver = _ENTITY_RESOLVERS.get(entity_type)
+    if resolver is None:
+        return None
+    return resolver(entity_id)
+
+
+@_register_resolver("datasets")
+def _resolve_dataset_folder(dataset_id):
+    success, kb = KnowledgebaseService.get_by_id(dataset_id)
+    if not success:
+        return None
+    # Find the folder with matching name, source_type, and tenant_id
+    folders = FileService.query(
+        name=kb.name,
+        source_type=FileSource.KNOWLEDGEBASE.value,
+        type="folder",
+        tenant_id=kb.tenant_id,
+    )
+    if folders:
+        return folders[0].id
+    return None
+
+
+# ── Route registration helper ─────────────────────────────────────────────
+
+def _register_commit_routes(prefix, param_name, resolver_type=None):
+    """Register all 8 commit endpoints for a given URL prefix.
+
+    Args:
+        prefix: URL prefix like '/folders/<folder_id>'
+        param_name: The URL parameter name (e.g. 'folder_id', 'dataset_id')
+        resolver_type: If set, resolve param_name → folder_id before calling logic
+    """
+    # Unique suffix for this call to prevent Blueprint endpoint name collisions
+    _route_suffix[0] += 1
+    _n = _route_suffix[0]
+
+    def _resolve(entity_id):
+        if resolver_type is None:
+            return entity_id  # already a folder_id
+        folder_id = _resolve_folder_id(resolver_type, entity_id)
+        if folder_id is None:
+            raise ValueError(f"Could not resolve {resolver_type} '{entity_id}' to a folder")
+        return folder_id
+
+    # ── Create commit ──────────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits', methods=['POST'], endpoint=f'create_commit_{_n}')  # noqa: F821
+    @login_required
+    @validate_request("message", "files")
+    async def create_commit(entity_id):
+        folder_id = _resolve(entity_id)
+        req = await get_request_json()
+        try:
+            commit = FileCommitService.create_commit(
+                folder_id=folder_id,
+                author_id=current_user.id,
+                message=req["message"],
+                file_changes=req["files"],
+            )
+            return get_json_result(data={
+                "id": commit.id,
+                "folder_id": commit.folder_id,
+                "parent_id": commit.parent_id,
+                "message": commit.message,
+                "author_id": commit.author_id,
+                "file_count": commit.file_count,
+                "tree_state": commit.tree_state,
+                "create_time": commit.create_time,
+            })
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── List commits ───────────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits', methods=['GET'], endpoint=f'list_commits_{_n}')  # noqa: F821
+    @login_required
+    async def list_commits(entity_id):
+        folder_id = _resolve(entity_id)
+        try:
+            page = int(request.args.get("page", 1))
+            page_size = int(request.args.get("page_size", 15))
+            order_by = request.args.get("order_by", "create_time")
+            desc = request.args.get("desc", "true").lower() != "false"
+            commits, total = FileCommitService.list_commits(folder_id, page, page_size, order_by, desc)
+            return get_json_result(data={
+                "total": total,
+                "page": page,
+                "page_size": page_size,
+                "commits": [{
+                    "id": c.id,
+                    "folder_id": c.folder_id,
+                    "parent_id": c.parent_id,
+                    "message": c.message,
+                    "author_id": c.author_id,
+                    "file_count": c.file_count,
+                    "create_time": c.create_time,
+                } for c in commits],
+            })
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── Get commit ─────────────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits/<commit_id>', methods=['GET'], endpoint=f'get_commit_{_n}')  # noqa: F821
+    @login_required
+    async def get_commit(entity_id, commit_id):
+        folder_id = _resolve(entity_id)
+        try:
+            commit = FileCommitService.get_commit(commit_id)
+            if not commit:
+                return get_data_error_result("Commit not found")
+            if commit.folder_id != folder_id:
+                return get_data_error_result("Commit not found in workspace")
+            items = FileCommitService.list_commit_files(commit_id)
+            return get_json_result(data={
+                "id": commit.id,
+                "folder_id": commit.folder_id,
+                "parent_id": commit.parent_id,
+                "message": commit.message,
+                "author_id": commit.author_id,
+                "file_count": commit.file_count,
+                "create_time": commit.create_time,
+                "files": [{
+                    "file_id": item.file_id,
+                    "operation": item.operation,
+                    "old_hash": item.old_hash,
+                    "new_hash": item.new_hash,
+                    "old_name": item.old_name,
+                    "new_name": item.new_name,
+                } for item in items],
+            })
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── List commit files ──────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits/<commit_id>/files', methods=['GET'], endpoint=f'list_commit_files_{_n}')  # noqa: F821
+    @login_required
+    async def list_commit_files(entity_id, commit_id):
+        folder_id = _resolve(entity_id)
+        try:
+            commit = FileCommitService.get_commit(commit_id)
+            if not commit:
+                return get_data_error_result("Commit not found")
+            if commit.folder_id != folder_id:
+                return get_data_error_result("Commit not found in workspace")
+            items = FileCommitService.list_commit_files(commit_id)
+            return get_json_result(data=[{
+                "id": item.id,
+                "file_id": item.file_id,
+                "operation": item.operation,
+                "old_hash": item.old_hash,
+                "new_hash": item.new_hash,
+                "old_location": item.old_location,
+                "new_location": item.new_location,
+                "old_name": item.old_name,
+                "new_name": item.new_name,
+            } for item in items])
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── Diff commits ───────────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits/diff', methods=['GET'], endpoint=f'diff_commits_{_n}')  # noqa: F821
+    @login_required
+    async def diff_commits(entity_id):
+        folder_id = _resolve(entity_id)
+        from_id = request.args.get("from")
+        to_id = request.args.get("to")
+        if not from_id or not to_id:
+            return get_data_error_result("'from' and 'to' parameters are required")
+        try:
+            from_commit = FileCommitService.get_commit(from_id)
+            to_commit = FileCommitService.get_commit(to_id)
+            if not from_commit or not to_commit:
+                return get_data_error_result("Commit not found")
+            if from_commit.folder_id != folder_id or to_commit.folder_id != folder_id:
+                return get_data_error_result("Commit not found in workspace")
+            diff = FileCommitService.diff_commits(from_id, to_id)
+            return get_json_result(data=diff)
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── Get uncommitted changes ────────────────────────────────────────────
+    @manager.route(f'{prefix}/changes', methods=['GET'], endpoint=f'get_uncommitted_changes_{_n}')  # noqa: F821
+    @login_required
+    async def get_uncommitted_changes(entity_id):
+        folder_id = _resolve(entity_id)
+        try:
+            changes = FileCommitService.get_uncommitted_changes(folder_id)
+            return get_json_result(data=changes)
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── Get commit tree ────────────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits/<commit_id>/tree', methods=['GET'], endpoint=f'get_commit_tree_{_n}')  # noqa: F821
+    @login_required
+    async def get_commit_tree(entity_id, commit_id):
+        folder_id = _resolve(entity_id)
+        try:
+            commit = FileCommitService.get_commit(commit_id)
+            if not commit:
+                return get_data_error_result("Commit not found")
+            if commit.folder_id != folder_id:
+                return get_data_error_result("Commit not found in workspace")
+            tree = FileCommitService.get_commit_tree(commit_id)
+            return get_json_result(data=tree)
+        except Exception as e:
+            return server_error_response(e)
+
+    # ── Get commit file content ────────────────────────────────────────────
+    @manager.route(f'{prefix}/commits/<commit_id>/files/<file_id>/content', methods=['GET'], endpoint=f'get_commit_file_content_{_n}')  # noqa: F821
+    @login_required
+    async def get_commit_file_content(entity_id, commit_id, file_id):
+        folder_id = _resolve(entity_id)
+        try:
+            commit = FileCommitService.get_commit(commit_id)
+            if not commit:
+                return get_data_error_result("Commit not found")
+            if commit.folder_id != folder_id:
+                return get_data_error_result("Commit not found in workspace")
+            content = FileCommitService.get_commit_file_content(folder_id, commit_id, file_id)
+            if content is None:
+                return get_data_error_result("File not found in this commit")
+            return get_json_result(data={"content": content.decode("utf-8", errors="replace")})
+        except Exception as e:
+            return server_error_response(e)
+
+    # Expose handlers at module level for direct testing.
+    _g = globals()
+    _g['create_commit'] = create_commit
+    _g['list_commits'] = list_commits
+    _g['get_commit'] = get_commit
+    _g['list_commit_files'] = list_commit_files
+    _g['diff_commits'] = diff_commits
+    _g['get_uncommitted_changes'] = get_uncommitted_changes
+    _g['get_commit_tree'] = get_commit_tree
+    _g['get_commit_file_content'] = get_commit_file_content
+
+# ── Register routes for all entity types ──────────────────────────────────
+# All URL patterns use <entity_id> as the consistent param name.
+# For /folders/ entity_id IS the folder_id directly.
+# For other entity types entity_id is resolved via _resolve_folder_id().
+# Register datasets first, workspace second, folders last —
+# the last call's handlers overwrite module-level names for test access.
+_register_commit_routes('/datasets/<entity_id>', 'entity_id', resolver_type='datasets')
+_register_commit_routes('/workspace/<entity_id>', 'entity_id')  # alias — workspace_id == folder_id
+_register_commit_routes('/folders/<entity_id>', 'entity_id')  # direct — entity_id == folder_id (wins)
+# /memories and /skills routes are not mounted until resolvers are implemented.
+
+
+# ── File version history (shared across all entity types) ─────────────────
+@manager.route('/files/<file_id>/versions', methods=['GET'])  # noqa: F821
+@login_required
+async def get_file_version_history(file_id):
+    try:
+        versions = FileCommitService.get_file_version_history(file_id)
+        return get_json_result(data=versions)
+    except Exception as e:
+        return server_error_response(e)

api/apps/restful_apis/langfuse_api.py

@@ -23,7 +23,7 @@ from api.db.services.langfuse_service import TenantLangfuseService
 from api.utils.api_utils import get_error_data_result, get_json_result, get_request_json, server_error_response, validate_request
 
 
-@manager.route("/api_key", methods=["POST", "PUT"])  # noqa: F821
+@manager.route("/langfuse/api-key", methods=["POST", "PUT"])  # noqa: F821
 @login_required
 @validate_request("secret_key", "public_key", "host")
 async def set_api_key():
@@ -58,7 +58,7 @@ async def set_api_key():
             return server_error_response(e)
 
 
-@manager.route("/api_key", methods=["GET"])  # noqa: F821
+@manager.route("/langfuse/api-key", methods=["GET"])  # noqa: F821
 @login_required
 @validate_request()
 def get_api_key():
@@ -82,7 +82,7 @@ def get_api_key():
     return get_json_result(data=langfuse_entry)
 
 
-@manager.route("/api_key", methods=["DELETE"])  # noqa: F821
+@manager.route("/langfuse/api-key", methods=["DELETE"])  # noqa: F821
 @login_required
 @validate_request()
 def delete_api_key():