diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 5945298aac..5e0d03d2a1 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -14,7 +14,9 @@ on: # The only difference between pull_request and pull_request_target is the context in which the workflow runs: # — pull_request_target workflows use the workflow files from the default branch, and secrets are available. # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable. - pull_request: + # We use pull_request_target so that GITEE_TOKEN (and other repo secrets) are available when building + # ragflow:nightly from a fork PR, while still running the full test suite on PRs labeled with `ci`. + pull_request_target: types: [synchronize, labeled] paths-ignore: - 'docs/**' @@ -22,6 +24,8 @@ on: - '*.mdx' schedule: - cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00) + # Allow maintainers to trigger a full build+test run manually (secrets are available). + workflow_dispatch: # https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: @@ -33,7 +37,7 @@ jobs: name: ragflow_preflight # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution # https://github.com/orgs/community/discussions/26261 - if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} runs-on: [ "self-hosted", "ragflow-test" ] outputs: http_api_test_level: ${{ steps.test_level.outputs.http_api_test_level }} @@ -54,11 +58,12 @@ jobs: ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} fetch-depth: 0 fetch-tags: true + allow-unsafe-pr-checkout: true - name: Check workflow duplication if: ${{ !cancelled() && !failure() }} run: | - if [[ ${GITHUB_EVENT_NAME} != "pull_request" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then + if [[ ${GITHUB_EVENT_NAME} != "pull_request" && ${GITHUB_EVENT_NAME} != "pull_request_target" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then HEAD=$(git rev-parse HEAD) # Find a PR that introduced a given commit gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}" @@ -83,7 +88,7 @@ jobs: fi fi fi - elif [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then + elif [[ ${GITHUB_EVENT_NAME} == "pull_request" || ${GITHUB_EVENT_NAME} == "pull_request_target" ]]; then PR_NUMBER=${{ github.event.pull_request.number }} PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER} # Calculate the hash of the current workspace content @@ -179,7 +184,7 @@ jobs: has_python=true has_web=true else - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then CHANGED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }}) else BASE_SHA="${{ github.event.before }}" @@ -238,7 +243,7 @@ jobs: - name: Prepare Python test environment if: steps.detect_changes.outputs.has_python_changes == 'true' run: | - git config --global url."https://oauth2:796df937ae36d0014a35aa8ab2b02609@gitee.com/".insteadOf "https://gitee.com/" + git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/" uv sync --python 3.13 --group test --frozen uv pip install -e sdk/python @@ -255,7 +260,7 @@ jobs: ragflow_tests_infinity: name: ragflow_tests_infinity (${{ matrix.api_proxy_scheme }}) needs: ragflow_preflight - if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} runs-on: [ "self-hosted", "ragflow-test" ] strategy: fail-fast: false @@ -283,6 +288,7 @@ jobs: ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} fetch-depth: 0 fetch-tags: true + allow-unsafe-pr-checkout: true - name: Build ragflow go server if: env.API_PROXY_SCHEME == 'go' @@ -342,17 +348,22 @@ jobs: RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock mkdir -p "$(dirname "${BUILD_LOCK_FILE}")" + # Pass the Gitee personal access token to the Docker build as a BuildKit secret + # (mounted inside the builder at /run/secrets/gitee_token, never persisted in any layer). + GITEE_TOKEN_FILE="$(mktemp)" + printf '%s' '${{ secrets.GITEE_TOKEN }}' > "${GITEE_TOKEN_FILE}" ( flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" ##sudo docker pull ubuntu:24.04 - sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=0 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} . + sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile -t ${RAGFLOW_IMAGE} . ) 9>"${BUILD_LOCK_FILE}" + rm -f "${GITEE_TOKEN_FILE}" - name: Prepare Python test environment if: env.API_PROXY_SCHEME != 'web' run: | - git config --global url."https://oauth2:796df937ae36d0014a35aa8ab2b02609@gitee.com/".insteadOf "https://gitee.com/" + git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/" uv sync --python 3.13 --group test --frozen uv pip install -e sdk/python @@ -829,7 +840,7 @@ jobs: ragflow_tests_elasticsearch: name: ragflow_tests_elasticsearch (${{ matrix.api_proxy_scheme }}) needs: ragflow_preflight - if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} + if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }} runs-on: [ "self-hosted", "ragflow-test" ] strategy: fail-fast: false @@ -857,6 +868,7 @@ jobs: ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} fetch-depth: 0 fetch-tags: true + allow-unsafe-pr-checkout: true - name: Build ragflow go server if: env.API_PROXY_SCHEME == 'go' @@ -916,17 +928,22 @@ jobs: RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} BUILD_LOCK_FILE=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/docker-build.lock mkdir -p "$(dirname "${BUILD_LOCK_FILE}")" + # Pass the Gitee personal access token to the Docker build as a BuildKit secret + # (mounted inside the builder at /run/secrets/gitee_token, never persisted in any layer). + GITEE_TOKEN_FILE="$(mktemp)" + printf '%s' '${{ secrets.GITEE_TOKEN }}' > "${GITEE_TOKEN_FILE}" ( flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" ##sudo docker pull ubuntu:24.04 - sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=0 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} . + sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile -t ${RAGFLOW_IMAGE} . ) 9>"${BUILD_LOCK_FILE}" + rm -f "${GITEE_TOKEN_FILE}" - name: Prepare Python test environment if: env.API_PROXY_SCHEME != 'web' run: | - git config --global url."https://oauth2:796df937ae36d0014a35aa8ab2b02609@gitee.com/".insteadOf "https://gitee.com/" + git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/" uv sync --python 3.13 --group test --frozen uv pip install -e sdk/python diff --git a/Dockerfile b/Dockerfile index b088564c04..77e8bd5116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,9 +50,15 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \ fonts-freefont-ttf fonts-noto-cjk postgresql-client # Download resource from GitHub to /usr/share/infinity -RUN mkdir -p /usr/share/infinity/resource && \ +RUN --mount=type=secret,id=gitee_token \ + mkdir -p /usr/share/infinity/resource && \ if [ "$NEED_MIRROR" == "1" ]; then \ - git clone --depth 1 --single-branch https://gitee.com/infiniflow/resource /tmp/resource; \ + GITEE_TOKEN=$(cat /run/secrets/gitee_token 2>/dev/null || echo ""); \ + if [ -n "$GITEE_TOKEN" ]; then \ + git clone --depth 1 --single-branch "https://oauth2:${GITEE_TOKEN}@gitee.com/infiniflow/resource" /tmp/resource; \ + else \ + git clone --depth 1 --single-branch https://github.com/infiniflow/resource.git /tmp/resource; \ + fi; \ else \ git clone --depth 1 --single-branch https://github.com/infiniflow/resource.git /tmp/resource; \ fi && \