From 92ae4f67697f610d3575e7450ff4304b7e851fbb Mon Sep 17 00:00:00 2001 From: maoyifeng Date: Thu, 16 Jul 2026 13:52:29 +0800 Subject: [PATCH] Fix ci: run ci port confict,pull network timeout (#16979) ### Summary 1. fix ci port confict 2. fix pull image network timeout --- .github/workflows/sep-tests.yml | 152 ++++++++++++++++++++++---------- 1 file changed, 104 insertions(+), 48 deletions(-) diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml index 0b3c0a91a9..29323c0c47 100644 --- a/.github/workflows/sep-tests.yml +++ b/.github/workflows/sep-tests.yml @@ -332,7 +332,7 @@ jobs: ( flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" - sudo docker pull ubuntu:24.04 + ##sudo docker pull ubuntu:24.04 sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} . ) 9>"${BUILD_LOCK_FILE}" @@ -359,7 +359,14 @@ jobs: # Determine runner number (default to 1 if not found) RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true) - RUNNER_NUM=${RUNNER_NUM:-1} + if [[ -z "${RUNNER_NUM:-}" ]]; then + # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates. + if [[ -n "${RUNNER_NAME:-}" ]]; then + RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 )) + else + RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 )) + fi + fi # Engine-specific offset partitions keep concurrent engine jobs from # choosing the same host ports when they land on the same self-hosted runner. @@ -371,15 +378,28 @@ jobs: infinity) PARTITION_BASE=31000 ;; *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;; esac - PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks - mkdir -p "${PORT_LOCK_DIR}" + # Runners run inside docker containers that share the host docker daemon + # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners: + # each container has its own /tmp and its own network namespace, and `ss` cannot + # see host ports bound by other runners' compose stacks. Use the shared daemon for + # both primitives: + # - offset mutex = a docker network created atomically on the shared daemon + # (docker network create is NOT idempotent: it fails on name conflict, which + # is exactly the cross-runner test-and-set we need); + # - port availability = docker ps published ports (sees every container on the + # shared daemon regardless of which runner started it). + # No file-system sharing required, no host netns required. + PORT_GUARD_PREFIX=ragflow-ci-portguard port_offset_available() { local offset=$1 - local base port + local base port used + # Enumerate host-published tcp ports across ALL containers on the shared daemon. + # Format example: "0.0.0.0:38217->4222/tcp". We extract the left side. + used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | tr -d '->' | sort -u) for base in "${PORT_BASES[@]}"; do port=$((base + offset)) - if ss -ltnH "sport = :${port}" | grep -q .; then + if grep -qx "${port}" <<< "${used}"; then return 1 fi done @@ -387,31 +407,37 @@ jobs: } cleanup_stale_port_locks() { - local now stale_after lock lock_ts + local now stale_after net ts now=$(date -u +%s) - stale_after=$((6 * 60 * 60)) - for lock in "${PORT_LOCK_DIR}"/*.lock; do - [[ -e "${lock}" ]] || continue - lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true) - if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then - rm -f "${lock}" + stale_after=$((2 * 60 * 60)) + for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do + ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true) + if [[ "${ts}" =~ ^[0-9]+$ ]] && (( now - ts > stale_after )); then + sudo docker network rm "${net}" >/dev/null 2>&1 || true fi done } reserve_port_offset() { - local attempt candidate reservation + local attempt candidate guard cleanup_stale_port_locks - for attempt in $(seq 0 59); do + for attempt in $(seq 0 5999); do candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) )) - reservation="${PORT_LOCK_DIR}/${candidate}.lock" - if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then - if port_offset_available "${candidate}"; then - PORT_OFFSET=${candidate} - PORT_RESERVATION=${reservation} - return 0 - fi - rm -f "${reservation}" + guard="${PORT_GUARD_PREFIX}-${candidate}" + # Already held by some runner on this shared daemon. + if sudo docker network inspect "${guard}" >/dev/null 2>&1; then + continue + fi + if ! port_offset_available "${candidate}"; then + continue + fi + # Atomic cross-runner test-and-set: docker network create fails if the name + # already exists on the shared daemon, so even if two runners race here, only + # one wins; the loser retries the next candidate. + if sudo docker network create --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then + PORT_OFFSET=${candidate} + PORT_RESERVATION="${guard}" + return 0 fi done return 1 @@ -716,7 +742,9 @@ jobs: sudo docker rmi -f ${RAGFLOW_IMAGE} fi if [[ -n ${PORT_RESERVATION:-} ]]; then - rm -f "${PORT_RESERVATION}" + # PORT_RESERVATION is now a docker network name on the shared daemon (created by the + # prepare step as a cross-runner port-offset mutex). Remove it to free the offset. + sudo docker network rm "${PORT_RESERVATION}" >/dev/null 2>&1 || true fi @@ -814,7 +842,7 @@ jobs: ( flock -w 10800 9 || { echo "Timed out waiting for the shared Docker build slot" >&2; exit 1; } echo "Acquired Docker build slot for ${DOC_ENGINE}/${API_PROXY_SCHEME}" - sudo docker pull ubuntu:24.04 + ##sudo docker pull ubuntu:24.04 sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} . ) 9>"${BUILD_LOCK_FILE}" @@ -841,7 +869,14 @@ jobs: # Determine runner number (default to 1 if not found) RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true) - RUNNER_NUM=${RUNNER_NUM:-1} + if [[ -z "${RUNNER_NUM:-}" ]]; then + # GitHub self-hosted runners always expose RUNNER_NAME; hash it to spread candidates. + if [[ -n "${RUNNER_NAME:-}" ]]; then + RUNNER_NUM=$(( $(printf '%d' 0x$(echo -n "${RUNNER_NAME}" | md5sum | cut -c1-4)) % 1000 )) + else + RUNNER_NUM=$(( GITHUB_RUN_ID % 1000 )) + fi + fi # Engine-specific offset partitions keep concurrent engine jobs from # choosing the same host ports when they land on the same self-hosted runner. @@ -853,15 +888,28 @@ jobs: infinity) PARTITION_BASE=31000 ;; *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;; esac - PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks - mkdir -p "${PORT_LOCK_DIR}" + # Runners run inside docker containers that share the host docker daemon + # (/var/run/docker.sock), so neither /tmp nor `ss` can coordinate across runners: + # each container has its own /tmp and its own network namespace, and `ss` cannot + # see host ports bound by other runners' compose stacks. Use the shared daemon for + # both primitives: + # - offset mutex = a docker network created atomically on the shared daemon + # (docker network create is NOT idempotent: it fails on name conflict, which + # is exactly the cross-runner test-and-set we need); + # - port availability = docker ps published ports (sees every container on the + # shared daemon regardless of which runner started it). + # No file-system sharing required, no host netns required. + PORT_GUARD_PREFIX=ragflow-ci-portguard port_offset_available() { local offset=$1 - local base port + local base port used + # Enumerate host-published tcp ports across ALL containers on the shared daemon. + # Format example: "0.0.0.0:38217->4222/tcp". We extract the left side. + used=$(sudo docker ps --format '{{.Ports}}' | grep -oE '[0-9]+->' | tr -d '->' | sort -u) for base in "${PORT_BASES[@]}"; do port=$((base + offset)) - if ss -ltnH "sport = :${port}" | grep -q .; then + if grep -qx "${port}" <<< "${used}"; then return 1 fi done @@ -869,31 +917,37 @@ jobs: } cleanup_stale_port_locks() { - local now stale_after lock lock_ts + local now stale_after net ts now=$(date -u +%s) - stale_after=$((6 * 60 * 60)) - for lock in "${PORT_LOCK_DIR}"/*.lock; do - [[ -e "${lock}" ]] || continue - lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true) - if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then - rm -f "${lock}" + stale_after=$((2 * 60 * 60)) + for net in $(sudo docker network ls --filter "label=ragflow-ci-created" --format '{{.Name}}'); do + ts=$(sudo docker network inspect --format '{{ index .Labels "ragflow-ci-created" }}' "${net}" 2>/dev/null || true) + if [[ "${ts}" =~ ^[0-9]+$ ]] && (( now - ts > stale_after )); then + sudo docker network rm "${net}" >/dev/null 2>&1 || true fi done } reserve_port_offset() { - local attempt candidate reservation + local attempt candidate guard cleanup_stale_port_locks - for attempt in $(seq 0 59); do + for attempt in $(seq 0 5999); do candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) )) - reservation="${PORT_LOCK_DIR}/${candidate}.lock" - if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then - if port_offset_available "${candidate}"; then - PORT_OFFSET=${candidate} - PORT_RESERVATION=${reservation} - return 0 - fi - rm -f "${reservation}" + guard="${PORT_GUARD_PREFIX}-${candidate}" + # Already held by some runner on this shared daemon. + if sudo docker network inspect "${guard}" >/dev/null 2>&1; then + continue + fi + if ! port_offset_available "${candidate}"; then + continue + fi + # Atomic cross-runner test-and-set: docker network create fails if the name + # already exists on the shared daemon, so even if two runners race here, only + # one wins; the loser retries the next candidate. + if sudo docker network create --label ragflow-ci-created="$(date -u +%s)" "${guard}" >/dev/null 2>&1; then + PORT_OFFSET=${candidate} + PORT_RESERVATION="${guard}" + return 0 fi done return 1 @@ -1193,5 +1247,7 @@ jobs: sudo docker rmi -f ${RAGFLOW_IMAGE} fi if [[ -n ${PORT_RESERVATION:-} ]]; then - rm -f "${PORT_RESERVATION}" + # PORT_RESERVATION is now a docker network name on the shared daemon (created by the + # prepare step as a cross-runner port-offset mutex). Remove it to free the offset. + sudo docker network rm "${PORT_RESERVATION}" >/dev/null 2>&1 || true fi