#!/usr/bin/env python3 # PEP 723 metadata # /// script # requires-python = ">=3.10" # dependencies = [ # "nltk", # "huggingface-hub" # ] # /// # This script downloads every artifact that the `infiniflow/ragflow_deps` # Docker image bakes in. Run it from anywhere — the `__main__` block # chdir's into this file's own directory, so all outputs land under # `ragflow_deps/` regardless of the caller's CWD. # # Build-context relationship: `ragflow_deps/Dockerfile` is built with # `ragflow_deps/` as its build context, so the files written here MUST # sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume # top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`, # `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-`). # # Typical workflow: # # uv run ragflow_deps/download_deps.py # download # cd ragflow_deps # docker build -f Dockerfile -t infiniflow/ragflow_deps . # # The main `Dockerfile` (built from the project root) pulls this image # via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and # is unaffected by where these files live locally. import argparse import os import urllib.request from typing import Union import nltk from huggingface_hub import snapshot_download def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]: if use_china_mirrors: return [ "http://mirrors.tuna.tsinghua.edu.cn/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", "http://mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar", "https://repo.huaweicloud.com/repository/maven/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5", "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", ["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"], ["https://registry.npmmirror.com/-/binary/chrome-for-testing/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"], "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz", "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz", # stagehand-server-v3 Node.js SEA binaries (used by Browser # component in local mode). # # The stagehand-go Go module (pinned in go.mod) and the # stagehand-server binary (this release) are LOOSELY # MATCHED — both stay on the v3.x line and remain # protocol-compatible. The two version numbers do NOT # track each other: the Go SDK is at v3.21.0 while the # current latest server release is v3.7.2. # # On every go.mod bump, refresh this URL to the current # latest server release. There is no version # correspondence to maintain; "both on v3.x" is the # compatibility contract. "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", ] else: return [ "http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb", "http://ports.ubuntu.com/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_arm64.deb", "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar", "https://repo1.maven.org/maven2/org/apache/tika/tika-server-standard/3.3.0/tika-server-standard-3.3.0.jar.md5", "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken", ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chrome-linux64.zip", "chrome-linux64-121-0-6167-85"], ["https://storage.googleapis.com/chrome-for-testing-public/121.0.6167.85/linux64/chromedriver-linux64.zip", "chromedriver-linux64-121-0-6167-85"], "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-x86_64-unknown-linux-gnu.tar.gz", "https://github.com/astral-sh/uv/releases/download/0.9.16/uv-aarch64-unknown-linux-gnu.tar.gz", # stagehand-server-v3 Node.js SEA binaries (used by Browser # component in local mode). # # The stagehand-go Go module (pinned in go.mod) and the # stagehand-server binary (this release) are LOOSELY # MATCHED — both stay on the v3.x line and remain # protocol-compatible. The two version numbers do NOT # track each other: the Go SDK is at v3.21.0 while the # current latest server release is v3.7.2. # # On every go.mod bump, refresh this URL to the current # latest server release. There is no version # correspondence to maintain; "both on v3.x" is the # compatibility contract. "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64", "https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64", ] repos = [ "InfiniFlow/text_concat_xgb_v1.0", "InfiniFlow/deepdoc", ] def download_model(repository_id): local_directory = os.path.abspath(os.path.join("huggingface.co", repository_id)) os.makedirs(local_directory, exist_ok=True) snapshot_download(repo_id=repository_id, local_dir=local_directory) if __name__ == "__main__": # Anchor CWD to this file's directory so all relative outputs # (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land # at the top of ragflow_deps/ regardless of where the user invokes # the script from. This is the build context for `ragflow_deps/Dockerfile`. os.chdir(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support") parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads") args = parser.parse_args() urls = get_urls(args.china_mirrors) # Some mirrors (e.g. archive.ubuntu.com) reject the default urllib # User-Agent with HTTP 403, so install an opener with a browser-like UA. opener = urllib.request.build_opener() opener.addheaders = [("User-Agent", "Mozilla/5.0")] urllib.request.install_opener(opener) for url in urls: download_url = url[0] if isinstance(url, list) else url filename = url[1] if isinstance(url, list) else url.split("/")[-1] print(f"Downloading {filename} from {download_url}...") if not os.path.exists(filename): urllib.request.urlretrieve(download_url, filename) local_dir = os.path.abspath("nltk_data") for data in ["wordnet", "punkt", "punkt_tab"]: print(f"Downloading nltk {data}...") nltk.download(data, download_dir=local_dir) for repo_id in repos: print(f"Downloading huggingface repo {repo_id}...") download_model(repo_id)