mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-28 03:38:11 +08:00
161 lines
7.6 KiB
Python
161 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# PEP 723 metadata
|
|
# /// script
|
|
# requires-python = ">=3.10"
|
|
# dependencies = [
|
|
# "nltk",
|
|
# "huggingface-hub"
|
|
# ]
|
|
# ///
|
|
|
|
# This script downloads every artifact that the `infiniflow/ragflow_deps`
|
|
# Docker image bakes in. Run it from anywhere — the `__main__` block
|
|
# chdir's into this file's own directory, so all outputs land under
|
|
# `ragflow_deps/` regardless of the caller's CWD.
|
|
#
|
|
# Build-context relationship: `ragflow_deps/Dockerfile` is built with
|
|
# `ragflow_deps/` as its build context, so the files written here MUST
|
|
# sit at the top of `ragflow_deps/`. The Dockerfile's COPY lines assume
|
|
# top-level paths (`huggingface.co`, `nltk_data`, `cl100k_base.tiktoken`,
|
|
# `*.deb`, `*.jar`, `*.tar.gz`, `stagehand-server-v3-linux-<arch>`).
|
|
#
|
|
# Typical workflow:
|
|
#
|
|
# uv run python3 ragflow_deps/download_deps.py # download
|
|
# cd ragflow_deps
|
|
# docker build -f Dockerfile -t infiniflow/ragflow_deps .
|
|
#
|
|
# The main `Dockerfile` (built from the project root) pulls this image
|
|
# via `--mount=type=bind,from=infiniflow/ragflow_deps:latest,...` and
|
|
# is unaffected by where these files live locally.
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import requests
|
|
from typing import Union
|
|
|
|
|
|
def get_urls(use_china_mirrors=False) -> list[Union[str, list[str]]]:
|
|
if use_china_mirrors:
|
|
return [
|
|
# stagehand-server-v3 Node.js SEA binaries (used by Browser
|
|
# component in local mode).
|
|
#
|
|
# The stagehand-go Go module (pinned in go.mod) and the
|
|
# stagehand-server binary (this release) are LOOSELY
|
|
# MATCHED — both stay on the v3.x line and remain
|
|
# protocol-compatible. The two version numbers do NOT
|
|
# track each other: the Go SDK is at v3.21.0 while the
|
|
# current latest server release is v3.7.2.
|
|
#
|
|
# On every go.mod bump, refresh this URL to the current
|
|
# latest server release. There is no version
|
|
# correspondence to maintain; "both on v3.x" is the
|
|
# compatibility contract.
|
|
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
|
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
|
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
|
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
|
# network access during CI.
|
|
["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
|
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
|
["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.3/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
|
]
|
|
else:
|
|
return [
|
|
# stagehand-server-v3 Node.js SEA binaries (used by Browser
|
|
# component in local mode).
|
|
#
|
|
# The stagehand-go Go module (pinned in go.mod) and the
|
|
# stagehand-server binary (this release) are LOOSELY
|
|
# MATCHED — both stay on the v3.x line and remain
|
|
# protocol-compatible. The two version numbers do NOT
|
|
# track each other: the Go SDK is at v3.21.0 while the
|
|
# current latest server release is v3.7.2.
|
|
#
|
|
# On every go.mod bump, refresh this URL to the current
|
|
# latest server release. There is no version
|
|
# correspondence to maintain; "both on v3.x" is the
|
|
# compatibility contract.
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
|
|
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
|
|
# Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
|
|
# Used by build.sh's check_*_deps functions — pre-downloaded to avoid
|
|
# network access during CI.
|
|
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
|
|
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
|
|
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.3/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
|
|
]
|
|
|
|
|
|
def download_with_progress(url, filename):
|
|
response = requests.get(url, stream=True)
|
|
total_size = int(response.headers.get("content-length", 0))
|
|
block_size = 1024
|
|
|
|
with open(filename, "wb") as file:
|
|
downloaded = 0
|
|
for data in response.iter_content(block_size):
|
|
file.write(data)
|
|
downloaded += len(data)
|
|
|
|
if total_size > 0:
|
|
progress = (downloaded / total_size) * 100
|
|
sys.stdout.write(f"\rProgress: {progress:.1f}% ({downloaded}/{total_size} bytes)")
|
|
sys.stdout.flush()
|
|
|
|
print()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Anchor CWD to this file's directory so all relative outputs
|
|
# (huggingface.co/, nltk_data/, *.deb, *.jar, *.tar.gz, etc.) land
|
|
# at the top of ragflow_deps/ regardless of where the user invokes
|
|
# the script from. This is the build context for `ragflow_deps/Dockerfile`.
|
|
os.chdir(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
parser = argparse.ArgumentParser(description="Download dependencies with optional China mirror support")
|
|
parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads")
|
|
args = parser.parse_args()
|
|
|
|
urls = get_urls(args.china_mirrors)
|
|
|
|
# Some mirrors (e.g. archive.ubuntu.com) reject the default urllib
|
|
# User-Agent with HTTP 403, so install an opener with a browser-like UA.
|
|
# opener = urllib.request.build_opener()
|
|
# opener.addheaders = [("User-Agent", "Mozilla/5.0")]
|
|
# urllib.request.install_opener(opener)
|
|
|
|
for url in urls:
|
|
download_url = url[0] if isinstance(url, list) else url
|
|
filename = url[1] if isinstance(url, list) else url.split("/")[-1]
|
|
print(f"Downloading {filename} from {download_url}...")
|
|
if not os.path.exists(filename):
|
|
download_with_progress(download_url, filename)
|
|
|
|
# Extract native static libraries to ~/ragflow-native-libs for Go build.
|
|
# Ensures build.sh can find them without network access.
|
|
native_deps_dir = os.path.expanduser("~/ragflow-native-libs")
|
|
extractions = [
|
|
("pdfium-linux-x64-static.tgz", "pdfium-static"),
|
|
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
|
|
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
|
|
]
|
|
import tarfile
|
|
|
|
for archive, subdir in extractions:
|
|
archive_path = os.path.join(os.getcwd(), archive)
|
|
if not os.path.isfile(archive_path):
|
|
print(f" Skipping extraction: {archive} not found")
|
|
continue
|
|
target = os.path.join(native_deps_dir, subdir)
|
|
if os.path.isdir(target):
|
|
print(f" ✓ {subdir} already extracted to {target}")
|
|
continue
|
|
os.makedirs(target, exist_ok=True)
|
|
print(f" Extracting {archive} → {target}")
|
|
with tarfile.open(archive_path) as tf:
|
|
tf.extractall(target)
|