fix ci unit test not found in installed package 'wordnet' (#19251)

api/validation.py

@@ -33,8 +33,16 @@ python_version_validation()
 
 # Download nltk data
 def download_nltk_data():
+    import os
+
+    # NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in.
+    os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
     import nltk
 
+    # NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` package, so
+    # both must be present or tokenization-backed paths raise LookupError.
+    nltk.download("omw-1.4", halt_on_error=False, quiet=True)
     nltk.download("wordnet", halt_on_error=False, quiet=True)
     nltk.download("punkt_tab", halt_on_error=False, quiet=True)
 

ragflow_deps/download_deps.py

@@ -35,6 +35,10 @@ import os
 import shutil
 import urllib.request
 
+# NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in; the
+# runners sit behind a proxy, so allow proxied fetches before importing nltk.
+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
 import nltk
 from huggingface_hub import snapshot_download
 
@@ -254,7 +258,9 @@ if __name__ == "__main__":
         print(f"  Skipping onnxruntime static check: no .a found under {ort_static_dir}")
 
     local_dir = os.path.abspath("nltk_data")
-    for data in ["wordnet", "punkt", "punkt_tab"]:
+    # NLTK >=3.8.2 gates `wordnet` behind `omw-1.4`; both must be provisioned
+    # or tokenization-backed paths raise LookupError at runtime.
+    for data in ["omw-1.4", "wordnet", "punkt", "punkt_tab"]:
         print(f"Downloading nltk {data}...")
         nltk.download(data, download_dir=local_dir)
 

run_tests.py

@@ -181,7 +181,10 @@ EXAMPLES:
 
     def build_pytest_command(self) -> List[str]:
         """Build the pytest command arguments"""
-        cmd = ["pytest"]
+        # Use the current interpreter (the venv python) to run pytest as a
+        # module so the run works even when the `pytest` console script is not
+        # on PATH (e.g. the venv was not explicitly activated).
+        cmd = [self.python, "-m", "pytest"]
         if self.test_path:
             test_target = Path(self.test_path)
             if not test_target.is_absolute():

test/unit_test/conftest.py

@@ -29,22 +29,42 @@ when present, and download only what is still missing.
 
 import os
 
+# NLTK >=3.10 refuses proxied downloads (SSRF guard, CWE-918) by default. The
+# CI runners and some dev boxes sit behind a proxy, so opt in to proxied fetches
+# before importing nltk; otherwise `nltk.download` fails with a Security
+# Violation. Must be set before the `import nltk` below so pathsec reads it.
+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
 import nltk
+import warnings
 
 # Reuse data already fetched by download_deps.py (the directory the app exports
-# as NLTK_DATA) so provisioned environments do not download it again.
+# as NLTK_DATA) so provisioned environments do not download it again. Create it
+# if absent so the fallback download below lands in a repo-local, reproducible
+# location instead of a shared home directory.
 _REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
 _LOCAL_NLTK_DATA = os.path.join(_REPO_ROOT, "ragflow_deps", "nltk_data")
-if os.path.isdir(_LOCAL_NLTK_DATA) and _LOCAL_NLTK_DATA not in nltk.data.path:
+os.makedirs(_LOCAL_NLTK_DATA, exist_ok=True)
+if _LOCAL_NLTK_DATA not in nltk.data.path:
     nltk.data.path.insert(0, _LOCAL_NLTK_DATA)
 
 # (download name, resource path used by nltk.data.find)
+# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data
+# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises
+# LookupError at load time; `omw-1.4` must also be present.
 _REQUIRED_NLTK_DATA = (
     ("punkt_tab", "tokenizers/punkt_tab"),
     ("wordnet", "corpora/wordnet"),
+    ("omw-1.4", "corpora/omw-1.4"),
 )
 for _name, _find_path in _REQUIRED_NLTK_DATA:
     try:
         nltk.data.find(_find_path)
     except LookupError:
-        nltk.download(_name, quiet=True)
+        # On shared CI runners the download dir is often group-writable, which
+        # makes NLTK emit a "non-private download directory" UserWarning. pytest
+        # escalates warnings to errors (filterwarnings = error), so suppress it
+        # for the duration of the download and fetch into the repo-local dir.
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", UserWarning)
+            nltk.download(_name, download_dir=_LOCAL_NLTK_DATA, quiet=True)