mirror of
https://github.com/cocoindex-io/cocoindex-code.git
synced 2026-09-14 16:39:38 +08:00
6f84edca2e
* perf(docker): split install into stable deps + per-release layers; add GHA cache
Dockerfile previously installed cocoindex-code, cocoindex, torch,
sentence-transformers, and all transitive deps in one RUN. Any change to
the source tree (via COPY . /ccc-src) invalidated that single layer,
forcing a full re-install — ~1 GB of wheels for torch + friends — on
every release. Under QEMU for the arm64 cross-build this was slow
enough to be painful.
Split into two stages:
- `deps`: install cocoindex + cocoindex-code[default] from PyPI. Cache
key is just the RUN command string, so this layer is reused across
releases until we bump the pins.
- `builder`: overlay the release version via
`CCC_INSTALL_SPEC=/ccc-src[default]` with `--no-deps
--force-reinstall` — only the cocoindex-code package is touched; the
heavy deps layer stays untouched.
Also add BuildKit layer cache (`type=gha`) to the publish-docker job so
the deps layer persists across workflow runs, not just within a single
build.
* feat(docker,packaging): slim/full image variants; rename [default]→[full] extra
Build two Docker image variants per release:
- slim (:latest, default) — ~450 MB. LiteLLM-only. cocoindex + cocoindex-code
without sentence-transformers. Targets cloud-backed embeddings.
- full (:full) — ~5 GB. Bundles sentence-transformers + torch +
a pre-baked default model. Targets offline-ready local embeddings.
Dockerfile gains a CCC_VARIANT build arg that gates stage 1's
sentence-transformers install and stage 3's model bake. Release workflow
matrices on {slim, full}; each variant has its own GHA cache scope so
layer reuse works across releases without the variants evicting each
other.
Also rename the PyPI `[default]` umbrella extra to `[full]` so pip and
Docker names match. `[embeddings-local]` remains the canonical primary
extra (the one that specifically pulls in sentence-transformers); `[full]`
is its umbrella alias that may bundle additional optional niceties later.
CLI hints that point at missing sentence-transformers continue to name
`[embeddings-local]` directly — the most specific pointer for that case.
README documents both image variants with a comparison table and narrows
the Mac-on-Docker MPS note to only :full users (slim + LiteLLM is
unaffected).
123 lines
3.6 KiB
Python
123 lines
3.6 KiB
Python
"""Shared fixtures for Dockerized end-to-end tests.
|
|
|
|
All tests here are gated behind the ``docker_e2e`` pytest marker AND a
|
|
``skipif not docker_available()`` so that missing Docker on the host skips
|
|
cleanly instead of failing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
import uuid
|
|
from collections.abc import Iterator
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile"
|
|
FIXTURE_PROJECT = REPO_ROOT / "tests" / "e2e_docker_fixtures" / "sample_project"
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_image() -> str:
|
|
"""Build the image once per test session, installing cocoindex-code from the
|
|
local source tree (not PyPI) so tests exercise the current changes. Returns the tag.
|
|
"""
|
|
# Tests exercise the `full` variant so `ccc init -f` in non-TTY mode can
|
|
# fall back to sentence-transformers (the slim variant requires
|
|
# `--litellm-model`, which would add setup boilerplate to every test).
|
|
tag = "cocoindex-code:pytest"
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"build",
|
|
"-f",
|
|
str(DOCKERFILE),
|
|
"--build-arg",
|
|
"CCC_VARIANT=full",
|
|
"--build-arg",
|
|
"CCC_INSTALL_SPEC=/ccc-src[full]",
|
|
"-t",
|
|
tag,
|
|
str(REPO_ROOT),
|
|
],
|
|
check=True,
|
|
)
|
|
return tag
|
|
|
|
|
|
@pytest.fixture()
|
|
def fixture_workspace(tmp_path: Path) -> Path:
|
|
"""A fresh copy of the sample project, bind-mountable into the container.
|
|
|
|
Each test gets its own copy so that one test's index state / settings
|
|
don't bleed into another.
|
|
"""
|
|
dst = tmp_path / "workspace"
|
|
shutil.copytree(FIXTURE_PROJECT, dst)
|
|
return dst
|
|
|
|
|
|
@pytest.fixture()
|
|
def container(
|
|
docker_image: str,
|
|
fixture_workspace: Path,
|
|
) -> Iterator[str]:
|
|
"""Start a fresh container with the sample project bind-mounted at /workspace.
|
|
|
|
Uses an anonymous cocoindex-data volume so each test starts with a clean
|
|
DB / model cache (the image's copy-up populates the cache from the
|
|
baked-in path).
|
|
"""
|
|
name = f"ccc-e2e-{uuid.uuid4().hex[:12]}"
|
|
host_ws = str(fixture_workspace)
|
|
try:
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"-d",
|
|
"--rm",
|
|
"--name",
|
|
name,
|
|
"-v",
|
|
f"{host_ws}:/workspace",
|
|
"-v",
|
|
"/var/cocoindex", # anonymous volume per-test
|
|
"-e",
|
|
f"COCOINDEX_CODE_HOST_PATH_MAPPING=/workspace={host_ws}",
|
|
docker_image,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
# Poll for the daemon socket so we know startup finished.
|
|
_wait_for_daemon_ready(name, timeout=30.0)
|
|
yield name
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
|
|
|
|
|
|
def _wait_for_daemon_ready(container_name: str, timeout: float) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
container_name,
|
|
"sh",
|
|
"-c",
|
|
"test -S /var/run/cocoindex_code/daemon.sock",
|
|
],
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
if result.returncode == 0:
|
|
return
|
|
time.sleep(0.5)
|
|
raise TimeoutError(f"Daemon in {container_name} did not become ready within {timeout}s")
|