Files
Jiangzhou 6f84edca2e feat(docker): slim/full image variants + cached deps layer + [full] extra rename (#138)
* perf(docker): split install into stable deps + per-release layers; add GHA cache

Dockerfile previously installed cocoindex-code, cocoindex, torch,
sentence-transformers, and all transitive deps in one RUN. Any change to
the source tree (via COPY . /ccc-src) invalidated that single layer,
forcing a full re-install — ~1 GB of wheels for torch + friends — on
every release. Under QEMU for the arm64 cross-build this was slow
enough to be painful.

Split into two stages:
- `deps`: install cocoindex + cocoindex-code[default] from PyPI. Cache
  key is just the RUN command string, so this layer is reused across
  releases until we bump the pins.
- `builder`: overlay the release version via
  `CCC_INSTALL_SPEC=/ccc-src[default]` with `--no-deps
  --force-reinstall` — only the cocoindex-code package is touched; the
  heavy deps layer stays untouched.

Also add BuildKit layer cache (`type=gha`) to the publish-docker job so
the deps layer persists across workflow runs, not just within a single
build.

* feat(docker,packaging): slim/full image variants; rename [default]→[full] extra

Build two Docker image variants per release:
- slim (:latest, default) — ~450 MB. LiteLLM-only. cocoindex + cocoindex-code
  without sentence-transformers. Targets cloud-backed embeddings.
- full (:full)            — ~5 GB. Bundles sentence-transformers + torch +
  a pre-baked default model. Targets offline-ready local embeddings.

Dockerfile gains a CCC_VARIANT build arg that gates stage 1's
sentence-transformers install and stage 3's model bake. Release workflow
matrices on {slim, full}; each variant has its own GHA cache scope so
layer reuse works across releases without the variants evicting each
other.

Also rename the PyPI `[default]` umbrella extra to `[full]` so pip and
Docker names match. `[embeddings-local]` remains the canonical primary
extra (the one that specifically pulls in sentence-transformers); `[full]`
is its umbrella alias that may bundle additional optional niceties later.
CLI hints that point at missing sentence-transformers continue to name
`[embeddings-local]` directly — the most specific pointer for that case.

README documents both image variants with a comparison table and narrows
the Mac-on-Docker MPS note to only :full users (slim + LiteLLM is
unaffected).
2026-04-14 16:25:42 -07:00

123 lines
3.6 KiB
Python

"""Shared fixtures for Dockerized end-to-end tests.
All tests here are gated behind the ``docker_e2e`` pytest marker AND a
``skipif not docker_available()`` so that missing Docker on the host skips
cleanly instead of failing.
"""
from __future__ import annotations
import shutil
import subprocess
import time
import uuid
from collections.abc import Iterator
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[2]
DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile"
FIXTURE_PROJECT = REPO_ROOT / "tests" / "e2e_docker_fixtures" / "sample_project"
@pytest.fixture(scope="session")
def docker_image() -> str:
"""Build the image once per test session, installing cocoindex-code from the
local source tree (not PyPI) so tests exercise the current changes. Returns the tag.
"""
# Tests exercise the `full` variant so `ccc init -f` in non-TTY mode can
# fall back to sentence-transformers (the slim variant requires
# `--litellm-model`, which would add setup boilerplate to every test).
tag = "cocoindex-code:pytest"
subprocess.run(
[
"docker",
"build",
"-f",
str(DOCKERFILE),
"--build-arg",
"CCC_VARIANT=full",
"--build-arg",
"CCC_INSTALL_SPEC=/ccc-src[full]",
"-t",
tag,
str(REPO_ROOT),
],
check=True,
)
return tag
@pytest.fixture()
def fixture_workspace(tmp_path: Path) -> Path:
"""A fresh copy of the sample project, bind-mountable into the container.
Each test gets its own copy so that one test's index state / settings
don't bleed into another.
"""
dst = tmp_path / "workspace"
shutil.copytree(FIXTURE_PROJECT, dst)
return dst
@pytest.fixture()
def container(
docker_image: str,
fixture_workspace: Path,
) -> Iterator[str]:
"""Start a fresh container with the sample project bind-mounted at /workspace.
Uses an anonymous cocoindex-data volume so each test starts with a clean
DB / model cache (the image's copy-up populates the cache from the
baked-in path).
"""
name = f"ccc-e2e-{uuid.uuid4().hex[:12]}"
host_ws = str(fixture_workspace)
try:
subprocess.run(
[
"docker",
"run",
"-d",
"--rm",
"--name",
name,
"-v",
f"{host_ws}:/workspace",
"-v",
"/var/cocoindex", # anonymous volume per-test
"-e",
f"COCOINDEX_CODE_HOST_PATH_MAPPING=/workspace={host_ws}",
docker_image,
],
check=True,
capture_output=True,
)
# Poll for the daemon socket so we know startup finished.
_wait_for_daemon_ready(name, timeout=30.0)
yield name
finally:
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
def _wait_for_daemon_ready(container_name: str, timeout: float) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = subprocess.run(
[
"docker",
"exec",
container_name,
"sh",
"-c",
"test -S /var/run/cocoindex_code/daemon.sock",
],
capture_output=True,
check=False,
)
if result.returncode == 0:
return
time.sleep(0.5)
raise TimeoutError(f"Daemon in {container_name} did not become ready within {timeout}s")