mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-08-25 02:20:18 +08:00
Rebuild stale asset records when the assets system is enabled
Alembic migrates the shape of the assets tables. Nothing migrated their meaning, so a row could be structurally current and still hold values a superseded rule computed, with no maintenance path that ever repaired it. loader_path is the clearest case: the column was added to existing databases with no backfill, and it is only ever written when a reference is first created. The scan computes it for paths it has not seen before, so every reference older than the column serves a null loader path forever -- while the API tells clients to prefer loader_path over name. A semantics version now records which generation of the derivation logic produced a database's rows, tracked separately from the Alembic schema version because the two move independently. Reset steps are numbered and applied in order from the stored version, each stamped only once it finishes, so an interrupted run resumes instead of half-applying. The first step re-derives what a file's location implies -- loader_path, the backend tags a path carries, and whether the file is there -- and leaves everything else alone. It reads no file contents: verify_file_unchanged says whether a row's recorded hash and size still describe the file, and a file that has moved on is handed to the existing needs_verify path rather than re-read, so an untouched model library costs one stat per file and no hashing. Manual tags, user metadata, previews, deletions and job ids are never touched, nor are references whose file is gone or whose path falls outside every root this install currently knows about. It runs at the start of a scan, before anything reads or extends those rows. A database already at the current version costs one indexed row read.
This commit is contained in:
Simon Pinfold
672
tests-unit/assets_test/test_semantics_reset.py
Normal file
672
tests-unit/assets_test/test_semantics_reset.py
Normal file
@@ -0,0 +1,672 @@
|
||||
"""Tests for the asset semantics reset (app/assets/semantics).
|
||||
|
||||
Runs standalone against in-memory SQLite:
|
||||
|
||||
pytest tests-unit/assets_test/test_semantics_reset.py --noconftest
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
import app.assets.semantics as semantics
|
||||
from app.assets.database.models import (
|
||||
Asset,
|
||||
AssetReference,
|
||||
AssetReferenceTag,
|
||||
AssetSemanticsVersion,
|
||||
Base,
|
||||
Tag,
|
||||
)
|
||||
from app.assets.database.queries.semantics import get_semantics_version
|
||||
from app.assets.semantics import run_pending_semantics_steps
|
||||
from app.assets.semantics.reproject_derived import reproject_derived_state
|
||||
from app.assets.semantics.step import SemanticsStep, SemanticsStepInterrupted
|
||||
from app.assets.services.file_utils import get_mtime_ns
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def autoclean_unit_test_assets():
|
||||
"""Override the package autouse fixture; these tests need no server."""
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session_factory():
|
||||
"""A session factory over one shared in-memory database.
|
||||
|
||||
StaticPool keeps every session on the same connection, so the batching walk
|
||||
sees its own committed writes across sessions.
|
||||
"""
|
||||
engine = create_engine(
|
||||
"sqlite:///:memory:",
|
||||
poolclass=StaticPool,
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
Base.metadata.create_all(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
with (
|
||||
patch("app.assets.semantics.reproject_derived.create_session", factory),
|
||||
patch("app.assets.semantics.create_session", factory),
|
||||
patch("app.assets.semantics.can_create_session", return_value=True),
|
||||
):
|
||||
yield factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session(session_factory) -> Session:
|
||||
"""A session for the test's own reads and writes."""
|
||||
with session_factory() as sess:
|
||||
yield sess
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def comfy_dirs():
|
||||
"""Point every asset root at a throwaway tree with one model category."""
|
||||
with tempfile.TemporaryDirectory() as base:
|
||||
dirs = {
|
||||
name: Path(base) / name
|
||||
for name in ("checkpoints", "loras", "input", "output", "temp", "elsewhere")
|
||||
}
|
||||
for directory in dirs.values():
|
||||
directory.mkdir()
|
||||
with (
|
||||
patch("folder_paths.get_input_directory", return_value=str(dirs["input"])),
|
||||
patch(
|
||||
"folder_paths.get_output_directory", return_value=str(dirs["output"])
|
||||
),
|
||||
patch("folder_paths.get_temp_directory", return_value=str(dirs["temp"])),
|
||||
patch(
|
||||
"app.assets.services.path_utils.get_comfy_models_folders",
|
||||
return_value=[
|
||||
("checkpoints", [str(dirs["checkpoints"])], {".safetensors"}),
|
||||
("loras", [str(dirs["loras"])], {".safetensors"}),
|
||||
],
|
||||
),
|
||||
):
|
||||
yield dirs
|
||||
|
||||
|
||||
def _write(directory: Path, name: str, content: bytes = b"\x00" * 100) -> str:
|
||||
path = directory / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(content)
|
||||
return os.path.abspath(str(path))
|
||||
|
||||
|
||||
def _register(
|
||||
session: Session,
|
||||
file_path: str,
|
||||
ref_id: str,
|
||||
*,
|
||||
loader_path: str | None = None,
|
||||
asset_hash: str | None = "",
|
||||
size_bytes: int = 100,
|
||||
mtime_ns: int | None = None,
|
||||
is_missing: bool = False,
|
||||
needs_verify: bool = False,
|
||||
user_metadata: dict | None = None,
|
||||
preview_id: str | None = None,
|
||||
deleted_at: datetime | None = None,
|
||||
job_id: str | None = None,
|
||||
tags: dict[str, str] | None = None,
|
||||
) -> AssetReference:
|
||||
"""Insert an Asset + AssetReference (+ tags as {name: origin}) and commit.
|
||||
|
||||
``asset_hash=""`` means "any unique hash"; assets.hash is unique-indexed.
|
||||
"""
|
||||
if asset_hash == "":
|
||||
asset_hash = f"blake3:{ref_id}"
|
||||
if mtime_ns is None:
|
||||
mtime_ns = get_mtime_ns(os.stat(file_path, follow_symlinks=True))
|
||||
|
||||
session.add(Asset(id=f"asset-{ref_id}", hash=asset_hash, size_bytes=size_bytes))
|
||||
session.flush()
|
||||
ref = AssetReference(
|
||||
id=ref_id,
|
||||
asset_id=f"asset-{ref_id}",
|
||||
name=os.path.basename(file_path),
|
||||
owner_id="",
|
||||
file_path=file_path,
|
||||
loader_path=loader_path,
|
||||
mtime_ns=mtime_ns,
|
||||
is_missing=is_missing,
|
||||
needs_verify=needs_verify,
|
||||
user_metadata=user_metadata,
|
||||
preview_id=preview_id,
|
||||
deleted_at=deleted_at,
|
||||
job_id=job_id,
|
||||
)
|
||||
session.add(ref)
|
||||
session.flush()
|
||||
|
||||
for tag_name, origin in (tags or {}).items():
|
||||
session.merge(Tag(name=tag_name))
|
||||
session.add(
|
||||
AssetReferenceTag(
|
||||
asset_reference_id=ref_id, tag_name=tag_name, origin=origin
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return ref
|
||||
|
||||
|
||||
def _tags(session: Session, ref_id: str) -> dict[str, str]:
|
||||
rows = session.execute(
|
||||
select(AssetReferenceTag.tag_name, AssetReferenceTag.origin).where(
|
||||
AssetReferenceTag.asset_reference_id == ref_id
|
||||
)
|
||||
).all()
|
||||
return {name: origin for name, origin in rows}
|
||||
|
||||
|
||||
def _snapshot(session: Session) -> list[tuple]:
|
||||
"""Every field the reset could plausibly touch, for equality across runs."""
|
||||
session.expire_all()
|
||||
refs = session.execute(select(AssetReference).order_by(AssetReference.id)).scalars()
|
||||
rows = [
|
||||
(
|
||||
ref.id,
|
||||
ref.file_path,
|
||||
ref.loader_path,
|
||||
ref.is_missing,
|
||||
ref.needs_verify,
|
||||
ref.mtime_ns,
|
||||
ref.name,
|
||||
ref.preview_id,
|
||||
str(ref.user_metadata),
|
||||
ref.deleted_at,
|
||||
ref.job_id,
|
||||
tuple(sorted(_tags(session, ref.id).items())),
|
||||
)
|
||||
for ref in refs
|
||||
]
|
||||
assets = session.execute(select(Asset).order_by(Asset.id)).scalars()
|
||||
rows.extend((asset.id, asset.hash, asset.size_bytes) for asset in assets)
|
||||
return rows
|
||||
|
||||
|
||||
class TestLoaderPathReprojection:
|
||||
def test_null_loader_path_is_backfilled(self, session, comfy_dirs):
|
||||
"""The drift 0006 left behind: a column added with no backfill."""
|
||||
path = _write(comfy_dirs["checkpoints"], "flux/model.safetensors")
|
||||
_register(session, path, "ref-1", loader_path=None)
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
ref = session.get(AssetReference, "ref-1")
|
||||
assert ref.loader_path == "flux/model.safetensors"
|
||||
|
||||
def test_stale_loader_path_is_rewritten(self, session, comfy_dirs):
|
||||
path = _write(comfy_dirs["input"], "sub/photo.png")
|
||||
_register(session, path, "ref-1", loader_path="checkpoints/sub/photo.png")
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-1").loader_path == "sub/photo.png"
|
||||
assert summary.loader_paths_rewritten == 1
|
||||
|
||||
def test_correct_loader_path_is_left_alone(self, session, comfy_dirs):
|
||||
path = _write(comfy_dirs["loras"], "style.safetensors")
|
||||
_register(session, path, "ref-1", loader_path="style.safetensors")
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
assert summary.loader_paths_rewritten == 0
|
||||
|
||||
def test_unloadable_extension_loses_its_loader_path(self, session, comfy_dirs):
|
||||
"""The current rule gives no loader path to a file its category cannot load."""
|
||||
path = _write(comfy_dirs["checkpoints"], "notes.txt")
|
||||
_register(session, path, "ref-1", loader_path="notes.txt")
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-1").loader_path is None
|
||||
|
||||
def test_reference_without_file_path_is_skipped(self, session, comfy_dirs):
|
||||
session.add(Asset(id="asset-api", hash="blake3:abc", size_bytes=10))
|
||||
session.flush()
|
||||
session.add(
|
||||
AssetReference(
|
||||
id="ref-api", asset_id="asset-api", name="api.png", owner_id=""
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
assert summary.scanned == 0
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-api").loader_path is None
|
||||
|
||||
|
||||
class TestHashPreservation:
|
||||
def test_unchanged_file_keeps_its_hash_and_is_not_read(self, session, comfy_dirs):
|
||||
path = _write(comfy_dirs["checkpoints"], "big.safetensors")
|
||||
_register(session, path, "ref-1", asset_hash="blake3:original", size_bytes=100)
|
||||
|
||||
with patch(
|
||||
"app.assets.services.hashing.compute_blake3_hash",
|
||||
side_effect=AssertionError("the reset must never re-hash"),
|
||||
):
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(Asset, "asset-ref-1").hash == "blake3:original"
|
||||
assert summary.unchanged_files == 1
|
||||
assert summary.changed_files == 0
|
||||
|
||||
def test_changed_file_keeps_its_hash_and_is_flagged_for_verify(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
"""A file that moved on is handed to the existing verify path, not re-read."""
|
||||
path = _write(comfy_dirs["checkpoints"], "big.safetensors")
|
||||
_register(
|
||||
session,
|
||||
path,
|
||||
"ref-1",
|
||||
asset_hash="blake3:original",
|
||||
mtime_ns=1,
|
||||
size_bytes=100,
|
||||
)
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(Asset, "asset-ref-1").hash == "blake3:original"
|
||||
assert session.get(Asset, "asset-ref-1").size_bytes == 100
|
||||
assert session.get(AssetReference, "ref-1").needs_verify is True
|
||||
assert summary.changed_files == 1
|
||||
|
||||
def test_changed_file_still_gets_its_path_state_reprojected(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
path = _write(comfy_dirs["loras"], "style.safetensors")
|
||||
_register(session, path, "ref-1", loader_path=None, mtime_ns=1)
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-1").loader_path == "style.safetensors"
|
||||
|
||||
|
||||
class TestIntentIsPreserved:
|
||||
def test_manual_tags_metadata_preview_and_deletion_survive(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
path = _write(comfy_dirs["checkpoints"], "curated.safetensors")
|
||||
other = _write(comfy_dirs["input"], "thumb.png")
|
||||
_register(session, other, "ref-preview")
|
||||
deleted = datetime(2026, 1, 2, 3, 4, 5)
|
||||
_register(
|
||||
session,
|
||||
path,
|
||||
"ref-1",
|
||||
loader_path=None,
|
||||
user_metadata={"note": "hand written", "filename": "kept.safetensors"},
|
||||
preview_id="ref-preview",
|
||||
deleted_at=deleted,
|
||||
job_id="job-42",
|
||||
tags={"favourite": "manual", "uploaded": "upload"},
|
||||
)
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
ref = session.get(AssetReference, "ref-1")
|
||||
assert ref.user_metadata == {
|
||||
"note": "hand written",
|
||||
"filename": "kept.safetensors",
|
||||
}
|
||||
assert ref.preview_id == "ref-preview"
|
||||
assert ref.deleted_at == deleted
|
||||
assert ref.job_id == "job-42"
|
||||
assert ref.name == "curated.safetensors"
|
||||
tags = _tags(session, "ref-1")
|
||||
assert tags["favourite"] == "manual"
|
||||
assert tags["uploaded"] == "upload"
|
||||
# ...while the derived state around them was still brought forward.
|
||||
assert ref.loader_path == "curated.safetensors"
|
||||
|
||||
def test_manual_tag_inside_the_derived_vocabulary_is_not_removed(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
"""A person may tag an input file 'models'; that is their business."""
|
||||
path = _write(comfy_dirs["input"], "photo.png")
|
||||
_register(session, path, "ref-1", tags={"models": "manual"})
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
assert _tags(session, "ref-1")["models"] == "manual"
|
||||
|
||||
|
||||
class TestTagReprojection:
|
||||
def test_missing_derived_tags_are_added(self, session, comfy_dirs):
|
||||
path = _write(comfy_dirs["checkpoints"], "model.safetensors")
|
||||
_register(session, path, "ref-1", tags={})
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
assert _tags(session, "ref-1") == {
|
||||
"models": "automatic",
|
||||
"model_type:checkpoints": "automatic",
|
||||
}
|
||||
|
||||
def test_superseded_automatic_tag_is_removed(self, session, comfy_dirs):
|
||||
"""An older rule tagged by directory alone; extensions now gate the tag."""
|
||||
path = _write(comfy_dirs["checkpoints"], "model.safetensors")
|
||||
_register(
|
||||
session,
|
||||
path,
|
||||
"ref-1",
|
||||
tags={
|
||||
"models": "automatic",
|
||||
"model_type:checkpoints": "automatic",
|
||||
"model_type:loras": "automatic",
|
||||
},
|
||||
)
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
assert "model_type:loras" not in _tags(session, "ref-1")
|
||||
assert "model_type:checkpoints" in _tags(session, "ref-1")
|
||||
|
||||
def test_automatic_tag_outside_the_vocabulary_is_left_alone(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
"""'missing' is automatic but not path-derived; the scanner owns it."""
|
||||
path = _write(comfy_dirs["checkpoints"], "model.safetensors")
|
||||
_register(session, path, "ref-1", tags={"missing": "automatic"})
|
||||
|
||||
reproject_derived_state()
|
||||
|
||||
assert "missing" in _tags(session, "ref-1")
|
||||
|
||||
|
||||
class TestFileState:
|
||||
def test_present_file_is_unflagged_as_missing(self, session, comfy_dirs):
|
||||
path = _write(comfy_dirs["temp"], "preview.png")
|
||||
_register(session, path, "ref-1", is_missing=True)
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-1").is_missing is False
|
||||
assert summary.missing_flags_cleared == 1
|
||||
|
||||
def test_absent_file_is_left_to_the_scanner(self, session, comfy_dirs):
|
||||
path = os.path.join(str(comfy_dirs["checkpoints"]), "gone.safetensors")
|
||||
_register(
|
||||
session,
|
||||
path,
|
||||
"ref-1",
|
||||
mtime_ns=1,
|
||||
loader_path="stale/gone.safetensors",
|
||||
is_missing=True,
|
||||
)
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
ref = session.get(AssetReference, "ref-1")
|
||||
assert ref.is_missing is True
|
||||
assert ref.loader_path == "stale/gone.safetensors"
|
||||
assert summary.absent_files == 1
|
||||
|
||||
def test_path_outside_every_known_root_is_left_alone(self, session, comfy_dirs):
|
||||
"""A root missing from the config must not strip tags off its assets."""
|
||||
path = _write(comfy_dirs["elsewhere"], "model.safetensors")
|
||||
_register(
|
||||
session,
|
||||
path,
|
||||
"ref-1",
|
||||
loader_path="model.safetensors",
|
||||
tags={"models": "automatic", "model_type:checkpoints": "automatic"},
|
||||
)
|
||||
|
||||
summary = reproject_derived_state()
|
||||
|
||||
session.expire_all()
|
||||
assert session.get(AssetReference, "ref-1").loader_path == "model.safetensors"
|
||||
assert _tags(session, "ref-1") == {
|
||||
"models": "automatic",
|
||||
"model_type:checkpoints": "automatic",
|
||||
}
|
||||
assert summary.unclassified_paths == 1
|
||||
|
||||
|
||||
class TestIdempotence:
|
||||
def test_empty_database_is_a_no_op(self, session, comfy_dirs):
|
||||
summary = reproject_derived_state()
|
||||
|
||||
assert summary.scanned == 0
|
||||
assert summary.loader_paths_rewritten == 0
|
||||
assert summary.tags_added == 0
|
||||
|
||||
def test_second_run_changes_nothing(self, session, comfy_dirs):
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["checkpoints"], "a/model.safetensors"),
|
||||
"ref-1",
|
||||
loader_path=None,
|
||||
tags={"model_type:loras": "automatic", "favourite": "manual"},
|
||||
)
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["input"], "pasted/clip.png"),
|
||||
"ref-2",
|
||||
loader_path="wrong.png",
|
||||
is_missing=True,
|
||||
)
|
||||
_register(
|
||||
session,
|
||||
os.path.join(str(comfy_dirs["output"]), "gone.png"),
|
||||
"ref-3",
|
||||
mtime_ns=1,
|
||||
)
|
||||
|
||||
reproject_derived_state()
|
||||
after_first = _snapshot(session)
|
||||
|
||||
second = reproject_derived_state()
|
||||
assert _snapshot(session) == after_first
|
||||
assert second.loader_paths_rewritten == 0
|
||||
assert second.tags_added == 0
|
||||
assert second.tags_removed == 0
|
||||
assert second.missing_flags_cleared == 0
|
||||
|
||||
def test_statements_chunked_by_bind_param_limit_stay_correct(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
"""The bind-param chunking must not drop or duplicate a tag."""
|
||||
for index in range(4):
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["checkpoints"], f"m{index}.safetensors"),
|
||||
f"ref-{index}",
|
||||
tags={"model_type:loras": "automatic"},
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.assets.database.queries.semantics.MAX_BIND_PARAMS", 2),
|
||||
patch("app.assets.database.queries.common.MAX_BIND_PARAMS", 2),
|
||||
):
|
||||
reproject_derived_state()
|
||||
|
||||
for index in range(4):
|
||||
assert _tags(session, f"ref-{index}") == {
|
||||
"models": "automatic",
|
||||
"model_type:checkpoints": "automatic",
|
||||
}
|
||||
|
||||
def test_walk_crosses_batch_boundaries(self, session, comfy_dirs):
|
||||
for index in range(7):
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["checkpoints"], f"m{index:02d}.safetensors"),
|
||||
f"ref-{index:02d}",
|
||||
loader_path=None,
|
||||
)
|
||||
|
||||
with patch("app.assets.semantics.reproject_derived._BATCH_SIZE", 2):
|
||||
summary = reproject_derived_state()
|
||||
|
||||
assert summary.scanned == 7
|
||||
session.expire_all()
|
||||
assert all(
|
||||
session.get(AssetReference, f"ref-{index:02d}").loader_path
|
||||
== f"m{index:02d}.safetensors"
|
||||
for index in range(7)
|
||||
)
|
||||
|
||||
|
||||
class TestRunner:
|
||||
def test_pending_step_runs_and_stamps(self, session, comfy_dirs):
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["checkpoints"], "model.safetensors"),
|
||||
"ref-1",
|
||||
loader_path=None,
|
||||
)
|
||||
|
||||
assert run_pending_semantics_steps() == 1
|
||||
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == semantics.CURRENT_SEMANTICS_VERSION
|
||||
assert session.get(AssetReference, "ref-1").loader_path == "model.safetensors"
|
||||
|
||||
def test_stamped_database_does_not_walk_again(self, session, comfy_dirs):
|
||||
run_pending_semantics_steps()
|
||||
|
||||
with patch(
|
||||
"app.assets.semantics.reproject_derived.get_file_backed_references_page",
|
||||
side_effect=AssertionError("a stamped database must not be walked"),
|
||||
):
|
||||
assert run_pending_semantics_steps() == 0
|
||||
|
||||
def test_stamp_is_not_advanced_when_a_step_raises(self, session, comfy_dirs):
|
||||
def _explode(_interrupt_check):
|
||||
raise RuntimeError("step failed")
|
||||
|
||||
with patch.object(
|
||||
semantics,
|
||||
"SEMANTICS_STEPS",
|
||||
(SemanticsStep(version=1, description="explodes", apply=_explode),),
|
||||
):
|
||||
assert run_pending_semantics_steps() == 0
|
||||
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == 0
|
||||
assert session.get(AssetSemanticsVersion, 1) is None
|
||||
|
||||
def test_stamp_is_not_advanced_when_a_step_is_interrupted(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
def _interrupted(_interrupt_check):
|
||||
raise SemanticsStepInterrupted("stopped")
|
||||
|
||||
with patch.object(
|
||||
semantics,
|
||||
"SEMANTICS_STEPS",
|
||||
(SemanticsStep(version=1, description="interrupts", apply=_interrupted),),
|
||||
):
|
||||
assert run_pending_semantics_steps() == 0
|
||||
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == 0
|
||||
|
||||
def test_earlier_steps_stay_stamped_when_a_later_one_fails(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
def _ok(_interrupt_check):
|
||||
return "fine"
|
||||
|
||||
def _explode(_interrupt_check):
|
||||
raise RuntimeError("step failed")
|
||||
|
||||
with patch.object(
|
||||
semantics,
|
||||
"SEMANTICS_STEPS",
|
||||
(
|
||||
SemanticsStep(version=1, description="ok", apply=_ok),
|
||||
SemanticsStep(version=2, description="explodes", apply=_explode),
|
||||
),
|
||||
):
|
||||
assert run_pending_semantics_steps() == 1
|
||||
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == 1
|
||||
|
||||
def test_steps_below_the_stored_version_are_skipped(self, session, comfy_dirs):
|
||||
applied: list[int] = []
|
||||
|
||||
def _record(version):
|
||||
def _apply(_interrupt_check):
|
||||
applied.append(version)
|
||||
return version
|
||||
|
||||
return _apply
|
||||
|
||||
steps = (
|
||||
SemanticsStep(version=1, description="one", apply=_record(1)),
|
||||
SemanticsStep(version=2, description="two", apply=_record(2)),
|
||||
)
|
||||
with patch.object(semantics, "SEMANTICS_STEPS", steps[:1]):
|
||||
run_pending_semantics_steps()
|
||||
with patch.object(semantics, "SEMANTICS_STEPS", steps):
|
||||
run_pending_semantics_steps()
|
||||
|
||||
assert applied == [1, 2]
|
||||
|
||||
def test_interrupted_walk_leaves_committed_work_and_resumes(
|
||||
self, session, comfy_dirs
|
||||
):
|
||||
for index in range(4):
|
||||
_register(
|
||||
session,
|
||||
_write(comfy_dirs["checkpoints"], f"m{index}.safetensors"),
|
||||
f"ref-{index}",
|
||||
loader_path=None,
|
||||
)
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def _stop_after_first_batch() -> bool:
|
||||
calls["n"] += 1
|
||||
return calls["n"] > 1
|
||||
|
||||
with (
|
||||
patch("app.assets.semantics.reproject_derived._BATCH_SIZE", 2),
|
||||
patch.object(
|
||||
semantics,
|
||||
"SEMANTICS_STEPS",
|
||||
(
|
||||
SemanticsStep(
|
||||
version=1,
|
||||
description="reproject",
|
||||
apply=reproject_derived_state,
|
||||
),
|
||||
),
|
||||
),
|
||||
):
|
||||
assert run_pending_semantics_steps(_stop_after_first_batch) == 0
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == 0
|
||||
assert session.get(AssetReference, "ref-0").loader_path is not None
|
||||
assert session.get(AssetReference, "ref-3").loader_path is None
|
||||
|
||||
assert run_pending_semantics_steps() == 1
|
||||
|
||||
session.expire_all()
|
||||
assert get_semantics_version(session) == 1
|
||||
assert session.get(AssetReference, "ref-3").loader_path == "m3.safetensors"
|
||||
@@ -23,6 +23,7 @@ def mock_dependencies():
|
||||
"""Mock all external dependencies for isolated testing."""
|
||||
with (
|
||||
patch("app.assets.seeder.dependencies_available", return_value=True),
|
||||
patch("app.assets.seeder.run_pending_semantics_steps", return_value=0),
|
||||
patch("app.assets.seeder.sync_root_safely", return_value=set()),
|
||||
patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),
|
||||
patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),
|
||||
@@ -454,6 +455,7 @@ class TestSeederMarkMissing:
|
||||
|
||||
with (
|
||||
patch("app.assets.seeder.dependencies_available", return_value=True),
|
||||
patch("app.assets.seeder.run_pending_semantics_steps", return_value=0),
|
||||
patch("app.assets.seeder.get_owned_prefixes", return_value=["/models"]),
|
||||
patch("app.assets.seeder.mark_missing_outside_prefixes_safely", side_effect=track_mark),
|
||||
patch("app.assets.seeder.sync_temp_references_safely"),
|
||||
@@ -475,6 +477,7 @@ class TestSeederMarkMissing:
|
||||
):
|
||||
with (
|
||||
patch("app.assets.seeder.dependencies_available", return_value=True),
|
||||
patch("app.assets.seeder.run_pending_semantics_steps", return_value=0),
|
||||
patch("app.assets.seeder.get_owned_prefixes", return_value=["/models"]),
|
||||
patch("app.assets.seeder.mark_missing_outside_prefixes_safely", return_value=0),
|
||||
patch("app.assets.seeder.sync_temp_references_safely") as sync_temp,
|
||||
@@ -494,6 +497,73 @@ class TestSeederMarkMissing:
|
||||
)
|
||||
|
||||
|
||||
class TestSeederSemanticsReset:
|
||||
"""The scan brings stale rows forward before it reads or extends them."""
|
||||
|
||||
def test_semantics_reset_runs_before_any_scan_work(
|
||||
self, fresh_seeder: _AssetSeeder
|
||||
):
|
||||
call_order = []
|
||||
|
||||
with (
|
||||
patch("app.assets.seeder.dependencies_available", return_value=True),
|
||||
patch(
|
||||
"app.assets.seeder.run_pending_semantics_steps",
|
||||
side_effect=lambda interrupt_check=None: call_order.append("reset"),
|
||||
),
|
||||
patch("app.assets.seeder.get_owned_prefixes", return_value=["/models"]),
|
||||
patch(
|
||||
"app.assets.seeder.mark_missing_outside_prefixes_safely",
|
||||
side_effect=lambda prefixes: call_order.append("prune") or 0,
|
||||
),
|
||||
patch(
|
||||
"app.assets.seeder.sync_temp_references_safely",
|
||||
side_effect=lambda: call_order.append("sync_temp"),
|
||||
),
|
||||
patch(
|
||||
"app.assets.seeder.sync_root_safely",
|
||||
side_effect=lambda root: call_order.append("sync") or set(),
|
||||
),
|
||||
patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),
|
||||
patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),
|
||||
patch("app.assets.seeder.insert_asset_specs", return_value=0),
|
||||
patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),
|
||||
patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),
|
||||
):
|
||||
fresh_seeder.start(roots=("models",), prune_first=True)
|
||||
fresh_seeder.wait(timeout=5.0)
|
||||
|
||||
assert call_order[0] == "reset", (
|
||||
"reprojection must finish before the scan reads or extends those rows"
|
||||
)
|
||||
|
||||
def test_semantics_reset_can_be_cancelled(self, fresh_seeder: _AssetSeeder):
|
||||
captured = {}
|
||||
|
||||
with (
|
||||
patch("app.assets.seeder.dependencies_available", return_value=True),
|
||||
patch(
|
||||
"app.assets.seeder.run_pending_semantics_steps",
|
||||
side_effect=lambda interrupt_check=None: captured.update(
|
||||
check=interrupt_check
|
||||
),
|
||||
),
|
||||
patch("app.assets.seeder.sync_root_safely", return_value=set()),
|
||||
patch("app.assets.seeder.collect_paths_for_roots", return_value=[]),
|
||||
patch("app.assets.seeder.build_asset_specs", return_value=([], set(), 0)),
|
||||
patch("app.assets.seeder.insert_asset_specs", return_value=0),
|
||||
patch("app.assets.seeder.get_unenriched_assets_for_roots", return_value=[]),
|
||||
patch("app.assets.seeder.enrich_assets_batch", return_value=(0, 0)),
|
||||
):
|
||||
fresh_seeder.start(roots=("models",))
|
||||
fresh_seeder.wait(timeout=5.0)
|
||||
|
||||
assert captured.get("check") is not None
|
||||
assert captured["check"]() is False
|
||||
fresh_seeder._cancel_event.set()
|
||||
assert captured["check"]() is True
|
||||
|
||||
|
||||
class TestSeederPhases:
|
||||
"""Test phased scanning behavior."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user