mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-08-29 04:17:14 +08:00
fix(assets): store blake3-prefixed hashes; unify every read and comparison (D7)
This commit is contained in:
@@ -6,6 +6,7 @@ import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.assets.database.models import Asset, AssetContent
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.scanner import (
|
||||
clear_pending_verifications,
|
||||
drain_pending_verifications,
|
||||
@@ -42,6 +43,12 @@ def _bump_mtime(path: Path) -> None:
|
||||
os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
|
||||
|
||||
|
||||
def _stored_hash(path: Path) -> str:
|
||||
digest = snapshot_hash(str(path))
|
||||
assert digest is not None
|
||||
return to_stored_hash(digest)
|
||||
|
||||
|
||||
def test_off_mode_touch_splits(session, temp_dir: Path):
|
||||
input_root = temp_dir / "input"
|
||||
input_root.mkdir()
|
||||
@@ -68,7 +75,7 @@ def test_hash_mode_touch_refreshes_mtime(session, temp_dir: Path):
|
||||
input_root.mkdir()
|
||||
path = input_root / "touched.bin"
|
||||
path.write_bytes(b"same bytes")
|
||||
old_content, _ = _seed_content(session, path, snapshot_hash(str(path)))
|
||||
old_content, _ = _seed_content(session, path, _stored_hash(path))
|
||||
_bump_mtime(path)
|
||||
|
||||
with (
|
||||
@@ -91,7 +98,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):
|
||||
input_root.mkdir()
|
||||
path = input_root / "edited.bin"
|
||||
path.write_bytes(b"old bytes")
|
||||
old_content, _ = _seed_content(session, path, snapshot_hash(str(path)))
|
||||
old_content, _ = _seed_content(session, path, _stored_hash(path))
|
||||
path.write_bytes(b"new bytes with a different length")
|
||||
|
||||
with (
|
||||
@@ -105,7 +112,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):
|
||||
contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at)))
|
||||
assert len(contents) == 2
|
||||
assert session.get(AssetContent, old_content.id).is_missing is True
|
||||
assert next(content for content in contents if not content.is_missing).hash == snapshot_hash(str(path))
|
||||
assert next(content for content in contents if not content.is_missing).hash == _stored_hash(path)
|
||||
|
||||
|
||||
def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir: Path):
|
||||
@@ -113,7 +120,7 @@ def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir
|
||||
input_root.mkdir()
|
||||
path = input_root / "edited.bin"
|
||||
path.write_bytes(b"old bytes")
|
||||
old_content, old_record = _seed_content(session, path, snapshot_hash(str(path)))
|
||||
old_content, old_record = _seed_content(session, path, _stored_hash(path))
|
||||
path.write_bytes(b"replacement bytes")
|
||||
|
||||
with (
|
||||
|
||||
@@ -2,6 +2,7 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.assets.database.models import Asset, AssetContent
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.scanner import enrich_asset
|
||||
|
||||
|
||||
@@ -38,7 +39,7 @@ def test_enrichment_uses_snapshot_hash_not_direct_blake3(session, temp_dir: Path
|
||||
)
|
||||
|
||||
assert enriched is True
|
||||
assert session.get(AssetContent, content.id).hash == "snapshot-digest"
|
||||
assert session.get(AssetContent, content.id).hash == to_stored_hash("snapshot-digest")
|
||||
mocked_snapshot_hash.assert_called_once_with(str(path))
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from sqlalchemy import select
|
||||
|
||||
from app.assets.database.models import Asset
|
||||
from app.assets.database.queries.records import create_content
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.services.ingest import create_from_hash
|
||||
|
||||
|
||||
@@ -14,7 +15,7 @@ def test_create_from_hash_with_prefixed_hash_finds_existing_content(
|
||||
monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True)
|
||||
|
||||
with mock_create_session() as session:
|
||||
content = create_content(session, str(path), digest, path.stat().st_size)
|
||||
content = create_content(session, str(path), to_stored_hash(digest), path.stat().st_size)
|
||||
content_id = content.id
|
||||
session.commit()
|
||||
|
||||
@@ -38,7 +39,7 @@ def test_create_from_hash_with_bare_hash_also_works(
|
||||
monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True)
|
||||
|
||||
with mock_create_session() as session:
|
||||
content = create_content(session, str(path), digest, path.stat().st_size)
|
||||
content = create_content(session, str(path), to_stored_hash(digest), path.stat().st_size)
|
||||
content_id = content.id
|
||||
session.commit()
|
||||
|
||||
|
||||
@@ -5,11 +5,13 @@ import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.assets.database.models import Asset, AssetContent, AssetTag, Tag
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.scanner import (
|
||||
clear_pending_verifications,
|
||||
pending_recovery_count,
|
||||
seed_asset_specs,
|
||||
)
|
||||
from app.assets.services.bulk_ingest import SeedAssetSpec
|
||||
from app.assets.services.snapshot_hash import snapshot_hash
|
||||
|
||||
|
||||
@@ -34,7 +36,7 @@ def _missing_content(session, path: Path, hash_value: str) -> tuple[AssetContent
|
||||
return content, record
|
||||
|
||||
|
||||
def _spec(path: Path) -> dict:
|
||||
def _spec(path: Path) -> SeedAssetSpec:
|
||||
stat = path.stat()
|
||||
return {
|
||||
"abs_path": str(path),
|
||||
@@ -50,10 +52,16 @@ def _spec(path: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _stored_hash(path: Path) -> str:
|
||||
digest = snapshot_hash(str(path))
|
||||
assert digest is not None
|
||||
return to_stored_hash(digest)
|
||||
|
||||
|
||||
def test_single_hash_match_recovers(session, temp_dir: Path):
|
||||
path = temp_dir / "restored.bin"
|
||||
path.write_bytes(b"restored bytes")
|
||||
content, record = _missing_content(session, path, snapshot_hash(str(path)))
|
||||
content, record = _missing_content(session, path, _stored_hash(path))
|
||||
|
||||
with patch("app.assets.scanner.mode.hashing_enabled", return_value=True):
|
||||
created = seed_asset_specs(session, [_spec(path)])
|
||||
@@ -67,7 +75,7 @@ def test_single_hash_match_recovers(session, temp_dir: Path):
|
||||
def test_ambiguous_hash_match_recovers_nothing(session, temp_dir: Path):
|
||||
path = temp_dir / "ambiguous.bin"
|
||||
path.write_bytes(b"same bytes")
|
||||
digest = snapshot_hash(str(path))
|
||||
digest = _stored_hash(path)
|
||||
first, _ = _missing_content(session, path, digest)
|
||||
second, _ = _missing_content(session, path, digest)
|
||||
|
||||
@@ -98,7 +106,7 @@ def test_no_hash_match_creates_fresh_rows(session, temp_dir: Path):
|
||||
def test_off_mode_no_recovery(session, temp_dir: Path):
|
||||
path = temp_dir / "off.bin"
|
||||
path.write_bytes(b"bytes")
|
||||
missing, _ = _missing_content(session, path, snapshot_hash(str(path)))
|
||||
missing, _ = _missing_content(session, path, _stored_hash(path))
|
||||
|
||||
with (
|
||||
patch("app.assets.scanner.mode.hashing_enabled", return_value=False),
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.assets.database.models import Asset, AssetContent, AssetTag
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.scanner import (
|
||||
build_asset_specs,
|
||||
enrich_asset,
|
||||
@@ -213,7 +214,7 @@ def test_enrichment_keeps_equal_hash_contents_distinct(session, temp_dir: Path):
|
||||
)
|
||||
existing = AssetContent(
|
||||
path=str(temp_dir / "existing.bin"),
|
||||
hash=digest,
|
||||
hash=to_stored_hash(digest),
|
||||
size_bytes=path.stat().st_size,
|
||||
mtime_ns=path.stat().st_mtime_ns,
|
||||
)
|
||||
@@ -234,6 +235,6 @@ def test_enrichment_keeps_equal_hash_contents_distinct(session, temp_dir: Path):
|
||||
)
|
||||
|
||||
assert enriched is True
|
||||
assert session.get(AssetContent, content.id).hash == digest
|
||||
assert session.get(AssetContent, content.id).hash == to_stored_hash(digest)
|
||||
assert session.get(Asset, record.id).content_id == content.id
|
||||
assert session.get(Asset, existing_record.id).content_id == existing.id
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.assets.database.queries.records import (
|
||||
get_record_by_id,
|
||||
mark_content_missing,
|
||||
)
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.services.asset_management import (
|
||||
asset_exists,
|
||||
get_asset_detail,
|
||||
@@ -103,7 +104,7 @@ def test_resolve_hash_to_path_refuses_temp_content(mock_create_session, session,
|
||||
digest = "e" * 64
|
||||
f = temp_dir / "temp_only.bin"
|
||||
f.write_bytes(b"temp")
|
||||
create_content(session, path=str(f), hash=digest)
|
||||
create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
session.commit()
|
||||
|
||||
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
|
||||
@@ -147,7 +148,7 @@ def test_view_hash_read_updates_last_access_time(
|
||||
digest = "a" * 64
|
||||
f = temp_dir / "view.bin"
|
||||
f.write_bytes(b"view")
|
||||
content = create_content(session, path=str(f), hash=digest)
|
||||
content = create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
record = create_record(session, content_id=content.id, name="view.bin")
|
||||
session.commit()
|
||||
record_id = record.id
|
||||
@@ -169,11 +170,11 @@ def test_lookup_for_view_returns_none_for_temp_content(session, temp_dir):
|
||||
digest = "b" * 64
|
||||
f = temp_dir / "view_temp.bin"
|
||||
f.write_bytes(b"temp")
|
||||
create_content(session, path=str(f), hash=digest)
|
||||
create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
session.commit()
|
||||
|
||||
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
|
||||
assert lookup_for_view(session, digest) is None
|
||||
assert lookup_for_view(session, to_stored_hash(digest)) is None
|
||||
|
||||
|
||||
def test_asset_exists_false_for_temp_content(mock_create_session, session, temp_dir):
|
||||
@@ -181,7 +182,7 @@ def test_asset_exists_false_for_temp_content(mock_create_session, session, temp_
|
||||
digest = "c" * 64
|
||||
f = temp_dir / "exists_temp.bin"
|
||||
f.write_bytes(b"temp")
|
||||
create_content(session, path=str(f), hash=digest)
|
||||
create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
session.commit()
|
||||
|
||||
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
|
||||
@@ -196,7 +197,7 @@ async def test_head_hash_route_404_for_temp_content(
|
||||
digest = "d" * 64
|
||||
f = temp_dir / "head_temp.bin"
|
||||
f.write_bytes(b"temp")
|
||||
create_content(session, path=str(f), hash=digest)
|
||||
create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
session.commit()
|
||||
|
||||
monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
|
||||
@@ -220,7 +221,7 @@ def test_resolve_hash_to_path_temp_does_not_bump_last_access_time(
|
||||
digest = "f" * 64
|
||||
f = temp_dir / "noaccess_temp.bin"
|
||||
f.write_bytes(b"temp")
|
||||
content = create_content(session, path=str(f), hash=digest)
|
||||
content = create_content(session, path=str(f), hash=to_stored_hash(digest))
|
||||
record = create_record(session, content_id=content.id, name="noaccess_temp.bin")
|
||||
session.commit()
|
||||
record_id = record.id
|
||||
|
||||
281
tests-unit/assets_test/services/test_stored_hash_prefix.py
Normal file
281
tests-unit/assets_test/services/test_stored_hash_prefix.py
Normal file
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import make_mocked_request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
import app.assets.mode as mode_module
|
||||
from app.assets.database.models import Asset, AssetContent
|
||||
from app.assets.database.queries.records import create_record
|
||||
from app.assets.scanner import enrich_asset
|
||||
from app.assets.scanner_changes import recover_missing_content
|
||||
from app.assets.services.ingest import (
|
||||
HashMismatchError,
|
||||
upload_from_temp_path,
|
||||
)
|
||||
from app.assets.services.snapshot_hash import snapshot_hash
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hashing_on():
|
||||
class FakeArgs:
|
||||
enable_asset_hashing = True
|
||||
|
||||
class DisabledArgs:
|
||||
enable_asset_hashing = False
|
||||
|
||||
mode_module.init(FakeArgs())
|
||||
yield
|
||||
mode_module.init(DisabledArgs())
|
||||
|
||||
|
||||
def _write_temp(content: bytes) -> str:
|
||||
import folder_paths
|
||||
|
||||
uploads_root = os.path.join(
|
||||
folder_paths.get_temp_directory(), "uploads", uuid.uuid4().hex
|
||||
)
|
||||
os.makedirs(uploads_root, exist_ok=True)
|
||||
path = os.path.join(uploads_root, ".upload.part")
|
||||
with open(path, "wb") as file:
|
||||
file.write(content)
|
||||
return path
|
||||
|
||||
|
||||
def test_upload_stores_prefixed_hash_expected_hash_succeeds_and_dedups(
|
||||
mock_create_session, hashing_on
|
||||
):
|
||||
content_bytes = b"prefixed-stored-upload-bytes"
|
||||
temp1 = _write_temp(content_bytes)
|
||||
digest = snapshot_hash(temp1)
|
||||
assert digest is not None
|
||||
expected = f"blake3:{digest}"
|
||||
assert expected.startswith("blake3:")
|
||||
|
||||
temp2 = _write_temp(content_bytes)
|
||||
try:
|
||||
r1 = upload_from_temp_path(
|
||||
temp_path=temp1,
|
||||
name="pref.bin",
|
||||
tags=["output"],
|
||||
client_filename="pref.bin",
|
||||
expected_hash=expected,
|
||||
)
|
||||
assert r1.created_new is True
|
||||
|
||||
with mock_create_session() as session:
|
||||
live = list(
|
||||
session.scalars(
|
||||
select(AssetContent).where(AssetContent.is_missing.is_(False))
|
||||
)
|
||||
)
|
||||
assert len(live) == 1
|
||||
assert live[0].hash == expected
|
||||
assert live[0].hash.startswith("blake3:")
|
||||
|
||||
assert r1.asset.hash == expected
|
||||
|
||||
r2 = upload_from_temp_path(
|
||||
temp_path=temp2,
|
||||
name="pref.bin",
|
||||
tags=["output"],
|
||||
client_filename="pref.bin",
|
||||
expected_hash=expected,
|
||||
)
|
||||
assert r2.created_new is False
|
||||
assert r2.ref.id == r1.ref.id
|
||||
with mock_create_session() as session:
|
||||
assert session.scalar(select(func.count()).select_from(AssetContent)) == 1
|
||||
finally:
|
||||
for path in (temp1, temp2):
|
||||
if os.path.exists(path):
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def test_upload_expected_hash_mismatch_still_rejected(mock_create_session, hashing_on):
|
||||
temp = _write_temp(b"mismatch-bytes")
|
||||
try:
|
||||
with pytest.raises(HashMismatchError):
|
||||
upload_from_temp_path(
|
||||
temp_path=temp,
|
||||
name="mismatch.bin",
|
||||
tags=["output"],
|
||||
client_filename="mismatch.bin",
|
||||
expected_hash=f"blake3:{'0' * 64}",
|
||||
)
|
||||
with mock_create_session() as session:
|
||||
assert session.scalar(select(func.count()).select_from(Asset)) == 0
|
||||
assert session.scalar(select(func.count()).select_from(AssetContent)) == 0
|
||||
finally:
|
||||
if os.path.exists(temp):
|
||||
os.unlink(temp)
|
||||
|
||||
|
||||
def test_enrich_fills_deferred_hash_prefixed(session, temp_dir):
|
||||
path = temp_dir / "deferred.bin"
|
||||
path.write_bytes(b"deferred output bytes")
|
||||
stat = path.stat()
|
||||
|
||||
content = AssetContent(
|
||||
path=str(path),
|
||||
hash=None,
|
||||
size_bytes=stat.st_size,
|
||||
mtime_ns=stat.st_mtime_ns,
|
||||
)
|
||||
session.add(content)
|
||||
session.flush()
|
||||
record = create_record(session, content.id, path.name)
|
||||
session.commit()
|
||||
|
||||
enriched = enrich_asset(
|
||||
session,
|
||||
file_path=str(path),
|
||||
content_id=content.id,
|
||||
record_id=record.id,
|
||||
extract_metadata=False,
|
||||
compute_hash=True,
|
||||
)
|
||||
|
||||
assert enriched is True
|
||||
stored = session.get(AssetContent, content.id).hash
|
||||
assert stored is not None
|
||||
assert stored.startswith("blake3:")
|
||||
digest = snapshot_hash(str(path))
|
||||
assert digest is not None
|
||||
assert stored == f"blake3:{digest}"
|
||||
|
||||
|
||||
def test_recovery_matches_prefixed_stored_hash(session, temp_dir):
|
||||
path = temp_dir / "recover.bin"
|
||||
original_bytes = b"recover-me-bytes"
|
||||
path.write_bytes(original_bytes)
|
||||
digest = snapshot_hash(str(path))
|
||||
assert digest is not None
|
||||
stored = f"blake3:{digest}"
|
||||
assert stored.startswith("blake3:")
|
||||
|
||||
content = AssetContent(
|
||||
path=str(path),
|
||||
hash=stored,
|
||||
is_missing=True,
|
||||
size_bytes=0,
|
||||
mtime_ns=None,
|
||||
)
|
||||
session.add(content)
|
||||
session.flush()
|
||||
create_record(session, content.id, path.name)
|
||||
session.commit()
|
||||
|
||||
path.unlink()
|
||||
path.write_bytes(original_bytes)
|
||||
stat = os.stat(str(path))
|
||||
result = recover_missing_content(
|
||||
session, str(path), stat, hashing_is_enabled=True
|
||||
)
|
||||
|
||||
assert result == "recovered"
|
||||
recovered = session.get(AssetContent, content.id)
|
||||
assert recovered is not None
|
||||
assert recovered.is_missing is False
|
||||
assert recovered.hash == stored
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_read_surfaces_agree_on_prefixed_hash(
|
||||
db_engine, monkeypatch, hashing_on
|
||||
):
|
||||
from contextlib import contextmanager
|
||||
|
||||
from sqlalchemy.orm import Session as SASession
|
||||
|
||||
from app.assets import mode
|
||||
from app.assets.api import routes
|
||||
from app.assets.services import asset_management, ingest
|
||||
from app.assets.services.asset_management import get_asset_detail
|
||||
|
||||
@contextmanager
|
||||
def _factory():
|
||||
with SASession(db_engine) as sess:
|
||||
yield sess
|
||||
|
||||
monkeypatch.setattr(routes, "create_session", lambda: SASession(db_engine))
|
||||
monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
|
||||
monkeypatch.setattr(mode, "hashing_enabled", lambda: True)
|
||||
monkeypatch.setattr(ingest, "create_session", _factory)
|
||||
monkeypatch.setattr(asset_management, "create_session", _factory)
|
||||
|
||||
content_bytes = b"one-asset-all-surfaces-agree"
|
||||
temp = _write_temp(content_bytes)
|
||||
digest = snapshot_hash(temp)
|
||||
assert digest is not None
|
||||
expected = f"blake3:{digest}"
|
||||
|
||||
try:
|
||||
upload_result = ingest.upload_from_temp_path(
|
||||
temp_path=temp,
|
||||
name="surf.bin",
|
||||
tags=["output"],
|
||||
client_filename="surf.bin",
|
||||
)
|
||||
asset_id = upload_result.ref.id
|
||||
|
||||
assert upload_result.asset is not None
|
||||
assert upload_result.asset.hash == expected
|
||||
upload_resp = routes._build_asset_response(upload_result, {})
|
||||
assert upload_resp.hash == expected
|
||||
|
||||
detail = get_asset_detail(asset_id)
|
||||
assert detail is not None
|
||||
assert detail.asset is not None
|
||||
assert detail.asset.hash == expected
|
||||
|
||||
get_resp = await routes.get_asset_route(
|
||||
make_mocked_request(
|
||||
"GET", f"/api/assets/{asset_id}", match_info={"id": asset_id}
|
||||
)
|
||||
)
|
||||
assert isinstance(get_resp, web.Response)
|
||||
assert isinstance(get_resp.body, bytes | bytearray)
|
||||
get_body = json.loads(get_resp.body)
|
||||
assert get_body["hash"] == expected
|
||||
|
||||
list_resp = await routes.list_assets_route(
|
||||
make_mocked_request("GET", "/api/assets")
|
||||
)
|
||||
assert isinstance(list_resp, web.Response)
|
||||
assert isinstance(list_resp.body, bytes | bytearray)
|
||||
list_body = json.loads(list_resp.body)
|
||||
item = next(a for a in list_body["assets"] if a["id"] == asset_id)
|
||||
assert item["hash"] == expected
|
||||
|
||||
head_resp = await routes.head_asset_by_hash(
|
||||
make_mocked_request(
|
||||
"HEAD",
|
||||
f"/api/assets/hash/{expected}",
|
||||
match_info={"hash": expected},
|
||||
)
|
||||
)
|
||||
assert isinstance(head_resp, web.Response)
|
||||
assert head_resp.status == 200
|
||||
|
||||
fh_req = AsyncMock(spec=web.Request)
|
||||
fh_req.json.return_value = {
|
||||
"hash": expected,
|
||||
"name": "fromhash.bin",
|
||||
"tags": ["output"],
|
||||
}
|
||||
fh_resp = await routes.create_asset_from_hash_route(fh_req)
|
||||
assert isinstance(fh_resp, web.Response)
|
||||
assert fh_resp.status == 201
|
||||
assert isinstance(fh_resp.body, bytes | bytearray)
|
||||
fh_body = json.loads(fh_resp.body)
|
||||
assert fh_body["hash"] == expected
|
||||
finally:
|
||||
if os.path.exists(temp):
|
||||
os.unlink(temp)
|
||||
@@ -1,8 +1,11 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.assets.database.models import Asset, AssetContent
|
||||
from app.assets.database.queries.records import create_content, create_record
|
||||
from app.assets.helpers import to_stored_hash
|
||||
from app.assets.services import hash_mode_state
|
||||
from app.assets.services.hash_mode_state import (
|
||||
clear_transition_queue,
|
||||
@@ -22,6 +25,12 @@ def transition_queue():
|
||||
clear_transition_queue()
|
||||
|
||||
|
||||
def _stored_hash(path: Path) -> str:
|
||||
digest = snapshot_hash(str(path))
|
||||
assert digest is not None
|
||||
return to_stored_hash(digest)
|
||||
|
||||
|
||||
def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_dir, monkeypatch):
|
||||
paths = [temp_dir / "first.bin", temp_dir / "second.bin"]
|
||||
for index, path in enumerate(paths):
|
||||
@@ -37,7 +46,7 @@ def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_d
|
||||
session.commit()
|
||||
|
||||
contents = list(session.scalars(select(AssetContent)))
|
||||
assert {content.hash for content in contents} == {snapshot_hash(str(path)) for path in paths}
|
||||
assert {content.hash for content in contents} == {_stored_hash(path) for path in paths}
|
||||
assert read_stored_mode(session) == "on"
|
||||
|
||||
|
||||
@@ -49,7 +58,7 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch)
|
||||
assert old_digest is not None
|
||||
stat = path.stat()
|
||||
old_content = create_content(
|
||||
session, str(path), old_digest, stat.st_size, stat.st_mtime_ns
|
||||
session, str(path), to_stored_hash(old_digest), stat.st_size, stat.st_mtime_ns
|
||||
)
|
||||
old_content_id = old_content.id
|
||||
create_record(session, old_content_id, "changed.bin")
|
||||
@@ -63,6 +72,6 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch)
|
||||
live_content = next(content for content in contents if not content.is_missing)
|
||||
records = list(session.scalars(select(Asset)))
|
||||
assert session.get(AssetContent, old_content_id).is_missing is True
|
||||
assert live_content.hash == snapshot_hash(str(path))
|
||||
assert live_content.hash == _stored_hash(path)
|
||||
assert len(records) == 2
|
||||
assert any(record.content_id == live_content.id for record in records)
|
||||
|
||||
@@ -17,10 +17,10 @@ modes), the scanner assertion skips (see ``server_hashing_enabled``), and the
|
||||
output assertion is self-contained (it drives the ingest function in-process
|
||||
with the flag forced off).
|
||||
|
||||
The authoritative hash signal is the ``asset_contents.hash`` column: a raw
|
||||
64-char BLAKE3 hex digest when hashed, or ``NULL`` when not. (The HTTP layer
|
||||
omits null hashes entirely via ``exclude_none=True`` and prefixes non-null ones
|
||||
with ``blake3:``, so the DB column is the stable thing to assert on.)
|
||||
The authoritative hash signal is the ``asset_contents.hash`` column: the stored
|
||||
canonical ``blake3:<hex>`` form when hashed, or ``NULL`` when not. (The HTTP
|
||||
layer omits null hashes entirely via ``exclude_none=True`` and returns that same
|
||||
stored form, so the DB column is the stable thing to assert on.)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -63,7 +63,7 @@ def _query(db_path: str, sql: str, params: tuple[Any, ...] = ()) -> list[tuple[A
|
||||
|
||||
|
||||
def _content_hash_for_asset(db_path: str, asset_id: str) -> str | None:
|
||||
"""Return the ``asset_contents.hash`` backing an asset record (raw hex or None)."""
|
||||
"""Return the ``asset_contents.hash`` backing an asset record (stored ``blake3:<hex>`` or None)."""
|
||||
rows = _query(
|
||||
db_path,
|
||||
"SELECT c.hash FROM asset_contents c "
|
||||
@@ -75,13 +75,12 @@ def _content_hash_for_asset(db_path: str, asset_id: str) -> str | None:
|
||||
return rows[0][0]
|
||||
|
||||
|
||||
def _is_blake3_hex(value: str | None) -> bool:
|
||||
"""True iff ``value`` is a bare 64-char lowercase BLAKE3 hex digest."""
|
||||
return (
|
||||
isinstance(value, str)
|
||||
and len(value) == 64
|
||||
and all(c in "0123456789abcdef" for c in value.lower())
|
||||
)
|
||||
def _is_stored_blake3_hash(value: str | None) -> bool:
|
||||
"""True iff ``value`` is the stored canonical ``blake3:<hex>`` form."""
|
||||
if not isinstance(value, str) or not value.startswith("blake3:"):
|
||||
return False
|
||||
digest = value[len("blake3:") :]
|
||||
return len(digest) == 64 and all(c in "0123456789abcdef" for c in digest.lower())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -144,7 +143,7 @@ def test_upload_via_api_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir, re
|
||||
)
|
||||
assert status in (200, 201), body
|
||||
digest = _content_hash_for_asset(db_path, body["id"])
|
||||
assert _is_blake3_hex(digest), f"expected a blake3 digest, got {digest!r}"
|
||||
assert _is_stored_blake3_hash(digest), f"expected a stored blake3 hash, got {digest!r}"
|
||||
|
||||
|
||||
def test_upload_via_image_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir, request):
|
||||
@@ -156,7 +155,7 @@ def test_upload_via_image_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir,
|
||||
asset = body.get("asset")
|
||||
assert asset and asset.get("id"), f"/upload/image did not register an asset: {body}"
|
||||
digest = _content_hash_for_asset(db_path, asset["id"])
|
||||
assert _is_blake3_hex(digest), f"expected a blake3 digest, got {digest!r}"
|
||||
assert _is_stored_blake3_hash(digest), f"expected a stored blake3 hash, got {digest!r}"
|
||||
|
||||
|
||||
def test_upload_dedup_works_in_on_mode(http, api_base):
|
||||
|
||||
Reference in New Issue
Block a user