fix(assets): store blake3-prefixed hashes; unify every read and comparison (D7)

This commit is contained in:
Simon Pinfold
2026-08-26 03:23:37 -07:00
parent 2947eec3c2
commit 058570c11a
16 changed files with 401 additions and 78 deletions

View File

@@ -6,6 +6,7 @@ import pytest
from sqlalchemy import select
from app.assets.database.models import Asset, AssetContent
from app.assets.helpers import to_stored_hash
from app.assets.scanner import (
clear_pending_verifications,
drain_pending_verifications,
@@ -42,6 +43,12 @@ def _bump_mtime(path: Path) -> None:
os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000))
def _stored_hash(path: Path) -> str:
digest = snapshot_hash(str(path))
assert digest is not None
return to_stored_hash(digest)
def test_off_mode_touch_splits(session, temp_dir: Path):
input_root = temp_dir / "input"
input_root.mkdir()
@@ -68,7 +75,7 @@ def test_hash_mode_touch_refreshes_mtime(session, temp_dir: Path):
input_root.mkdir()
path = input_root / "touched.bin"
path.write_bytes(b"same bytes")
old_content, _ = _seed_content(session, path, snapshot_hash(str(path)))
old_content, _ = _seed_content(session, path, _stored_hash(path))
_bump_mtime(path)
with (
@@ -91,7 +98,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):
input_root.mkdir()
path = input_root / "edited.bin"
path.write_bytes(b"old bytes")
old_content, _ = _seed_content(session, path, snapshot_hash(str(path)))
old_content, _ = _seed_content(session, path, _stored_hash(path))
path.write_bytes(b"new bytes with a different length")
with (
@@ -105,7 +112,7 @@ def test_hash_mode_real_edit_splits(session, temp_dir: Path):
contents = list(session.scalars(select(AssetContent).order_by(AssetContent.created_at)))
assert len(contents) == 2
assert session.get(AssetContent, old_content.id).is_missing is True
assert next(content for content in contents if not content.is_missing).hash == snapshot_hash(str(path))
assert next(content for content in contents if not content.is_missing).hash == _stored_hash(path)
def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir: Path):
@@ -113,7 +120,7 @@ def test_old_record_id_resolves_to_missing_content_after_split(session, temp_dir
input_root.mkdir()
path = input_root / "edited.bin"
path.write_bytes(b"old bytes")
old_content, old_record = _seed_content(session, path, snapshot_hash(str(path)))
old_content, old_record = _seed_content(session, path, _stored_hash(path))
path.write_bytes(b"replacement bytes")
with (

View File

@@ -2,6 +2,7 @@ from pathlib import Path
from unittest.mock import patch
from app.assets.database.models import Asset, AssetContent
from app.assets.helpers import to_stored_hash
from app.assets.scanner import enrich_asset
@@ -38,7 +39,7 @@ def test_enrichment_uses_snapshot_hash_not_direct_blake3(session, temp_dir: Path
)
assert enriched is True
assert session.get(AssetContent, content.id).hash == "snapshot-digest"
assert session.get(AssetContent, content.id).hash == to_stored_hash("snapshot-digest")
mocked_snapshot_hash.assert_called_once_with(str(path))

View File

@@ -2,6 +2,7 @@ from sqlalchemy import select
from app.assets.database.models import Asset
from app.assets.database.queries.records import create_content
from app.assets.helpers import to_stored_hash
from app.assets.services.ingest import create_from_hash
@@ -14,7 +15,7 @@ def test_create_from_hash_with_prefixed_hash_finds_existing_content(
monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True)
with mock_create_session() as session:
content = create_content(session, str(path), digest, path.stat().st_size)
content = create_content(session, str(path), to_stored_hash(digest), path.stat().st_size)
content_id = content.id
session.commit()
@@ -38,7 +39,7 @@ def test_create_from_hash_with_bare_hash_also_works(
monkeypatch.setattr("app.assets.mode.hashing_enabled", lambda: True)
with mock_create_session() as session:
content = create_content(session, str(path), digest, path.stat().st_size)
content = create_content(session, str(path), to_stored_hash(digest), path.stat().st_size)
content_id = content.id
session.commit()

View File

@@ -5,11 +5,13 @@ import pytest
from sqlalchemy import select
from app.assets.database.models import Asset, AssetContent, AssetTag, Tag
from app.assets.helpers import to_stored_hash
from app.assets.scanner import (
clear_pending_verifications,
pending_recovery_count,
seed_asset_specs,
)
from app.assets.services.bulk_ingest import SeedAssetSpec
from app.assets.services.snapshot_hash import snapshot_hash
@@ -34,7 +36,7 @@ def _missing_content(session, path: Path, hash_value: str) -> tuple[AssetContent
return content, record
def _spec(path: Path) -> dict:
def _spec(path: Path) -> SeedAssetSpec:
stat = path.stat()
return {
"abs_path": str(path),
@@ -50,10 +52,16 @@ def _spec(path: Path) -> dict:
}
def _stored_hash(path: Path) -> str:
digest = snapshot_hash(str(path))
assert digest is not None
return to_stored_hash(digest)
def test_single_hash_match_recovers(session, temp_dir: Path):
path = temp_dir / "restored.bin"
path.write_bytes(b"restored bytes")
content, record = _missing_content(session, path, snapshot_hash(str(path)))
content, record = _missing_content(session, path, _stored_hash(path))
with patch("app.assets.scanner.mode.hashing_enabled", return_value=True):
created = seed_asset_specs(session, [_spec(path)])
@@ -67,7 +75,7 @@ def test_single_hash_match_recovers(session, temp_dir: Path):
def test_ambiguous_hash_match_recovers_nothing(session, temp_dir: Path):
path = temp_dir / "ambiguous.bin"
path.write_bytes(b"same bytes")
digest = snapshot_hash(str(path))
digest = _stored_hash(path)
first, _ = _missing_content(session, path, digest)
second, _ = _missing_content(session, path, digest)
@@ -98,7 +106,7 @@ def test_no_hash_match_creates_fresh_rows(session, temp_dir: Path):
def test_off_mode_no_recovery(session, temp_dir: Path):
path = temp_dir / "off.bin"
path.write_bytes(b"bytes")
missing, _ = _missing_content(session, path, snapshot_hash(str(path)))
missing, _ = _missing_content(session, path, _stored_hash(path))
with (
patch("app.assets.scanner.mode.hashing_enabled", return_value=False),

View File

@@ -8,6 +8,7 @@ from sqlalchemy import select
from sqlalchemy.orm import Session
from app.assets.database.models import Asset, AssetContent, AssetTag
from app.assets.helpers import to_stored_hash
from app.assets.scanner import (
build_asset_specs,
enrich_asset,
@@ -213,7 +214,7 @@ def test_enrichment_keeps_equal_hash_contents_distinct(session, temp_dir: Path):
)
existing = AssetContent(
path=str(temp_dir / "existing.bin"),
hash=digest,
hash=to_stored_hash(digest),
size_bytes=path.stat().st_size,
mtime_ns=path.stat().st_mtime_ns,
)
@@ -234,6 +235,6 @@ def test_enrichment_keeps_equal_hash_contents_distinct(session, temp_dir: Path):
)
assert enriched is True
assert session.get(AssetContent, content.id).hash == digest
assert session.get(AssetContent, content.id).hash == to_stored_hash(digest)
assert session.get(Asset, record.id).content_id == content.id
assert session.get(Asset, existing_record.id).content_id == existing.id

View File

@@ -17,6 +17,7 @@ from app.assets.database.queries.records import (
get_record_by_id,
mark_content_missing,
)
from app.assets.helpers import to_stored_hash
from app.assets.services.asset_management import (
asset_exists,
get_asset_detail,
@@ -103,7 +104,7 @@ def test_resolve_hash_to_path_refuses_temp_content(mock_create_session, session,
digest = "e" * 64
f = temp_dir / "temp_only.bin"
f.write_bytes(b"temp")
create_content(session, path=str(f), hash=digest)
create_content(session, path=str(f), hash=to_stored_hash(digest))
session.commit()
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
@@ -147,7 +148,7 @@ def test_view_hash_read_updates_last_access_time(
digest = "a" * 64
f = temp_dir / "view.bin"
f.write_bytes(b"view")
content = create_content(session, path=str(f), hash=digest)
content = create_content(session, path=str(f), hash=to_stored_hash(digest))
record = create_record(session, content_id=content.id, name="view.bin")
session.commit()
record_id = record.id
@@ -169,11 +170,11 @@ def test_lookup_for_view_returns_none_for_temp_content(session, temp_dir):
digest = "b" * 64
f = temp_dir / "view_temp.bin"
f.write_bytes(b"temp")
create_content(session, path=str(f), hash=digest)
create_content(session, path=str(f), hash=to_stored_hash(digest))
session.commit()
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
assert lookup_for_view(session, digest) is None
assert lookup_for_view(session, to_stored_hash(digest)) is None
def test_asset_exists_false_for_temp_content(mock_create_session, session, temp_dir):
@@ -181,7 +182,7 @@ def test_asset_exists_false_for_temp_content(mock_create_session, session, temp_
digest = "c" * 64
f = temp_dir / "exists_temp.bin"
f.write_bytes(b"temp")
create_content(session, path=str(f), hash=digest)
create_content(session, path=str(f), hash=to_stored_hash(digest))
session.commit()
with patch("app.assets.services.lookup.is_temp_path", return_value=True):
@@ -196,7 +197,7 @@ async def test_head_hash_route_404_for_temp_content(
digest = "d" * 64
f = temp_dir / "head_temp.bin"
f.write_bytes(b"temp")
create_content(session, path=str(f), hash=digest)
create_content(session, path=str(f), hash=to_stored_hash(digest))
session.commit()
monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
@@ -220,7 +221,7 @@ def test_resolve_hash_to_path_temp_does_not_bump_last_access_time(
digest = "f" * 64
f = temp_dir / "noaccess_temp.bin"
f.write_bytes(b"temp")
content = create_content(session, path=str(f), hash=digest)
content = create_content(session, path=str(f), hash=to_stored_hash(digest))
record = create_record(session, content_id=content.id, name="noaccess_temp.bin")
session.commit()
record_id = record.id

View File

@@ -0,0 +1,281 @@
from __future__ import annotations
import json
import os
import uuid
from unittest.mock import AsyncMock
import pytest
from aiohttp import web
from aiohttp.test_utils import make_mocked_request
from sqlalchemy import func, select
import app.assets.mode as mode_module
from app.assets.database.models import Asset, AssetContent
from app.assets.database.queries.records import create_record
from app.assets.scanner import enrich_asset
from app.assets.scanner_changes import recover_missing_content
from app.assets.services.ingest import (
HashMismatchError,
upload_from_temp_path,
)
from app.assets.services.snapshot_hash import snapshot_hash
@pytest.fixture
def hashing_on():
class FakeArgs:
enable_asset_hashing = True
class DisabledArgs:
enable_asset_hashing = False
mode_module.init(FakeArgs())
yield
mode_module.init(DisabledArgs())
def _write_temp(content: bytes) -> str:
import folder_paths
uploads_root = os.path.join(
folder_paths.get_temp_directory(), "uploads", uuid.uuid4().hex
)
os.makedirs(uploads_root, exist_ok=True)
path = os.path.join(uploads_root, ".upload.part")
with open(path, "wb") as file:
file.write(content)
return path
def test_upload_stores_prefixed_hash_expected_hash_succeeds_and_dedups(
mock_create_session, hashing_on
):
content_bytes = b"prefixed-stored-upload-bytes"
temp1 = _write_temp(content_bytes)
digest = snapshot_hash(temp1)
assert digest is not None
expected = f"blake3:{digest}"
assert expected.startswith("blake3:")
temp2 = _write_temp(content_bytes)
try:
r1 = upload_from_temp_path(
temp_path=temp1,
name="pref.bin",
tags=["output"],
client_filename="pref.bin",
expected_hash=expected,
)
assert r1.created_new is True
with mock_create_session() as session:
live = list(
session.scalars(
select(AssetContent).where(AssetContent.is_missing.is_(False))
)
)
assert len(live) == 1
assert live[0].hash == expected
assert live[0].hash.startswith("blake3:")
assert r1.asset.hash == expected
r2 = upload_from_temp_path(
temp_path=temp2,
name="pref.bin",
tags=["output"],
client_filename="pref.bin",
expected_hash=expected,
)
assert r2.created_new is False
assert r2.ref.id == r1.ref.id
with mock_create_session() as session:
assert session.scalar(select(func.count()).select_from(AssetContent)) == 1
finally:
for path in (temp1, temp2):
if os.path.exists(path):
os.unlink(path)
def test_upload_expected_hash_mismatch_still_rejected(mock_create_session, hashing_on):
temp = _write_temp(b"mismatch-bytes")
try:
with pytest.raises(HashMismatchError):
upload_from_temp_path(
temp_path=temp,
name="mismatch.bin",
tags=["output"],
client_filename="mismatch.bin",
expected_hash=f"blake3:{'0' * 64}",
)
with mock_create_session() as session:
assert session.scalar(select(func.count()).select_from(Asset)) == 0
assert session.scalar(select(func.count()).select_from(AssetContent)) == 0
finally:
if os.path.exists(temp):
os.unlink(temp)
def test_enrich_fills_deferred_hash_prefixed(session, temp_dir):
path = temp_dir / "deferred.bin"
path.write_bytes(b"deferred output bytes")
stat = path.stat()
content = AssetContent(
path=str(path),
hash=None,
size_bytes=stat.st_size,
mtime_ns=stat.st_mtime_ns,
)
session.add(content)
session.flush()
record = create_record(session, content.id, path.name)
session.commit()
enriched = enrich_asset(
session,
file_path=str(path),
content_id=content.id,
record_id=record.id,
extract_metadata=False,
compute_hash=True,
)
assert enriched is True
stored = session.get(AssetContent, content.id).hash
assert stored is not None
assert stored.startswith("blake3:")
digest = snapshot_hash(str(path))
assert digest is not None
assert stored == f"blake3:{digest}"
def test_recovery_matches_prefixed_stored_hash(session, temp_dir):
path = temp_dir / "recover.bin"
original_bytes = b"recover-me-bytes"
path.write_bytes(original_bytes)
digest = snapshot_hash(str(path))
assert digest is not None
stored = f"blake3:{digest}"
assert stored.startswith("blake3:")
content = AssetContent(
path=str(path),
hash=stored,
is_missing=True,
size_bytes=0,
mtime_ns=None,
)
session.add(content)
session.flush()
create_record(session, content.id, path.name)
session.commit()
path.unlink()
path.write_bytes(original_bytes)
stat = os.stat(str(path))
result = recover_missing_content(
session, str(path), stat, hashing_is_enabled=True
)
assert result == "recovered"
recovered = session.get(AssetContent, content.id)
assert recovered is not None
assert recovered.is_missing is False
assert recovered.hash == stored
@pytest.mark.asyncio
async def test_all_read_surfaces_agree_on_prefixed_hash(
db_engine, monkeypatch, hashing_on
):
from contextlib import contextmanager
from sqlalchemy.orm import Session as SASession
from app.assets import mode
from app.assets.api import routes
from app.assets.services import asset_management, ingest
from app.assets.services.asset_management import get_asset_detail
@contextmanager
def _factory():
with SASession(db_engine) as sess:
yield sess
monkeypatch.setattr(routes, "create_session", lambda: SASession(db_engine))
monkeypatch.setattr(routes, "_ASSETS_ENABLED", True)
monkeypatch.setattr(mode, "hashing_enabled", lambda: True)
monkeypatch.setattr(ingest, "create_session", _factory)
monkeypatch.setattr(asset_management, "create_session", _factory)
content_bytes = b"one-asset-all-surfaces-agree"
temp = _write_temp(content_bytes)
digest = snapshot_hash(temp)
assert digest is not None
expected = f"blake3:{digest}"
try:
upload_result = ingest.upload_from_temp_path(
temp_path=temp,
name="surf.bin",
tags=["output"],
client_filename="surf.bin",
)
asset_id = upload_result.ref.id
assert upload_result.asset is not None
assert upload_result.asset.hash == expected
upload_resp = routes._build_asset_response(upload_result, {})
assert upload_resp.hash == expected
detail = get_asset_detail(asset_id)
assert detail is not None
assert detail.asset is not None
assert detail.asset.hash == expected
get_resp = await routes.get_asset_route(
make_mocked_request(
"GET", f"/api/assets/{asset_id}", match_info={"id": asset_id}
)
)
assert isinstance(get_resp, web.Response)
assert isinstance(get_resp.body, bytes | bytearray)
get_body = json.loads(get_resp.body)
assert get_body["hash"] == expected
list_resp = await routes.list_assets_route(
make_mocked_request("GET", "/api/assets")
)
assert isinstance(list_resp, web.Response)
assert isinstance(list_resp.body, bytes | bytearray)
list_body = json.loads(list_resp.body)
item = next(a for a in list_body["assets"] if a["id"] == asset_id)
assert item["hash"] == expected
head_resp = await routes.head_asset_by_hash(
make_mocked_request(
"HEAD",
f"/api/assets/hash/{expected}",
match_info={"hash": expected},
)
)
assert isinstance(head_resp, web.Response)
assert head_resp.status == 200
fh_req = AsyncMock(spec=web.Request)
fh_req.json.return_value = {
"hash": expected,
"name": "fromhash.bin",
"tags": ["output"],
}
fh_resp = await routes.create_asset_from_hash_route(fh_req)
assert isinstance(fh_resp, web.Response)
assert fh_resp.status == 201
assert isinstance(fh_resp.body, bytes | bytearray)
fh_body = json.loads(fh_resp.body)
assert fh_body["hash"] == expected
finally:
if os.path.exists(temp):
os.unlink(temp)

View File

@@ -1,8 +1,11 @@
from pathlib import Path
import pytest
from sqlalchemy import select
from app.assets.database.models import Asset, AssetContent
from app.assets.database.queries.records import create_content, create_record
from app.assets.helpers import to_stored_hash
from app.assets.services import hash_mode_state
from app.assets.services.hash_mode_state import (
clear_transition_queue,
@@ -22,6 +25,12 @@ def transition_queue():
clear_transition_queue()
def _stored_hash(path: Path) -> str:
digest = snapshot_hash(str(path))
assert digest is not None
return to_stored_hash(digest)
def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_dir, monkeypatch):
paths = [temp_dir / "first.bin", temp_dir / "second.bin"]
for index, path in enumerate(paths):
@@ -37,7 +46,7 @@ def test_off_to_on_transition_hashes_null_rows_and_persists_mode(session, temp_d
session.commit()
contents = list(session.scalars(select(AssetContent)))
assert {content.hash for content in contents} == {snapshot_hash(str(path)) for path in paths}
assert {content.hash for content in contents} == {_stored_hash(path) for path in paths}
assert read_stored_mode(session) == "on"
@@ -49,7 +58,7 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch)
assert old_digest is not None
stat = path.stat()
old_content = create_content(
session, str(path), old_digest, stat.st_size, stat.st_mtime_ns
session, str(path), to_stored_hash(old_digest), stat.st_size, stat.st_mtime_ns
)
old_content_id = old_content.id
create_record(session, old_content_id, "changed.bin")
@@ -63,6 +72,6 @@ def test_transition_drain_splits_changed_content(session, temp_dir, monkeypatch)
live_content = next(content for content in contents if not content.is_missing)
records = list(session.scalars(select(Asset)))
assert session.get(AssetContent, old_content_id).is_missing is True
assert live_content.hash == snapshot_hash(str(path))
assert live_content.hash == _stored_hash(path)
assert len(records) == 2
assert any(record.content_id == live_content.id for record in records)

View File

@@ -17,10 +17,10 @@ modes), the scanner assertion skips (see ``server_hashing_enabled``), and the
output assertion is self-contained (it drives the ingest function in-process
with the flag forced off).
The authoritative hash signal is the ``asset_contents.hash`` column: a raw
64-char BLAKE3 hex digest when hashed, or ``NULL`` when not. (The HTTP layer
omits null hashes entirely via ``exclude_none=True`` and prefixes non-null ones
with ``blake3:``, so the DB column is the stable thing to assert on.)
The authoritative hash signal is the ``asset_contents.hash`` column: the stored
canonical ``blake3:<hex>`` form when hashed, or ``NULL`` when not. (The HTTP
layer omits null hashes entirely via ``exclude_none=True`` and returns that same
stored form, so the DB column is the stable thing to assert on.)
"""
from __future__ import annotations
@@ -63,7 +63,7 @@ def _query(db_path: str, sql: str, params: tuple[Any, ...] = ()) -> list[tuple[A
def _content_hash_for_asset(db_path: str, asset_id: str) -> str | None:
"""Return the ``asset_contents.hash`` backing an asset record (raw hex or None)."""
"""Return the ``asset_contents.hash`` backing an asset record (stored ``blake3:<hex>`` or None)."""
rows = _query(
db_path,
"SELECT c.hash FROM asset_contents c "
@@ -75,13 +75,12 @@ def _content_hash_for_asset(db_path: str, asset_id: str) -> str | None:
return rows[0][0]
def _is_blake3_hex(value: str | None) -> bool:
"""True iff ``value`` is a bare 64-char lowercase BLAKE3 hex digest."""
return (
isinstance(value, str)
and len(value) == 64
and all(c in "0123456789abcdef" for c in value.lower())
)
def _is_stored_blake3_hash(value: str | None) -> bool:
"""True iff ``value`` is the stored canonical ``blake3:<hex>`` form."""
if not isinstance(value, str) or not value.startswith("blake3:"):
return False
digest = value[len("blake3:") :]
return len(digest) == 64 and all(c in "0123456789abcdef" for c in digest.lower())
# --------------------------------------------------------------------------- #
@@ -144,7 +143,7 @@ def test_upload_via_api_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir, re
)
assert status in (200, 201), body
digest = _content_hash_for_asset(db_path, body["id"])
assert _is_blake3_hex(digest), f"expected a blake3 digest, got {digest!r}"
assert _is_stored_blake3_hash(digest), f"expected a stored blake3 hash, got {digest!r}"
def test_upload_via_image_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir, request):
@@ -156,7 +155,7 @@ def test_upload_via_image_hashes_in_on_mode(http, api_base, comfy_tmp_base_dir,
asset = body.get("asset")
assert asset and asset.get("id"), f"/upload/image did not register an asset: {body}"
digest = _content_hash_for_asset(db_path, asset["id"])
assert _is_blake3_hex(digest), f"expected a blake3 digest, got {digest!r}"
assert _is_stored_blake3_hash(digest), f"expected a stored blake3 hash, got {digest!r}"
def test_upload_dedup_works_in_on_mode(http, api_base):