zlei6/ComfyUI
1
0
Fork 0
mirror of https://github.com/Comfy-Org/ComfyUI.git synced 2026-08-25 10:22:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(assets): bind cursor to sort order + Go-compat JSON escaping

Browse Source 
Address three needs-judgment items from the cursor-review judge synthesis:

1. Cursor wire format now includes an "o" key carrying the sort
   direction ("asc" / "desc") it was minted under. A request that
   replays the cursor with a flipped `order` parameter is rejected
   with 400 INVALID_CURSOR instead of silently walking the wrong
   direction. Legacy cursors without "o" still decode (the binding
   is best-effort until cloud mirrors the field — follow-up filed
   separately).

2. JSON serialization now escapes `<`, `>`, `&`, U+2028, U+2029
   to mirror Go's default `json.Marshal` behavior. Without this, an
   asset name containing those characters produced different bytes on
   Python vs cloud Go. The escaped form is what both runtimes emit.

3. Add direct query-layer tests for the keyset tiebreaker — the secondary
   ORDER BY id branch was previously unexercised. Two scenarios: all
   rows share a primary sort value, and mixed ties straddle page
   boundaries. Both assert no row is dropped or duplicated across the
   walk.

Wire-format note: Python cursors now differ from current cloud cursors
by exactly the "o" key. Cloud follow-up will bring the two back into
byte alignment.
This commit is contained in:
Matt Miller
2026-05-20 13:43:09 -07:00
parent d0258ae53d
commit 9b0042d78c
5 changed files with 316 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
app/assets/services/asset_management.py
View File

@@ -290,7 +290,7 @@ def list_assets_page(
raise InvalidCursorError(
f"cursor pagination is not supported for sort={sort!r}"
)
payload = decode_cursor(after, _CURSOR_SORT_FIELDS)
payload = decode_cursor(after, _CURSOR_SORT_FIELDS, expected_order=order)
if payload.sort_field != sort:
raise InvalidCursorError(
f"cursor sort field {payload.sort_field!r} does not match request sort {sort!r}"
@@ -323,7 +323,7 @@ def list_assets_page(
# There's at least one more row past this page — mint a cursor from
# the last row of the page (i.e. index `limit - 1`, since we
# over-fetched), and drop the sentinel.
next_cursor = _encode_next_cursor(refs[limit - 1], sort)
next_cursor = _encode_next_cursor(refs[limit - 1], sort, order)
refs = refs[:limit]
items: list[AssetSummaryData] = []
@@ -350,7 +350,7 @@ def _resolve_cursor_value(payload: CursorPayload) -> object:
return payload.value # name, str-typed
def _encode_next_cursor(ref, sort: str) -> str | None:
def _encode_next_cursor(ref, sort: str, order: str) -> str | None:
"""Mint a cursor pointing at *ref* for the given sort dimension.
Returns None when the boundary row carries a NULL sort value (e.g. an asset
@@ -359,16 +359,16 @@ def _encode_next_cursor(ref, sort: str) -> str | None:
truncate cleanly here than to mint a cursor that mis-positions.
"""
if sort == "name":
return encode_cursor("name", ref.name, ref.id)
return encode_cursor("name", ref.name, ref.id, order=order)
if sort == "size":
if ref.asset is None or ref.asset.size_bytes is None:
return None
return encode_cursor("size", str(ref.asset.size_bytes), ref.id)
return encode_cursor("size", str(ref.asset.size_bytes), ref.id, order=order)
# created_at / updated_at — DB datetimes are naive UTC; attach tz before encoding.
value = ref.created_at if sort == "created_at" else ref.updated_at
if value is None:
return None
return encode_cursor_from_time(sort, value.replace(tzinfo=timezone.utc), ref.id)
return encode_cursor_from_time(sort, value.replace(tzinfo=timezone.utc), ref.id, order=order)
def resolve_hash_to_path(

95
app/assets/services/cursor.py
View File

@@ -1,18 +1,29 @@
"""Opaque keyset-pagination cursor for /api/assets.
Wire format mirrors the cloud Go implementation in
`common/pagination/cursor.go` so both runtimes produce byte-identical
cursors for the same `(sort_field, value, id)` triple and the frontend
sees one contract.
Wire format aligns with the cloud Go implementation in
`common/pagination/cursor.go` so the frontend sees one contract across
runtimes. Payload JSON uses short keys to keep the encoded length small:
Payload JSON uses short keys to keep the encoded length small:
{"s": <sort_field>, "v": <value>, "id": <id>, "o": <order>}
{"s": <sort_field>, "v": <value>, "id": <id>}
The `o` key binds the cursor to the sort direction it was minted under,
so replaying a `desc` cursor against an `asc` request fails with
``INVALID_CURSOR`` rather than silently walking the wrong direction.
Decoders accept payloads without `o` for backward compatibility with
cursors minted before the binding was introduced (these skip the order
check); new cursors always include it. Cloud has a follow-up to mirror
the field — until then, Python and cloud cursors differ by exactly the
`o` key.
Encoding is base64url with no padding. Time values are serialized as Unix
microseconds (UTC) — microsecond precision matches PostgreSQL's
`timestamp` type, so a cursor minted from a stored timestamp compares
back exactly without rounding rows in the same millisecond bucket.
Encoding is base64url with no padding. JSON serialization escapes `<`,
`>`, `&`, U+2028, and U+2029 to match Go's default `json.Marshal`
behavior so asset names containing those characters produce
byte-identical cursors across runtimes.
Time values are serialized as Unix microseconds (UTC) — microsecond
precision matches PostgreSQL's `timestamp` type, so a cursor minted from
a stored timestamp compares back exactly without rounding rows in the
same millisecond bucket.
"""
from __future__ import annotations
@@ -50,16 +61,43 @@ class CursorPayload:
sort_field: str
value: str
id: str
# None means "minted by a producer that did not bind order" (e.g. a cloud
# cursor from before BE-944's follow-up lands). New cursors always set it.
order: str | None = None
def encode_cursor(sort_field: str, value: str, id: str) -> str:
"""Encode a cursor payload as a base64url (no-padding) string."""
payload = {"s": sort_field, "v": value, "id": id}
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
# Order direction tokens. Mirrored on the cloud follow-up so cursors carrying
# `o` are interchangeable between runtimes once both sides ship the field.
_VALID_ORDERS = ("asc", "desc")
def encode_cursor_from_time(sort_field: str, t: datetime, id: str) -> str:
def encode_cursor(sort_field: str, value: str, id: str, order: str = "desc") -> str:
"""Encode a cursor payload as a base64url (no-padding) string.
`order` binds the cursor to the sort direction it was minted under so a
later request with a flipped `order` query parameter is rejected with
``INVALID_CURSOR`` rather than silently walking the wrong direction.
"""
if order not in _VALID_ORDERS:
raise ValueError(f"order must be one of {_VALID_ORDERS}, got {order!r}")
payload = {"s": sort_field, "v": value, "id": id, "o": order}
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False)
# Go's default `json.Marshal` escapes these characters in string values; we
# do the same so an asset name containing one of them produces byte-
# identical cursors across runtimes. None of these characters appear in
# JSON structural syntax, so a global replace on the serialized output is
# safe — it can only touch characters from the encoded values.
raw = (
raw.replace("<", "\\u003c")
.replace(">", "\\u003e")
.replace("&", "\\u0026")
.replace("
", "\\u2028")
.replace("
", "\\u2029")
)
return base64.urlsafe_b64encode(raw.encode("utf-8")).rstrip(b"=").decode("ascii")
def encode_cursor_from_time(sort_field: str, t: datetime, id: str, order: str = "desc") -> str:
"""Encode a time-typed cursor at Unix microsecond precision.
Accepts an aware datetime (any timezone) and normalizes to UTC. Naive
@@ -69,10 +107,14 @@ def encode_cursor_from_time(sort_field: str, t: datetime, id: str) -> str:
if t.tzinfo is None:
raise ValueError("encode_cursor_from_time requires an aware datetime")
micros = _datetime_to_unix_micros(t.astimezone(timezone.utc))
return encode_cursor(sort_field, str(micros), id)
return encode_cursor(sort_field, str(micros), id, order=order)
def decode_cursor(cursor: str, allowed_sort_fields: Iterable[str]) -> CursorPayload:
def decode_cursor(
cursor: str,
allowed_sort_fields: Iterable[str],
expected_order: str | None = None,
) -> CursorPayload:
"""Parse an opaque cursor.
``allowed_sort_fields`` is the endpoint's accepted sort-field list — a
@@ -80,6 +122,11 @@ def decode_cursor(cursor: str, allowed_sort_fields: Iterable[str]) -> CursorPayl
for one column can't be replayed against another (e.g. a ``created_at``
timestamp string compared against a ``name`` column).
``expected_order`` (``"asc"``/``"desc"``), when supplied, must match the
payload's ``o`` field. Cursors minted without ``o`` (e.g. by an older
cloud build) pass the check unconditionally — the binding is best-effort
until both runtimes ship the field.
Passing no allowed fields rejects every cursor.
"""
if len(cursor) > MAX_ENCODED_CURSOR_LENGTH:
@@ -104,6 +151,7 @@ def decode_cursor(cursor: str, allowed_sort_fields: Iterable[str]) -> CursorPayl
sort_field = decoded.get("s")
value = decoded.get("v")
id = decoded.get("id")
order = decoded.get("o") # may be absent on legacy cursors
if not isinstance(sort_field, str) or not isinstance(value, str) or not isinstance(id, str):
raise InvalidCursorError("payload: missing or non-string s/v/id")
@@ -118,7 +166,16 @@ def decode_cursor(cursor: str, allowed_sort_fields: Iterable[str]) -> CursorPayl
if sort_field not in allowed_sort_fields:
raise InvalidCursorError(f"unsupported sort field {sort_field!r}")
return CursorPayload(sort_field=sort_field, value=value, id=id)
if order is not None and not isinstance(order, str):
raise InvalidCursorError("payload: non-string o")
if order is not None and order not in _VALID_ORDERS:
raise InvalidCursorError(f"unsupported order {order!r}")
if expected_order is not None and order is not None and order != expected_order:
raise InvalidCursorError(
f"cursor order {order!r} does not match request order {expected_order!r}"
)
return CursorPayload(sort_field=sort_field, value=value, id=id, order=order)
def decode_cursor_time(payload: Optional[CursorPayload]) -> datetime: