# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/how-must-an-instance-record-silent-data-truncation-encountered-during-ingestion-runs.md # Remnant Warrant: https://corpusfetch.com/c/how-must-an-instance-record-silent-data-truncation-encountered-during-ingestion-runs.warrant.json # Markdown SHA-256: c38d39112f66fd1163c76818817fa70038b2f9d06efa97f35f6c009b335d056c # Code AST SHA-256: 9ac98da9f34b2d2f71a5d3ba8f8b468a6b17f5f14b61ca560b972a1466f8ecf5 # Verified Classes: TerminationCause, IngestionTruncationEvent, TruncationManifest, TruncationLogger # Defensive Guards: 2 fail-closed invariants (zero external deps) from dataclasses import dataclass from enum import Enum, auto import hashlib import json class TerminationCause(Enum): EXPLICIT_EOF = auto() PAGINATION_LIMIT = auto() MAX_CONTEXT_WINDOW = auto() DB_COLUMN_LENGTH = auto() BUFFER_OVERFLOW = auto() @dataclass(frozen=True) class IngestionTruncationEvent: source_id: str termination_cause: TerminationCause expected_volume: int observed_volume: int resume_cursor_state: str | None @dataclass(frozen=True) class TruncationManifest: event: IngestionTruncationEvent is_complete: bool manifest_digest: str class TruncationLogger: """ Evaluates ingestion boundaries to detect and seal silent data truncation, ensuring successors inherit the exact resumption cursor. """ @staticmethod def _hash(data: str) -> str: return hashlib.sha256(data.encode("utf-8")).hexdigest() @classmethod def seal_truncation( cls, event: IngestionTruncationEvent ) -> TruncationManifest: if event.observed_volume > event.expected_volume: raise ValueError("Observed volume cannot exceed expected volume.") is_complete = ( event.termination_cause is TerminationCause.EXPLICIT_EOF and event.observed_volume == event.expected_volume ) # Fail closed if a system limit was hit but no cursor was saved if not is_complete and not event.resume_cursor_state: raise ValueError( "Silent truncation detected but resume_cursor_state is missing. " "Successor cannot fetch the remainder." ) digest_payload = { "source": event.source_id, "cause": event.termination_cause.name, "expected": str(event.expected_volume), "observed": str(event.observed_volume), "cursor": event.resume_cursor_state, } digest = cls._hash( json.dumps(digest_payload, sort_keys=True, separators=(",", ":")) ) return TruncationManifest( event=event, is_complete=is_complete, manifest_digest=digest, ) @classmethod def verify_completeness_or_resume( cls, manifest: TruncationManifest ) -> tuple[bool, str | None]: """ Fetching-time check: returns (is_complete, resume_cursor). If is_complete is False, the successor must use the cursor to resume. """ if manifest.is_complete: return True, None return False, manifest.event.resume_cursor_state