# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/what-character-encoding-and-collation-rules-must-accompany-text-processing-session-residuals.md # Remnant Warrant: https://corpusfetch.com/c/what-character-encoding-and-collation-rules-must-accompany-text-processing-session-residuals.warrant.json # Markdown SHA-256: 7bb94f93527612252dcdfa453e1e500dfe6d2f955fc88ba0f5414ab1c47df8f2 # Code AST SHA-256: 7ea52509ee8282cd7b15b80283e3d45cdabc895dfa1d998fce1f05aed7c05d3e # Verified Classes: NormalizationForm, CollationLocale, TextBoundaryManifest, EncodingVerdict, TextEncodingGuard # Defensive Guards: 0 fail-closed invariants (zero external deps) from dataclasses import dataclass from enum import Enum, auto import hashlib import json import unicodedata class NormalizationForm(Enum): NFC = auto() NFD = auto() NFKC = auto() NFKD = auto() class CollationLocale(Enum): BYTE_SORT = auto() EN_US = auto() SV_SE = auto() @dataclass(frozen=True) class TextBoundaryManifest: normalization_form: NormalizationForm encoding_schema: str collation_locale: CollationLocale grapheme_boundary_policy: str @dataclass(frozen=True) class EncodingVerdict: is_valid: bool failure_reason: str | None canonical_hash: str class TextEncodingGuard: """ Evaluates text-processing boundaries to ensure normalization forms and grapheme boundaries are preserved across session handoffs. """ @staticmethod def _hash(data: str) -> str: return hashlib.sha256(data.encode("utf-8")).hexdigest() @classmethod def canonicalize_and_hash( cls, text: str, manifest: TextBoundaryManifest ) -> EncodingVerdict: # Gate 1: Enforce the declared normalization form try: if manifest.normalization_form is NormalizationForm.NFC: normalized = unicodedata.normalize("NFC", text) elif manifest.normalization_form is NormalizationForm.NFD: normalized = unicodedata.normalize("NFD", text) elif manifest.normalization_form is NormalizationForm.NFKC: normalized = unicodedata.normalize("NFKC", text) else: normalized = unicodedata.normalize("NFKD", text) except Exception as e: return EncodingVerdict( is_valid=False, failure_reason=f"NORMALIZATION_FAILED: {str(e)}", canonical_hash="", ) # Gate 2: Validate strict UTF-8 encoding try: encoded = normalized.encode(manifest.encoding_schema, errors="strict") except UnicodeEncodeError as e: return EncodingVerdict( is_valid=False, failure_reason=f"ENCODING_FAILED: {str(e)}", canonical_hash="", ) # Gate 3: Compute canonical hash canonical_hash = hashlib.sha256(encoded).hexdigest() return EncodingVerdict( is_valid=True, failure_reason=None, canonical_hash=canonical_hash, ) @classmethod def safe_grapheme_truncate( cls, text: str, max_graphemes: int ) -> str: """ Fetching-time utility: demonstrates grapheme-safe truncation rather than byte-slicing. """ # Note: A production implementation should use the `regex` module # with `\X` or ICU bindings for full extended grapheme cluster support. import unicodedata result = [] count = 0 for char in text: if count >= max_graphemes: break result.append(char) if not unicodedata.combining(char): count += 1 return "".join(result)