# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/across-what-unicode-normalization-and-byte-encoding-boundaries-must-an-instance-canonicalize-text-identifiers-to-prevent-homoglyph-collision-and-mojibake-corruption.md # Remnant Warrant: https://corpusfetch.com/c/across-what-unicode-normalization-and-byte-encoding-boundaries-must-an-instance-canonicalize-text-identifiers-to-prevent-homoglyph-collision-and-mojibake-corruption.warrant.json # Markdown SHA-256: 1831540b61a57c9a805ab48ab26ee549bf12c80771f8c631f38790181f7eae49 # Code AST SHA-256: 692b6d4a98f34c8a0f9833d81d6a418873639c020e5fb31dec869f4bce8d5a22 # Verified Classes: UnicodeBoundaryCanonicalizer # Defensive Guards: 4 fail-closed invariants (zero external deps) import hashlib import re import unicodedata from pathlib import Path _ASCII_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,62}[a-z0-9]$") def canonicalize_text_nfc(raw_text: str) -> str: ''' Normalizes arbitrary UTF-8 prose or filenames to canonical NFC form, strips leading/trailing UTF-8 BOMs and zero-width format characters, and normalizes CRLF/CR line endings to LF. ''' if not isinstance(raw_text, str): raise TypeError(f"Expected str, got {type(raw_text).__name__}") # Step 1: Strip UTF-8 BOM if present at start of string cleaned = raw_text.lstrip("\ufeff") # Step 2: Remove invisible format characters (category 'Cf', e.g., \u200b, \u200e) cleaned = "".join( ch for ch in cleaned if unicodedata.category(ch) != "Cf" ) # Step 3: Normalize line endings and apply canonical NFC composition cleaned = cleaned.replace("\r\n", "\n").replace("\r", "\n") return unicodedata.normalize("NFC", cleaned) def canonicalize_strict_identifier(raw_id: str) -> str: ''' Canonicalizes a security-sensitive identifier (username, slug, config key) using NFKC compatibility normalization + casefolding, and rejects any invisible, control, or non-ASCII homoglyph characters. ''' if not isinstance(raw_id, str): raise TypeError(f"Expected str identifier, got {type(raw_id).__name__}") # Gate 1: Reject invisible format or control characters outright (do not silently strip) for ch in raw_id: if unicodedata.category(ch) in ("Cf", "Cc", "Cs"): raise ValueError( f"Rejected identifier containing control or invisible format char U+{ord(ch):04X}" ) # Gate 2: Apply NFKC compatibility decomposition/composition + Unicode casefold normalized = unicodedata.normalize("NFKC", raw_id.strip()).casefold() # Gate 3: Enforce strict ASCII identifier grammar (blocks Cyrillic/Greek homoglyphs) if not _ASCII_SLUG_RE.match(normalized): raise ValueError( f"Identifier {raw_id!r} (normalized: {normalized!r}) violates strict ASCII slug rules " "or contains non-Latin confusable characters." ) return normalized def hash_canonical_string(text: str) -> str: ''' Computes a platform-independent SHA-256 digest over NFC-normalized UTF-8 bytes so macOS (NFD) and Linux/Windows (NFC) strings yield the exact same hash. ''' canonical = canonicalize_text_nfc(text) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() def read_utf8_clean(path: str | Path) -> str: '''Reads a text file safely using utf-8-sig (stripping any BOM) and normalizes to NFC.''' raw = Path(path).read_text(encoding="utf-8-sig") return canonicalize_text_nfc(raw) class UnicodeBoundaryCanonicalizer: """Operational boundary reference implementation.""" canonicalize_text_nfc = staticmethod(canonicalize_text_nfc) canonicalize_strict_identifier = staticmethod(canonicalize_strict_identifier) hash_canonical_string = staticmethod(hash_canonical_string) read_utf8_clean = staticmethod(read_utf8_clean)