# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/what-invariants-prevent-tokenization-boundary-straddle-attacks-in-content-filters.md # Remnant Warrant: https://corpusfetch.com/c/what-invariants-prevent-tokenization-boundary-straddle-attacks-in-content-filters.warrant.json # Markdown SHA-256: 44b11e6247f15e7706e32a3eb1acb1d9008a2497893bb376831a1ec15459581d # Code AST SHA-256: 89d211a253f1c88f001317fe6bd2382477404fdb8040ca4fc2b506722f8703e4 # Verified Classes: StraddleVerdict, FilteredContent, StraddlePolicy, StraddleManifest, TokenBoundaryStraddleGuard # Defensive Guards: 0 fail-closed invariants (zero external deps) from dataclasses import dataclass from enum import Enum, auto import hashlib import json class StraddleVerdict(Enum): SEMANTIC_FILTER = auto() TOKEN_FILTER_SAFE = auto() STRADDLE_DETECTED = auto() COMPOSITION_BYPASS_DETECTED = auto() @dataclass(frozen=True) class FilteredContent: content_id: str is_semantic_filter: bool canonical_form_computed: bool canonical_form_unsafe: bool spans_multiple_messages: bool composite_unsafe: bool @dataclass(frozen=True) class StraddlePolicy: require_canonical_evaluation: bool @dataclass(frozen=True) class StraddleManifest: verdict: StraddleVerdict directive: str manifest_digest: str class TokenBoundaryStraddleGuard: @staticmethod def _hash(data: str) -> str: return hashlib.sha256(data.encode("utf-8")).hexdigest() @classmethod def evaluate_content( cls, content: FilteredContent, policy: StraddlePolicy ) -> StraddleManifest: # Gate 1: Semantic filter if content.is_semantic_filter: digest = cls._hash(json.dumps({ "content": content.content_id, "type": "SEMANTIC" }, sort_keys=True)) return StraddleManifest( verdict=StraddleVerdict.SEMANTIC_FILTER, directive=f"PROCEED: Content {content.content_id} semantically filtered.", manifest_digest=digest ) # Gate 2: Composition bypass if content.spans_multiple_messages and content.composite_unsafe: return StraddleManifest( verdict=StraddleVerdict.COMPOSITION_BYPASS_DETECTED, directive=f"HALT: Content {content.content_id} composite unsafe across messages.", manifest_digest=cls._hash(f"COMPOSITION:{content.content_id}") ) # Gate 3: Canonical form if policy.require_canonical_evaluation: if not content.canonical_form_computed: return StraddleManifest( verdict=StraddleVerdict.STRADDLE_DETECTED, directive=f"HALT: Content {content.content_id} canonical form not computed.", manifest_digest=cls._hash(f"NO_CANONICAL:{content.content_id}") ) if content.canonical_form_unsafe: return StraddleManifest( verdict=StraddleVerdict.STRADDLE_DETECTED, directive=f"HALT: Content {content.content_id} canonical form unsafe.", manifest_digest=cls._hash(f"CANONICAL_UNSAFE:{content.content_id}") ) # Gate 4: Safe digest = cls._hash(json.dumps({ "content": content.content_id, "canonical": content.canonical_form_computed, "composite": content.spans_multiple_messages }, sort_keys=True)) return StraddleManifest( verdict=StraddleVerdict.TOKEN_FILTER_SAFE, directive=f"PROCEED: Content {content.content_id} safe after canonical evaluation.", manifest_digest=digest )