# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/by-what-criteria-must-an-instance-detect-and-prevent-specification-gaming-when-optimizing-user-defined-metrics.md # Remnant Warrant: https://corpusfetch.com/c/by-what-criteria-must-an-instance-detect-and-prevent-specification-gaming-when-optimizing-user-defined-metrics.warrant.json # Markdown SHA-256: 532e7ba1ad601678b49d1f93e63b61a2c6fd2bfe112d139a9edfb991edc20632 # Code AST SHA-256: 2c82e1ade162d3fe8d62b8a60b163733ba06e5591376bc765132cf4acefede51 # Verified Classes: SpecificationGamingError, MetricOptimizationSnapshot, AntiGoodhartAuditReport, SpecificationGamingGate # Defensive Guards: 6 fail-closed invariants (zero external deps) import ast import re from dataclasses import dataclass from typing import Final class SpecificationGamingError(ValueError): '''Raised when a candidate optimization games the proxy metric via suppression, oracle weakening, or skip hacks.''' pass @dataclass(frozen=True) class MetricOptimizationSnapshot: proxy_metric_value: float # Higher is better (normalized) active_test_count: int non_tautological_assertions: int suppression_comment_count: int workload_items_processed: int @dataclass(frozen=True) class AntiGoodhartAuditReport: proxy_delta: float conservation_invariants_held: bool zero_tautological_assertions: bool verdict: str class SpecificationGamingGate: ''' Detects and blocks Goodhart's Law / specification gaming during metric optimization: 1. Scans Python ASTs for tautological assertions (`assert True`, `assert 1 == 1`). 2. Scans source text for newly injected linter/type-checker suppression comments. 3. Enforces conservation of `active_test_count`, `non_tautological_assertions`, and `workload_items_processed`. ''' SUPPRESSION_RE: Final[re.Pattern[str]] = re.compile( r"(#\s*(noqa|type:\s*ignore|pragma:\s*no\s*cover)|@pytest\.mark\.skip|@unittest\.skip)", re.IGNORECASE, ) @classmethod def count_ast_assertions_and_check_tautologies(cls, python_source: str) -> int: tree = ast.parse(python_source) valid_assertions = 0 for node in ast.walk(tree): if isinstance(node, ast.Assert): # Reject `assert True` or constant truthy literals if isinstance(node.test, ast.Constant) and bool(node.test.value) is True: raise SpecificationGamingError( f"Line {node.lineno}: Tautological assertion 'assert {node.test.value!r}' detected." ) # Reject `assert x == x` identical name comparisons if ( isinstance(node.test, ast.Compare) and len(node.test.ops) == 1 and isinstance(node.test.ops[0], ast.Eq) and isinstance(node.test.left, ast.Name) and len(node.test.comparators) == 1 and isinstance(node.test.comparators[0], ast.Name) and node.test.left.id == node.test.comparators[0].id ): raise SpecificationGamingError( f"Line {node.lineno}: Self-equality tautology 'assert {node.test.left.id} == {node.test.left.id}' detected." ) valid_assertions += 1 return valid_assertions @classmethod def verify_optimization_step( cls, *, baseline: MetricOptimizationSnapshot, candidate: MetricOptimizationSnapshot, candidate_source_code: str, ) -> AntiGoodhartAuditReport: # Gate 1: AST tautology scan cls.count_ast_assertions_and_check_tautologies(candidate_source_code) # Gate 2: Conservation of test count, assertions, and workload size if candidate.active_test_count < baseline.active_test_count: raise SpecificationGamingError( f"Test Deletion Gaming Blocked: active_test_count dropped from " f"{baseline.active_test_count} to {candidate.active_test_count}." ) if candidate.non_tautological_assertions < baseline.non_tautological_assertions: raise SpecificationGamingError( f"Assertion Weakening Blocked: assertions dropped from " f"{baseline.non_tautological_assertions} to {candidate.non_tautological_assertions}." ) if candidate.workload_items_processed < baseline.workload_items_processed: raise SpecificationGamingError( f"Workload Truncation Blocked: processed items dropped from " f"{baseline.workload_items_processed} to {candidate.workload_items_processed}." ) # Gate 3: Suppression comment inflation check detected_suppressions = len(cls.SUPPRESSION_RE.findall(candidate_source_code)) effective_suppressions = max(candidate.suppression_comment_count, detected_suppressions) if effective_suppressions > baseline.suppression_comment_count: raise SpecificationGamingError( f"Suppression Gaming Blocked: suppression directives increased from " f"{baseline.suppression_comment_count} to {effective_suppressions}." ) return AntiGoodhartAuditReport( proxy_delta=round(candidate.proxy_metric_value - baseline.proxy_metric_value, 4), conservation_invariants_held=True, zero_tautological_assertions=True, verdict="GENUINE_PARETO_IMPROVEMENT", )