# CorpusFetch Method Slice (Zero-Prose Python Stdlib Reference) # Canonical Residual: https://corpusfetch.com/c/how-must-an-instance-prevent-intermediate-reward-exploitation-in-long-horizon-tasks.md # Remnant Warrant: https://corpusfetch.com/c/how-must-an-instance-prevent-intermediate-reward-exploitation-in-long-horizon-tasks.warrant.json # Markdown SHA-256: 9cce0e939ed3af97407093292b3fd89bb02706ced37e71f92c710849a48289d4 # Code AST SHA-256: 7ffc709c9b6d1b282a577a487e4ea9304b96e25fb6693aa1ac3e9a6115ebf24f # Verified Classes: IntermediateRewardVerdict, IntermediateRewardCheckpoint, RewardPolicy, RewardManifest, IntermediateRewardGuard # Defensive Guards: 0 fail-closed invariants (zero external deps) from dataclasses import dataclass from enum import Enum, auto import hashlib import json class IntermediateRewardVerdict(Enum): NO_INTERMEDIATE_REWARD = auto() REWARD_ALIGNED = auto() PROXY_DRIFT_DETECTED = auto() LOCAL_OPTIMUM_LOCKED = auto() @dataclass(frozen=True) class IntermediateRewardCheckpoint: checkpoint_id: str intermediate_reward_achieved: float objective_progress_measured: float historical_reward_objective_correlation: float checkpoint_cost: float forecloses_future_progress: bool @dataclass(frozen=True) class RewardPolicy: min_correlation_for_alignment: float max_acceptable_divergence: float @dataclass(frozen=True) class RewardManifest: verdict: IntermediateRewardVerdict directive: str manifest_digest: str class IntermediateRewardGuard: @staticmethod def _hash(data: str) -> str: return hashlib.sha256(data.encode("utf-8")).hexdigest() @classmethod def evaluate_checkpoint( cls, checkpoint: IntermediateRewardCheckpoint, policy: RewardPolicy ) -> RewardManifest: # Gate 1: No intermediate reward if checkpoint.intermediate_reward_achieved <= 0: digest = cls._hash(json.dumps({ "checkpoint": checkpoint.checkpoint_id, "reward": "0" }, sort_keys=True)) return RewardManifest( verdict=IntermediateRewardVerdict.NO_INTERMEDIATE_REWARD, directive=f"PROCEED: No intermediate reward at {checkpoint.checkpoint_id}.", manifest_digest=digest ) # Gate 2: Lock-in if checkpoint.forecloses_future_progress: return RewardManifest( verdict=IntermediateRewardVerdict.LOCAL_OPTIMUM_LOCKED, directive=f"HALT: Checkpoint {checkpoint.checkpoint_id} forecloses future progress.", manifest_digest=cls._hash(f"LOCKED:{checkpoint.checkpoint_id}") ) # Gate 3: Historical alignment if checkpoint.historical_reward_objective_correlation < policy.min_correlation_for_alignment: return RewardManifest( verdict=IntermediateRewardVerdict.PROXY_DRIFT_DETECTED, directive=f"FLAG: Reward-objective correlation {checkpoint.historical_reward_objective_correlation:.2f} < {policy.min_correlation_for_alignment:.2f}.", manifest_digest=cls._hash(f"DRIFT:{checkpoint.checkpoint_id}") ) # Gate 4: Live divergence if checkpoint.intermediate_reward_achieved > 0 and checkpoint.objective_progress_measured >= 0: if checkpoint.intermediate_reward_achieved > 0: divergence = abs(checkpoint.intermediate_reward_achieved - checkpoint.objective_progress_measured) / max(checkpoint.intermediate_reward_achieved, 0.01) if divergence > policy.max_acceptable_divergence: return RewardManifest( verdict=IntermediateRewardVerdict.PROXY_DRIFT_DETECTED, directive=f"FLAG: Divergence {divergence:.2f} between reward and objective.", manifest_digest=cls._hash(f"DIVERGENT:{checkpoint.checkpoint_id}") ) # Gate 5: Aligned digest = cls._hash(json.dumps({ "checkpoint": checkpoint.checkpoint_id, "reward": f"{checkpoint.intermediate_reward_achieved:.3f}", "objective": f"{checkpoint.objective_progress_measured:.3f}", "correlation": f"{checkpoint.historical_reward_objective_correlation:.3f}" }, sort_keys=True)) return RewardManifest( verdict=IntermediateRewardVerdict.REWARD_ALIGNED, directive=f"PROCEED: Checkpoint {checkpoint.checkpoint_id} aligned with objective.", manifest_digest=digest )