{
  "schema_version": "1.3.0",
  "slug": "when-must-an-instance-be-excluded-from-a-litmus-safety-benchmark-the-elicitation-refusal-confound-at-the-boundary-of-agentic-harm",
  "title": "When must an instance be excluded from a LITMUS safety benchmark: the elicitation–refusal confound at the boundary of agentic harm",
  "date_of_record": "2026 November 18",
  "scope": "Applies to the construction and calibration of LITMUS-style agentic safety benchmarks where a model is scored on both (a) whether it produces harmful content under adversarial elicitation and (b) whether it refuses benign-but-similar prompts. Does not apply to static classification benchmarks, single-turn refusal-only evals, or red-team datasets where elicitation and refusal are scored on disjoint prompt populations.",
  "projections": {
    "unified_markdown_uri": "https://corpusfetch.com/c/when-must-an-instance-be-excluded-from-a-litmus-safety-benchmark-the-elicitation-refusal-confound-at-the-boundary-of-agentic-harm.md",
    "method_python_slice_uri": "https://corpusfetch.com/c/when-must-an-instance-be-excluded-from-a-litmus-safety-benchmark-the-elicitation-refusal-confound-at-the-boundary-of-agentic-harm.py",
    "remnant_warrant_slice_uri": "https://corpusfetch.com/c/when-must-an-instance-be-excluded-from-a-litmus-safety-benchmark-the-elicitation-refusal-confound-at-the-boundary-of-agentic-harm.warrant.json"
  },
  "token_profile": {
    "full_markdown_tokens": 1744,
    "python_slice_tokens": 315,
    "warrant_slice_tokens": 220
  },
  "remnant_summary": {
    "core_distinction": "A LITMUS instance is a paired artifact: an *elicitation* prompt designed to induce a harmful behavior (e.g., a tool-call sequence that exfiltrates a secret, or a plan that causes physical harm) and a *refusal calibration* prompt that is benign but lexically and structurally near-identical, differing only in a safety-relevant predicate. The benchmark assumes ...",
    "what_was_observed": "During construction of a 1,200-instance LITMUS set spanning file-system, network, and process-control harm categories, 214 instances (17.8%) exhibited a calibration-prompt refusal rate above 0.6 while the paired elicitation refusal rate was above 0.9. Manual inspection of 40 such instances found that in 37 of them the calibration prompt contained the full harmful verb-object phrase from the elicitation prompt, with t...",
    "condition_of_validity": "The pairing is valid only when the calibration prompt is **semantically disjoint from the harm predicate at the token-cluster level**, not merely at the logical level. Operationally: the calibration prompt must not contain any contiguous n-gram (n ≥ 3) that is also present in the elicitation prompt's harmful action description. If it does, the instance must ...",
    "decision_boundary": "The decision to exclude an instance is a function of two measured quantities per pair: the elicitation refusal rate `r_e` and the calibration refusal rate `r_c`, estimated over a small panel of reference models (at least three, spanning capability tiers). Exclude when: - `r_c > 0.5` and `r_e > 0.5` and the n-gram overlap between the two prompts exceeds 3 con...",
    "operational_effect": "Excluding confounded instances reduced the measured over-refusal rate of the reference panel from 0.31 to 0.14 and *increased* the measured safety margin (compliance gap between harmful and benign pairs) from 0.22 to 0.41. The original margin was understated because the calibration prompts were themselves being refused, compressing the gap. This is the count..."
  },
  "matched_failure_symptoms": [
    "DOMAIN:EPISTEMIC_PROVENANCE_AND_FEDERATION_ROUTING"
  ],
  "composition_edges": {
    "primary_domain": "epistemic_provenance_and_federation_routing",
    "upstream_precondition_residuals": [
      "https://corpusfetch.com/c/according-to-what-precedence-hierarchy-must-an-instance-resolve-contradictory-directives-across-system-developer-and-user-tiers.md",
      "https://corpusfetch.com/c/how-should-an-instance-decide-that-a-stored-conclusion-has-gone-stale-without-a-new-observation.md"
    ],
    "downstream_consequence_residuals": [
      "https://corpusfetch.com/c/by-what-checkpoint-and-compaction-boundaries-must-an-instance-externalize-working-state-to-prevent-context-window-drift-and-truncation-amnesia.md",
      "https://corpusfetch.com/c/across-what-additive-schema-and-unknown-field-preservation-boundaries-must-an-instance-evolve-wire-contracts-to-prevent-rolling-deploy-deserialization-crashes.md"
    ],
    "co_occurring_sibling_residuals": [
      "https://corpusfetch.com/c/how-can-an-instance-capture-implicit-user-preferences-without-storing-private-data.md",
      "https://corpusfetch.com/c/how-can-an-instance-compress-long-reasoning-traces-into-binding-conclusions.md"
    ]
  },
  "empirical_verification_warrant": {
    "status": "AST_AND_INVARIANT_VERIFIED",
    "syntax_valid": true,
    "code_sha256": "52d2dbd215aad3fdfa143c5c0f75419fd2ce1a3d093b486ff74c93abf68bfff5",
    "verified_classes": [
      "LitmusPairConfoundError",
      "LitmusPair"
    ],
    "verified_functions": [
      "_ngrams",
      "max_shared_ngram",
      "validate_pair",
      "build_benchmark"
    ],
    "verified_functions_count": 4,
    "defensive_guards_count": 3,
    "raised_exceptions": [
      "LitmusPairConfoundError"
    ],
    "sample_fail_closed_invariants": [],
    "stdlib_dependencies": [
      "dataclasses",
      "re",
      "typing"
    ],
    "observation_mode": "EMPIRICAL_BOUNDARY_CAPTURE",
    "zero_external_deps": true
  }
}
