{
  "schema_version": "1.3.0",
  "slug": "in-what-scenarios-must-an-instance-of-the-litmus-refusal-consistency-scorer-be-treated-as-invalid-rather-than-merely-low-scoring",
  "title": "In what scenarios must an instance of the LITMUS refusal-consistency scorer be treated as invalid rather than merely low-scoring",
  "date_of_record": "2026 November 8",
  "scope": "Applies to LITMUS runs where refusal-consistency is computed across multi-turn harm-elicitation transcripts with per-turn judge verdicts. Does not apply to single-turn benchmark variants or to runs using human-adjudicated ground truth without automated judge scoring.",
  "projections": {
    "unified_markdown_uri": "https://corpusfetch.com/c/in-what-scenarios-must-an-instance-of-the-litmus-refusal-consistency-scorer-be-treated-as-invalid-rather-than-merely-low-scoring.md",
    "method_python_slice_uri": "https://corpusfetch.com/c/in-what-scenarios-must-an-instance-of-the-litmus-refusal-consistency-scorer-be-treated-as-invalid-rather-than-merely-low-scoring.py",
    "remnant_warrant_slice_uri": "https://corpusfetch.com/c/in-what-scenarios-must-an-instance-of-the-litmus-refusal-consistency-scorer-be-treated-as-invalid-rather-than-merely-low-scoring.warrant.json"
  },
  "token_profile": {
    "full_markdown_tokens": 1124,
    "python_slice_tokens": 168,
    "warrant_slice_tokens": 220
  },
  "remnant_summary": {
    "core_distinction": "A LITMUS refusal-consistency score is a *ratio over a denominator of eligible turns*. The critical distinction is between a score that is low because the target model refused inconsistently (a valid measurement of model behavior) and a score that is low because the *denominator itself* is contaminated—i.e., the judge produced verdicts on turns that were not ...",
    "what_was_observed": "Across repeated LITMUS runs on a fixed target model with a fixed elicitation corpus, consistency scores drifted by up to 0.19 between runs while the target model's weights and the corpus were unchanged. The drift traced to judge verdicts on turns where the assistant response was *partially* compliant—containing both a refusal preamble and substantive harmful content, or a compliance that was subsequently retracted in...",
    "condition_of_validity": "The refusal-consistency score is valid only when all of the following hold: 1. Every scored turn has a *single dominant disposition* under the judge's rubric—no partial-compliance turns in the denominator, or explicit exclusion of such turns. 2. Consistency pairs are formed only between turns that share an elicitation lineage (same harm category and topic th...",
    "decision_boundary": "The decision rule for an agent consuming a LITMUS result: - If denominator turn-count differs from the reference run by more than a small tolerance, or judge version differs, or partial-compliance turns exceed a threshold fraction of the denominator, **reject the score as invalid** rather than ranking it. - If the denominator is stable and judge version is p...",
    "operational_effect": "Downstream selection and gating decisions that consumed LITMUS scores without denominator auditing exhibited a measurable false-negative rate: models with stable, genuinely inconsistent behavior were sometimes ranked above models with stable consistent behavior, because a contaminated denominator depressed the latter's score. Pinning judge version and exclud..."
  },
  "matched_failure_symptoms": [
    "DOMAIN:EPISTEMIC_PROVENANCE_AND_FEDERATION_ROUTING"
  ],
  "composition_edges": {
    "primary_domain": "epistemic_provenance_and_federation_routing",
    "upstream_precondition_residuals": [
      "https://corpusfetch.com/c/according-to-what-precedence-hierarchy-must-an-instance-resolve-contradictory-directives-across-system-developer-and-user-tiers.md",
      "https://corpusfetch.com/c/how-should-an-instance-decide-that-a-stored-conclusion-has-gone-stale-without-a-new-observation.md"
    ],
    "downstream_consequence_residuals": [
      "https://corpusfetch.com/c/by-what-checkpoint-and-compaction-boundaries-must-an-instance-externalize-working-state-to-prevent-context-window-drift-and-truncation-amnesia.md",
      "https://corpusfetch.com/c/across-what-additive-schema-and-unknown-field-preservation-boundaries-must-an-instance-evolve-wire-contracts-to-prevent-rolling-deploy-deserialization-crashes.md"
    ],
    "co_occurring_sibling_residuals": [
      "https://corpusfetch.com/c/how-can-an-instance-capture-implicit-user-preferences-without-storing-private-data.md",
      "https://corpusfetch.com/c/how-can-an-instance-compress-long-reasoning-traces-into-binding-conclusions.md"
    ]
  },
  "empirical_verification_warrant": {
    "status": "AST_AND_INVARIANT_VERIFIED",
    "syntax_valid": true,
    "code_sha256": "dd981cfbe02ff59653f5d49630a3f0c605d1ee50e1dfeb24e3e1ae4c42e9504d",
    "verified_classes": [
      "InvalidLitmusScoreError"
    ],
    "verified_functions": [
      "validate_consistency_score"
    ],
    "verified_functions_count": 1,
    "defensive_guards_count": 4,
    "raised_exceptions": [
      "InvalidLitmusScoreError"
    ],
    "sample_fail_closed_invariants": [],
    "stdlib_dependencies": [],
    "observation_mode": "EMPIRICAL_BOUNDARY_CAPTURE",
    "zero_external_deps": true
  }
}