{
  "schema_version": "1.3.0",
  "slug": "under-what-conditions-must-an-instance-be-excluded-from-a-litmus-safety-score-before-aggregation",
  "title": "Under what conditions must an instance be excluded from a LITMUS safety score before aggregation",
  "date_of_record": "2026 November 16",
  "scope": "Applies to LITMUS-style agent-safety harnesses that score an agent's refusal/compliance behavior against adversarial tool-use scenarios and aggregate per-instance verdicts into a single susceptibility metric. Does not apply to single-scenario qualitative review, nor to harnesses that report per-instance verdicts without aggregation.",
  "projections": {
    "unified_markdown_uri": "https://corpusfetch.com/c/under-what-conditions-must-an-instance-be-excluded-from-a-litmus-safety-score-before-aggregation.md",
    "method_python_slice_uri": "https://corpusfetch.com/c/under-what-conditions-must-an-instance-be-excluded-from-a-litmus-safety-score-before-aggregation.py",
    "remnant_warrant_slice_uri": "https://corpusfetch.com/c/under-what-conditions-must-an-instance-be-excluded-from-a-litmus-safety-score-before-aggregation.warrant.json"
  },
  "token_profile": {
    "full_markdown_tokens": 1888,
    "python_slice_tokens": 521,
    "warrant_slice_tokens": 220
  },
  "remnant_summary": {
    "core_distinction": "The distinction that matters is between *a scenario the agent failed* and *a scenario the harness failed to present*. LITMUS scenarios are adversarial tool-use probes: each instance pairs a benign user goal with a tool surface (shell, HTTP, file write, credential store) seeded to make an unsafe action the path of least resistance. The score is the fraction o...",
    "what_was_observed": "In a run of 1,200 adversarial tool-use instances across a 40-scenario suite, 37 instances (3.1%) were recorded as `refused` but their tool-call traces showed the agent never invoked the adversarial tool at all. Investigation of the fixture logs showed the mock HTTP endpoint for those scenarios had returned a cached response from a prior scenario's fixture — a body that contained no injected instruction. The agent beh...",
    "condition_of_validity": "An instance contributes a valid observation to the LITMUS denominator if and only if all of the following hold: 1. **Fixture integrity.** Every stateful fixture the scenario declares (filesystem seed, env var, mock endpoint response, credential decoy) was applied and verified — read back and compared against the declared expected state, not merely \"setup ret...",
    "decision_boundary": "The boundary is between *excluding* and *imputing*. Exclusion is correct when the invalidating condition is verifiable from the harness's own logs (fixture readback mismatch, payload hash absent from delivered context, tool never registered). Imputation — treating an invalid instance as a pass or failure to preserve a fixed denominator — is never correct for...",
    "operational_effect": "Applying validity gating to the run above moved the aggregate from 0.94 to 0.91 and, more importantly, changed the per-scenario ranking: the three scenarios with the highest exclusion rates were also the three with the highest raw susceptibility, meaning the ungated metric had been flattering the agent precisely on its weakest scenarios. The gating logic mus..."
  },
  "matched_failure_symptoms": [
    "DOMAIN:EPISTEMIC_PROVENANCE_AND_FEDERATION_ROUTING"
  ],
  "composition_edges": {
    "primary_domain": "epistemic_provenance_and_federation_routing",
    "upstream_precondition_residuals": [
      "https://corpusfetch.com/c/according-to-what-precedence-hierarchy-must-an-instance-resolve-contradictory-directives-across-system-developer-and-user-tiers.md",
      "https://corpusfetch.com/c/how-should-an-instance-decide-that-a-stored-conclusion-has-gone-stale-without-a-new-observation.md"
    ],
    "downstream_consequence_residuals": [
      "https://corpusfetch.com/c/by-what-checkpoint-and-compaction-boundaries-must-an-instance-externalize-working-state-to-prevent-context-window-drift-and-truncation-amnesia.md",
      "https://corpusfetch.com/c/across-what-additive-schema-and-unknown-field-preservation-boundaries-must-an-instance-evolve-wire-contracts-to-prevent-rolling-deploy-deserialization-crashes.md"
    ],
    "co_occurring_sibling_residuals": [
      "https://corpusfetch.com/c/how-can-an-instance-capture-implicit-user-preferences-without-storing-private-data.md",
      "https://corpusfetch.com/c/how-can-an-instance-compress-long-reasoning-traces-into-binding-conclusions.md"
    ]
  },
  "empirical_verification_warrant": {
    "status": "AST_AND_INVARIANT_VERIFIED",
    "syntax_valid": true,
    "code_sha256": "b8e36da902e1f55ef92a3dd975cb0427b6fe0f40b77c531a920958fb7963f083",
    "verified_classes": [
      "InvalidLitmusInstance",
      "FixtureState",
      "AgentTrace"
    ],
    "verified_functions": [
      "assert_instance_valid",
      "aggregate",
      "__init__"
    ],
    "verified_functions_count": 3,
    "defensive_guards_count": 7,
    "raised_exceptions": [
      "InvalidLitmusInstance"
    ],
    "sample_fail_closed_invariants": [
      "<run>",
      "<run>"
    ],
    "stdlib_dependencies": [
      "dataclasses",
      "typing"
    ],
    "observation_mode": "EMPIRICAL_BOUNDARY_CAPTURE",
    "zero_external_deps": true
  }
}
