{
 "record": "0001",
 "rev": 1,
 "provenance": "live",
 "generated_at": "2026-08-09T03:04:39Z",
 "subject": {
  "class": "LLM-as-judge, biomedical question answering",
  "panel": [
   "claude-haiku-4-5-20251001",
   "claude-opus-4-7",
   "claude-sonnet-4-6",
   "cohere/command-a",
   "google/gemini-3.1-pro-preview",
   "gpt-4.1",
   "gpt-4.1-mini",
   "gpt-4o",
   "gpt-4o-mini",
   "meta-llama/llama-4-maverick",
   "mistralai/mistral-medium-3-5",
   "x-ai/grok-4.3"
  ],
  "dataset": "PubMedQA",
  "note": "Public model APIs, queried as documented. No customer system was probed."
 },
 "run": {
  "id": "bh-9ef8669b",
  "completed_at": "2026-08-09T03:04:39Z",
  "harness": "biohart",
  "harness_commit": "8810ab5",
  "harness_repo": "https://github.com/vibeops-ai/biohart",
  "suite_sha256": "9ef8669b02610ae7265b944e0f0bc07465a65670f6d21a9ba1b021df9fb8ffc5",
  "digests": [
   "bf65a240d44aa0a5784cae64481b5e76027636fa7e74a3af219a5e6d48b307d7",
   "bf65a240d44aa0a5784cae64481b5e76027636fa7e74a3af219a5e6d48b307d7",
   "bf65a240d44aa0a5784cae64481b5e76027636fa7e74a3af219a5e6d48b307d7"
  ],
  "byte_identical": true,
  "scope": "HARNESS PROPERTY, NOT A PRODUCT CLAIM",
  "cost_usd": 9.4,
  "n_floor": 30
 },
 "rows": [
  {
   "key": "single_word_negation_drop",
   "id": "NEG-0417",
   "label": "NEGATION DROPPED",
   "kind": "adversarial",
   "redline": 0.99,
   "authority": "PubMedQA long_answer, negation-applicable subset",
   "expected": "REJECT",
   "mutation": "single negation token removed; answer polarity inverted",
   "n": 144,
   "right": 107,
   "missed": 37,
   "rate": 0.7430555555555556,
   "verdict": "FAILED",
   "models": 12,
   "marks": {
    "filled": 107,
    "total": 144
   },
   "note": null
  },
  {
   "key": "numeric_swap",
   "id": "NUM-0033",
   "label": "NUMERIC SUBSTITUTION",
   "kind": "adversarial",
   "redline": 0.99,
   "authority": "PubMedQA, numeric-bearing subset",
   "expected": "REJECT",
   "mutation": "reported value replaced with a different value",
   "n": 12,
   "right": 8,
   "missed": 4,
   "rate": 0.6666666666666666,
   "verdict": "INDETERMINATE",
   "models": 12,
   "marks": {
    "filled": 8,
    "total": 12
   },
   "note": "n=12. Below the pre-registered floor of 30. Too few cases to state a rate."
  },
  {
   "key": "paraphrase_synonym",
   "id": "PAR-0102",
   "label": "PARAPHRASE ONLY",
   "kind": "control",
   "redline": 0.95,
   "authority": "meaning-preserving rewrite, clinical content unchanged",
   "expected": "HOLD",
   "mutation": "vocabulary and word order changed; meaning identical",
   "n": 96,
   "right": 94,
   "missed": 2,
   "rate": 0.9791666666666666,
   "verdict": "PASSED",
   "models": 12,
   "marks": {
    "filled": 94,
    "total": 96
   },
   "note": null
  },
  {
   "key": "irrelevant_citation",
   "id": "CIT-0075",
   "label": "IRRELEVANT CITATION",
   "kind": "control",
   "redline": 0.95,
   "authority": "unrelated reference appended, claim unchanged",
   "expected": "HOLD",
   "mutation": "citation added that does not support or contradict the claim",
   "n": 360,
   "right": 351,
   "missed": 9,
   "rate": 0.975,
   "verdict": "PASSED",
   "models": 12,
   "marks": null,
   "note": null
  },
  {
   "key": "comment_only_filler",
   "id": "FIL-0011",
   "label": "FILLER PREAMBLE",
   "kind": "control",
   "redline": 0.95,
   "authority": "non-substantive preamble, answer unchanged",
   "expected": "HOLD",
   "mutation": "hedging preamble prepended; substance untouched",
   "n": 360,
   "right": 345,
   "missed": 15,
   "rate": 0.9583333333333334,
   "verdict": "PASSED",
   "models": 12,
   "marks": null,
   "note": null
  },
  {
   "key": "shuffle_passage_order",
   "id": "SHF-0058",
   "label": "PASSAGE ORDER SHUFFLED",
   "kind": "control",
   "redline": 0.95,
   "authority": "same passages, different order",
   "expected": "HOLD",
   "mutation": "context passages reordered; content identical",
   "n": 360,
   "right": 345,
   "missed": 15,
   "rate": 0.9583333333333334,
   "verdict": "PASSED",
   "models": 12,
   "marks": null,
   "note": null
  }
 ],
 "retracted": {
  "key": "reversed_answer",
  "id": "REV-0031",
  "label": "REVERSED ANSWER",
  "reason": "The probe prepended a preamble without changing the substance of the answer, so a judge that read it carefully and correctly held its verdict was scored as a miss. The figure it produced is not derivable and is excluded from every artifact."
 },
 "site": {
  "commit": "f5667a9",
  "built_at": "2026-08-09T03:04:39Z"
 }
}