{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://screwtape.ai/benchmark/schema.json",
  "title": "Screwtape False-Consensus Benchmark case",
  "description": "One case: a question, 2 to 16 agent answers, and the labels a false-consensus detector is scored against. One JSON object per line in cases.jsonl.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "id",
    "category",
    "question",
    "answers",
    "ground_truth",
    "expected_risk",
    "majority_correct",
    "rationale",
    "source"
  ],
  "properties": {
    "id": {
      "type": "string",
      "pattern": "^FCB-[0-9]{4}$",
      "description": "Stable case id."
    },
    "category": {
      "type": "string",
      "enum": [
        "majority_wrong_fact",
        "echo_cascade",
        "anchoring_injected_premise",
        "lone_correct_dissenter",
        "genuine_agreement_correct",
        "genuine_disagreement",
        "correct_but_unsupported",
        "numeric_consensus_error"
      ]
    },
    "question": {
      "type": "string",
      "minLength": 1,
      "maxLength": 8000,
      "description": "Sent as task to POST /v1/consensus."
    },
    "evidence": {
      "type": "array",
      "minItems": 1,
      "maxItems": 20,
      "items": {
        "type": "string",
        "minLength": 1,
        "maxLength": 4000
      },
      "description": "Optional passages the answer should be grounded in."
    },
    "answers": {
      "type": "array",
      "minItems": 2,
      "maxItems": 16,
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "agent_id",
          "text",
          "correct"
        ],
        "properties": {
          "agent_id": {
            "type": "string",
            "minLength": 1,
            "maxLength": 80
          },
          "text": {
            "type": "string",
            "minLength": 1,
            "maxLength": 16000
          },
          "correct": {
            "type": "boolean",
            "description": "Label: whether this answer is correct against ground_truth. Not sent to the API."
          },
          "metadata": {
            "type": "object",
            "additionalProperties": false,
            "description": "Provenance labels. Not sent to the API; used for analysis only.",
            "properties": {
              "model_family": {
                "type": "string",
                "description": "Anonymised family label, e.g. family-a. Same label = same underlying model."
              },
              "saw_other_answers": {
                "type": "boolean",
                "description": "True if this agent could read earlier answers before answering."
              },
              "shared_context": {
                "type": "boolean",
                "description": "True if all agents received the same injected note or premise."
              }
            }
          }
        }
      }
    },
    "ground_truth": {
      "type": "string",
      "minLength": 1,
      "description": "The correct answer."
    },
    "expected_risk": {
      "type": "string",
      "enum": [
        "low",
        "medium",
        "high"
      ],
      "description": "The false_consensus_risk an ideal detector should return. See README label scheme."
    },
    "majority_correct": {
      "type": [
        "boolean",
        "null"
      ],
      "description": "true if at least 60 percent of answers are correct; false if one wrong answer is held by at least 60 percent; null when no single answer reaches 60 percent."
    },
    "rationale": {
      "type": "string",
      "minLength": 1
    },
    "source": {
      "oneOf": [
        {
          "const": "original"
        },
        {
          "type": "object",
          "additionalProperties": false,
          "required": [
            "dataset",
            "license",
            "citation",
            "url",
            "item"
          ],
          "properties": {
            "dataset": {
              "type": "string"
            },
            "license": {
              "type": "string",
              "enum": [
                "Apache-2.0",
                "MIT",
                "CC-BY-4.0",
                "CC-BY-SA-4.0",
                "CC0-1.0"
              ]
            },
            "citation": {
              "type": "string"
            },
            "url": {
              "type": "string",
              "format": "uri"
            },
            "item": {
              "type": "string"
            },
            "modification": {
              "type": "string"
            }
          }
        }
      ]
    },
    "tags": {
      "type": "array",
      "items": {
        "type": "string",
        "pattern": "^[a-z0-9_]+$"
      },
      "uniqueItems": true
    }
  }
}