{
  "created_at": "2026-09-25T12:03:04.239073+00:00",
  "status": "failed_preregistered_quality_domain",
  "protocol_sha256": "9475e52b838ae6dd7c5dbb69ab9134f8de75adcc18503c12676828e891a5e078",
  "predictions_seal_sha256": "ab32d275e4680a38dbf3e374e48a7ea9185d8e5749ab8e17f66c30c4b6e5f9b3",
  "expected_rows": 809,
  "evaluation_results_published": false,
  "changes_to_protocol_or_predictions": false,
  "diagnostic": [
    {
      "model": "claude-3-haiku-20240307",
      "selected_rows": 809,
      "metrics": {
        "math_metric": {
          "count": 24,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "mcq_accuracy": {
          "count": 565,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "chess_accuracy": {
          "count": 13,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "exact_match": {
          "count": 62,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "code_accuracy": {
          "count": 38,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "meteor_score": {
          "count": 70,
          "min": -0.0330188679245283,
          "max": 1.0,
          "outside_0_1": 1
        },
        "superglue_clozetest": {
          "count": 5,
          "min": 0.0,
          "max": 0.0,
          "outside_0_1": 0
        },
        "superglue_exact_match": {
          "count": 32,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        }
      },
      "invalid_quality": [
        {
          "global_index": "NarrativeQA_4102",
          "metric": "meteor_score",
          "score": -0.0330188679245283
        }
      ],
      "invalid_cost": []
    },
    {
      "model": "gemini-2.0-flash-001",
      "selected_rows": 809,
      "metrics": {
        "math_metric": {
          "count": 24,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "mcq_accuracy": {
          "count": 565,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "chess_accuracy": {
          "count": 13,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "exact_match": {
          "count": 62,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "code_accuracy": {
          "count": 38,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "meteor_score": {
          "count": 70,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "superglue_clozetest": {
          "count": 5,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "superglue_exact_match": {
          "count": 32,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        }
      },
      "invalid_quality": [],
      "invalid_cost": []
    },
    {
      "model": "gpt-4o-mini",
      "selected_rows": 809,
      "metrics": {
        "math_metric": {
          "count": 24,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "mcq_accuracy": {
          "count": 565,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "chess_accuracy": {
          "count": 13,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "exact_match": {
          "count": 62,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "code_accuracy": {
          "count": 38,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "meteor_score": {
          "count": 70,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        },
        "superglue_clozetest": {
          "count": 5,
          "min": 0.0,
          "max": 0.0,
          "outside_0_1": 0
        },
        "superglue_exact_match": {
          "count": 32,
          "min": 0.0,
          "max": 1.0,
          "outside_0_1": 0
        }
      },
      "invalid_quality": [],
      "invalid_cost": []
    }
  ]
}