{
 "source_dir": "predicted_beam1m_k200_r2",
 "cutoff": "top_200",
 "questions": 700,
 "pass": 508,
 "pass_rate": 72.57,
 "avg_score": 0.6692,
 "by_type": {
  "abstention": {
   "n": 70,
   "pass": 43,
   "avg": 0.5857
  },
  "contradiction_resolution": {
   "n": 70,
   "pass": 31,
   "avg": 0.4179
  },
  "event_ordering": {
   "n": 70,
   "pass": 48,
   "avg": 0.5828
  },
  "information_extraction": {
   "n": 70,
   "pass": 58,
   "avg": 0.7866
  },
  "instruction_following": {
   "n": 70,
   "pass": 62,
   "avg": 0.8667
  },
  "knowledge_update": {
   "n": 70,
   "pass": 48,
   "avg": 0.6786
  },
  "multi_session_reasoning": {
   "n": 70,
   "pass": 54,
   "avg": 0.6927
  },
  "preference_following": {
   "n": 70,
   "pass": 69,
   "avg": 0.8738
  },
  "summarization": {
   "n": 70,
   "pass": 51,
   "avg": 0.643
  },
  "temporal_reasoning": {
   "n": 70,
   "pass": 44,
   "avg": 0.5643
  }
 },
 "pass_rule": "per-question score >= 0.5",
 "harness_metrics_file": {
  "top_200": {
   "overall": {
    "total": 700,
    "correct": 508,
    "errors": 0,
    "accuracy": 72.57142857142857,
    "avg_score": 0.6692114285714286
   },
   "by_question_type": {
    "abstention": {
     "total": 70,
     "correct": 43,
     "accuracy": 61.42857142857143,
     "avg_score": 0.5857142857142857
    },
    "contradiction_resolution": {
     "total": 70,
     "correct": 31,
     "accuracy": 44.285714285714285,
     "avg_score": 0.41785714285714287
    },
    "event_ordering": {
     "total": 70,
     "correct": 48,
     "accuracy": 68.57142857142857,
     "avg_score": 0.58285
    },
    "information_extraction": {
     "total": 70,
     "correct": 58,
     "accuracy": 82.85714285714286,
     "avg_score": 0.7866057142857142
    },
    "instruction_following": {
     "total": 70,
     "correct": 62,
     "accuracy": 88.57142857142857,
     "avg_score": 0.8666657142857143
    },
    "knowledge_update": {
     "total": 70,
     "correct": 48,
     "accuracy": 68.57142857142857,
     "avg_score": 0.6785714285714286
    },
    "multi_session_reasoning": {
     "total": 70,
     "correct": 54,
     "accuracy": 77.14285714285715,
     "avg_score": 0.6927028571428572
    },
    "preference_following": {
     "total": 70,
     "correct": 69,
     "accuracy": 98.57142857142858,
     "avg_score": 0.8738128571428572
    },
    "summarization": {
     "total": 70,
     "correct": 51,
     "accuracy": 72.85714285714285,
     "avg_score": 0.6430485714285714
    },
    "temporal_reasoning": {
     "total": 70,
     "correct": 44,
     "accuracy": 62.857142857142854,
     "avg_score": 0.5642857142857143
    }
   }
  }
 },
 "recomputed_from": "per-question judged rows in beam1m_top200_run2_judged.json"
}