{
 "source_dir": "predicted_beam1m_k200",
 "cutoff": "top_200",
 "questions": 700,
 "pass": 504,
 "pass_rate": 72.0,
 "avg_score": 0.6559,
 "by_type": {
  "abstention": {
   "n": 70,
   "pass": 38,
   "avg": 0.5214
  },
  "contradiction_resolution": {
   "n": 70,
   "pass": 35,
   "avg": 0.3875
  },
  "event_ordering": {
   "n": 70,
   "pass": 48,
   "avg": 0.5976
  },
  "information_extraction": {
   "n": 70,
   "pass": 59,
   "avg": 0.7897
  },
  "instruction_following": {
   "n": 70,
   "pass": 61,
   "avg": 0.8417
  },
  "knowledge_update": {
   "n": 70,
   "pass": 51,
   "avg": 0.7071
  },
  "multi_session_reasoning": {
   "n": 70,
   "pass": 52,
   "avg": 0.668
  },
  "preference_following": {
   "n": 70,
   "pass": 68,
   "avg": 0.8768
  },
  "summarization": {
   "n": 70,
   "pass": 51,
   "avg": 0.6372
  },
  "temporal_reasoning": {
   "n": 70,
   "pass": 41,
   "avg": 0.5321
  }
 },
 "pass_rule": "per-question score >= 0.5",
 "harness_metrics_file": {
  "top_200": {
   "overall": {
    "total": 700,
    "correct": 504,
    "errors": 0,
    "accuracy": 72.0,
    "avg_score": 0.6559171428571429
   },
   "by_question_type": {
    "abstention": {
     "total": 70,
     "correct": 38,
     "accuracy": 54.285714285714285,
     "avg_score": 0.5214285714285715
    },
    "contradiction_resolution": {
     "total": 70,
     "correct": 35,
     "accuracy": 50.0,
     "avg_score": 0.3875
    },
    "event_ordering": {
     "total": 70,
     "correct": 48,
     "accuracy": 68.57142857142857,
     "avg_score": 0.5975685714285714
    },
    "information_extraction": {
     "total": 70,
     "correct": 59,
     "accuracy": 84.28571428571429,
     "avg_score": 0.7897442857142857
    },
    "instruction_following": {
     "total": 70,
     "correct": 61,
     "accuracy": 87.14285714285714,
     "avg_score": 0.8416671428571428
    },
    "knowledge_update": {
     "total": 70,
     "correct": 51,
     "accuracy": 72.85714285714285,
     "avg_score": 0.7071428571428572
    },
    "multi_session_reasoning": {
     "total": 70,
     "correct": 52,
     "accuracy": 74.28571428571429,
     "avg_score": 0.6679828571428571
    },
    "preference_following": {
     "total": 70,
     "correct": 68,
     "accuracy": 97.14285714285714,
     "avg_score": 0.8767885714285715
    },
    "summarization": {
     "total": 70,
     "correct": 51,
     "accuracy": 72.85714285714285,
     "avg_score": 0.6372057142857143
    },
    "temporal_reasoning": {
     "total": 70,
     "correct": 41,
     "accuracy": 58.57142857142858,
     "avg_score": 0.5321428571428571
    }
   }
  }
 },
 "recomputed_from": "per-question judged rows in beam1m_top200_run1_judged.json"
}