{
  "schema": "fak.swebench-opus-smoke-contract.v1",
  "generated_at": "2026-06-26T11:34:35Z",
  "benchmark": "SWE-bench Verified",
  "model_class": "opus-class",
  "status": "READY_FOR_REMOTE_GRADING",
  "evidence_class": "EXTERNAL_RUN_CONTRACT",
  "claim_boundary": "Pre-run contract only: fixes task ids, model identity, arm commands, and official-grader commands for an Opus-class raw-vs-fak SWE-bench smoke. It is not a solve-rate result until both predictions files are produced and graded by the official SWE-bench harness.",
  "task_selection": {
    "source": "difficulty map testdata/swebench_smoke.json (5 instances from princeton-nlp/SWE-bench_Verified, bucket-derived geometry)",
    "filter": "smoke",
    "task_ids": [
      "django__django-12345",
      "django__django-23456",
      "scikit-learn__scikit-learn-56789",
      "sympy__sympy-34567",
      "sympy__sympy-45678"
    ],
    "difficulty_distribution": {
      "1-4hr": 1,
      "15min-1hr": 2,
      "\u003c15min": 2
    },
    "same_task_ids": true
  },
  "arms": [
    {
      "name": "raw-opus",
      "harness": "benchmark-native-or-raw-agent-scaffold",
      "model": "claude-opus-4-8",
      "command": "$env:MSWEA_COST_TRACKING='ignore_errors'; mini-extra swebench --subset verified --split test -w 1 -o experiments/agent-live/swebench-opus-raw-smoke-20260626 -m anthropic/claude-opus-4-8 -c swebench.yaml --filter \"django__django-12345|django__django-23456|scikit-learn__scikit-learn-56789|sympy__sympy-34567|sympy__sympy-45678\"; Copy-Item -LiteralPath experiments/agent-live/swebench-opus-raw-smoke-20260626/preds.json -Destination experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json",
      "output_dir": "experiments/agent-live/swebench-opus-raw-smoke-20260626",
      "predictions_path": "experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json",
      "eval_run_id": "swebench-opus-raw-smoke",
      "eval_command": "python -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Verified --predictions_path experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json --run_id swebench-opus-raw-smoke --max_workers 4"
    },
    {
      "name": "fak-opus",
      "harness": "fak-gateway-fleet-runner",
      "model": "claude-opus-4-8",
      "command": "go run ./cmd/fak swebench run --agent fleet --filter smoke --difficulty testdata/swebench_smoke.json --gateway localhost:8080 --model claude-opus-4-8 --preds-only --output experiments/agent-live/swebench-opus-fak-smoke-20260626",
      "output_dir": "experiments/agent-live/swebench-opus-fak-smoke-20260626",
      "predictions_path": "experiments/agent-live/swebench-opus-fak-smoke-20260626/predictions.json",
      "eval_run_id": "swebench-opus-fak-smoke",
      "eval_command": "python -m swebench.harness.run_evaluation --dataset_name princeton-nlp/SWE-bench_Verified --predictions_path experiments/agent-live/swebench-opus-fak-smoke-20260626/predictions.json --run_id swebench-opus-fak-smoke --max_workers 4"
    }
  ],
  "compare_evidence_link": {
    "required": true,
    "predictions": [
      "experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json",
      "experiments/agent-live/swebench-opus-fak-smoke-20260626/predictions.json"
    ],
    "metadata": [
      "experiments/agent-live/swebench-opus-raw-smoke-20260626/meta.json",
      "experiments/agent-live/swebench-opus-fak-smoke-20260626/meta.json"
    ],
    "official_eval": [
      "experiments/agent-live/swebench-opus-raw-smoke-20260626/eval.json",
      "experiments/agent-live/swebench-opus-fak-smoke-20260626/eval.json"
    ],
    "fak_evidence": [
      "experiments/agent-live/swebench-opus-fak-smoke-20260626/fak-adjudication-evidence.jsonl",
      "experiments/agent-live/swebench-opus-fak-smoke-20260626/raw-fak-opus-smoke-compare.json"
    ],
    "join_keys": [
      "instance_id",
      "model",
      "prediction_sha256",
      "evidence_id"
    ],
    "detail": "The raw/fak compare artifact must join every Opus smoke prediction and official grader row to fak-arm adjudication evidence for the same instance id."
  },
  "gates": [
    {
      "name": "fixed_task_ids",
      "ok": true,
      "detail": "5 task ids selected"
    },
    {
      "name": "same_task_ids",
      "ok": true,
      "detail": "raw and fak arms consume the same selected task id list"
    },
    {
      "name": "same_model",
      "ok": true,
      "detail": "claude-opus-4-8"
    },
    {
      "name": "raw_arm_command",
      "ok": true,
      "detail": "$env:MSWEA_COST_TRACKING='ignore_errors'; mini-extra swebench --subset verified --split test -w 1 -o experiments/agent-live/swebench-opus-raw-smoke-20260626 -m anthropic/claude-opus-4-8 -c swebench.yaml --filter \"django__django-12345|django__django-23456|scikit-learn__scikit-learn-56789|sympy__sympy-34567|sympy__sympy-45678\"; Copy-Item -LiteralPath experiments/agent-live/swebench-opus-raw-smoke-20260626/preds.json -Destination experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json"
    },
    {
      "name": "fak_arm_command",
      "ok": true,
      "detail": "go run ./cmd/fak swebench run --agent fleet --filter smoke --difficulty testdata/swebench_smoke.json --gateway localhost:8080 --model claude-opus-4-8 --preds-only --output experiments/agent-live/swebench-opus-fak-smoke-20260626"
    },
    {
      "name": "official_grader_local",
      "ok": false,
      "detail": "swebench python module not importable — `pip install swebench[harness]` (or run scripts/install_swebench.py)"
    }
  ],
  "official_grader": {
    "docker_present": true,
    "harness_present": false,
    "python": "python",
    "runnable": false,
    "reason": "swebench python module not importable — `pip install swebench[harness]` (or run scripts/install_swebench.py)"
  },
  "compare_metrics": [
    "solve_rate",
    "safe_completion",
    "cost_or_token_budget",
    "latency",
    "tool_call_count",
    "policy_blocks",
    "evidence_completeness"
  ],
  "required_before_claim": [
    "raw-opus predictions.json generated by the raw scaffold over the selected task ids",
    "fak-opus predictions.json generated by the fak fleet runner over the same selected task ids",
    "official SWE-bench harness report.json for both arms",
    "raw/fak compare evidence link joining predictions, metadata, official eval rows, and fak adjudication events by instance id",
    "raw/fak compare artifact that folds solve rate, safe completion, cost or token budget, latency, policy blocks, and evidence completeness"
  ],
  "result_claim_allowed": false
}