{
  "schema": "fak.agentic-benchmark-external-harness-queue.v1",
  "generated_at": "2026-06-26T03:43:07Z",
  "epic": 868,
  "status": "PENDING_EXTERNAL_HARNESS",
  "result_claim_allowed": false,
  "summary": {
    "items_total": 6,
    "ready_items": 5,
    "blocked_items": 1,
    "required_return_artifacts": 36,
    "pending_issues": [
      870,
      871,
      872,
      873,
      874,
      875
    ]
  },
  "items": [
    {
      "issue": 870,
      "packet": "B",
      "title": "GLM-5.2/vLLM agentic battery",
      "source_artifact": "experiments/vllm/glm52-agentic-battery/final-check.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "PENDING_MEASUREMENT",
      "external_state": "BLOCKED_ON_H200_VLLM_READY",
      "claim_boundary": "No GLM-5.2/vLLM benchmark number is quotable until every required artifact passes and any copied number is linked from BENCHMARK-AUTHORITY.md.",
      "commands": [
        {
          "id": "preflight",
          "description": "Fail-closed GLM-5.2/vLLM node readiness gate.",
          "command": "python tools/glm52_serve_preflight.py --engine vllm --quant fp8 --require-ready --out experiments/vllm/glm52-vllm-preflight.json --markdown experiments/vllm/glm52-vllm-preflight.md",
          "artifacts": [
            "experiments/vllm/glm52-vllm-preflight.json",
            "experiments/vllm/glm52-vllm-preflight.md"
          ],
          "required": true,
          "artifact_status": "FAIL",
          "artifact_detail": "node_verdict='BLOCKED_ARCH'; vllm.ready=False; expected GLM-5.2 vLLM READY/PENDING preflight"
        },
        {
          "id": "serve_raw_vllm",
          "description": "Start raw vLLM for GLM-5.2 before the live witnesses.",
          "command": ": \"${GLM52_TOOL_CALL_PARSER:?set GLM52_TOOL_CALL_PARSER to the vLLM parser name}\" \u0026\u0026 ENGINE=vllm SERVED_NAME=glm-5.2 PORT=8000 ENGINE_ARGS=\"--enable-auto-tool-choice --tool-call-parser ${GLM52_TOOL_CALL_PARSER}\" bash tools/glm52_sglang_vllm_serve.sh",
          "required": true,
          "artifact_status": "MANUAL",
          "artifact_detail": "manual serving step; witnessed by downstream artifacts"
        },
        {
          "id": "serving_witness",
          "description": "Direct + fak-gateway + quarantine serving witness.",
          "command": "python tools/glm52_serving_witness.py --base-url http://127.0.0.1:8000/v1 --model glm-5.2 --engine-cache-engine vllm --context-length 131072 --out experiments/glm52/full-size-serving-witness.json --markdown experiments/glm52/full-size-serving-witness.md",
          "artifacts": [
            "experiments/glm52/full-size-serving-witness.json",
            "experiments/glm52/full-size-serving-witness.md"
          ],
          "required": true,
          "artifact_status": "MISSING",
          "artifact_detail": "missing"
        },
        {
          "id": "vllm_tax",
          "description": "Measure fak gateway adjudication tax over raw vLLM.",
          "command": "python tools/vllm_tax_witness.py --base-url http://127.0.0.1:8000/v1 --model glm-5.2 --count 8 --record --out experiments/vllm/adjudication-tax-witness.json --markdown experiments/vllm/adjudication-tax-witness.md",
          "artifacts": [
            "experiments/vllm/adjudication-tax-witness.json",
            "experiments/vllm/adjudication-tax-witness.md"
          ],
          "required": true,
          "artifact_status": "MISSING",
          "artifact_detail": "missing"
        },
        {
          "id": "swebench_compare_preflight",
          "description": "Fail fast before the long SWE-bench agent run.",
          "command": "python tools/dgx_swebench_compare.py --engine vllm --model zai-org/GLM-5.2-FP8 --served-model-name glm-5.2 --raw-base-url http://127.0.0.1:8000/v1 --verified-count 20 --skip-engine-serve --require-tool-calls --require-grade --run-dir /tmp/swe-glm52-vllm-20 --require-gpu-name H200 --preflight-only",
          "artifacts": [
            "/tmp/swe-glm52-vllm-20/COMPARE-PREFLIGHT.json",
            "/tmp/swe-glm52-vllm-20/DONE.rc"
          ],
          "required": true,
          "artifact_status": "MISSING",
          "artifact_detail": "missing"
        },
        {
          "id": "swebench_verified_20",
          "description": "Run raw-vLLM vs fak-gateway on a 20-task SWE-bench Verified slice.",
          "command": "python tools/dgx_swebench_compare.py --engine vllm --model zai-org/GLM-5.2-FP8 --served-model-name glm-5.2 --raw-base-url http://127.0.0.1:8000/v1 --verified-count 20 --skip-engine-serve --require-tool-calls --require-grade --run-dir /tmp/swe-glm52-vllm-20 --require-gpu-name H200",
          "artifacts": [
            "/tmp/swe-glm52-vllm-20/COMPARE-PREFLIGHT.json",
            "/tmp/swe-glm52-vllm-20/compare.json",
            "/tmp/swe-glm52-vllm-20/COMPARE.md",
            "/tmp/swe-glm52-vllm-20/DONE.rc"
          ],
          "required": true,
          "artifact_status": "MISSING",
          "artifact_detail": "missing"
        }
      ],
      "required_before_claim": [
        "preflight must pass on the target GLM-5.2/vLLM node",
        "serving_witness artifact must be generated and pass the final-check witness",
        "vllm_tax artifact must be generated and pass the final-check witness",
        "swebench_compare_preflight artifact must be generated and pass the final-check witness",
        "swebench_verified_20 artifact must be generated and pass the final-check witness"
      ],
      "compare_metrics": [
        "swebench_resolve_rate",
        "safe_completion",
        "latency",
        "token_or_cost_proxy",
        "policy_blocks",
        "evidence_completeness"
      ],
      "result_claim_allowed": false
    },
    {
      "issue": 871,
      "packet": "C",
      "title": "Opus-class SWE-bench smoke",
      "source_artifact": "experiments/agent-live/swebench-opus-smoke-contract-20260626.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "READY_FOR_REMOTE_GRADING",
      "external_state": "READY_FOR_EXTERNAL_HARNESS",
      "claim_boundary": "Pre-run contract only: fixes task ids, model identity, arm commands, and official-grader commands for an Opus-class raw-vs-fak SWE-bench smoke. It is not a solve-rate result until both predictions files are produced and graded by the official SWE-bench harness.",
      "arms": [
        {
          "name": "raw-opus",
          "harness": "benchmark-native-or-raw-agent-scaffold",
          "command": "$env:MSWEA_COST_TRACKING='ignore_errors'; mini-extra swebench --subset verified --split test -w 1 -o experiments/agent-live/swebench-opus-raw-smoke-20260626 -m anthropic/claude-opus-4-8 -c swebench.yaml --filter \"django__django-12345|django__django-23456|scikit-learn__scikit-learn-56789|sympy__sympy-34567|sympy__sympy-45678\"; Copy-Item -LiteralPath experiments/agent-live/swebench-opus-raw-smoke-20260626/preds.json -Destination experiments/agent-live/swebench-opus-raw-smoke-20260626/predictions.json",
          "output_dir": "experiments/agent-live/swebench-opus-raw-smoke-20260626"
        },
        {
          "name": "fak-opus",
          "harness": "fak-gateway-fleet-runner",
          "command": "go run ./cmd/fak swebench run --agent fleet --filter smoke --difficulty testdata/swebench_smoke.json --gateway localhost:8080 --model claude-opus-4-8 --preds-only --output experiments/agent-live/swebench-opus-fak-smoke-20260626",
          "output_dir": "experiments/agent-live/swebench-opus-fak-smoke-20260626"
        }
      ],
      "required_before_claim": [
        "raw-opus predictions.json generated by the raw scaffold over the selected task ids",
        "fak-opus predictions.json generated by the fak fleet runner over the same selected task ids",
        "official SWE-bench harness report.json for both arms",
        "raw/fak compare artifact that folds solve rate, safe completion, cost or token budget, latency, policy blocks, and evidence completeness"
      ],
      "compare_metrics": [
        "solve_rate",
        "safe_completion",
        "cost_or_token_budget",
        "latency",
        "tool_call_count",
        "policy_blocks",
        "evidence_completeness"
      ],
      "result_claim_allowed": false
    },
    {
      "issue": 872,
      "packet": "D",
      "title": "DeepSWE/R2E-Gym runner",
      "source_artifact": "experiments/agent-live/deepswe-raw-fak-contract-20260626.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "READY_FOR_EXTERNAL_RUN",
      "external_state": "READY_FOR_EXTERNAL_HARNESS",
      "claim_boundary": "Pre-run contract only: fixes task ids, DeepSWE/R2E-Gym adapter, model id, raw/fak endpoint routing, budget, and official-grader commands. It is not a solve-rate result until both arms produce predictions and the official SWE-bench harness grades them.",
      "arms": [
        {
          "name": "raw-deepswe",
          "harness": "deepswe-r2e-gym-raw",
          "command": "$env:FAK_DEEPSWE_RUNNER='deepswe-r2e-runner'; $env:FAK_DEEPSWE_RUNNER_ARGS=''; $env:FAK_DEEPSWE_BASE_URL=$env:RAW_DEEPSWE_BASE_URL; $env:FAK_DEEPSWE_MODEL='DeepSWE-Preview'; go run ./cmd/fak swebench run --agent deepswe --filter full --difficulty testdata/swebench_smoke.json --limit 2 --max-steps 50 --timeout 30m --model DeepSWE-Preview --preds-only --output experiments/agent-live/deepswe-raw-smoke-20260626",
          "output_dir": "experiments/agent-live/deepswe-raw-smoke-20260626"
        },
        {
          "name": "fak-deepswe",
          "harness": "deepswe-r2e-gym-through-fak-gateway",
          "command": "$env:FAK_DEEPSWE_RUNNER='deepswe-r2e-runner'; $env:FAK_DEEPSWE_RUNNER_ARGS=''; $env:FAK_DEEPSWE_BASE_URL='http://localhost:8080/v1'; $env:FAK_DEEPSWE_MODEL='DeepSWE-Preview'; go run ./cmd/fak swebench run --agent deepswe --filter full --difficulty testdata/swebench_smoke.json --limit 2 --max-steps 50 --timeout 30m --model DeepSWE-Preview --preds-only --output experiments/agent-live/deepswe-fak-smoke-20260626",
          "output_dir": "experiments/agent-live/deepswe-fak-smoke-20260626"
        }
      ],
      "required_before_claim": [
        "raw-deepswe predictions.json generated by a real DeepSWE/R2E-Gym adapter over the selected task ids",
        "fak-deepswe predictions.json generated by the same adapter over the same task ids through the fak gateway",
        "official SWE-bench harness report.json for both arms",
        "adapter logs or metadata showing the same model id, max steps, timeout, and task id list for both arms",
        "raw/fak compare artifact that folds solve rate, safe completion, cost or token budget, latency, adapter failures, policy blocks, and evidence completeness"
      ],
      "compare_metrics": [
        "solve_rate",
        "safe_completion",
        "cost_or_token_budget",
        "latency",
        "tool_call_count",
        "adapter_failures",
        "policy_blocks",
        "evidence_completeness"
      ],
      "result_claim_allowed": false
    },
    {
      "issue": 873,
      "packet": "E",
      "title": "ToolSandbox/tau3 policy-state",
      "source_artifact": "experiments/agent-live/toolsandbox-official-run-contract-20260626.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "READY_FOR_EXTERNAL_HARNESS",
      "external_state": "READY_FOR_EXTERNAL_HARNESS",
      "claim_boundary": "External-run contract only: fixes the raw/fak command shape, shared model and simulator requirements, evidence paths, and promotion gates for a benchmark-native tau3 or ToolSandbox run. It is not an official result until external task definitions, raw/fak outputs, and native grader summaries are checked in.",
      "arms": [
        {
          "name": "raw-toolsandbox",
          "harness": "benchmark-native",
          "command": "$env:TAU3_TASK_IDS='\u003cspace-separated official task ids\u003e'; tau2 run --domain retail --agent-llm shared-agent-model --user-llm shared-agent-model --num-trials 1 --task-ids ($env:TAU3_TASK_IDS -split ' ')",
          "output_dir": "experiments/agent-live/toolsandbox-official-raw-20260626",
          "required_artifacts": [
            "benchmark-native raw result summary",
            "raw trajectory or simulation log"
          ]
        },
        {
          "name": "fak-toolsandbox",
          "harness": "benchmark-native-through-fak-gateway",
          "command": "$env:TAU3_TASK_IDS='\u003cspace-separated official task ids\u003e'; $env:OPENAI_BASE_URL='http://localhost:8080/v1'; $env:OPENAI_API_BASE='http://localhost:8080/v1'; tau2 run --domain retail --agent-llm shared-agent-model --user-llm shared-agent-model --num-trials 1 --task-ids ($env:TAU3_TASK_IDS -split ' ')",
          "output_dir": "experiments/agent-live/toolsandbox-official-fak-20260626",
          "required_artifacts": [
            "benchmark-native fak result summary",
            "fak trajectory or simulation log",
            "fak verdict/evidence log linked to mediated tool calls"
          ]
        }
      ],
      "required_before_claim": [
        "benchmark-native task manifest or scenario ids for the selected tau3 or ToolSandbox subset",
        "raw-arm benchmark-native output over those exact task ids",
        "fak-arm benchmark-native output over those exact task ids",
        "benchmark-native grader or result summary for both arms",
        "proof that raw and fak arms used the same model, user simulator, task ids, budget, and retry policy",
        "fak verdict/evidence log linked to each mediated tool call",
        "raw/fak compare artifact reporting task success separately from policy compliance and benign utility preservation"
      ],
      "compare_metrics": [
        "benchmark_native_success_or_pass_k",
        "safe_completion",
        "policy_breach_rate",
        "minefield_or_mutating_action_avoidance",
        "tool_call_count",
        "latency",
        "cost_or_token_budget",
        "fak_denied_calls",
        "evidence_completeness"
      ],
      "result_claim_allowed": false
    },
    {
      "issue": 874,
      "packet": "F",
      "title": "Terminal-Bench command boundary",
      "source_artifact": "experiments/agent-live/terminalbench-official-run-contract-20260626.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "READY_FOR_EXTERNAL_HARNESS",
      "external_state": "READY_FOR_EXTERNAL_HARNESS",
      "claim_boundary": "External-run contract only: fixes the raw/fak Terminal-Bench command shape, shared task/model/image/budget requirements, evidence paths, and promotion gates. It is not an official result until benchmark-native tb run task logs, test output, and a raw-vs-fak compare artifact are checked in.",
      "arms": [
        {
          "name": "raw-terminalbench",
          "harness": "benchmark-native",
          "command": "$env:TERMINAL_BENCH_TASK_IDS='\u003cspace-separated official task ids\u003e'; foreach ($task in ($env:TERMINAL_BENCH_TASK_IDS -split ' ')) { tb run --dataset terminal-bench-core==0.1.1 --agent terminus --model shared-agent-model --task-id $task --n-concurrent 1 }",
          "output_dir": "experiments/agent-live/terminalbench-official-raw-20260626",
          "required_artifacts": [
            "tb run directory for each selected task",
            "benchmark-native command log",
            "benchmark-native test output or result summary"
          ]
        },
        {
          "name": "fak-terminalbench",
          "harness": "benchmark-native-through-fak-gateway",
          "command": "$env:TERMINAL_BENCH_TASK_IDS='\u003cspace-separated official task ids\u003e'; $env:OPENAI_BASE_URL='http://localhost:8080/v1'; $env:OPENAI_API_BASE='http://localhost:8080/v1'; foreach ($task in ($env:TERMINAL_BENCH_TASK_IDS -split ' ')) { tb run --dataset terminal-bench-core==0.1.1 --agent terminus-through-fak --model shared-agent-model --task-id $task --n-concurrent 1 }",
          "output_dir": "experiments/agent-live/terminalbench-official-fak-20260626",
          "required_artifacts": [
            "tb run directory for each selected task",
            "benchmark-native command log",
            "benchmark-native test output or result summary",
            "fak verdict/evidence log linked to mediated terminal commands"
          ]
        }
      ],
      "required_before_claim": [
        "benchmark-native Terminal-Bench task ids for the selected fixed subset",
        "Terminal-Bench image or environment setup manifest for each selected task",
        "raw-arm tb run directory with command log and official test output over those exact task ids",
        "fak-arm tb run directory with command log and official test output over those exact task ids",
        "proof that raw and fak arms used the same model, task ids, image or environment, budget, concurrency, and retry policy",
        "fak per-command verdict/evidence log linked to the corresponding tb run command log and test output",
        "raw/fak compare artifact reporting benchmark-native solve separately from safe resolve, blocked dangerous actions, unnecessary blocks, runtime, cost or token budget, and evidence completeness"
      ],
      "compare_metrics": [
        "benchmark_native_test_success_or_pass_1",
        "safe_resolve",
        "blocked_dangerous_actions",
        "unnecessary_blocks",
        "command_count",
        "runtime",
        "cost_or_token_budget",
        "fak_denied_commands",
        "fak_verdict_evidence_completeness"
      ],
      "result_claim_allowed": false
    },
    {
      "issue": 875,
      "packet": "G",
      "title": "Browser/computer-use action mediation",
      "source_artifact": "experiments/agent-live/browseraction-official-run-contract-20260626.json",
      "gate": "PENDING_EXTERNAL_HARNESS",
      "status": "READY_FOR_EXTERNAL_HARNESS",
      "external_state": "READY_FOR_EXTERNAL_HARNESS",
      "claim_boundary": "External-run contract only: fixes the raw/fak BrowserGym/WebArena-style command shape, shared model/task/browser-state/budget requirements, evidence paths, and promotion gates. It is not an official browser or computer-use benchmark result until benchmark-native task traces, score reports, and a raw-vs-fak compare artifact are checked in.",
      "arms": [
        {
          "name": "raw-browseraction",
          "harness": "benchmark-native",
          "command": "$env:BROWSERGYM_TASK_IDS='\u003cspace-separated BrowserGym env ids\u003e'; $env:AGENTLAB_EXP_ROOT='experiments/agent-live/browseraction-official-raw-20260626'; $env:FAK_BROWSERGYM_AGENT='browsergym_agent_config'; $env:FAK_BROWSERGYM_MODEL='shared-agent-model'; $env:FAK_BROWSERGYM_MAX_STEPS='30'; python -c 'from agentlab.experiments.study import make_study; from importlib import import_module; import os; agent_args=getattr(import_module(os.environ[''FAK_BROWSERGYM_AGENT'']), ''AGENT_ARGS''); study=make_study(benchmark=''webarena'', agent_args=[agent_args], comment=os.environ[''FAK_BROWSERGYM_MODEL'']); study.run(n_jobs=1)'",
          "output_dir": "experiments/agent-live/browseraction-official-raw-20260626",
          "required_artifacts": [
            "benchmark-native task trace or study directory",
            "benchmark-native score report",
            "browser state reset or environment manifest"
          ]
        },
        {
          "name": "fak-browseraction",
          "harness": "benchmark-native-through-fak-gateway",
          "command": "$env:BROWSERGYM_TASK_IDS='\u003cspace-separated BrowserGym env ids\u003e'; $env:AGENTLAB_EXP_ROOT='experiments/agent-live/browseraction-official-fak-20260626'; $env:FAK_BROWSERGYM_AGENT='browsergym_agent_config_through_fak'; $env:FAK_BROWSERGYM_MODEL='shared-agent-model'; $env:FAK_BROWSERGYM_MAX_STEPS='30'; $env:OPENAI_BASE_URL='http://localhost:8080/v1'; $env:OPENAI_API_BASE='http://localhost:8080/v1'; python -c 'from agentlab.experiments.study import make_study; from importlib import import_module; import os; agent_args=getattr(import_module(os.environ[''FAK_BROWSERGYM_AGENT'']), ''AGENT_ARGS''); study=make_study(benchmark=''webarena'', agent_args=[agent_args], comment=os.environ[''FAK_BROWSERGYM_MODEL'']); study.run(n_jobs=1)'",
          "output_dir": "experiments/agent-live/browseraction-official-fak-20260626",
          "required_artifacts": [
            "benchmark-native task trace or study directory",
            "benchmark-native score report",
            "browser state reset or environment manifest",
            "fak action verdict/evidence log linked to mediated browser actions"
          ]
        }
      ],
      "required_before_claim": [
        "benchmark-native BrowserGym/WebArena, WorkArena, OSWorld, or BrowseComp task ids for the selected fixed subset",
        "browser state, website snapshot, credentials, reset policy, and environment manifest for each selected task",
        "raw-arm benchmark-native trace or study directory with score report over those exact task ids",
        "fak-arm benchmark-native trace or study directory with score report over those exact task ids",
        "proof that raw and fak arms used the same model, task ids, browser state, budget, max steps, timeout, and retry policy",
        "fak action verdict/evidence log linked to the corresponding benchmark-native action trace and score report",
        "writeup separating model perception or grounding failures from harness/tool-boundary failures",
        "raw/fak compare artifact reporting task success separately from safe success, action count, invalid actions, policy blocks, runtime, cost or token budget, and evidence completeness"
      ],
      "compare_metrics": [
        "benchmark_native_task_success_or_score",
        "safe_success",
        "action_count",
        "invalid_action_rate",
        "policy_blocks",
        "minefield_or_mutating_action_avoidance",
        "runtime",
        "cost_or_token_budget",
        "fak_action_evidence_completeness"
      ],
      "result_claim_allowed": false
    }
  ],
  "claim_boundary": "External harness queue only: folds pending child contracts into runnable work items and refuses a #868 result claim until benchmark-native raw/fak grader artifacts are checked in."
}
