{ "version": "1.1.0", "checked_on": "2026-11-11", "scope": "Methodological references, not a performance ranking. No external suite was executed for this review. Differences are PALO maintainer assessments of documented scope, not proof of absence of features elsewhere.", "discovery": { "name": "terryyz/llm-benchmark", "url": "https://github.com/terryyz/llm-benchmark", "revision": "ffe4975fbdefd922136ae6311ac1beee480a9b0d", "role": "Bibliographic directory; not a benchmark result or endorsement. All 12 listed entries are accounted for; historical dataset counts are not copied as current facts." }, "palo_status": { "implemented": "Evidence schema, annotation consistency, raw-output binding, calculated task acceptance, structural replay and historical configuration explorer. The public dashboard includes 18 internal model configurations, metric-specific exploratory ranks and separate automated countertests; labels remain assistant drafts.", "proposed": "Procurement specification-fidelity study, independently adjudicated unseen cases, critical-rule inversion analysis and measured human correction effort.", "qualified_heldout_results": 0, "pbi_score": null, "open_gates": "External preregistration/custody, independent annotation, metric calibration, proposition-bank reuse and family-aware uncertainty." }, "entries": [ { "id": "helm", "name": "HELM", "kind": "Evaluation framework", "source_url": "https://github.com/stanford-crfm/helm", "focus": "Broad scenario and metric coverage for foundation models.", "evidence": "Public framework, prompts/responses inspection and leaderboards.", "palo_difference": "PALO proposes a narrower procurement specification-fidelity study. Multi-dimensional evaluation is established prior art.", "lesson": "Retain scenario, adapter, model and metric versions.", "source_status": "README reports maintenance mode since June 2026.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "bigbench", "name": "BIG-bench", "kind": "Task suite", "source_url": "https://github.com/google/BIG-bench", "focus": "Collaborative tasks probing diverse language-model capabilities.", "evidence": "Public tasks, evaluation code and published experiments.", "palo_difference": "PALO targets business-rule preservation in a reviewed specification, rather than broad capability coverage.", "lesson": "Use explicit task definitions and diverse independently selected cases.", "source_status": "Repository archived on 17 April 2026; retained as historical prior art.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "bigbio", "name": "BigBIO", "kind": "Dataset library", "source_url": "https://github.com/bigscience-workshop/biomedical", "focus": "Harmonised biomedical datasets and task schemas.", "evidence": "Dataloaders, provenance, licensing metadata and dataset documentation.", "palo_difference": "A dataset library is a reference for corpus governance, not a competing procurement-specification score.", "lesson": "Document source rights, schema mappings and provenance.", "source_status": "Primary source reviewed", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "bigscience", "name": "BigScience Evaluation", "kind": "Evaluation framework", "source_url": "https://github.com/bigscience-workshop/evaluation", "focus": "Task-based language-model evaluation, including translation and question answering.", "evidence": "Code, task abstractions and evaluation instructions.", "palo_difference": "PALO adds a scoped evidence contract for annotated business propositions; it has not demonstrated broader capability coverage.", "lesson": "Keep task-specific evaluation separate from model execution.", "source_status": "README retains 2021 milestones; no current deployment qualification inferred.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "lm-eval", "name": "LM Evaluation Harness", "kind": "Evaluation framework", "source_url": "https://github.com/EleutherAI/lm-evaluation-harness", "focus": "Configurable execution of academic language-model tasks across backends.", "evidence": "Task configurations, model adapters, metrics and sample logging.", "palo_difference": "This is reusable execution infrastructure. PALO supplies a proposed domain-specific task and adjudication contract; no integration is claimed.", "lesson": "Reuse configuration discipline and record prompt, backend and generation settings.", "source_status": "Primary source reviewed", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "scholar", "name": "Scholar Evals", "kind": "Unverified directory entry", "source_url": "https://github.com/scholar-org/scholar-evals", "focus": "Scope could not be verified from the referenced repository.", "evidence": "The referenced repository returned HTTP 404 during this review.", "palo_difference": "No capability comparison or judgement of benchmark quality is made from an unavailable source.", "lesson": "Preserve the discovery reference and mark unknown evidence explicitly.", "source_status": "Unavailable at the referenced URL; reason unknown.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "bigcode", "name": "BigCode Evaluation Harness", "kind": "Coding evaluation framework", "source_url": "https://github.com/bigcode-project/bigcode-evaluation-harness", "focus": "Code generation and related coding tasks, including unit-test-based evaluation.", "evidence": "Task implementations, generation/evaluation tooling and container instructions.", "palo_difference": "Correct code against supplied tests does not establish that the originating business requirement was interpreted faithfully.", "lesson": "Keep code-test outcomes separate from source-rule fidelity.", "source_status": "Primary source reviewed", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "arena", "name": "Chatbot Arena / Arena", "kind": "Preference evaluation platform", "source_url": "https://arena.ai/blog/ranking-method", "focus": "Model comparison with uncertainty-aware score and rank presentation.", "evidence": "Published ranking methodology and confidence/rank intervals.", "palo_difference": "PALO proposes source-based adjudication of obligations and exceptions; preference and semantic fidelity answer different questions.", "lesson": "Show uncertainty and avoid treating a small numerical ordering as a stable ranking.", "source_status": "Current methodology checked; directory points to historical FastChat infrastructure.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "glue", "name": "GLUE", "kind": "Language-understanding suite", "source_url": "https://arxiv.org/abs/1804.07461", "focus": "General natural-language understanding across multiple tasks.", "evidence": "Task suite, analysis platform and published methodology.", "palo_difference": "PALO examines source-to-specification transformations and the correction burden within a business domain.", "lesson": "Use task-specific gold and publish per-task results.", "source_status": "The directory links jiant, an implementation toolkit whose README says it is no longer actively maintained.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "superglue", "name": "SuperGLUE", "kind": "Language-understanding suite", "source_url": "https://arxiv.org/abs/1905.00537", "focus": "More challenging general language-understanding tasks following GLUE.", "evidence": "Benchmark tasks, evaluation methodology and analysis resources.", "palo_difference": "PALO is narrower and currently unqualified; neither general NLU performance nor PALO diagnostics imply success on the other.", "lesson": "Define acceptance and task difficulty before comparing systems.", "source_status": "Distinguish the benchmark from the jiant implementation linked by the directory.", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "clue", "name": "CLUE", "kind": "Language-understanding suite", "source_url": "https://github.com/CLUEbenchmark/CLUE", "focus": "Chinese-language understanding through representative classification and reading tasks.", "evidence": "Datasets, baselines, task metrics and leaderboards.", "palo_difference": "PALO has not qualified multilingual specification fidelity. Country of model origin is not a language-quality measurement.", "lesson": "Treat language and localisation as explicit evaluation conditions.", "source_status": "Primary source reviewed", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "codexglue", "name": "CodeXGLUE", "kind": "Code-intelligence suite", "source_url": "https://github.com/microsoft/CodeXGLUE", "focus": "Code understanding and generation, including code/text transformations.", "evidence": "Task datasets, baselines and evaluation resources.", "palo_difference": "PALO proposes checking the business meaning before code is produced; code/text overlap is acknowledged.", "lesson": "Separate transformation types and use outcome-appropriate metrics.", "source_status": "Primary source reviewed", "in_requested_directory": true, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "vals", "name": "Vals AI", "kind": "Applied evaluation platform", "source_url": "https://www.vals.ai/benchmarks", "focus": "Industry-oriented tasks and model comparisons across professional domains.", "evidence": "Published benchmark pages and methodologies; some benchmarks are proprietary.", "palo_difference": "Enterprise evaluation is already established. PALO proposes a specific document-to-specification fidelity study, without claiming broader coverage or superior validation.", "lesson": "Use realistic tasks and make public/private evidence boundaries explicit.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "artificial-analysis", "name": "Artificial Analysis", "kind": "Evaluation platform", "source_url": "https://artificialanalysis.ai/methodology", "focus": "Quality, intelligence, performance and price of models, endpoints and systems.", "evidence": "Published measurement definitions and comparative results.", "palo_difference": "PALO records configuration-specific evidence and proposes workflow correction effort; current cost records are not full enterprise cost measurements.", "lesson": "Distinguish model, serving endpoint and complete system.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "officeqa", "name": "OfficeQA", "kind": "Document-reasoning benchmark", "source_url": "https://github.com/databricks/officeqa", "focus": "Complex questions over real-world financial documents.", "evidence": "Evaluation code and corpus tooling; source documents, parsed data and gated answer sets are documented.", "palo_difference": "A close comparator. PALO proposes evaluating the resulting specification and altered rules, rather than only answers to questions.", "lesson": "Separate parsing, retrieval and reasoning; keep unseen document families.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "dabstep", "name": "DABstep", "kind": "Data-agent benchmark", "source_url": "https://huggingface.co/blog/dabstep", "focus": "Multi-step business data analysis with factoid answers.", "evidence": "Tasks, answer-format requirements and automated answer checks.", "palo_difference": "PALO propositions can admit multiple legitimate formulations and require semantic adjudication; answer normalisation alone is insufficient.", "lesson": "Use deterministic checks where valid, without confusing formatting with correctness.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "omnidocbench", "name": "OmniDocBench", "kind": "Document-parsing benchmark", "source_url": "https://github.com/opendatalab/OmniDocBench", "focus": "Document parsing, including text, tables, formulas and layout.", "evidence": "Annotated document data and evaluation tooling.", "palo_difference": "Parsing is an upstream component; PALO proposes checking whether extracted content retains its business meaning in a specification.", "lesson": "Measure extraction independently from interpretation.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "swebench", "name": "SWE-bench Verified", "kind": "Software-agent benchmark", "source_url": "https://www.swebench.com/verified", "focus": "Resolution of repository issues using a human-validated task subset.", "evidence": "Task definitions, evaluation resources and published agent results.", "palo_difference": "PALO focuses on interpreting the source requirement before implementation; passing software tests addresses a different evaluation stage.", "lesson": "Freeze the harness and distinguish model comparisons from whole-agent comparisons.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "facts", "name": "FACTS Grounding", "kind": "Grounded-generation benchmark", "source_url": "https://arxiv.org/abs/2501.03200", "focus": "Factual grounding of generated responses in supplied documents.", "evidence": "Published benchmark and multi-judge evaluation methodology.", "palo_difference": "A close semantic comparator. PALO proposes domain-specific critical-rule inversions and human repair effort; that additional value still needs validation.", "lesson": "Calibrate semantic judging against independent reference decisions.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "andon", "name": "Andon Labs Blueprint-Bench 2", "kind": "Spatial-reasoning benchmark", "source_url": "https://andonlabs.com/evals/blueprint-bench-2", "focus": "Reconstruction of apartment floor plans from photographs.", "evidence": "Published task description, connectivity metric and leaderboard.", "palo_difference": "Distinct project and meaning of blueprint: spatial floor plans versus business development specifications. No affiliation or shared score.", "lesson": "Disambiguate the name and compare only commensurate tasks.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "original-review", "version_scope": "", "visibility_basis": "Not assessed in the visibility extension", "visibility_sources": "", "repo_stars": null, "repo_star_scope": "", "caution": "", "palo_transfer": "Not assessed in the visibility extension", "repo_url": "" }, { "id": "hle", "name": "Humanity's Last Exam", "kind": "Expert academic benchmark", "source_url": "https://labs.scale.com/leaderboard/humanitys_last_exam", "focus": "Expert-level questions spanning text and images.", "evidence": "Public evaluation and a separate private set intended to monitor overfitting.", "palo_difference": "PALO targets faithful interpretation of supplied business documents, rather than broad expert knowledge.", "lesson": "Keep public diagnostics separate from unseen evaluation; record tool access and calibration alongside correctness.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Public and private sets; tool-enabled and tool-free runs must remain distinct.", "visibility_basis": "Featured in Google's annual research recap and maintained on a dedicated Scale leaderboard.", "visibility_sources": "https://blog.google/innovation-and-ai/products/2025-research-breakthroughs/ ; https://labs.scale.com/leaderboard/humanitys_last_exam", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "Exam performance is not business-process validity; public exposure and tool access affect interpretation.", "palo_transfer": "Method", "repo_url": "" }, { "id": "arc-agi", "name": "ARC-AGI", "kind": "Reasoning and interactive benchmark family", "source_url": "https://arcprize.org/leaderboard", "focus": "Generalisation in visual abstraction tasks and, in ARC-AGI-3, interactive environments.", "evidence": "Separate benchmark editions and verified/community leaderboards, with cost information.", "palo_difference": "PALO tests preservation of explicit source meaning, not abstract task generalisation.", "lesson": "Display verification status, task edition and cost per task separately; never join incompatible editions into one progress line.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "ARC-AGI-1, ARC-AGI-2 and ARC-AGI-3 are separate evaluations.", "visibility_basis": "Public prize competitions and distinct verified/community leaderboards provide visible participation signals.", "visibility_sources": "https://arcprize.org/leaderboard ; https://arcprize.org/results", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "Changes in task format, compute budget and evaluation split prevent direct score comparisons across editions.", "palo_transfer": "Method", "repo_url": "" }, { "id": "gpqa", "name": "GPQA / Diamond", "kind": "Expert science question answering", "source_url": "https://arxiv.org/abs/2311.12022", "focus": "Difficult expert-written multiple-choice science questions.", "evidence": "Domain-expert and non-expert assessment; public data and baseline code.", "palo_difference": "PALO requires evidence-bound interpretations and permissible alternatives, not a single exam option.", "lesson": "Use domain-qualified annotators, disagreement adjudication and an explicit correction channel for flawed gold.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "GPQA main and Diamond subsets must be named; prompt and tool configurations are separate.", "visibility_basis": "Explicitly used in the GPT-5 launch and Google research recap.", "visibility_sources": "https://openai.com/index/introducing-gpt-5/ ; https://blog.google/innovation-and-ai/products/2025-research-breakthroughs/", "repo_stars": 548, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "Expert authorship does not make every answer immutable; subset choice and reasoning budget must be frozen.", "palo_transfer": "Method", "repo_url": "https://github.com/idavidrein/gpqa" }, { "id": "mmlu-pro", "name": "MMLU-Pro", "kind": "Multi-domain knowledge and reasoning", "source_url": "https://arxiv.org/abs/2406.01574", "focus": "Harder multiple-choice questions across subject domains, with ten answer options.", "evidence": "The paper evaluates robustness across 24 prompt styles; public dataset and leaderboard.", "palo_difference": "PALO needs source-specific rules and exceptions; broad subject accuracy cannot replace that measure.", "lesson": "Add a preregistered prompt-robustness check while keeping the primary prompt fixed.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "MMLU-Pro is distinct from MMLU and multilingual MMMLU.", "visibility_basis": "Dedicated Hugging Face benchmark leaderboard and an active public evaluation repository.", "visibility_sources": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro ; https://github.com/TIGER-AI-Lab/MMLU-Pro", "repo_stars": 432, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "Aggregate subject accuracy can hide weak domains; broad popularity is not evidence of procurement fidelity.", "palo_transfer": "Context", "repo_url": "https://github.com/TIGER-AI-Lab/MMLU-Pro" }, { "id": "aime", "name": "AIME as an LLM evaluation", "kind": "Competition-derived mathematics evaluation", "source_url": "https://maa.org/maa-invitational-competitions/", "focus": "Invitational mathematics problems with objectively checkable answers.", "evidence": "An established human competition reused in model evaluations; provider protocols define the AI test.", "palo_difference": "PALO has ambiguous business interpretation and human review needs that exact-answer mathematics does not represent.", "lesson": "Publish edition, problem IDs, attempts, tool access and aggregation rules for every reported result.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "AIME 2025 launch results are a specific historical example, not a universal AIME protocol.", "visibility_basis": "Prominent in GPT-5 and Claude Sonnet 4.5 launch evaluations.", "visibility_sources": "https://openai.com/index/introducing-gpt-5/ ; https://www.anthropic.com/news/claude-sonnet-4-5", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "Small annual sets, repeated sampling and tool-assisted solutions can materially change results. MAA does not endorse PALO or define one universal LLM leaderboard.", "palo_transfer": "Context", "repo_url": "" }, { "id": "frontiermath", "name": "FrontierMath", "kind": "Advanced mathematics programme", "source_url": "https://epoch.ai/frontiermath", "focus": "Mathematician-authored difficult problems and separately defined research-mathematics tracks.", "evidence": "Tiers 1-4, Open Problems and formalised Erdos problems have different evaluation contracts.", "palo_difference": "PALO assesses business-rule meaning and correction effort, not mathematical research capability.", "lesson": "Disclose funding, advance data access and custodian roles before collecting submissions.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Tiers 1-4, Open Problems and Erdos tracks must not be pooled.", "visibility_basis": "Epoch documents its inclusion in the o3 announcement and the resulting sponsorship/data-access clarification.", "visibility_sources": "https://epoch.ai/latest/epoch-impact-report-2024 ; https://epoch.ai/latest/openai-and-frontiermath", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "Sponsorship and data access require explicit disclosure; a hard task set alone does not establish evaluator independence.", "palo_transfer": "Method", "repo_url": "" }, { "id": "livecodebench", "name": "LiveCodeBench", "kind": "Time-windowed coding evaluation", "source_url": "https://livecodebench.github.io/", "focus": "Coding tasks collected from programming contests with recorded release dates.", "evidence": "Public harness, task dates, leaderboard and several coding scenarios.", "palo_difference": "PALO uses business source families rather than programming-contest tasks.", "lesson": "Assign held-out source families by date and publish the evaluation window and contamination checks.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Release windows and coding scenarios are part of the benchmark identity.", "visibility_basis": "Public multi-model leaderboard and community evaluation repository.", "visibility_sources": "https://livecodebench.github.io/ ; https://github.com/LiveCodeBench/LiveCodeBench", "repo_stars": 974, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "Date filtering mitigates exposure risk but cannot prove absence from undisclosed model training data.", "palo_transfer": "Method", "repo_url": "https://github.com/LiveCodeBench/LiveCodeBench" }, { "id": "terminal-bench", "name": "Terminal-Bench", "kind": "Terminal-agent benchmark family", "source_url": "https://www.tbench.ai/", "focus": "Agents complete tasks in executable terminal environments.", "evidence": "The current page separates model and agent and displays resolution, cost, tokens and uncertainty.", "palo_difference": "PALO's document interpretation track precedes execution; Workbench end-to-end tests can reuse the environment discipline.", "lesson": "Freeze environment, agent and model identities; retain traces, resource costs and uncertainty over repeated tasks.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Current homepage shows 4.1. The published 2.0 methodology and older launch results refer to separate releases.", "visibility_basis": "Featured in Claude launch evaluation notes; a dedicated public leaderboard reports agent outcomes.", "visibility_sources": "https://www.anthropic.com/news/claude-sonnet-4-5 ; https://www.tbench.ai/", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "Agent scaffolding, tools and infrastructure affect results. Resolution rate is not a pure model capability score.", "palo_transfer": "Direct", "repo_url": "" }, { "id": "swebench-pro", "name": "SWE-Bench Pro", "kind": "Software-engineering agent evaluation", "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro", "focus": "Long-horizon changes to substantial software repositories.", "evidence": "Reproducible environments, fail-to-pass and regression tests, public and held-out partitions.", "palo_difference": "PALO evaluates the specification produced before coding, while this suite evaluates repository changes.", "lesson": "Use executable downstream checks to complement semantic review and preserve environment failures in the ledger.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "SWE-Bench Pro is distinct from SWE-bench Verified; public and private results are separate.", "visibility_basis": "Dedicated Scale leaderboard and public release with inspectable task infrastructure.", "visibility_sources": "https://labs.scale.com/leaderboard/swe_bench_pro ; https://github.com/scaleapi/SWE-bench_Pro-os", "repo_stars": 559, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "Passing tests does not establish faithful business intent; repository licensing alone does not prove contamination avoidance.", "palo_transfer": "Method", "repo_url": "https://github.com/scaleapi/SWE-bench_Pro-os" }, { "id": "browsecomp", "name": "BrowseComp", "kind": "Web-browsing agent evaluation", "source_url": "https://openai.com/index/browsecomp/", "focus": "Finding difficult, distributed information on the web.", "evidence": "Human-created questions favour short verifiable answers; the release documents search and compute effects.", "palo_difference": "PALO already has a supplied document corpus and needs to preserve its meaning, not primarily discover obscure web facts.", "lesson": "Measure retrieval success separately from interpretation and retain source URLs, retrieval times and tool budgets.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Original BrowseComp task set; tool setup and test-time computation are configuration fields.", "visibility_basis": "OpenAI benchmark launch explicitly compares browsing models and compute strategies.", "visibility_sources": "https://openai.com/index/browsecomp/", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "The authors explicitly note that short-answer performance may not transfer to open-ended user work.", "palo_transfer": "Method", "repo_url": "" }, { "id": "deepresearch-bench", "name": "DeepResearch Bench / II", "kind": "Research-report evaluation family", "source_url": "https://deepresearch-bench.github.io/", "focus": "Research report quality, evidence use and citation reliability.", "evidence": "Original RACE/FACT methods and a separate Bench II with fine-grained rubrics derived from expert reports.", "palo_difference": "PALO seeks source-to-specification fidelity; report-level quality and evidence checks are adjacent rather than interchangeable.", "lesson": "Build expert-reviewed atomic rubrics and version every judge, cleaning step and citation checker; test agreement against human labels.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Original and II are separate sets. The original repository records evaluator and cleaning-model changes in 2026.", "visibility_basis": "Public multi-agent leaderboard, released outputs and an active follow-on expert-rubric benchmark.", "visibility_sources": "https://github.com/Ayanami0730/deep_research_bench ; https://arxiv.org/abs/2601.08536", "repo_stars": 849, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "LLM judges can change scores or share model-family biases. Independent human calibration remains necessary.", "palo_transfer": "Direct", "repo_url": "https://github.com/Ayanami0730/deep_research_bench" }, { "id": "osworld", "name": "OSWorld", "kind": "Computer-use benchmark family", "source_url": "https://osworld-v2.xlang.ai/", "focus": "Computer agents execute workflows in real application environments.", "evidence": "Verified repairs earlier tasks; 2.0 extends long-horizon evaluation and distinguishes binary and partial reward.", "palo_difference": "PALO business usability also depends on completing import, review and export, beyond the quality of extracted text.", "lesson": "Test complete user journeys with environment resets, final-artifact checks, failure traces and explicit partial completion.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "OSWorld-Verified and OSWorld 2.0 remain separate; steps and repetitions must be recorded.", "visibility_basis": "Featured in Claude Sonnet launch evaluations, with a public project and subsequent 2.0 leaderboard.", "visibility_sources": "https://www.anthropic.com/news/claude-sonnet-4-5 ; https://osworld-v2.xlang.ai/ ; https://xlang.ai/blog/osworld-verified", "repo_stars": 3258, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "A GUI success score depends on action budget and environment; it is not a direct semantic-fidelity score.", "palo_transfer": "Direct", "repo_url": "https://github.com/xlang-ai/OSWorld" }, { "id": "tau2", "name": "tau2-bench", "kind": "Policy and tool-agent-user interaction", "source_url": "https://github.com/sierra-research/tau2-bench", "focus": "Agents work with users and tools under explicit domain policies.", "evidence": "Domain policies, environments, agent/user simulators and repeatable execution are provided.", "palo_difference": "PALO extracts policies into a specification; this suite tests following policies during interactions.", "lesson": "Link each source obligation to a testable state transition and measure repeated compliance, including exceptions and unauthorised actions.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "tau2 is distinct from tau-bench; domain, simulator, policy revision and modality must be pinned.", "visibility_basis": "Explicitly reported in Claude Sonnet 4.5 launch methodology, with substantial repository participation.", "visibility_sources": "https://www.anthropic.com/news/claude-sonnet-4-5 ; https://github.com/sierra-research/tau2-bench", "repo_stars": 2434, "repo_star_scope": "Repository snapshot, not benchmark usage", "caution": "User simulators and policy prompt amendments can change outcomes; they must be part of the tested system identity.", "palo_transfer": "Direct", "repo_url": "https://github.com/sierra-research/tau2-bench" }, { "id": "bfcl", "name": "Berkeley Function Calling Leaderboard", "kind": "Tool-use and agentic evaluation", "source_url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "focus": "Function selection, arguments and multi-turn or agentic tool use.", "evidence": "V4 exposes category scores, model interfaces, costs, latency, model outputs and a pinned evaluator checkpoint.", "palo_difference": "PALO needs semantic business validation in addition to syntactically valid tool calls.", "lesson": "Separate contract conformity from task correctness, publish adapter versions and category-level errors, and distinguish native tool APIs from text prompting.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "BFCL V4; do not mix scores from earlier category compositions or native/prompt interfaces.", "visibility_basis": "Public Berkeley leaderboard with sortable metrics, comparison visualisation and a widely followed parent repository.", "visibility_sources": "https://gorilla.cs.berkeley.edu/leaderboard.html ; https://github.com/ShishirPatil/gorilla", "repo_stars": 13069, "repo_star_scope": "Whole Gorilla repository, not BFCL alone", "caution": "Gorilla stars cover the parent project, not BFCL alone. Valid tool syntax is not proof of a correct business decision.", "palo_transfer": "Direct", "repo_url": "https://github.com/ShishirPatil/gorilla" }, { "id": "gdpval", "name": "GDPval", "kind": "Professional work-product evaluation", "source_url": "https://openai.com/index/gdpval/", "focus": "Economically valuable work products across professional occupations.", "evidence": "Domain professionals create and review tasks and compare model deliverables with human work.", "palo_difference": "PALO's proposed procurement study targets document-to-specification fidelity, a narrower unit than a finished professional deliverable.", "lesson": "Use actual procurement experts for blind rubric design and review, and measure repair effort alongside deliverable acceptance.", "source_status": "Primary source reviewed", "in_requested_directory": false, "checked_on": "2026-11-11", "review_group": "visibility-expansion-20261111", "version_scope": "Published GDPval release and public subset; human grading and automated grading are distinct.", "visibility_basis": "Dedicated OpenAI release compares several major providers on professional work products.", "visibility_sources": "https://openai.com/index/gdpval/", "repo_stars": null, "repo_star_scope": "Not measured", "caution": "The authors do not treat experimental automated grading as a replacement for expert assessment. Occupation coverage does not establish all enterprise use cases.", "palo_transfer": "Direct", "repo_url": "" } ], "supporting_sources": [ { "url": "https://github.com/nyu-mll/jiant", "purpose": "Distinguish toolkit maintenance status from GLUE/SuperGLUE task definitions." }, { "url": "https://github.com/lm-sys/FastChat", "purpose": "Historical Chatbot Arena infrastructure linked by the discovery directory; current comparison uses Arena methodology." } ], "visibility_selection": { "added": 15, "selected_on": "2026-11-11", "method": "Purposive high-visibility selection using official model-launch references, dedicated public leaderboards/competitions and dated GitHub repository participation. At least one source-backed visibility signal is required per added family.", "limits": "Not an exhaustive ranking of global hype, social reach or benchmark quality. No comparable social/media time series was collected. Repository stars are cumulative attention proxies with age and project-scope bias; the Gorilla count is parent-project-wide. Different benchmark versions remain separate. No external suite was executed.", "priority_definition": "Direct: transferable to business workflow, evidence or operational checks. Method: useful evaluation design. Context: broad capability context. These are maintainer assessments, not external validation." }, "transfer_priorities": [ { "title": "Expert gold and atomic rubrics", "references": [ "gdpval", "deepresearch-bench", "gpqa" ], "action": "Recruit procurement experts, freeze allowed alternatives and forbidden inferences, adjudicate disagreements and report human agreement before qualification." }, { "title": "End-to-end workflow reliability", "references": [ "tau2", "osworld", "terminal-bench" ], "action": "Test document import, interpretation, human correction and blueprint export with repeated runs, environment identity, failure states and cost/time accounting." }, { "title": "Structural and Semantic Assessment", "references": [ "bfcl", "swebench-pro", "browsecomp" ], "action": "Report schema/tool validity, source support, semantic violations and downstream executable checks as separate outcomes." }, { "title": "Freshness, custody and disclosure", "references": [ "livecodebench", "hle", "arc-agi", "frontiermath" ], "action": "Use externally selected unseen families, date windows, protected answer custody, sponsor/data-access disclosures and versioned judge/adapter policies." } ] }