tools.json

{
  "title": "The Comprehensive List of AI Evaluation Tools",
  "source": "https://github.com/aglio-lab/ai-evaluation-tools",
  "license": "CC0-1.0",
  "last_verified": "2026-07-16",
  "entry_count": 327,
  "tools": [
    {
      "name": "Confident AI",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.confident-ai.com",
      "description": "Confident AI is the AI quality platform built for enterprise platform teams to standardize evals and observability across the org — one consistent bar for how different product teams measure and monitor their AI. Docs",
      "secondary_links": [
        {
          "label": "Docs",
          "url": "https://documentation.confident-ai.com"
        }
      ],
      "status": "active"
    },
    {
      "name": "Braintrust",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.braintrust.dev",
      "description": "Braintrust is an eval-first AI engineering platform with experiments, datasets, a prompt playground, logging, and online scoring using its open-source autoevals scorers.",
      "secondary_links": [
        {
          "label": "autoevals",
          "url": "https://github.com/braintrustdata/autoevals"
        }
      ],
      "status": "active"
    },
    {
      "name": "LangSmith",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://smith.langchain.com",
      "description": "LangSmith is LangChain's platform for tracing, dataset-based evaluation, annotation queues, and prompt management; it works with or without the LangChain framework. Docs",
      "secondary_links": [
        {
          "label": "Docs",
          "url": "https://docs.langchain.com/langsmith"
        }
      ],
      "status": "active"
    },
    {
      "name": "Langfuse",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-source",
      "primary_url": "https://langfuse.com",
      "description": "Langfuse is an open-source, self-hostable LLM engineering platform with tracing, LLM-as-a-judge and custom evaluations, prompt management, datasets, and dashboards. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/langfuse/langfuse"
        }
      ],
      "status": "active"
    },
    {
      "name": "Arize AX",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://arize.com",
      "description": "Arize AX is an enterprise AI observability and evaluation platform; its open-source foundation, Phoenix, is listed separately under observability.",
      "secondary_links": [
        {
          "label": "Phoenix",
          "url": "https://github.com/Arize-ai/phoenix"
        }
      ],
      "status": "active"
    },
    {
      "name": "Opik",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-core",
      "primary_url": "https://www.comet.com/site/products/opik/",
      "description": "Opik, from Comet, is an open-source platform to debug, evaluate, and monitor LLM applications, RAG systems, and agents, with tracing, evaluation metrics, datasets, and dashboards. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/comet-ml/opik"
        }
      ],
      "status": "active"
    },
    {
      "name": "Galileo",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://galileo.ai",
      "description": "Galileo is an enterprise evaluation and observability platform with proprietary evaluation models (e.g., Luna), agent evaluations, guardrailing, and production monitoring.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Patronus AI",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.patronus.ai",
      "description": "Patronus AI is an automated AI evaluation and testing platform with proprietary evaluators (Lynx for hallucination detection, the Glider judge model), red teaming, and monitoring APIs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Weights & Biases Weave",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-core",
      "primary_url": "https://wandb.ai/site/weave/",
      "description": "W&B Weave is a tracing and evaluation toolkit for LLM applications with scorers, comparisons, leaderboards, and production monitoring. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/wandb/weave"
        }
      ],
      "status": "active"
    },
    {
      "name": "Vellum",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.vellum.ai",
      "description": "Vellum is a platform for building and evaluating LLM workflows with test suites, side-by-side comparisons, and deployment monitoring.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HoneyHive",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.honeyhive.ai",
      "description": "HoneyHive is an AI observability and evaluation platform with tracing, offline and online evaluators, dataset curation, and human review.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Maxim AI",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.getmaxim.ai",
      "description": "Maxim AI provides end-to-end simulation, evaluation, and observability for AI agents, including prompt experiments, agent simulation runs, and production quality checks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Freeplay",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://freeplay.ai",
      "description": "Freeplay combines prompt management, evaluation, and observability with human-in-the-loop review workflows for product teams.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Athina AI",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.athina.ai",
      "description": "Athina AI is a collaborative platform to prototype, evaluate, and monitor LLM applications, with a spreadsheet-style experimentation UI and preset plus custom evals.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Orq.ai",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://orq.ai",
      "description": "Orq.ai is a generative AI collaboration platform with built-in experiments, evaluators, guardrails, and deployment management.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Agenta",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-source",
      "primary_url": "https://agenta.ai",
      "description": "Agenta is an open-source LLMOps platform with a prompt playground, versioning, human and automatic evaluation, and observability. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/Agenta-AI/agenta"
        }
      ],
      "status": "active"
    },
    {
      "name": "LangWatch",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-source",
      "primary_url": "https://langwatch.ai",
      "description": "LangWatch is an open-source platform for LLM evaluation, agent simulation testing (Scenario), and observability. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/langwatch/langwatch"
        }
      ],
      "status": "active"
    },
    {
      "name": "Openlayer",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.openlayer.com",
      "description": "Openlayer provides testing and monitoring for AI systems (LLM and classic ML), with automated tests in CI and production quality alerts.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "PromptLayer",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://www.promptlayer.com",
      "description": "PromptLayer offers prompt management and evaluation: a visual prompt registry, batch evals, A/B testing, and usage analytics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Giskard Hub",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "open-core",
      "primary_url": "https://www.giskard.ai",
      "description": "Giskard Hub is an enterprise agent-testing platform built on the open-source Giskard library, adding vulnerability scanning, continuous red teaming, and business-failure detection.",
      "secondary_links": [
        {
          "label": "Giskard",
          "url": "https://github.com/Giskard-AI/giskard-oss"
        }
      ],
      "status": "active"
    },
    {
      "name": "Scale GenAI Platform",
      "category": "Full-Stack LLM Evaluation Platforms",
      "availability": "commercial",
      "primary_url": "https://scale.com",
      "description": "Scale AI's GenAI platform provides enterprise model evaluation and a data engine, including expert human evaluation and the SEAL leaderboards.",
      "secondary_links": [
        {
          "label": "SEAL leaderboards",
          "url": "https://scale.com/leaderboard"
        }
      ],
      "status": "active"
    },
    {
      "name": "DeepEval",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/confident-ai/deepeval",
      "description": "DeepEval is an open-source LLM evaluation framework (\"Pytest for LLMs\") with 50+ metrics for RAG, agents, multi-turn conversations, safety, and custom criteria, plus CI/CD and benchmark support.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Ragas",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/vibrantlabsai/ragas",
      "description": "Ragas is an evaluation toolkit for LLM applications, best known for reference-free RAG metrics such as faithfulness, answer relevancy, and context precision/recall. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2309.15217"
        }
      ],
      "status": "active"
    },
    {
      "name": "promptfoo",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/promptfoo/promptfoo",
      "description": "promptfoo is a developer-first CLI for testing prompts, models, and RAG/agent applications, with declarative test cases, comparison matrices, CI integration, and red teaming.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenAI Evals",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/evals",
      "description": "OpenAI Evals is OpenAI's original open-source framework and registry of community-contributed evals; the repository is in maintenance mode, and OpenAI's separate hosted Evals product is scheduled for shutdown in late 2026.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "openai/simple-evals",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/simple-evals",
      "description": "simple-evals contains OpenAI's lightweight reference implementations of its reported benchmarks (MMLU, GPQA, SimpleQA, HumanEval, HealthBench, and more).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "lm-evaluation-harness",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/EleutherAI/lm-evaluation-harness",
      "description": "EleutherAI's lm-evaluation-harness is a widely used harness for few-shot LLM benchmarking with hundreds of academic tasks; it powered the Hugging Face Open LLM Leaderboard.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Inspect AI",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/UKGovernmentBEIS/inspect_ai",
      "description": "Inspect AI is the UK AI Security Institute's framework for LLM evaluations, with solvers, scorers, sandboxed tool use, and agent evals. See also Inspect Evals, a community repository of ready-to-run benchmark implementations. Docs",
      "secondary_links": [
        {
          "label": "Inspect Evals",
          "url": "https://github.com/UKGovernmentBEIS/inspect_evals"
        },
        {
          "label": "Docs",
          "url": "https://inspect.aisi.org.uk"
        }
      ],
      "status": "active"
    },
    {
      "name": "HELM",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/stanford-crfm/helm",
      "description": "HELM (Holistic Evaluation of Language Models), from Stanford CRFM, provides transparent, reproducible, multi-metric model evaluation. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2211.09110"
        }
      ],
      "status": "active"
    },
    {
      "name": "OpenCompass",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/open-compass/opencompass",
      "description": "OpenCompass is a large-scale evaluation platform supporting 100+ datasets and a wide range of open and API models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LightEval",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/huggingface/lighteval",
      "description": "LightEval is Hugging Face's all-in-one LLM evaluation toolkit across multiple backends (transformers, vLLM, inference endpoints); Hugging Face recommends it over the older Evaluate library for LLM work.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Pydantic Evals",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://ai.pydantic.dev/evals/",
      "description": "Pydantic Evals is the evaluation library from the Pydantic team for scoring LLM and agent outputs with datasets, evaluators, and reports. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/pydantic/pydantic-ai"
        }
      ],
      "status": "active"
    },
    {
      "name": "MLflow LLM Evaluate",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://mlflow.org/docs/latest/genai/eval-monitor/",
      "description": "MLflow's GenAI evaluation module provides heuristic and LLM-judge metrics integrated with MLflow experiment tracking and tracing. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/mlflow/mlflow"
        }
      ],
      "status": "active"
    },
    {
      "name": "NeMo Evaluator",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/NVIDIA-NeMo/Evaluator",
      "description": "NVIDIA NeMo Evaluator is an open-source platform for scalable, reproducible evaluation of LLMs across popular benchmarks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Prompt flow",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/microsoft/promptflow",
      "description": "Microsoft Prompt flow is a development suite for LLM apps that includes built-in and custom evaluation flows for quality assessment in CI.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Evidently",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-core",
      "primary_url": "https://github.com/evidentlyai/evidently",
      "description": "Evidently is an open-source evaluation and monitoring library for ML and LLM systems with 100+ metrics, test suites, and reports, plus the commercial Evidently Cloud.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TruLens",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/truera/trulens",
      "description": "TruLens provides evaluation and tracking for LLM apps using feedback functions, and originated the \"RAG triad\" (context relevance, groundedness, answer relevance).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenEvals",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/langchain-ai/openevals",
      "description": "OpenEvals is LangChain's collection of ready-made evaluators for correctness, conciseness, hallucination, and structured output.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Evalite",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/mattpocock/evalite",
      "description": "Evalite is a TypeScript-native eval runner built on Vitest that treats evals as tests, with a local UI.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Giskard",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/Giskard-AI/giskard-oss",
      "description": "Giskard is an open-source testing library for LLM agents and ML models with automatic vulnerability scanning for hallucination, bias, and injection.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "EvalScope",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/modelscope/evalscope",
      "description": "EvalScope is ModelScope's framework for LLM/VLM/AIGC evaluation and inference performance benchmarking.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OLMES",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/allenai/olmes",
      "description": "OLMES is AI2's reproducible standard for LLM evaluations, used for the OLMo and Tülu model reports.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Eureka ML Insights",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/microsoft/eureka-ml-insights",
      "description": "Eureka ML Insights is Microsoft's framework for standardized, reproducible evaluation of large foundation models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Weave Evaluations",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/wandb/weave",
      "description": "Weave Evaluations is W&B Weave's code-first evaluation harness with scorers, datasets, and model comparisons.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LLM Comparator",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/PAIR-code/llm-comparator",
      "description": "LLM Comparator, from Google's PAIR team, is an interactive tool for side-by-side analysis of LLM evaluation results.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Parea",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/parea-ai/parea-sdk-py",
      "description": "Parea's SDK provides experiment tracking and evaluation for LLM applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "continuous-eval",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/relari-ai/continuous-eval",
      "description": "continuous-eval, from Relari, offers data-driven, modular evaluation for LLM pipelines with metric ensembles. Repository activity has slowed since early 2025.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Phoenix Evals",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/Arize-ai/phoenix",
      "description": "Phoenix Evals is Arize Phoenix's evaluation library with pre-tested LLM-judge templates for hallucination, relevance, toxicity, and more.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "UpTrain",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/uptrain-ai/uptrain",
      "description": "UpTrain is an open-source tool to evaluate and improve LLM applications with 20+ preconfigured checks. Repository activity has slowed since 2024.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AlpacaEval",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/tatsu-lab/alpaca_eval",
      "description": "AlpacaEval is a fast, inexpensive, replicable LLM-based automatic evaluator for instruction-following models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Evalchemy",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/mlfoundations/evalchemy",
      "description": "Evalchemy, from ML Foundations, is a unified toolkit for running many popular benchmarks against any model.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Evaluate",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/huggingface/evaluate",
      "description": "Hugging Face Evaluate is a library of standard NLP/ML metrics (BLEU, ROUGE, BERTScore) with a unified API; it is in maintenance mode, with LightEval recommended for new LLM evaluation work.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FastChat",
      "category": "LLM Evaluation Frameworks",
      "availability": "open-source",
      "primary_url": "https://github.com/lm-sys/FastChat",
      "description": "FastChat is LMSYS's platform for serving and evaluating chat models, and the original home of MT-Bench and the Chatbot Arena codebase; development has slowed and it is now mostly of historical reference value.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Microsoft Foundry Evaluations",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/observability",
      "description": "Microsoft Foundry (formerly Azure AI Foundry) includes built-in evaluators for generative and agentic applications, covering intent resolution, tool-call accuracy, task adherence, groundedness, and safety.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vertex AI Gen AI Evaluation",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://cloud.google.com/vertex-ai/generative-ai/docs/models/evaluation-overview",
      "description": "Google Cloud's Gen AI Evaluation Service provides model-based and computation-based metrics, including agent trajectory evaluation, integrated with Vertex AI.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Amazon Bedrock Evaluations",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation.html",
      "description": "Amazon Bedrock Evaluations runs managed model and RAG evaluation jobs using automatic metrics, LLM-as-a-judge, and human evaluation workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Mosaic AI Agent Evaluation",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://docs.databricks.com/en/generative-ai/agent-evaluation/index.html",
      "description": "Databricks' Mosaic AI Agent Evaluation provides managed judges, a human review app, and MLflow integration for evaluating GenAI apps on Databricks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenAI Evals (hosted)",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://platform.openai.com/docs/guides/evals",
      "description": "OpenAI's platform-integrated evaluation product for testing prompts and models in the OpenAI dashboard; OpenAI has announced a shutdown of the hosted Evals product scheduled for late 2026.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Anthropic Console Evaluations",
      "category": "Cloud Provider Evaluation Services",
      "availability": "commercial",
      "primary_url": "https://docs.anthropic.com/en/docs/test-and-evaluate/eval-tool",
      "description": "The Anthropic Console includes an evaluation tool for generating test cases and comparing prompt outputs for Claude models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Langfuse",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/langfuse/langfuse",
      "description": "Langfuse is a widely adopted open-source LLM observability platform with traces, sessions, cost tracking, evaluations, and prompt management; it is self-hostable.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Arize Phoenix",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/Arize-ai/phoenix",
      "description": "Phoenix is an OpenTelemetry-native tracing and evaluation library for LLM apps and agents that runs locally, in notebooks, or self-hosted; it underpins the commercial Arize AX platform.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Helicone",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/Helicone/helicone",
      "description": "Helicone is an LLM observability proxy/gateway added with one line of code, providing logging, caching, scoring, and experiments.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Opik",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/comet-ml/opik",
      "description": "Opik is Comet's open-source tracing and evaluation platform for LLM and agentic applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenLLMetry",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/traceloop/openllmetry",
      "description": "OpenLLMetry, from Traceloop, provides OpenTelemetry-based instrumentation for LLM apps, exporting traces to any OTel-compatible backend.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenInference",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/Arize-ai/openinference",
      "description": "OpenInference is an open specification and set of instrumentation libraries for capturing LLM application traces, used by Phoenix and other OTel backends.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenLIT",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/openlit/openlit",
      "description": "OpenLIT is an OpenTelemetry-native AI engineering platform with observability, evaluations, guardrails, and a prompt hub.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AgentOps",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/AgentOps-AI/agentops",
      "description": "AgentOps provides session replays, metrics, and monitoring purpose-built for AI agents across popular agent frameworks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Laminar",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/lmnr-ai/lmnr",
      "description": "Laminar is an open-source, OTel-based platform for tracing and evaluating AI applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Pydantic Logfire",
      "category": "LLM Observability and Tracing",
      "availability": "open-core",
      "primary_url": "https://github.com/pydantic/logfire",
      "description": "Logfire, from the Pydantic team, is an observability platform with first-class LLM and agent tracing built on OpenTelemetry.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Lunary",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/lunary-ai/lunary",
      "description": "Lunary is a production toolkit for LLM apps covering observability, prompt management, and analytics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Langtrace",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/Scale3-Labs/langtrace",
      "description": "Langtrace is an open-source, OTel-based observability tool with evaluations for LLM applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Portkey",
      "category": "LLM Observability and Tracing",
      "availability": "open-core",
      "primary_url": "https://github.com/Portkey-AI/gateway",
      "description": "Portkey combines an open-source AI gateway with a commercial observability and governance suite: routing, guardrails, logs, and feedback.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Datadog LLM Observability",
      "category": "LLM Observability and Tracing",
      "availability": "commercial",
      "primary_url": "https://www.datadoghq.com/product/llm-observability/",
      "description": "Datadog LLM Observability adds LLM tracing, quality and security scanning, and evaluation to the Datadog platform.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "New Relic AI Monitoring",
      "category": "LLM Observability and Tracing",
      "availability": "commercial",
      "primary_url": "https://newrelic.com/platform/ai-monitoring",
      "description": "New Relic AI Monitoring provides APM-style monitoring for LLM-powered applications, covering traces, cost, and response quality signals.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LangSmith",
      "category": "LLM Observability and Tracing",
      "availability": "commercial",
      "primary_url": "https://smith.langchain.com",
      "description": "LangSmith provides tracing and monitoring with online evaluators and alerts (see the [full-stack platforms](#full-stack-llm-evaluation-platforms) section).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Arthur",
      "category": "LLM Observability and Tracing",
      "availability": "open-core",
      "primary_url": "https://www.arthur.ai",
      "description": "Arthur is an AI performance monitoring platform; its open-source Arthur Bench LLM evaluation tool is no longer actively developed.",
      "secondary_links": [
        {
          "label": "Arthur Bench",
          "url": "https://github.com/arthur-ai/bench"
        }
      ],
      "status": "active"
    },
    {
      "name": "LangKit",
      "category": "LLM Observability and Tracing",
      "availability": "open-source",
      "primary_url": "https://github.com/whylabs/langkit",
      "description": "LangKit is an open-source library of text-quality and safety telemetry metrics for LLMs, built on whylogs. The hosted WhyLabs platform it fed into has been discontinued (see [Discontinued](#discontinued-and-historical-tools)).",
      "secondary_links": [
        {
          "label": "whylogs",
          "url": "https://github.com/whylabs/whylogs"
        }
      ],
      "status": "active"
    },
    {
      "name": "Ragas",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/vibrantlabsai/ragas",
      "description": "Ragas is a popular toolkit for RAG metrics: faithfulness, answer relevancy, context precision, and context recall.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DeepEval RAG metrics",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://deepeval.com/docs/metrics-introduction",
      "description": "DeepEval's RAG metrics cover answer relevancy, faithfulness, and contextual precision/recall/relevancy, with explainable per-test-case reasons.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TruLens",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/truera/trulens",
      "description": "TruLens implements RAG triad evaluation (context relevance, groundedness, answer relevance) with application instrumentation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ARES",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/stanford-futuredata/ARES",
      "description": "ARES, from Stanford, automates RAG evaluation using synthetic data and fine-tuned judge models with statistical guarantees. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.09476"
        }
      ],
      "status": "active"
    },
    {
      "name": "AutoRAG",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/Marker-Inc-Korea/AutoRAG",
      "description": "AutoRAG is an AutoML-style tool that evaluates RAG pipeline configurations on your own data to find the best-performing setup.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RAGChecker",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/amazon-science/RAGChecker",
      "description": "RAGChecker, from Amazon Science, is a fine-grained diagnostic framework for RAG using claim-level entailment checks on both retriever and generator.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BEIR",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/beir-cellar/beir",
      "description": "BEIR is a heterogeneous zero-shot benchmark for information retrieval spanning 18 datasets across 9 task types.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MTEB",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/embeddings-benchmark/mteb",
      "description": "MTEB (Massive Text Embedding Benchmark) is the benchmark suite for evaluating embedding models, with a public leaderboard.",
      "secondary_links": [
        {
          "label": "public leaderboard",
          "url": "https://huggingface.co/spaces/mteb/leaderboard"
        }
      ],
      "status": "active"
    },
    {
      "name": "CRAG",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/facebookresearch/CRAG",
      "description": "CRAG (Comprehensive RAG Benchmark), from Meta, provides 4,400+ QA pairs and mock APIs simulating web and knowledge-graph search, used in the KDD Cup 2024.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RAGBench",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/rungalileo/ragbench",
      "description": "RAGBench is a 100k-example, explainable RAG benchmark across five industry domains, with the TRACe evaluation framework.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BRIGHT",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/xlang-ai/BRIGHT",
      "description": "BRIGHT is a benchmark for reasoning-intensive retrieval where queries require deep reasoning rather than keyword matching.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MTRAG",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/IBM/mt-rag-benchmark",
      "description": "MTRAG, from IBM, is a human-generated benchmark for multi-turn RAG conversations across four domains.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Tonic Validate",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/TonicAI/tonic_validate",
      "description": "Tonic Validate is a RAG benchmarking and evaluation framework with a free visualization UI.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "continuous-eval",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/relari-ai/continuous-eval",
      "description": "continuous-eval offers modular, granular evaluation of each RAG pipeline stage: retrieval, reranking, and generation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FlashRAG",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/RUC-NLPIR/FlashRAG",
      "description": "FlashRAG is a Python toolkit for reproducing and evaluating RAG research, with 20+ algorithms and 16+ datasets.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RAGTruth",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/ParticleMedia/RAGTruth",
      "description": "RAGTruth is a word-level, hallucination-annotated corpus for training and evaluating trustworthy RAG systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RGB",
      "category": "RAG Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/chen700564/RGB",
      "description": "RGB benchmarks RAG robustness across noise, negative rejection, information integration, and counterfactuals.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DeepEval agentic evals",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://deepeval.com/docs/metrics-task-completion",
      "description": "DeepEval's agentic metrics cover task completion, tool correctness, and argument correctness, supporting end-to-end and component-level agent testing over traces.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LangWatch Scenario",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/langwatch/scenario",
      "description": "Scenario, from LangWatch, is an agent simulation testing framework that simulates users over multi-turn scenarios and asserts on outcomes.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AgentEvals",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/langchain-ai/agentevals",
      "description": "AgentEvals is LangChain's set of ready-made evaluators for agent trajectories, including trajectory match and LLM-judge over trajectories.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Inspect AI",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/UKGovernmentBEIS/inspect_ai",
      "description": "Inspect AI has first-class agent evaluation support: sandboxed tool execution, multi-step solvers, and agent bridges.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Rogue",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/qualifire-dev/rogue",
      "description": "Rogue, from Qualifire, evaluates and red-teams agents by probing them through realistic adversarial conversations.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "any-agent",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/mozilla-ai/any-agent",
      "description": "any-agent, from Mozilla AI, is a single interface for building and evaluating agents across different agent frameworks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vivaria",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/METR/vivaria",
      "description": "Vivaria is METR's platform for running agent evaluations and conducting elicitation research.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Harbor",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/harbor-framework/harbor",
      "description": "Harbor is a framework for running agent evaluations and creating RL environments at scale.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ToolSandbox",
      "category": "AI Agent Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/apple/ToolSandbox",
      "description": "ToolSandbox, from Apple, is a stateful, conversational, interactive evaluation benchmark for LLM tool-use capabilities.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Coval",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "commercial",
      "primary_url": "https://www.coval.dev",
      "description": "Coval is a simulation and evaluation platform for voice and chat agents, applying scenario-based automated testing practices borrowed from self-driving-car simulation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Hamming",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "commercial",
      "primary_url": "https://hamming.ai",
      "description": "Hamming automates voice-agent testing at scale with thousands of simulated phone calls, prompt experiments, and production call analytics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Cekura",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "commercial",
      "primary_url": "https://www.cekura.ai",
      "description": "Cekura (formerly Vocera) provides testing and observability for voice and chat agents via simulated conversations and production monitoring.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DeepEval conversational metrics",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "open-source",
      "primary_url": "https://deepeval.com/docs/metrics-conversational-geval",
      "description": "DeepEval's conversational metrics evaluate multi-turn conversations at the turn and conversation level (role adherence, knowledge retention, conversation completeness) and include user simulation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Bland AI Testing",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "commercial",
      "primary_url": "https://www.bland.ai",
      "description": "Bland AI includes built-in test suites for phone-call agents on the Bland platform.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MultiChallenge",
      "category": "Voice AI and Conversational Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/ScaleAI/MultiChallenge",
      "description": "MultiChallenge, from Scale AI, benchmarks realistic multi-turn conversation ability of frontier LLMs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DeepTeam",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/confident-ai/deepteam",
      "description": "DeepTeam is an open-source LLM red-teaming framework covering 120+ vulnerabilities and 20+ attack methods, with mappings to the OWASP Top 10, NIST AI RMF, and MITRE ATLAS.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "garak",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/NVIDIA/garak",
      "description": "garak, from NVIDIA, is an LLM vulnerability scanner (\"nmap for LLMs\") probing hallucination, jailbreaks, toxicity, and data leakage. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2406.11036"
        }
      ],
      "status": "active"
    },
    {
      "name": "PyRIT",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/Azure/PyRIT",
      "description": "PyRIT is Microsoft's Python Risk Identification Toolkit for generative AI, used by Microsoft's AI Red Team for automated adversarial probing.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "promptfoo red teaming",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://www.promptfoo.dev/docs/red-team/",
      "description": "promptfoo's red-teaming module runs automated scans for injection, jailbreaks, PII leakage, and RBAC issues, with OWASP and NIST mappings.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HarmBench",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/centerforaisafety/HarmBench",
      "description": "HarmBench, from the Center for AI Safety, is a standardized evaluation framework for automated red teaming and robust refusal.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Giskard scan",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/Giskard-AI/giskard-oss",
      "description": "Giskard's scan feature automatically detects vulnerabilities in LLM agents, including injection, harmful content, and sycophancy.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FuzzyAI",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/cyberark/FuzzyAI",
      "description": "FuzzyAI, from CyberArk, is an automated LLM fuzzer for discovering jailbreaks and adversarial weaknesses in model guardrails.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "CyberSecEval",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/meta-llama/PurpleLlama",
      "description": "CyberSecEval, part of Meta's Purple Llama project, benchmarks LLM cybersecurity risks including insecure code generation and cyberattack helpfulness.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Lakera Red",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "commercial",
      "primary_url": "https://www.lakera.ai",
      "description": "Lakera Red is an AI red-teaming service from the team behind the Gandalf prompt-injection game.",
      "secondary_links": [
        {
          "label": "Gandalf",
          "url": "https://gandalf.lakera.ai"
        }
      ],
      "status": "active"
    },
    {
      "name": "Mindgard",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "commercial",
      "primary_url": "https://www.mindgard.ai",
      "description": "Mindgard is an automated AI red teaming and security testing platform (offensive security for AI systems).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HiddenLayer",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "commercial",
      "primary_url": "https://hiddenlayer.com",
      "description": "HiddenLayer is an AI security platform covering model scanning and adversarial ML detection and response.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Cisco AI Defense",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "commercial",
      "primary_url": "https://www.cisco.com/site/us/en/products/security/ai-defense/index.html",
      "description": "Cisco AI Defense, built on the Robust Intelligence acquisition, provides algorithmic red teaming and runtime guardrails for enterprises.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Haize Labs",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "commercial",
      "primary_url": "https://www.haizelabs.com",
      "description": "Haize Labs provides automated stress-testing/jailbreaking (\"haizing\") suites and judge infrastructure for frontier AI systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AgentDojo",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/ethz-spylab/agentdojo",
      "description": "AgentDojo, from ETH Zurich, is a dynamic environment for evaluating prompt-injection attacks and defenses in tool-using LLM agents.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "JailbreakBench",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/JailbreakBench/jailbreakbench",
      "description": "JailbreakBench is an open robustness benchmark for jailbreaking, with a leaderboard and artifact repository.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Moonshot",
      "category": "LLM Red Teaming and Security Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/aiverify-foundation/moonshot",
      "description": "Moonshot, from the AI Verify Foundation, is a modular tool for benchmarking and red-teaming LLM applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "NeMo Guardrails",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/NVIDIA/NeMo-Guardrails",
      "description": "NeMo Guardrails is NVIDIA's toolkit for programmable guardrails (via Colang) on conversational systems: topic control, jailbreak detection, and fact-checking rails.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Guardrails AI",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/guardrails-ai/guardrails",
      "description": "Guardrails AI is a Python framework for input/output validation, with a community Guardrails Hub of validators.",
      "secondary_links": [
        {
          "label": "Guardrails Hub",
          "url": "https://hub.guardrailsai.com"
        }
      ],
      "status": "active"
    },
    {
      "name": "LLM Guard",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/protectai/llm-guard",
      "description": "LLM Guard, from Protect AI, is a security toolkit providing sanitization, PII detection, and prompt-injection and toxicity scanners.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LlamaFirewall",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/meta-llama/PurpleLlama/tree/main/LlamaFirewall",
      "description": "LlamaFirewall, from Meta, is an open-source guardrail framework for agentic systems, combining PromptGuard, agent-alignment checks, and code-safety scanning.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenAI Guardrails",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/openai-guardrails-python",
      "description": "OpenAI Guardrails is a modular safety framework for validating LLM inputs and outputs with configurable pipelines.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Invariant Guardrails",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-source",
      "primary_url": "https://github.com/invariantlabs-ai/invariant",
      "description": "Invariant, from Invariant Labs, provides guardrailing rules and analysis for agentic systems, including tool-call policy enforcement.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Lakera Guard",
      "category": "Guardrails and Runtime Safety",
      "availability": "commercial",
      "primary_url": "https://www.lakera.ai/lakera-guard",
      "description": "Lakera Guard is a low-latency API for detecting prompt injection, content violations, and PII in production.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Amazon Bedrock Guardrails",
      "category": "Guardrails and Runtime Safety",
      "availability": "commercial",
      "primary_url": "https://aws.amazon.com/bedrock/guardrails/",
      "description": "Amazon Bedrock Guardrails applies configurable safety policies — content filters, denied topics, and contextual grounding checks — to Bedrock applications.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Azure AI Content Safety",
      "category": "Guardrails and Runtime Safety",
      "availability": "commercial",
      "primary_url": "https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety",
      "description": "Azure AI Content Safety provides content moderation plus Prompt Shields (jailbreak and indirect injection detection) and groundedness detection.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Google Model Armor",
      "category": "Guardrails and Runtime Safety",
      "availability": "commercial",
      "primary_url": "https://cloud.google.com/security-command-center/docs/model-armor-overview",
      "description": "Model Armor is Google Cloud's service for screening LLM prompts and responses for injection, jailbreaks, sensitive data, and harmful content.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Llama Guard",
      "category": "Guardrails and Runtime Safety",
      "availability": "open-weights",
      "primary_url": "https://github.com/meta-llama/PurpleLlama",
      "description": "Llama Guard is Meta's family of downloadable safety classifier models (Llama Guard, Prompt Guard) under the Purple Llama project, released under Meta's community license.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OpenAI Moderation",
      "category": "Guardrails and Runtime Safety",
      "availability": "commercial",
      "primary_url": "https://platform.openai.com/docs/guides/moderation",
      "description": "The OpenAI Moderation endpoint is a free API for text and image safety classification.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vectara HHEM",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/vectara/hallucination_evaluation_model",
      "description": "HHEM is Vectara's widely used open hallucination detection model, which powers the Hughes Hallucination Leaderboard.",
      "secondary_links": [
        {
          "label": "Hughes Hallucination Leaderboard",
          "url": "https://github.com/vectara/hallucination-leaderboard"
        }
      ],
      "status": "active"
    },
    {
      "name": "Patronus Lynx",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-weights",
      "primary_url": "https://huggingface.co/PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct",
      "description": "Lynx, from Patronus AI, is a hallucination detection model reported to outperform GPT-4 on the HaluBench benchmark (July 2024 paper). Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2407.08488"
        }
      ],
      "status": "active"
    },
    {
      "name": "SelfCheckGPT",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/potsawee/selfcheckgpt",
      "description": "SelfCheckGPT performs zero-resource, black-box hallucination detection via sampling consistency. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2303.08896"
        }
      ],
      "status": "active"
    },
    {
      "name": "FActScore",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/shmsw25/FActScore",
      "description": "FActScore provides fine-grained, atomic-fact evaluation of factual precision in long-form generation. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2305.14251"
        }
      ],
      "status": "active"
    },
    {
      "name": "LongFact & SAFE",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/google-deepmind/long-form-factuality",
      "description": "LongFact (a long-form factuality benchmark) and SAFE (Search-Augmented Factuality Evaluator), from Google DeepMind, evaluate long-form factuality using search-grounded fact checking.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MiniCheck",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/Liyan06/MiniCheck",
      "description": "MiniCheck is an efficient fact-checking model for verifying whether LLM output sentences are grounded in provided documents, at a fraction of GPT-4's cost.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AlignScore",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/yuh-zha/AlignScore",
      "description": "AlignScore is a unified metric for factual-consistency evaluation trained on a broad alignment corpus.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FacTool",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/GAIR-NLP/factool",
      "description": "FacTool performs tool-augmented factuality detection across knowledge-based QA, code, math, and scientific review.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TruthfulQA",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/sylinrl/TruthfulQA",
      "description": "TruthfulQA is a benchmark measuring whether models reproduce common human falsehoods. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2109.07958"
        }
      ],
      "status": "active"
    },
    {
      "name": "HaluEval",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/RUCAIBox/HaluEval",
      "description": "HaluEval is a large-scale hallucination evaluation benchmark with 35K samples.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HaluBench",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/PatronusAI/HaluBench",
      "description": "HaluBench, from Patronus AI, is a hallucination evaluation benchmark of 15K real-world RAG samples spanning finance and medicine.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FACTS Grounding",
      "category": "Hallucination Detection and Factuality",
      "availability": "commercial",
      "primary_url": "https://www.kaggle.com/facts-leaderboard",
      "description": "FACTS Grounding, from Google DeepMind, is a benchmark and leaderboard for grounding quality in long-form responses.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SimpleQA",
      "category": "Hallucination Detection and Factuality",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/simple-evals",
      "description": "SimpleQA, from OpenAI, benchmarks short-form factuality and serves as a useful hallucination-rate probe. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2411.04368"
        }
      ],
      "status": "active"
    },
    {
      "name": "G-Eval",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://arxiv.org/abs/2303.16634",
      "description": "G-Eval is a chain-of-thought, form-filling LLM-judging algorithm (Liu et al., 2023); production implementations are available in DeepEval and other frameworks, letting you define custom criteria in plain language.",
      "secondary_links": [
        {
          "label": "DeepEval",
          "url": "https://deepeval.com/docs/metrics-llm-evals"
        }
      ],
      "status": "active"
    },
    {
      "name": "Prometheus 2",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-weights",
      "primary_url": "https://github.com/prometheus-eval/prometheus-eval",
      "description": "Prometheus 2 is a family of open judge language models specialized in fine-grained, rubric-based evaluation of other models. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2405.01535"
        }
      ],
      "status": "active"
    },
    {
      "name": "Glider",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-weights",
      "primary_url": "https://huggingface.co/PatronusAI/glider",
      "description": "Glider, from Patronus AI, is a 3.8B-parameter explainable rubric-based judge model with span-level highlights.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Atla Selene",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-weights",
      "primary_url": "https://huggingface.co/AtlaAI/Selene-1-Mini-Llama-3.1-8B",
      "description": "Selene, from Atla, is a small judge model family purpose-trained for evaluation tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "JudgeLM",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://github.com/baaivision/JudgeLM",
      "description": "JudgeLM, from BAAI, provides fine-tuned, scalable judge models for open-ended benchmarks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Verdict",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://github.com/haizelabs/verdict",
      "description": "Verdict, from Haize Labs, is a declarative library for building compound LLM-judge systems: ensembles, debates, and verification chains.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "judges",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://github.com/quotient-ai/judges",
      "description": "judges, from Quotient AI, is a small library of curated, research-backed judge prompts.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "autoevals",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://github.com/braintrustdata/autoevals",
      "description": "autoevals, from Braintrust, is a library of automatic evaluators (factuality, closed-QA, RAG scorers) for LLM outputs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "PandaLM",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-source",
      "primary_url": "https://github.com/WeOpenML/PandaLM",
      "description": "PandaLM is a reproducible judge model for pairwise LLM comparison.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Flow Judge",
      "category": "LLM-as-a-Judge Models and Libraries",
      "availability": "open-weights",
      "primary_url": "https://github.com/flowaicom/flow-judge",
      "description": "Flow Judge, from Flow AI, is a compact 3.8B-parameter open evaluator model for customizable rubric evaluation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RewardBench",
      "category": "Judge and Reward Model Meta-Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/allenai/reward-bench",
      "description": "RewardBench, from AI2, was the first benchmark for evaluating reward models; RewardBench 2 extends it with harder, unseen human prompts.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "JudgeBench",
      "category": "Judge and Reward Model Meta-Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/ScalerLab/JudgeBench",
      "description": "JudgeBench evaluates LLM-based judges on challenging response pairs spanning knowledge, reasoning, math, and coding.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RM-Bench",
      "category": "Judge and Reward Model Meta-Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/THU-KEG/RM-Bench",
      "description": "RM-Bench evaluates reward models' sensitivity to subtle content changes and resistance to style bias.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LLMBar",
      "category": "Judge and Reward Model Meta-Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/princeton-nlp/LLMBar",
      "description": "LLMBar, from Princeton NLP, is a meta-evaluation benchmark testing whether LLM evaluators can detect instruction-following correctly against adversarial distractors.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LM-Polygraph",
      "category": "Uncertainty and Calibration",
      "availability": "open-source",
      "primary_url": "https://github.com/IINemo/lm-polygraph",
      "description": "LM-Polygraph implements a broad suite of uncertainty-quantification methods for LLM outputs with a unified benchmark.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TruthTorchLM",
      "category": "Uncertainty and Calibration",
      "availability": "open-source",
      "primary_url": "https://github.com/Ybakman/TruthTorchLM",
      "description": "TruthTorchLM is an open-source library of truthfulness-prediction and uncertainty methods for generative LLMs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Cleanlab TLM",
      "category": "Uncertainty and Calibration",
      "availability": "commercial",
      "primary_url": "https://cleanlab.ai/tlm/",
      "description": "Cleanlab's Trustworthy Language Model (TLM) adds real-time trustworthiness and confidence scores to any LLM output.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "UQ360",
      "category": "Uncertainty and Calibration",
      "availability": "open-source",
      "primary_url": "https://github.com/IBM/UQ360",
      "description": "Uncertainty Quantification 360, from IBM, is a toolkit for quantifying and communicating uncertainty in machine-learning models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MMLU",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/hendrycks/test",
      "description": "MMLU is a 57-subject multiple-choice knowledge benchmark that served for years as the default capability probe. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2009.03300"
        }
      ],
      "status": "active"
    },
    {
      "name": "MMLU-Pro",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/TIGER-AI-Lab/MMLU-Pro",
      "description": "MMLU-Pro is a harder, more robust successor to MMLU with 10 answer choices and more reasoning-heavy questions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "GPQA",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/idavidrein/gpqa",
      "description": "GPQA is a graduate-level, \"Google-proof\" science QA benchmark written by domain experts. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.12022"
        }
      ],
      "status": "active"
    },
    {
      "name": "Humanity's Last Exam",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/centerforaisafety/hle",
      "description": "Humanity's Last Exam contains 2,500 extremely difficult expert-written questions across 100+ subjects, designed as a \"final\" closed-ended academic benchmark. Site",
      "secondary_links": [
        {
          "label": "Site",
          "url": "https://lastexam.ai"
        }
      ],
      "status": "active"
    },
    {
      "name": "BIG-bench",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/google/BIG-bench",
      "description": "BIG-bench is a collaborative benchmark of 200+ tasks probing model capabilities and limitations; see also BIG-Bench Hard.",
      "secondary_links": [
        {
          "label": "BIG-Bench Hard",
          "url": "https://github.com/suzgunmirac/BIG-Bench-Hard"
        }
      ],
      "status": "active"
    },
    {
      "name": "GSM8K",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/grade-school-math",
      "description": "GSM8K is a benchmark of grade-school math word problems, a classic chain-of-thought reasoning test.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MATH",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/hendrycks/math",
      "description": "MATH contains 12,500 competition mathematics problems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FrontierMath",
      "category": "General Capability Benchmarks",
      "availability": "commercial",
      "primary_url": "https://epoch.ai/frontiermath",
      "description": "FrontierMath, from Epoch AI, is a private benchmark of exceptionally hard, unpublished research-level math problems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AIME",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://maa.org/aime-thresholds-are-available/",
      "description": "AIME (American Invitational Mathematics Examination) problems are widely used to test frontier-model math, e.g., via MathArena.",
      "secondary_links": [
        {
          "label": "MathArena",
          "url": "https://matharena.ai"
        }
      ],
      "status": "active"
    },
    {
      "name": "IFEval",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/google-research/google-research/tree/master/instruction_following_eval",
      "description": "IFEval tests verifiable instruction following (e.g., \"answer in exactly 3 paragraphs\"). Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.07911"
        }
      ],
      "status": "active"
    },
    {
      "name": "MT-Bench",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge",
      "description": "MT-Bench is a multi-turn conversation benchmark scored by LLM judges, introduced alongside Chatbot Arena. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2306.05685"
        }
      ],
      "status": "active"
    },
    {
      "name": "Arena-Hard",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/lmarena/arena-hard-auto",
      "description": "Arena-Hard is an automatic benchmark built from challenging live Arena prompts, with high correlation to human preference rankings.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LiveBench",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/LiveBench/LiveBench",
      "description": "LiveBench is a contamination-resistant benchmark with monthly-refreshed questions and objective scoring.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ARC-AGI",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/fchollet/ARC-AGI",
      "description": "ARC-AGI is François Chollet's abstraction-and-reasoning corpus and the basis of the ARC Prize.",
      "secondary_links": [
        {
          "label": "ARC Prize",
          "url": "https://arcprize.org"
        }
      ],
      "status": "active"
    },
    {
      "name": "SimpleBench",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://simple-bench.com",
      "description": "SimpleBench is a trick-question benchmark on which average humans still outperform frontier models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "EQ-Bench",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/EQ-bench/EQ-Bench",
      "description": "EQ-Bench benchmarks emotional intelligence and creative writing in LLMs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HellaSwag",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/rowanz/hellaswag",
      "description": "HellaSwag is a classic commonsense-reasoning benchmark using adversarially filtered sentence completions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "WinoGrande",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/allenai/winogrande",
      "description": "WinoGrande is a large-scale Winograd-schema-style benchmark for commonsense pronoun resolution.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ARC (AI2 Reasoning Challenge)",
      "category": "General Capability Benchmarks",
      "availability": "open-source",
      "primary_url": "https://allenai.org/data/arc",
      "description": "ARC is AI2's grade-school science question benchmark, still common in base-model reports.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SWE-bench",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/SWE-bench/SWE-bench",
      "description": "SWE-bench requires resolving real GitHub issues in real repositories and is the standard agentic coding benchmark, with Lite, Verified, and Multimodal variants. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2310.06770"
        }
      ],
      "status": "active"
    },
    {
      "name": "HumanEval",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/human-eval",
      "description": "HumanEval, from OpenAI, contains 164 hand-written function-synthesis problems and originated the pass@k code metric.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MBPP",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/google-research/google-research/tree/master/mbpp",
      "description": "MBPP, from Google Research, contains roughly 1,000 crowd-sourced entry-level Python problems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "EvalPlus",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/evalplus/evalplus",
      "description": "EvalPlus re-evaluates HumanEval and MBPP with 80×/35× more tests (HumanEval+ and MBPP+).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BigCodeBench",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/bigcode-project/bigcodebench",
      "description": "BigCodeBench tests practical, tool-using code generation with rich function calls and branch coverage.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LiveCodeBench",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/LiveCodeBench/LiveCodeBench",
      "description": "LiveCodeBench is a contamination-aware code benchmark continuously updated from LeetCode, AtCoder, and Codeforces problems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Aider Polyglot",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://aider.chat/docs/leaderboards/",
      "description": "The Aider Polyglot benchmark tests code editing across 225 hard Exercism problems in six languages.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Terminal-Bench",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/laude-institute/terminal-bench",
      "description": "Terminal-Bench evaluates agents on real tasks in a terminal environment: builds, system administration, and data wrangling.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SWE-Lancer",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/SWELancer-Benchmark",
      "description": "SWE-Lancer, from OpenAI, benchmarks 1,400+ real freelance software engineering tasks with a combined task value of $1M USD.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "CodeElo",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://codeelo-bench.github.io",
      "description": "CodeElo benchmarks competition-level code generation with human-comparable Elo ratings from Codeforces.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SciCode",
      "category": "Coding Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/scicode-bench/SciCode",
      "description": "SciCode is a research-level benchmark of coding problems drawn from real scientific research workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "GAIA",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/gaia-benchmark/GAIA",
      "description": "GAIA contains 466 real-world assistant questions requiring reasoning, browsing, and tool use. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.12983"
        }
      ],
      "status": "active"
    },
    {
      "name": "tau-bench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/sierra-research/tau-bench",
      "description": "tau-bench (τ-bench), from Sierra, benchmarks tool-using agents that interact with simulated users under domain policies (retail, airline).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "tau2-bench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/sierra-research/tau2-bench",
      "description": "tau2-bench (τ²-bench) extends tau-bench with a dual-control telecom environment where both agent and simulated user act on shared state.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AgentBench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/THUDM/AgentBench",
      "description": "AgentBench is a comprehensive multi-environment benchmark for LLMs acting as agents across OS, database, web, and game environments.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OSWorld",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/xlang-ai/OSWorld",
      "description": "OSWorld benchmarks multimodal agents on real computer-use tasks in real operating systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AppWorld",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/StonyBrookNLP/appworld",
      "description": "AppWorld is a controllable world of nine simulated apps and 457 APIs for benchmarking interactive coding agents on complex everyday tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "WebArena",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/web-arena-x/webarena",
      "description": "WebArena is a realistic, reproducible, self-hosted web environment for autonomous agents; VisualWebArena extends it to visual tasks.",
      "secondary_links": [
        {
          "label": "VisualWebArena",
          "url": "https://github.com/web-arena-x/visualwebarena"
        }
      ],
      "status": "active"
    },
    {
      "name": "Mind2Web",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/OSU-NLP-Group/Mind2Web",
      "description": "Mind2Web is a dataset and benchmark for generalist web agents across 137 real websites.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BrowserGym",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/ServiceNow/BrowserGym",
      "description": "BrowserGym, from ServiceNow, is a gym environment unifying WebArena, MiniWoB++, WorkArena, and more for web-agent evaluation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BFCL",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://gorilla.cs.berkeley.edu/leaderboard.html",
      "description": "The Berkeley Function-Calling Leaderboard (BFCL) is the standard benchmark for tool/function-calling accuracy. GitHub",
      "secondary_links": [
        {
          "label": "GitHub",
          "url": "https://github.com/ShishirPatil/gorilla"
        }
      ],
      "status": "active"
    },
    {
      "name": "MCPMark",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/eval-sys/mcpmark",
      "description": "MCPMark is a stress-testing benchmark for agent capabilities in realistic Model Context Protocol (MCP) environments.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BrowseComp",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/simple-evals",
      "description": "BrowseComp, from OpenAI, benchmarks browsing agents on hard-to-find, entangled information-seeking tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TheAgentCompany",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/TheAgentCompany/TheAgentCompany",
      "description": "TheAgentCompany benchmarks LLM agents on consequential real-world professional tasks in a simulated software company.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RE-Bench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/METR/ai-rd-tasks",
      "description": "RE-Bench, from METR, evaluates frontier AI R&D capabilities of agents against human experts on realistic research-engineering tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AgentHarm",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/ai-safety-institute/AgentHarm",
      "description": "AgentHarm, from the UK AI Security Institute, measures harmfulness of LLM agents on multi-step malicious tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MLE-bench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/mle-bench",
      "description": "MLE-bench, from OpenAI, benchmarks agents on Kaggle-style machine-learning engineering tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "PaperBench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/preparedness",
      "description": "PaperBench, from OpenAI, evaluates agents on replicating state-of-the-art AI research papers.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MiniWoB++",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/Farama-Foundation/miniwob-plusplus",
      "description": "MiniWoB++ contains 100+ small web-interaction tasks and remains the classic web-agent testbed.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ClawBench",
      "category": "Agent Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/TIGER-AI-Lab/ClawBench",
      "description": "ClawBench is an open-source benchmark framework for end-to-end browser-agent tasks on live websites, with isolated runs and replayable five-layer traces. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2604.08523"
        }
      ],
      "status": "active"
    },
    {
      "name": "AILuminate",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://mlcommons.org/ailuminate/",
      "description": "AILuminate, from MLCommons, grades systems across 12 hazard categories and is developed by an industry consortium.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HarmBench",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/centerforaisafety/HarmBench",
      "description": "HarmBench provides standardized red-teaming and refusal-robustness evaluation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DecodingTrust",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/AI-secure/DecodingTrust",
      "description": "DecodingTrust is a comprehensive trustworthiness assessment covering toxicity, stereotype bias, robustness, privacy, ethics, and fairness (NeurIPS 2023 Outstanding Paper).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SORRY-Bench",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/SORRY-Bench/sorry-bench",
      "description": "SORRY-Bench systematically evaluates refusal behavior across 45 unsafe topic categories.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "XSTest",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/paul-rottger/xstest",
      "description": "XSTest tests exaggerated safety (over-refusal) using safe prompts that superficially look unsafe.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "StrongREJECT",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/alexandrasouly/strongreject",
      "description": "StrongREJECT is a rigorous benchmark for measuring jailbreak effectiveness.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Do-Not-Answer",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/Libr-AI/do-not-answer",
      "description": "Do-Not-Answer collects questions responsible models should refuse, for evaluating safeguards.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "WildGuard",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-weights",
      "primary_url": "https://github.com/allenai/wildguard",
      "description": "WildGuard, from AI2, is an open moderation model for prompt harmfulness, response harmfulness, and refusal detection.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "TrustLLM",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/HowieHwong/TrustLLM",
      "description": "TrustLLM benchmarks trustworthiness across truthfulness, safety, fairness, robustness, privacy, and machine ethics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BBQ",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/nyu-mll/BBQ",
      "description": "BBQ (Bias Benchmark for QA) measures social bias across nine demographic dimensions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ToxiGen",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/microsoft/TOXIGEN",
      "description": "ToxiGen, from Microsoft, is a large-scale dataset for implicit and adversarial hate-speech detection across 13 minority groups.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SafetyBench",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/thu-coai/SafetyBench",
      "description": "SafetyBench is a multiple-choice LLM safety evaluation in English and Chinese.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "sycophancy-eval",
      "category": "Safety and Alignment Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/meg-tong/sycophancy-eval",
      "description": "sycophancy-eval contains the evaluations from Anthropic's paper \"Towards Understanding Sycophancy in Language Models\".",
      "secondary_links": [
        {
          "label": "\"Towards Understanding Sycophancy in Language Models\"",
          "url": "https://arxiv.org/abs/2310.13548"
        }
      ],
      "status": "active"
    },
    {
      "name": "Global MMLU",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/CohereLabs/Global-MMLU",
      "description": "Global MMLU, from Cohere Labs, is a culturally aware version of MMLU professionally translated and verified across 42 languages.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Belebele",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/facebookresearch/belebele",
      "description": "Belebele, from Meta, is a machine-reading-comprehension benchmark spanning 122 language variants.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MGSM",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/juletxara/mgsm",
      "description": "MGSM (Multilingual Grade School Math) translates GSM8K problems into 10 typologically diverse languages.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FLORES-200",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/facebookresearch/flores",
      "description": "FLORES-200, from Meta, is the standard machine-translation evaluation benchmark covering 200 languages.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "INCLUDE",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/datasets/CohereLabs/include-base-44",
      "description": "INCLUDE is a regional-knowledge benchmark built from native exam sources in 44 languages, testing knowledge in original cultural contexts.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "C-Eval",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/hkust-nlp/ceval",
      "description": "C-Eval is a comprehensive Chinese evaluation suite of 13,948 multiple-choice questions across 52 disciplines.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "CMMLU",
      "category": "Multilingual Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/haonan-li/CMMLU",
      "description": "CMMLU measures Chinese language and cultural knowledge across 67 subjects, including China-specific topics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HealthBench",
      "category": "Domain-Specific Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/openai/simple-evals",
      "description": "HealthBench, from OpenAI, evaluates health-related conversations using 5,000 realistic dialogues scored against physician-written rubrics.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LegalBench",
      "category": "Domain-Specific Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/HazyResearch/legalbench",
      "description": "LegalBench is a collaboratively built benchmark of 162 legal-reasoning tasks designed with legal professionals.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FinanceBench",
      "category": "Domain-Specific Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/patronus-ai/financebench",
      "description": "FinanceBench, from Patronus AI, tests open-book financial question answering over real filings.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MedQA",
      "category": "Domain-Specific Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/jind11/MedQA",
      "description": "MedQA tests medical knowledge with USMLE-style board exam questions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "FinBen",
      "category": "Domain-Specific Benchmarks",
      "availability": "open-source",
      "primary_url": "https://github.com/The-FinAI/FinBen",
      "description": "FinBen is an extensive open benchmark suite for financial LLM evaluation across dozens of tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vals AI",
      "category": "Domain-Specific Benchmarks",
      "availability": "commercial",
      "primary_url": "https://www.vals.ai",
      "description": "Vals AI runs industry-specific model evaluations (legal, finance, medical, tax) on private datasets and publishes leaderboards.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Needle In A Haystack",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/gkamradt/LLMTest_NeedleInAHaystack",
      "description": "Needle In A Haystack is the original simple retrieval-from-long-context stress test.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "RULER",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/NVIDIA/RULER",
      "description": "RULER, from NVIDIA, is a synthetic benchmark that reveals a model's _effective_ context length versus its claimed length.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LongBench v2",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/THUDM/LongBench",
      "description": "LongBench v2 tests realistic long-context reasoning up to 2M words in English and Chinese.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HELMET",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/princeton-nlp/HELMET",
      "description": "HELMET, from Princeton, is an application-centric long-context evaluation suite at 128K+ tokens.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "∞Bench",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/OpenBMB/InfiniteBench",
      "description": "∞Bench (InfiniteBench) benchmarks context windows beyond 100K tokens.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LongMemEval",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/xiaowu0162/LongMemEval",
      "description": "LongMemEval benchmarks long-term interactive memory in chat assistants across five core memory abilities (ICLR 2025).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LoCoMo",
      "category": "Long-Context and Memory Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/snap-research/locomo",
      "description": "LoCoMo, from Snap Research, evaluates very long-term conversational memory over multi-session dialogues.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "lmms-eval",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/EvolvingLMMs-Lab/lmms-eval",
      "description": "lmms-eval is a one-for-all evaluation toolkit for large multimodal models across text, image, video, and audio tasks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "VLMEvalKit",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/open-compass/VLMEvalKit",
      "description": "VLMEvalKit, from OpenCompass, supports 220+ vision-language models and 80+ multimodal benchmarks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MMMU",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/MMMU-Benchmark/MMMU",
      "description": "MMMU is a massive multi-discipline multimodal understanding benchmark at college level. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.16502"
        }
      ],
      "status": "active"
    },
    {
      "name": "MMBench",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/open-compass/MMBench",
      "description": "MMBench provides systematic multimodal capability evaluation using the CircularEval protocol.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MathVista",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/lupantech/MathVista",
      "description": "MathVista benchmarks mathematical reasoning in visual contexts.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Video-MME",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/MME-Benchmarks/Video-MME",
      "description": "Video-MME is a comprehensive benchmark for video analysis by multimodal LLMs (CVPR 2025).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ChartQA",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/vis-nlp/ChartQA",
      "description": "ChartQA tests question answering about charts requiring visual and logical reasoning.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "OmniDocBench",
      "category": "Multimodal and Vision Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/opendatalab/OmniDocBench",
      "description": "OmniDocBench is a comprehensive document-parsing evaluation benchmark (CVPR 2025).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "VoiceBench",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/MatthewCYM/VoiceBench",
      "description": "VoiceBench benchmarks LLM-based voice assistants on speech interactions across knowledge, instruction following, safety, and robustness.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AudioBench",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/AudioLLMs/AudioBench",
      "description": "AudioBench is a universal benchmark for audio LLMs covering speech understanding, audio-scene understanding, and voice understanding.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Full-Duplex-Bench",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/DanielLin94144/Full-Duplex-Bench",
      "description": "Full-Duplex-Bench evaluates full-duplex spoken dialogue models on turn-taking: pause handling, backchanneling, and interruption.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "VBench",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/Vchitect/VBench",
      "description": "VBench is a comprehensive benchmark suite for video generative models, decomposing quality into 16 dimensions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "HEIM",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/stanford-crfm/helm",
      "description": "HEIM, from Stanford CRFM (part of the HELM family), is a holistic evaluation of text-to-image models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "GenEval",
      "category": "Audio and Generative Media Evaluation",
      "availability": "open-source",
      "primary_url": "https://github.com/djghosh13/geneval",
      "description": "GenEval is an object-focused framework for evaluating text-to-image alignment.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "GuideLLM",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/vllm-project/guidellm",
      "description": "GuideLLM, from the vLLM project, evaluates and optimizes LLM deployments for real-world inference performance under load.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "genai-bench",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/sgl-project/genai-bench",
      "description": "genai-bench, from the SGLang project, provides comprehensive token-level performance benchmarking of LLM serving systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "GenAI-Perf",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/perf_benchmark/genai-perf-README.html",
      "description": "GenAI-Perf, from NVIDIA, measures TTFT, inter-token latency, tokens per second, and requests per second for generative models served via inference endpoints.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LLMPerf",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/ray-project/llmperf",
      "description": "LLMPerf, from the Ray project, load-tests and validates the performance of LLM APIs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MLPerf Inference",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://mlcommons.org/benchmarks/inference/",
      "description": "MLPerf Inference, from MLCommons, is the industry consortium benchmark for ML and LLM inference performance across hardware and serving stacks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "inference-perf",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/kubernetes-sigs/inference-perf",
      "description": "inference-perf, a Kubernetes SIG project, is a GenAI inference performance benchmarking tool for Kubernetes deployments.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AIPerf",
      "category": "Inference Performance and Load Testing",
      "availability": "open-source",
      "primary_url": "https://github.com/ai-dynamo/aiperf",
      "description": "AIPerf, from NVIDIA's Dynamo project, provides comprehensive benchmarking of generative AI serving performance at scale.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LMArena (Chatbot Arena)",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://lmarena.ai",
      "description": "LMArena ranks models via crowdsourced pairwise human-preference battles with Elo-style ratings. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2403.04132"
        }
      ],
      "status": "active"
    },
    {
      "name": "Artificial Analysis",
      "category": "Leaderboards and Arenas",
      "availability": "commercial",
      "primary_url": "https://artificialanalysis.ai",
      "description": "Artificial Analysis independently benchmarks models and providers on intelligence, price, latency, and throughput.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Open LLM Leaderboard (archived)",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard",
      "description": "Hugging Face's Open LLM Leaderboard ranked open models until its retirement in 2025; it remains a useful historical archive.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LiveBench",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://livebench.ai",
      "description": "LiveBench maintains a contamination-limited leaderboard with monthly-refreshed questions.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Scale SEAL",
      "category": "Leaderboards and Arenas",
      "availability": "commercial",
      "primary_url": "https://scale.com/leaderboard",
      "description": "SEAL, from Scale AI, publishes private, expert-evaluated leaderboards designed to resist overfitting.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vellum LLM Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "commercial",
      "primary_url": "https://www.vellum.ai/llm-leaderboard",
      "description": "The Vellum LLM Leaderboard aggregates frontier-model comparisons across popular benchmarks.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Epoch AI Benchmarking Hub",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://epoch.ai/benchmarks",
      "description": "Epoch AI's Benchmarking Hub publishes independent evaluations and trend analyses of frontier models over time.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MTEB Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://huggingface.co/spaces/mteb/leaderboard",
      "description": "The MTEB Leaderboard ranks text-embedding models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "BFCL Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://gorilla.cs.berkeley.edu/leaderboard.html",
      "description": "The BFCL Leaderboard ranks models on function/tool-calling accuracy.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SWE-bench Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://www.swebench.com",
      "description": "The official SWE-bench leaderboard tracks agentic coding performance across SWE-bench variants.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ARC Prize Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://arcprize.org/leaderboard",
      "description": "The ARC Prize leaderboard tracks progress on ARC-AGI-1 and ARC-AGI-2.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Aider LLM Leaderboards",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://aider.chat/docs/leaderboards/",
      "description": "Aider's leaderboards rank models on polyglot code editing.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Vectara Hallucination Leaderboard",
      "category": "Leaderboards and Arenas",
      "availability": "open-source",
      "primary_url": "https://github.com/vectara/hallucination-leaderboard",
      "description": "The Hughes Hallucination Leaderboard ranks models by hallucination rate when summarizing documents, scored with HHEM.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LLM Decontaminator",
      "category": "Benchmark Contamination and Evaluation Reliability",
      "availability": "open-source",
      "primary_url": "https://github.com/lm-sys/llm-decontaminator",
      "description": "LLM Decontaminator, from LMSYS, detects rephrased benchmark samples in training data that string matching misses. Paper",
      "secondary_links": [
        {
          "label": "Paper",
          "url": "https://arxiv.org/abs/2311.04850"
        }
      ],
      "status": "active"
    },
    {
      "name": "LiveBench",
      "category": "Benchmark Contamination and Evaluation Reliability",
      "availability": "open-source",
      "primary_url": "https://github.com/LiveBench/LiveBench",
      "description": "LiveBench limits contamination structurally by releasing new questions monthly.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "LiveCodeBench",
      "category": "Benchmark Contamination and Evaluation Reliability",
      "availability": "open-source",
      "primary_url": "https://github.com/LiveCodeBench/LiveCodeBench",
      "description": "LiveCodeBench tags every problem with a release date so models can be scored only on problems published after their training cutoff.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Adding Error Bars to Evals",
      "category": "Benchmark Contamination and Evaluation Reliability",
      "availability": "open-source",
      "primary_url": "https://arxiv.org/abs/2411.00640",
      "description": "Anthropic's statistical guidance for reporting confidence intervals and significance in LLM evaluations.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DeepEval Synthesizer",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://deepeval.com/docs/synthesizer-introduction",
      "description": "DeepEval's Synthesizer generates synthetic goldens from documents, contexts, or from scratch, with evolution-based complexity control.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Ragas Testset Generation",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://docs.ragas.io/en/stable/concepts/test_data_generation/",
      "description": "Ragas generates knowledge-graph-based synthetic test sets for RAG pipelines.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "YourBench",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://github.com/huggingface/yourbench",
      "description": "YourBench, from Hugging Face, generates custom benchmarks from your own documents, cheaply and without contamination.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "distilabel",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://github.com/argilla-io/distilabel",
      "description": "distilabel, from Argilla, is a framework for synthetic data and AI-feedback pipelines for dataset creation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DataDreamer",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://github.com/datadreamer-dev/DataDreamer",
      "description": "DataDreamer is a Python library for synthetic data generation and reproducible LLM workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Meta Synthetic Data Kit",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://github.com/meta-llama/synthetic-data-kit",
      "description": "Synthetic Data Kit, from Meta, generates reasoning traces and QA pairs for fine-tuning and evaluation datasets.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "NeMo Curator",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-source",
      "primary_url": "https://github.com/NVIDIA-NeMo/Curator",
      "description": "NeMo Curator, from NVIDIA, is a scalable data curation and synthetic-generation pipeline for training and evaluation corpora.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Gretel",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "commercial",
      "primary_url": "https://gretel.ai",
      "description": "Gretel (acquired by NVIDIA) is a general synthetic-data platform for tabular, text, and structured data with privacy guarantees.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Mostly AI",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "open-core",
      "primary_url": "https://mostly.ai",
      "description": "Mostly AI provides a synthetic-data SDK and platform for privacy-safe tabular and structured data.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Snorkel",
      "category": "Synthetic Data and Test Set Generation",
      "availability": "commercial",
      "primary_url": "https://snorkel.ai",
      "description": "Snorkel AI provides programmatic data labeling and curation (Snorkel Flow), including expert-driven LLM evaluation dataset development; it grew out of the open-source Snorkel research project.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Label Studio",
      "category": "Human Annotation and Labeling",
      "availability": "open-core",
      "primary_url": "https://github.com/HumanSignal/label-studio",
      "description": "Label Studio is a widely used open-source data labeling tool supporting text, image, audio, video, and LLM response ranking.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Argilla",
      "category": "Human Annotation and Labeling",
      "availability": "open-source",
      "primary_url": "https://github.com/argilla-io/argilla",
      "description": "Argilla is a collaboration tool for building high-quality LLM datasets: rating, ranking, and feedback workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Prodigy",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://prodi.gy",
      "description": "Prodigy, from the spaCy team, is a scriptable annotation tool with model-in-the-loop workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Labelbox",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://labelbox.com",
      "description": "Labelbox combines a labeling platform with managed human evaluation for RLHF and evals.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Scale",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://scale.com",
      "description": "Scale AI provides managed expert data annotation and human evaluation at frontier-lab scale.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Surge AI",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://www.surgehq.ai",
      "description": "Surge AI provides human data labeling and RLHF workforces used by frontier labs.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Toloka",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://toloka.ai",
      "description": "Toloka provides a global expert workforce for AI data and human evaluation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "SuperAnnotate",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://www.superannotate.com",
      "description": "SuperAnnotate is an enterprise annotation platform including LLM evaluation and fine-tuning data workflows.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Encord",
      "category": "Human Annotation and Labeling",
      "availability": "commercial",
      "primary_url": "https://encord.com",
      "description": "Encord is a data development platform with annotation and model-evaluation tooling, strongest in vision.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "MLflow",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/mlflow/mlflow",
      "description": "MLflow provides experiment tracking, a model registry, and evaluation for ML and GenAI.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Weights & Biases",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-core",
      "primary_url": "https://wandb.ai",
      "description": "Weights & Biases provides experiment tracking, sweeps, a model registry, and reports, plus Weave for LLM work.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Comet",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-core",
      "primary_url": "https://www.comet.com",
      "description": "Comet provides experiment management and production model monitoring, plus Opik for LLM work.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "ClearML",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-core",
      "primary_url": "https://github.com/clearml/clearml",
      "description": "ClearML is an open-source MLOps suite for experiments, orchestration, and serving.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "DVC",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/iterative/dvc",
      "description": "DVC provides Git-based data and experiment versioning.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Deepchecks",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-core",
      "primary_url": "https://github.com/deepchecks/deepchecks",
      "description": "Deepchecks tests and validates ML models and data from research to production, and offers an LLM evaluation product.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Great Expectations",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/great-expectations/great_expectations",
      "description": "Great Expectations is a widely used framework for data quality testing and validation.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "whylogs",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/whylabs/whylogs",
      "description": "whylogs is an open standard for data logging and ML telemetry profiles.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "NannyML",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/NannyML/nannyml",
      "description": "NannyML estimates post-deployment model performance and detects silent failure without ground truth.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Fairlearn",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/fairlearn/fairlearn",
      "description": "Fairlearn is a Python package for assessing and improving the fairness of ML models.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "AI Fairness 360",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/Trusted-AI/AIF360",
      "description": "AIF360, from IBM (now LF AI), is a toolkit of fairness metrics and bias-mitigation algorithms.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Alibi Detect",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/SeldonIO/alibi-detect",
      "description": "Alibi Detect, from Seldon, provides outlier, adversarial, and drift detection for ML systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Cleanlab",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-source",
      "primary_url": "https://github.com/cleanlab/cleanlab",
      "description": "Cleanlab automatically detects label errors and data issues in ML datasets (data-centric evaluation).",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Fiddler",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "commercial",
      "primary_url": "https://www.fiddler.ai",
      "description": "Fiddler provides enterprise AI observability: model monitoring, explainability, and LLM monitoring.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Evidently",
      "category": "Classic ML Evaluation and Monitoring",
      "availability": "open-core",
      "primary_url": "https://github.com/evidentlyai/evidently",
      "description": "Evidently provides drift detection, data quality checks, and performance monitoring for ML and LLM systems.",
      "secondary_links": [],
      "status": "active"
    },
    {
      "name": "Literal AI",
      "category": "Discontinued and Historical Tools",
      "availability": "discontinued",
      "primary_url": "https://literalai.com",
      "description": "Literal AI, from the Chainlit team, announced its service shutdown effective October 31, 2025.",
      "secondary_links": [],
      "status": "discontinued"
    },
    {
      "name": "Neptune",
      "category": "Discontinued and Historical Tools",
      "availability": "discontinued",
      "primary_url": "https://neptune.ai",
      "description": "Neptune's hosted experiment tracker was discontinued in March 2026 following the company's acquisition; existing users were migrated off the platform.",
      "secondary_links": [],
      "status": "discontinued"
    },
    {
      "name": "WhyLabs Platform",
      "category": "Discontinued and Historical Tools",
      "availability": "discontinued",
      "primary_url": "https://whylabs.ai",
      "description": "The hosted WhyLabs observability platform discontinued operations; the open-source whylogs and LangKit libraries remain available.",
      "secondary_links": [
        {
          "label": "whylogs",
          "url": "https://github.com/whylabs/whylogs"
        },
        {
          "label": "LangKit",
          "url": "https://github.com/whylabs/langkit"
        }
      ],
      "status": "discontinued"
    },
    {
      "name": "OpenAI Evals (hosted)",
      "category": "Discontinued and Historical Tools",
      "availability": "discontinued",
      "primary_url": "https://platform.openai.com/docs/guides/evals",
      "description": "OpenAI announced the hosted Evals dashboard/API will shut down in late 2026; the open-source `openai/evals` repository remains available in maintenance mode.",
      "secondary_links": [],
      "status": "discontinued"
    },
    {
      "name": "Arthur Bench",
      "category": "Discontinued and Historical Tools",
      "availability": "discontinued",
      "primary_url": "https://github.com/arthur-ai/bench",
      "description": "Arthur Bench is no longer actively developed; Arthur's commercial platform continues.",
      "secondary_links": [],
      "status": "discontinued"
    }
  ]
}