_benchmark_profiles.json

{
  "default": "asr-publication-v2",
  "roles": {
    "langid": "langid-publication-v1",
    "vad": "vad-publication-v1"
  },
  "profiles": {
    "asr-publication-v2": {
      "description": "The complete benchmark set published for transcription models. v2: accuracy on L40S; batch_size is the recommended setting for new runs, and a cell is satisfied at any batch size (the row records the one used). A target may declare max_model_gib: weak reference machines (a base M4, a Ryzen 4750U host) are then required to run only models whose reference publication footprint fits, and larger models are published on the strong reference alone.",
      "accuracy": [
        {
          "dataset": "librispeech",
          "split": "test-clean",
          "languages": "english-if-supported",
          "quants": "all-downloads",
          "batch_size": 8,
          "sort_by_length": true,
          "timestamps": "none",
          "gpu": "L40S",
          "backend": "cuda"
        },
        {
          "dataset": "fleurs",
          "split": "test",
          "languages": "supported-intersect-fleurs",
          "quants": [
            "Q8_0"
          ],
          "batch_size": 8,
          "sort_by_length": true,
          "timestamps": "none",
          "gpu": "L40S",
          "backend": "cuda"
        }
      ],
      "speed": {
        "quants": [
          "Q8_0",
          "Q4_K_M"
        ],
        "samples": [
          "jfk",
          "dots"
        ],
        "iterations": 3,
        "warmup": 1,
        "targets": [
          {
            "machine": "m4",
            "display": "Apple M4",
            "backends": [
              "cpu",
              "metal"
            ],
            "max_model_gib": 5
          },
          {
            "machine": "m4-max",
            "display": "Apple M4 Max",
            "backends": [
              "cpu",
              "metal"
            ]
          },
          {
            "machine": "ryzen-4750u",
            "display": "AMD Ryzen 7 PRO 4750U (Radeon RADV RENOIR)",
            "backends": [
              "cpu",
              "vulkan"
            ],
            "cooldown_tctl_c": 55.0,
            "max_model_gib": 5
          }
        ],
        "single_language_samples": {
          "pattern": [
            "{lang}-short",
            "{lang}-long"
          ],
          "reason": "A variant that supports exactly one non-English language is benched on that language at the same two clip lengths as jfk/dots. English jfk/dots decode out of distribution on a single-language fine-tune and can loop until the position cap, and whether that happens differs between CPU and GPU, so the figure would not be comparable."
        }
      }
    },
    "langid-publication-v1": {
      "role": "langid",
      "description": "The benchmark set published for language ID (LANGID role) models. Accuracy: open-set top-1 over every label, the macro mean over the FLEURS test languages of scripts/langid/ingest.py, on the first crop_s seconds of each clip without silence trimming, C++ on CPU, every shipped quant; scored by scripts/langid/score.py from one scripts/langid/run.py sweep (all crops), which it also compares against the reference sweep (--ref) for the row's agreement. Speed: scripts/langid/bench.py through the Python binding; a sample <clip>-<N>s is the first N seconds of samples/<clip>.wav.",
      "accuracy": [
        {
          "dataset": "fleurs",
          "split": "test",
          "languages": "pooled",
          "metric": "accuracy",
          "quants": "all-downloads",
          "batch_size": 1,
          "timestamps": "none",
          "backend": "cpu",
          "crop_s": "5",
          "pooled_languages": [
            "en",
            "zh",
            "de",
            "fr",
            "es",
            "pt",
            "ja",
            "ko",
            "ru",
            "cs",
            "sk",
            "no",
            "da",
            "id",
            "ms"
          ]
        }
      ],
      "speed": {
        "quants": "all-downloads",
        "samples": [
          "ru-long-10s",
          "ru-long-30s"
        ],
        "iterations": 20,
        "warmup": 3,
        "targets": [
          {
            "machine": "m4",
            "display": "Apple M4",
            "backends": [
              "cpu",
              "metal"
            ],
            "max_model_gib": 5
          },
          {
            "machine": "m4-max",
            "display": "Apple M4 Max",
            "backends": [
              "cpu",
              "metal"
            ]
          },
          {
            "machine": "ryzen-4750u",
            "display": "AMD Ryzen 7 PRO 4750U (Radeon RADV RENOIR)",
            "backends": [
              "cpu",
              "vulkan"
            ],
            "cooldown_tctl_c": 55.0,
            "max_model_gib": 5
          }
        ]
      }
    },
    "vad-publication-v1": {
      "role": "vad",
      "description": "VAD speed only, no accuracy suite (numerical parity is the porting gate). scripts/vad/bench.py through the Python binding, CPU, one thread, every shipped quant. <clip>: transcribe_vad_run over the whole clip. <clip>-<N>ms: one pass of transcribe_vad_stream_feed over the clip in N ms chunks; a cell is the mean time per feed call.",
      "accuracy": [],
      "speed": {
        "quants": "all-downloads",
        "samples": [
          "love-loss-32ms",
          "love-loss-128ms",
          "love-loss-512ms",
          "jfk",
          "love-loss"
        ],
        "iterations": 10,
        "warmup": 3,
        "targets": [
          {
            "machine": "m4",
            "display": "Apple M4",
            "backends": [
              "cpu"
            ],
            "max_model_gib": 5
          }
        ]
      }
    }
  }
}