{
  "updated": "2026-10-18",
  "benchmark": {
    "name": "Freya-TR-Eval",
    "dataset": "freyavoice/freya-tr-eval",
    "sentences": 495,
    "paper": "https://arxiv.org/abs/2607.09530",
    "asr": "Whisper-large-v3 (faster-whisper, beam 5, language=tr)",
    "band_match_hz": 8000
  },
  "anchors": [
    {
      "label": "Human speech (FLEURS-tr)",
      "metric": "wer",
      "value": 9.7,
      "note": "Same band-matched pipeline on real recordings. Different corpus, so read as a scale reference, not a ceiling."
    }
  ],
  "models": [
    {
      "id": "ema-lightning",
      "name": "EMA Lightning",
      "variant": "canberkkkkkk/ema-lightning 1.0.1",
      "params_m": 8.6,
      "params_label": "8.6M",
      "voice": "Single voice",
      "link": "https://huggingface.co/canberkkkkkk/ema-lightning",
      "wer": 1.04,
      "wer_dp": 2,
      "wer_sd": 0.17,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "Flow-matching DiT",
      "wer_ci": [
        0.72,
        1.39
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.04 ± 0.17. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [0.72, 1.39].",
      "notes": "Plain and compiled (.lightning()) runs gave identical transcripts. Freya's own run (seeds 0 to 2) scored 1.00 [0.70, 1.33]; files in results/."
    },
    {
      "id": "antalia-2-mini",
      "name": "Antalia-2 Mini",
      "variant": "cloud0day3/antalia-mini 1.0.1",
      "params_m": 7.62,
      "params_label": "7.62M",
      "voice": "Default settings",
      "link": "https://huggingface.co/cloud0day3/antalia-mini",
      "wer": 1.09,
      "wer_dp": 2,
      "wer_sd": 0.12,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-10",
      "notes": "Run by PatientDesk AI, who make this system. Version 1.0.1 as reported by PatientDesk (1.09 ± 0.12). Its per-clip transcripts are not published, so this number could not be checked; the published transcripts are from 1.0.0 and recompute to 1.05 ± 0.07."
    },
    {
      "id": "alania-2",
      "name": "Alania-2",
      "variant": "Production API, voice Öncel",
      "params_m": 488,
      "params_label": "488M",
      "voice": "Voice Öncel (best of 11)",
      "link": "https://speech.patientdesk.ai/en/benchmarks",
      "wer": 1.15,
      "wer_dp": 2,
      "wer_sd": 0.22,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-12",
      "wer_ci": [
        0.83,
        1.51
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.15 ± 0.22. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [0.83, 1.51].",
      "notes": "Run by PatientDesk AI, who make this system. Öncel is the lowest-WER of 11 built-in voices. The default voice (F1) scored 1.44 ± 0.15 on 7 October (transcripts published) and 1.38 ± 0.18 on 12 October (as reported). Release alania-v2.1."
    },
    {
      "id": "trendyol-tts",
      "name": "Trendyol-TTS",
      "variant": "Trendyol/Trendyol-TTS (cfg 2.0, 16 steps)",
      "params_m": 2380,
      "params_label": "2.38B",
      "voice": "No reference clip (card defaults)",
      "link": "https://huggingface.co/Trendyol/Trendyol-TTS",
      "wer": 1.16,
      "wer_dp": 2,
      "wer_sd": 0.16,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "VoxCPM2 + merged LoRA",
      "wer_ci": [
        0.82,
        1.53
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.16 ± 0.16. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [0.82, 1.53]."
    },
    {
      "id": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash TTS",
      "variant": "Vertex AI, voice Kore",
      "params_m": null,
      "params_label": "API",
      "voice": "Voice Kore",
      "link": null,
      "wer": 1.23,
      "wer_dp": 2,
      "wer_sd": 0.1,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        0.93,
        1.56
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.23 ± 0.10. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [0.93, 1.56]."
    },
    {
      "id": "gemini-3.8-flash-lite",
      "name": "Gemini 3.8 Flash-Lite TTS",
      "variant": "Vertex AI, voice Kore",
      "params_m": null,
      "params_label": "API",
      "voice": "Voice Kore",
      "link": null,
      "wer": 1.34,
      "wer_dp": 2,
      "wer_sd": 0.19,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        1.01,
        1.7
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.34 ± 0.19. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [1.01, 1.70]."
    },
    {
      "id": "anka-tts",
      "name": "Anka TTS",
      "variant": "anka-tts v0.1",
      "params_m": 336,
      "params_label": "336M",
      "voice": "Built-in male voice",
      "link": "https://pypi.org/project/anka-tts/",
      "wer": 1.42,
      "wer_dp": 2,
      "wer_sd": 0.06,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        1.05,
        1.82
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.42 ± 0.06. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [1.05, 1.82]."
    },
    {
      "id": "chatterbox-mtl",
      "name": "Chatterbox Multilingual",
      "variant": "ResembleAI/chatterbox",
      "params_m": 500,
      "params_label": "500M",
      "voice": "Zero-shot cloner",
      "link": "https://huggingface.co/ResembleAI/chatterbox",
      "wer": 1.91,
      "wer_dp": 2,
      "wer_sd": 0.35,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        1.52,
        2.35
      ],
      "ci_note": "Mean ± SD over 3 takes: 1.91 ± 0.35. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [1.52, 2.35].",
      "notes": "Reference clip: Anka TTS's packaged male voice."
    },
    {
      "id": "piper-tr-dfki",
      "name": "Piper",
      "variant": "tr_TR-dfki-medium",
      "params_m": 16,
      "params_label": "16M",
      "voice": "Single voice",
      "link": "https://huggingface.co/rhasspy/piper-voices/tree/main/tr/tr_TR/dfki/medium",
      "wer": 3.41,
      "wer_dp": 2,
      "wer_sd": 0.21,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "VITS",
      "wer_ci": [
        2.87,
        3.98
      ],
      "ci_note": "Mean ± SD over 3 takes: 3.41 ± 0.21. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [2.87, 3.98].",
      "notes": "Run on CPU. The FreyaTTS paper (Table 2) reported 4.4."
    },
    {
      "id": "xtts-v2",
      "name": "XTTS-v2",
      "variant": "coqui/XTTS-v2 (multilingual)",
      "params_m": 470,
      "params_label": "470M",
      "voice": "Zero-shot cloner",
      "link": "https://huggingface.co/coqui/XTTS-v2",
      "wer": 3.67,
      "wer_dp": 2,
      "wer_sd": 0.37,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "Autoregressive",
      "wer_ci": [
        3.16,
        4.22
      ],
      "ci_note": "Mean ± SD over 3 takes: 3.67 ± 0.37. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [3.16, 4.22].",
      "notes": "Reference clip: Anka TTS's packaged male voice. The FreyaTTS paper (Table 2) reported 11.1 with a different reference clip; cloner scores depend on the clip."
    },
    {
      "id": "alania-1",
      "name": "Alania-1",
      "variant": "Production API",
      "params_m": 3300,
      "params_label": "3.3B",
      "voice": "Default voice",
      "link": "https://speech.patientdesk.ai/en/benchmarks",
      "wer": 5.99,
      "wer_dp": 2,
      "wer_sd": 0.47,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        5.3,
        6.71
      ],
      "ci_note": "Mean ± SD over 3 takes: 5.99 ± 0.47. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [5.30, 6.71].",
      "notes": "Run by PatientDesk AI, who make this system."
    },
    {
      "id": "antalia-1",
      "name": "Antalia-1",
      "variant": "cloud0day3/antalia-1 (release recipe)",
      "params_m": 305,
      "params_label": "305M",
      "voice": "Release recipe speaker",
      "link": "https://huggingface.co/cloud0day3/antalia-1",
      "wer": 6.28,
      "wer_dp": 2,
      "wer_sd": 0.27,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "wer_ci": [
        5.68,
        6.9
      ],
      "ci_note": "Mean ± SD over 3 takes: 6.28 ± 0.27. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [5.68, 6.90].",
      "notes": "Run by PatientDesk AI, who make this system. One seed per take, no best-of-n."
    },
    {
      "id": "mms-tts-tur",
      "name": "MMS-TTS",
      "variant": "facebook/mms-tts-tur",
      "params_m": 36,
      "params_label": "36M",
      "voice": "Single voice",
      "link": "https://huggingface.co/facebook/mms-tts-tur",
      "wer": 6.38,
      "wer_dp": 2,
      "wer_sd": 0.19,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "VITS",
      "wer_ci": [
        5.69,
        7.1
      ],
      "ci_note": "Mean ± SD over 3 takes: 6.38 ± 0.19. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [5.69, 7.10].",
      "notes": "The FreyaTTS paper (Table 2) reported 6.8."
    },
    {
      "id": "coqui-glowtts-tr",
      "name": "Coqui GlowTTS",
      "variant": "tts_models/tr/common-voice/glow-tts",
      "params_m": 28,
      "params_label": "28M",
      "voice": "Single voice",
      "link": "https://github.com/coqui-ai/TTS",
      "wer": 10.93,
      "wer_dp": 2,
      "wer_sd": 0.0,
      "takes": 3,
      "source": "PatientDesk re-run",
      "source_tag": "PatientDesk",
      "source_link": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun",
      "evaluated": "2026-10-07",
      "family": "Glow flow",
      "wer_ci": [
        9.75,
        12.12
      ],
      "ci_note": "Mean ± SD over 3 takes: 10.93 ± 0.00. 95% sentence-bootstrap interval (10,000 resamples, pooled takes), computed here from PatientDesk's published transcripts: [9.75, 12.12].",
      "notes": "Input lowercased the Turkish way (I to ı, İ to i): the model's symbol set has no capitals and drops them. Deterministic, so all three takes are identical. The FreyaTTS paper (Table 2) reported 12.1."
    },
    {
      "id": "freyatts",
      "name": "FreyaTTS-small",
      "variant": "freyavoice/Freya-TTS",
      "params_m": 183.2,
      "params_label": "183.2M",
      "voice": "Single voice",
      "link": "https://huggingface.co/freyavoice/Freya-TTS",
      "wer": 8.0,
      "wer_dp": 1,
      "source": "Paper, Table 2",
      "source_tag": "Paper",
      "evaluated": "2026-07-22",
      "family": "Flow-matching DiT",
      "wer_ci": [
        6.8,
        9.1
      ],
      "ci_note": "Sentence-level bootstrap (10,000 resamples) on an independent re-synthesis: WER 7.9 [6.8, 9.1].",
      "notes": "As reported in the FreyaTTS paper (arXiv:2607.09530, Table 2). PatientDesk's re-run of the released checkpoint scored 12.02.",
      "freya": true
    },
    {
      "id": "speecht5-tr",
      "name": "SpeechT5",
      "variant": "Turkish Common Voice fine-tune",
      "params_m": 144,
      "params_label": "144M",
      "family": "Autoregressive",
      "voice": "Speaker embedding",
      "link": "https://huggingface.co/Omarrran/turkish_finetuned_speecht5_tts",
      "wer": 83.4,
      "source": "Paper, Table 2",
      "evaluated": "2026-07-22",
      "source_tag": "Paper",
      "wer_dp": 1
    },
    {
      "id": "f5-tts-tr",
      "name": "F5-TTS",
      "variant": "marduk-ra/F5-TTS-Turkish",
      "params_m": 336,
      "params_label": "336M",
      "family": "Flow-matching DiT",
      "voice": "Zero-shot cloner",
      "link": "https://huggingface.co/marduk-ra/F5-TTS-Turkish",
      "wer": 24.3,
      "notes": "Given one fixed shared reference clip; cloner scores can depend on that choice.",
      "source": "Paper, Table 2",
      "evaluated": "2026-07-22",
      "source_tag": "Paper",
      "wer_dp": 1
    }
  ],
  "sources": {
    "PatientDesk": {
      "label": "PatientDesk AI re-run of Freya-TR-Eval (3 takes, 7 to 12 October 2026)",
      "url": "https://huggingface.co/datasets/PatientdeskAI/freya-tr-eval-rerun"
    },
    "Paper": {
      "label": "arXiv:2607.09530, Table 2",
      "url": "https://arxiv.org/abs/2607.09530"
    }
  }
}