feat: models test and fixes

This commit is contained in:
leonardotrapani
2026-02-02 01:13:33 +01:00
parent 2729882b9d
commit 195f9f5115
34 changed files with 507 additions and 154 deletions
+2 -1
View File
@@ -6,7 +6,8 @@ on:
branches: [main, develop] branches: [main, develop]
pull_request: pull_request:
branches: [main, develop] branches: [main, develop]
workflow_call: {} # <-- makes this workflow reusable workflow_dispatch: {} # manual trigger
workflow_call: {} # reusable workflow
jobs: jobs:
test: test:
+61
View File
@@ -0,0 +1,61 @@
# .github/workflows/e2e.yml
name: E2E Tests
on:
workflow_dispatch:
inputs:
timeout:
description: 'Per-model timeout (e.g. 60s)'
required: false
default: '60s'
jobs:
test-models:
name: Test All Models
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.24"
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y \
build-essential \
pkg-config \
libasound2-dev \
libpulse-dev \
libpipewire-0.3-dev
- name: Download dependencies
run: go mod download
- name: Build binary
env:
CGO_ENABLED: 1
run: go build -o hyprvoice ./cmd/hyprvoice
- name: Run test-models
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
run: |
./hyprvoice test-models \
--timeout=${{ inputs.timeout }} \
--output=test-models-report.json
- name: Upload report
uses: actions/upload-artifact@v4
if: always()
with:
name: test-models-report
path: test-models-report.json
retention-days: 30
+10 -7
View File
@@ -29,31 +29,27 @@ All supported speech-to-text providers and models:
- `whisper-large-v3` - `whisper-large-v3`
- `whisper-large-v3-turbo` - `whisper-large-v3-turbo`
- `distil-whisper-large-v3-en` (English only)
### Mistral (cloud) ### Mistral (cloud)
- `voxtral-mini-latest` - `voxtral-mini-latest`
- `voxtral-mini-2507`
### ElevenLabs (cloud) ### ElevenLabs (cloud)
- `scribe_v1` (batch) - `scribe_v1` (batch)
- `scribe_v2` (batch) - `scribe_v2` (batch)
- `scribe_v1-streaming` - `scribe_v2_realtime` (streaming)
- `scribe_v2-streaming`
### whisper-cpp (local) ### whisper-cpp (local)
- English-only: `tiny.en`, `base.en`, `small.en`, `medium.en` - English-only: `tiny.en`, `base.en`, `small.en`, `medium.en`
- Multilingual: `tiny`, `base`, `small`, `medium`, `large-v3` - Multilingual: `tiny`, `base`, `small`, `medium`, `large-v1`, `large-v2`, `large-v3`, `large-v3-turbo`
### Deepgram (cloud) ### Deepgram (cloud)
- `flux-general-en`
- `nova-3` - `nova-3`
- `nova-3-general`
- `nova-2` - `nova-2`
- `nova-2-general`
Language coverage: 57 languages overall; Deepgram models cover a subset; English-only models are labeled above. Language coverage: 57 languages overall; Deepgram models cover a subset; English-only models are labeled above.
@@ -120,6 +116,13 @@ hyprvoice model download base.en
hyprvoice model remove base.en hyprvoice model remove base.en
``` ```
### Model testing (E2E)
```bash
hyprvoice test-models
hyprvoice test-models --audio /path/to/sample.wav --output test-models.json
```
### Service management ### Service management
```bash ```bash
+12 -2
View File
@@ -4,6 +4,8 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"log"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"sort" "sort"
@@ -38,6 +40,7 @@ func init() {
onboardingCmd(), onboardingCmd(),
configureCmd(), configureCmd(),
modelCmd(), modelCmd(),
testModelsCmd(),
) )
} }
@@ -166,7 +169,7 @@ func runConfigure(onboarding bool) error {
var cfg *config.Config var cfg *config.Config
var err error var err error
if onboarding { if onboarding {
cfg, err = config.Load() cfg, err = loadConfigQuiet()
if err != nil { if err != nil {
if errors.Is(err, config.ErrConfigNotFound) { if errors.Is(err, config.ErrConfigNotFound) {
cfg = config.DefaultConfig() cfg = config.DefaultConfig()
@@ -175,7 +178,7 @@ func runConfigure(onboarding bool) error {
} }
} }
} else { } else {
cfg, err = config.Load() cfg, err = loadConfigQuiet()
if err != nil { if err != nil {
return fmt.Errorf("failed to load config: %w", err) return fmt.Errorf("failed to load config: %w", err)
} }
@@ -213,6 +216,13 @@ func runConfigure(onboarding bool) error {
return nil return nil
} }
func loadConfigQuiet() (*config.Config, error) {
prev := log.Writer()
log.SetOutput(io.Discard)
defer log.SetOutput(prev)
return config.Load()
}
func showNextSteps(cfg *config.Config, onboarding bool) { func showNextSteps(cfg *config.Config, onboarding bool) {
// Check if service is running // Check if service is running
serviceRunning := false serviceRunning := false
+20 -14
View File
@@ -108,10 +108,12 @@ language = "" # Empty for auto-detect, or "en", "es", "fr", et
Transcription using Mistral's Voxtral API, excellent for European languages: Transcription using Mistral's Voxtral API, excellent for European languages:
Note: Mistral's API supports streaming responses, but it is not real-time audio streaming. Hyprvoice treats Voxtral as batch-only.
```toml ```toml
[transcription] [transcription]
provider = "mistral-transcription" provider = "mistral-transcription"
model = "voxtral-mini-latest" # Or "voxtral-mini-2507" model = "voxtral-mini-latest"
language = "" # Empty for auto-detect language = "" # Empty for auto-detect
``` ```
@@ -122,7 +124,7 @@ Transcription using ElevenLabs' Scribe API with 57+ language support:
```toml ```toml
[transcription] [transcription]
provider = "elevenlabs" provider = "elevenlabs"
model = "scribe_v1" # Or "scribe_v2" for lower latency model = "scribe_v1" # Or "scribe_v2" for lower latency (batch)
language = "" # Empty for auto-detect language = "" # Empty for auto-detect
``` ```
@@ -148,9 +150,9 @@ language = "" # Empty for auto-detect
**Features:** **Features:**
- All models are streaming-only - Flux: streaming-only, English with turn detection
- Nova-3: 42 languages, best accuracy - Nova-3: 42 languages, best accuracy (batch+streaming)
- Nova-2: 33 languages, faster with filler word detection - Nova-2: 33 languages, faster with filler word detection (batch+streaming)
- Excellent for real-time transcription and live captions - Excellent for real-time transcription and live captions
### Local Transcription (whisper-cpp) ### Local Transcription (whisper-cpp)
@@ -182,7 +184,10 @@ threads = 0 # 0 = auto (uses NumCPU - 1)
| `base` | 142MB | 57 languages | Daily multilingual use | | `base` | 142MB | 57 languages | Daily multilingual use |
| `small` | 466MB | 57 languages | Better multilingual | | `small` | 466MB | 57 languages | Better multilingual |
| `medium` | 1.5GB | 57 languages | Great accuracy | | `medium` | 1.5GB | 57 languages | Great accuracy |
| `large-v1` | 2.9GB | 57 languages | Best accuracy |
| `large-v2` | 2.9GB | 57 languages | Best accuracy |
| `large-v3` | 3GB | 57 languages | Best accuracy | | `large-v3` | 3GB | 57 languages | Best accuracy |
| `large-v3-turbo` | 1.6GB | 57 languages | Faster large-v3 |
**Threads configuration:** **Threads configuration:**
@@ -195,12 +200,13 @@ threads = 0 # 0 = auto (uses NumCPU - 1)
For real-time transcription, use streaming models: For real-time transcription, use streaming models:
```toml ```toml
# ElevenLabs streaming # ElevenLabs streaming (realtime only)
[transcription] [transcription]
provider = "elevenlabs" provider = "elevenlabs"
model = "scribe_v1-streaming" # Or "scribe_v2-streaming" for <150ms latency model = "scribe_v2_realtime"
streaming = true
# Deepgram streaming (all models are streaming) # Deepgram streaming (all models support streaming)
[transcription] [transcription]
provider = "deepgram" provider = "deepgram"
model = "nova-3" model = "nova-3"
@@ -215,8 +221,8 @@ model = "gpt-4o-realtime-preview"
| Provider | Model | Latency | Languages | | Provider | Model | Latency | Languages |
|----------|-------|---------|-----------| |----------|-------|---------|-----------|
| ElevenLabs | `scribe_v1-streaming` | Low | 57+ | | ElevenLabs | `scribe_v2_realtime` | <150ms | 57+ |
| ElevenLabs | `scribe_v2-streaming` | <150ms | 57+ | | Deepgram | `flux-general-en` | Very Low | en |
| Deepgram | `nova-3` | Low | 42 | | Deepgram | `nova-3` | Low | 42 |
| Deepgram | `nova-2` | Very Low | 33 | | Deepgram | `nova-2` | Very Low | 33 |
| OpenAI | `gpt-4o-realtime-preview` | Low | 57 | | OpenAI | `gpt-4o-realtime-preview` | Low | 57 |
@@ -257,7 +263,6 @@ Some models only support English. When configuring via `hyprvoice configure`, on
| Provider | Model | | Provider | Model |
|----------|-------| |----------|-------|
| Groq | `distil-whisper-large-v3-en` |
| whisper-cpp | `tiny.en`, `base.en`, `small.en`, `medium.en` | | whisper-cpp | `tiny.en`, `base.en`, `small.en`, `medium.en` |
**Deepgram models** support fewer languages than the full 57 - see [providers.md](./providers.md#deepgram-language-support). **Deepgram models** support fewer languages than the full 57 - see [providers.md](./providers.md#deepgram-language-support).
@@ -270,8 +275,8 @@ Some models only support English. When configuring via `hyprvoice configure`, on
```toml ```toml
# This combination will be rejected at validation: # This combination will be rejected at validation:
[transcription] [transcription]
provider = "groq-transcription" provider = "whisper-cpp"
model = "distil-whisper-large-v3-en" # English only! model = "base.en" # English only!
language = "es" # Error: model does not support Spanish language = "es" # Error: model does not support Spanish
``` ```
@@ -623,7 +628,8 @@ You can customize notification text via the `[notifications.messages]` section:
[transcription] [transcription]
provider = "elevenlabs" provider = "elevenlabs"
model = "scribe_v2-streaming" # <150ms latency model = "scribe_v2_realtime" # <150ms latency
streaming = true
language = "" # Auto-detect language = "" # Auto-detect
[llm] [llm]
+15 -17
View File
@@ -11,7 +11,7 @@ This guide helps you choose the right transcription provider for your use case.
| **Mistral** | Cloud | 2 | 57 | No | Fast | Good | Pay per use | | **Mistral** | Cloud | 2 | 57 | No | Fast | Good | Pay per use |
| **ElevenLabs** | Cloud | 4 | 57+ | Yes | Fast | Excellent | Pay per use | | **ElevenLabs** | Cloud | 4 | 57+ | Yes | Fast | Excellent | Pay per use |
| **Deepgram** | Cloud | 4 | 33-42 | Yes | Very Fast | Excellent | Pay per use | | **Deepgram** | Cloud | 4 | 33-42 | Yes | Very Fast | Excellent | Pay per use |
| **whisper-cpp** | Local | 9 | 57 (4 EN-only) | No | Varies | Excellent | Free | | **whisper-cpp** | Local | 12 | 57 (4 EN-only) | No | Varies | Excellent | Free |
### OpenAI ### OpenAI
@@ -32,7 +32,6 @@ Extremely fast inference using specialized hardware. OpenAI-compatible API.
**Models:** **Models:**
- `whisper-large-v3` - Full Whisper v3, best accuracy - `whisper-large-v3` - Full Whisper v3, best accuracy
- `whisper-large-v3-turbo` - Faster with slightly lower accuracy - `whisper-large-v3-turbo` - Faster with slightly lower accuracy
- `distil-whisper-large-v3-en` - **English only**, fastest option
**Best for:** Speed-critical applications, English-only use cases, budget-conscious users **Best for:** Speed-critical applications, English-only use cases, budget-conscious users
@@ -42,7 +41,8 @@ European provider with Voxtral transcription models.
**Models:** **Models:**
- `voxtral-mini-latest` - Latest Voxtral, recommended - `voxtral-mini-latest` - Latest Voxtral, recommended
- `voxtral-mini-2507` - Stable version from July 2025
**Notes:** Mistral's streaming responses are not real-time audio streaming; hyprvoice treats Voxtral as batch-only.
**Best for:** European data residency requirements, Mistral ecosystem users **Best for:** European data residency requirements, Mistral ecosystem users
@@ -52,9 +52,8 @@ Known for voice synthesis, also offers excellent transcription via Scribe.
**Models:** **Models:**
- `scribe_v1` - 90+ languages, best accuracy (batch) - `scribe_v1` - 90+ languages, best accuracy (batch)
- `scribe_v2` - Lower latency, real-time optimized (batch) - `scribe_v2` - Lower latency (batch)
- `scribe_v1-streaming` - Real-time transcription - `scribe_v2_realtime` - Streaming-only realtime endpoint
- `scribe_v2-streaming` - Real-time with <150ms latency
**Best for:** Applications needing both TTS and STT, ultra-low latency streaming **Best for:** Applications needing both TTS and STT, ultra-low latency streaming
@@ -63,10 +62,11 @@ Known for voice synthesis, also offers excellent transcription via Scribe.
Streaming-first provider with Nova models. Excellent for real-time applications. Streaming-first provider with Nova models. Excellent for real-time applications.
**Models:** **Models:**
- `flux-general-en` - Streaming with turn detection (English)
- `nova-3` - Best accuracy, 42 languages - `nova-3` - Best accuracy, 42 languages
- `nova-3-general` - Same as nova-3
- `nova-2` - Fast, 33 languages, filler word detection - `nova-2` - Fast, 33 languages, filler word detection
- `nova-2-general` - Same as nova-2
**Notes:** Flux is English-only.
**Language Support:** Nova-3 supports 42 languages, Nova-2 supports 33 languages. Not all 57 languages from the master list are available. **Language Support:** Nova-3 supports 42 languages, Nova-2 supports 33 languages. Not all 57 languages from the master list are available.
@@ -93,7 +93,10 @@ Run Whisper models locally on your machine. No API keys, no network latency, com
| `base` | 142MB | Fast | Good | | `base` | 142MB | Fast | Good |
| `small` | 466MB | Medium | Better | | `small` | 466MB | Medium | Better |
| `medium` | 1.5GB | Slow | Great | | `medium` | 1.5GB | Slow | Great |
| `large-v1` | 2.9GB | Slowest | Best |
| `large-v2` | 2.9GB | Slowest | Best |
| `large-v3` | 3GB | Slowest | Best | | `large-v3` | 3GB | Slowest | Best |
| `large-v3-turbo` | 1.6GB | Slower | Great |
**Best for:** Privacy-sensitive applications, offline use, avoiding API costs **Best for:** Privacy-sensitive applications, offline use, avoiding API costs
@@ -121,14 +124,11 @@ Need complete privacy?
└─ Need real-time streaming? └─ Need real-time streaming?
├─ Yes ├─ Yes
│ └─ Latency critical (<150ms)? │ └─ Latency critical (<150ms)?
│ ├─ Yes → ElevenLabs scribe_v2-streaming │ ├─ Yes → ElevenLabs scribe_v2_realtime (streaming)
│ └─ No → Deepgram nova-3 or OpenAI realtime │ └─ No → Deepgram nova-3 or OpenAI realtime
└─ No (batch) └─ No (batch)
└─ Need fastest response? └─ Need fastest response?
├─ Yes ├─ Yes → Groq whisper-large-v3-turbo
│ └─ English only?
│ ├─ Yes → Groq distil-whisper-large-v3-en
│ └─ No → Groq whisper-large-v3-turbo
└─ No └─ No
└─ Need highest accuracy? └─ Need highest accuracy?
├─ Yes → OpenAI gpt-4o-transcribe or Groq whisper-large-v3 ├─ Yes → OpenAI gpt-4o-transcribe or Groq whisper-large-v3
@@ -140,10 +140,9 @@ Need complete privacy?
| Use Case | Recommended Provider | Model | | Use Case | Recommended Provider | Model |
|----------|---------------------|-------| |----------|---------------------|-------|
| General dictation | OpenAI | whisper-1 | | General dictation | OpenAI | whisper-1 |
| Fast English | Groq | distil-whisper-large-v3-en |
| Fast multilingual | Groq | whisper-large-v3-turbo | | Fast multilingual | Groq | whisper-large-v3-turbo |
| Live captions | Deepgram | nova-3 | | Live captions | Deepgram | nova-3 |
| Ultra-low latency | ElevenLabs | scribe_v2-streaming | | Ultra-low latency | ElevenLabs | scribe_v2_realtime (streaming) |
| Offline/privacy | whisper-cpp | base.en or base | | Offline/privacy | whisper-cpp | base.en or base |
| High accuracy | OpenAI | gpt-4o-transcribe | | High accuracy | OpenAI | gpt-4o-transcribe |
@@ -155,7 +154,7 @@ All providers support **auto-detect mode** (recommended for most users) which au
### Full Language Support (57 languages) ### Full Language Support (57 languages)
OpenAI, Groq (except distil model), Mistral, ElevenLabs, and whisper-cpp multilingual models support all 57 languages: OpenAI, Groq, Mistral, ElevenLabs, and whisper-cpp multilingual models support all 57 languages:
Afrikaans, Arabic, Armenian, Azerbaijani, Belarusian, Bosnian, Bulgarian, Catalan, Chinese, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, Galician, German, Greek, Hebrew, Hindi, Hungarian, Icelandic, Indonesian, Italian, Japanese, Kannada, Kazakh, Korean, Latvian, Lithuanian, Macedonian, Malay, Marathi, Maori, Nepali, Norwegian, Persian, Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovenian, Spanish, Swahili, Swedish, Tagalog, Tamil, Thai, Turkish, Ukrainian, Urdu, Vietnamese, Welsh Afrikaans, Arabic, Armenian, Azerbaijani, Belarusian, Bosnian, Bulgarian, Catalan, Chinese, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, Galician, German, Greek, Hebrew, Hindi, Hungarian, Icelandic, Indonesian, Italian, Japanese, Kannada, Kazakh, Korean, Latvian, Lithuanian, Macedonian, Malay, Marathi, Maori, Nepali, Norwegian, Persian, Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovenian, Spanish, Swahili, Swedish, Tagalog, Tamil, Thai, Turkish, Ukrainian, Urdu, Vietnamese, Welsh
@@ -165,7 +164,6 @@ These models only support English but are faster:
| Provider | Model | | Provider | Model |
|----------|-------| |----------|-------|
| Groq | `distil-whisper-large-v3-en` |
| whisper-cpp | `tiny.en`, `base.en`, `small.en`, `medium.en` | | whisper-cpp | `tiny.en`, `base.en`, `small.en`, `medium.en` |
If you select an English-only model with a non-English language, hyprvoice will: If you select an English-only model with a non-English language, hyprvoice will:
+3 -1
View File
@@ -1966,10 +1966,12 @@ timeout = "5m"
[transcription] [transcription]
provider = "openai" provider = "openai"
api_key = "test-key"
model = "whisper-1" model = "whisper-1"
language = "es" language = "es"
[providers.openai]
api_key = "test-key"
[injection] [injection]
backends = ["clipboard"] backends = ["clipboard"]
ydotool_timeout = "5s" ydotool_timeout = "5s"
+1 -1
View File
@@ -321,7 +321,7 @@ keywords = []
# - "openai": OpenAI Whisper API (cloud-based, excellent accuracy) # - "openai": OpenAI Whisper API (cloud-based, excellent accuracy)
# - "groq-transcription": Groq Whisper API (very fast, models: whisper-large-v3, whisper-large-v3-turbo) # - "groq-transcription": Groq Whisper API (very fast, models: whisper-large-v3, whisper-large-v3-turbo)
# - "mistral-transcription": Mistral Voxtral API (excellent for European languages, model: voxtral-mini-latest) # - "mistral-transcription": Mistral Voxtral API (excellent for European languages, model: voxtral-mini-latest)
# - "elevenlabs": ElevenLabs Scribe API (99 languages, models: scribe_v1, scribe_v2) # - "elevenlabs": ElevenLabs Scribe API (99 languages, models: scribe_v1, scribe_v2, scribe_v2_realtime)
# #
# LLM providers (for post-processing): # LLM providers (for post-processing):
# - "openai": GPT models (gpt-4o-mini recommended for cost/quality balance) # - "openai": GPT models (gpt-4o-mini recommended for cost/quality balance)
+3
View File
@@ -28,7 +28,10 @@ var models = []ModelInfo{
{ID: "base", Name: "Base", Filename: "ggml-base.bin", Size: "142MB", SizeBytes: 142_000_000, Multilingual: true}, {ID: "base", Name: "Base", Filename: "ggml-base.bin", Size: "142MB", SizeBytes: 142_000_000, Multilingual: true},
{ID: "small", Name: "Small", Filename: "ggml-small.bin", Size: "466MB", SizeBytes: 466_000_000, Multilingual: true}, {ID: "small", Name: "Small", Filename: "ggml-small.bin", Size: "466MB", SizeBytes: 466_000_000, Multilingual: true},
{ID: "medium", Name: "Medium", Filename: "ggml-medium.bin", Size: "1.5GB", SizeBytes: 1_500_000_000, Multilingual: true}, {ID: "medium", Name: "Medium", Filename: "ggml-medium.bin", Size: "1.5GB", SizeBytes: 1_500_000_000, Multilingual: true},
{ID: "large-v1", Name: "Large V1", Filename: "ggml-large-v1.bin", Size: "2.9GB", SizeBytes: 2_900_000_000, Multilingual: true},
{ID: "large-v2", Name: "Large V2", Filename: "ggml-large-v2.bin", Size: "2.9GB", SizeBytes: 2_900_000_000, Multilingual: true},
{ID: "large-v3", Name: "Large V3", Filename: "ggml-large-v3.bin", Size: "3GB", SizeBytes: 3_000_000_000, Multilingual: true}, {ID: "large-v3", Name: "Large V3", Filename: "ggml-large-v3.bin", Size: "3GB", SizeBytes: 3_000_000_000, Multilingual: true},
{ID: "large-v3-turbo", Name: "Large V3 Turbo", Filename: "ggml-large-v3-turbo.bin", Size: "1.6GB", SizeBytes: 1_600_000_000, Multilingual: true},
} }
// modelByID maps model ID to ModelInfo for quick lookup // modelByID maps model ID to ModelInfo for quick lookup
+5 -5
View File
@@ -108,8 +108,8 @@ func TestGetModel(t *testing.T) {
func TestListModels(t *testing.T) { func TestListModels(t *testing.T) {
models := ListModels() models := ListModels()
if len(models) != 9 { if len(models) != 12 {
t.Errorf("ListModels() returned %d models, want 9", len(models)) t.Errorf("ListModels() returned %d models, want 12", len(models))
} }
// verify known models exist // verify known models exist
@@ -118,7 +118,7 @@ func TestListModels(t *testing.T) {
ids[m.ID] = true ids[m.ID] = true
} }
expected := []string{"tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large-v3"} expected := []string{"tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large-v1", "large-v2", "large-v3", "large-v3-turbo"}
for _, id := range expected { for _, id := range expected {
if !ids[id] { if !ids[id] {
t.Errorf("ListModels() missing model %s", id) t.Errorf("ListModels() missing model %s", id)
@@ -128,8 +128,8 @@ func TestListModels(t *testing.T) {
func TestListMultilingualModels(t *testing.T) { func TestListMultilingualModels(t *testing.T) {
models := ListMultilingualModels() models := ListMultilingualModels()
if len(models) != 5 { if len(models) != 8 {
t.Errorf("ListMultilingualModels() returned %d models, want 5", len(models)) t.Errorf("ListMultilingualModels() returned %d models, want 8", len(models))
} }
for _, m := range models { for _, m := range models {
+6 -3
View File
@@ -16,6 +16,10 @@ func (p *DeepgramProvider) ValidateAPIKey(key string) bool {
return len(key) > 0 return len(key) > 0
} }
func (p *DeepgramProvider) APIKeyURL() string {
return "https://console.deepgram.com/project/keys"
}
func (p *DeepgramProvider) IsLocal() bool { func (p *DeepgramProvider) IsLocal() bool {
return false return false
} }
@@ -23,7 +27,6 @@ func (p *DeepgramProvider) IsLocal() bool {
func (p *DeepgramProvider) Models() []Model { func (p *DeepgramProvider) Models() []Model {
// https://developers.deepgram.com/docs/models-languages-overview // https://developers.deepgram.com/docs/models-languages-overview
nova3Langs := deepgramNova3Languages nova3Langs := deepgramNova3Languages
// https://developers.deepgram.com/docs/models-languages-overview
nova2Langs := deepgramNova2Languages nova2Langs := deepgramNova2Languages
docsURL := "https://developers.deepgram.com/docs/language" docsURL := "https://developers.deepgram.com/docs/language"
@@ -32,7 +35,7 @@ func (p *DeepgramProvider) Models() []Model {
{ {
ID: "nova-3", ID: "nova-3",
Name: "Nova-3", Name: "Nova-3",
Description: "Best accuracy, 40+ languages", Description: "Best accuracy; streaming available for faster response",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: true, SupportsStreaming: true,
@@ -46,7 +49,7 @@ func (p *DeepgramProvider) Models() []Model {
{ {
ID: "nova-2", ID: "nova-2",
Name: "Nova-2", Name: "Nova-2",
Description: "Fast, 30+ languages, filler words", Description: "Cheaper legacy model; still solid accuracy",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: true, SupportsStreaming: true,
+1 -4
View File
@@ -29,11 +29,8 @@ func TestDeepgramProvider_Models(t *testing.T) {
t.Errorf("Models() returned %d models, want 2", len(models)) t.Errorf("Models() returned %d models, want 2", len(models))
} }
// all models should support both batch and streaming // all models should support both streaming and batch
for _, m := range models { for _, m := range models {
if !m.SupportsBatch {
t.Errorf("model %s should support batch", m.ID)
}
if !m.SupportsStreaming { if !m.SupportsStreaming {
t.Errorf("model %s should support streaming", m.ID) t.Errorf("model %s should support streaming", m.ID)
} }
+15 -5
View File
@@ -16,6 +16,10 @@ func (p *ElevenLabsProvider) ValidateAPIKey(key string) bool {
return len(key) > 0 return len(key) > 0
} }
func (p *ElevenLabsProvider) APIKeyURL() string {
return "https://elevenlabs.io/app/settings/api-keys"
}
func (p *ElevenLabsProvider) IsLocal() bool { func (p *ElevenLabsProvider) IsLocal() bool {
return false return false
} }
@@ -29,40 +33,46 @@ func (p *ElevenLabsProvider) Models() []Model {
{ {
ID: "scribe_v1", ID: "scribe_v1",
Name: "Scribe v1", Name: "Scribe v1",
Description: "90+ languages, best accuracy", Description: "Most accurate; best for precision-critical work",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
Local: false, Local: false,
AdapterType: AdapterElevenLabs, AdapterType: AdapterElevenLabs,
StreamingAdapter: AdapterElevenLabsStream,
SupportedLanguages: allLangs, SupportedLanguages: allLangs,
Endpoint: &EndpointConfig{BaseURL: "https://api.elevenlabs.io", Path: "/v1/speech-to-text"}, Endpoint: &EndpointConfig{BaseURL: "https://api.elevenlabs.io", Path: "/v1/speech-to-text"},
StreamingEndpoint: &EndpointConfig{BaseURL: "wss://api.elevenlabs.io", Path: "/v1/speech-to-text/realtime"},
DocsURL: docsURL, DocsURL: docsURL,
}, },
{ {
ID: "scribe_v2", ID: "scribe_v2",
Name: "Scribe v2", Name: "Scribe v2",
Description: "Lower latency batch transcription", Description: "Faster processing with good accuracy",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
Local: false, Local: false,
AdapterType: AdapterElevenLabs, AdapterType: AdapterElevenLabs,
StreamingAdapter: AdapterElevenLabsStream,
SupportedLanguages: allLangs, SupportedLanguages: allLangs,
Endpoint: &EndpointConfig{BaseURL: "https://api.elevenlabs.io", Path: "/v1/speech-to-text"}, Endpoint: &EndpointConfig{BaseURL: "https://api.elevenlabs.io", Path: "/v1/speech-to-text"},
StreamingEndpoint: &EndpointConfig{BaseURL: "wss://api.elevenlabs.io", Path: "/v1/speech-to-text/realtime"},
DocsURL: docsURL, DocsURL: docsURL,
}, },
{ {
ID: "scribe_v2_realtime", ID: "scribe_v2_realtime",
Name: "Scribe v2 Realtime", Name: "Scribe v2 Realtime",
Description: "Real-time streaming, <150ms latency", Description: "Instant words as you speak; faster but costs more",
Type: Transcription, Type: Transcription,
SupportsBatch: false, SupportsBatch: false,
SupportsStreaming: true, SupportsStreaming: true,
Local: false, Local: false,
AdapterType: AdapterElevenLabsStream, AdapterType: AdapterElevenLabs,
StreamingAdapter: AdapterElevenLabsStream,
SupportedLanguages: allLangs, SupportedLanguages: allLangs,
Endpoint: &EndpointConfig{BaseURL: "wss://api.elevenlabs.io", Path: "/v1/speech-to-text/realtime"}, Endpoint: &EndpointConfig{BaseURL: "https://api.elevenlabs.io", Path: "/v1/speech-to-text"},
StreamingEndpoint: &EndpointConfig{BaseURL: "wss://api.elevenlabs.io", Path: "/v1/speech-to-text/realtime"},
DocsURL: docsURL, DocsURL: docsURL,
}, },
} }
+8 -15
View File
@@ -17,6 +17,10 @@ func (p *GroqProvider) ValidateAPIKey(key string) bool {
return strings.HasPrefix(key, "gsk_") return strings.HasPrefix(key, "gsk_")
} }
func (p *GroqProvider) APIKeyURL() string {
return "https://console.groq.com/keys"
}
func (p *GroqProvider) IsLocal() bool { func (p *GroqProvider) IsLocal() bool {
return false return false
} }
@@ -31,7 +35,7 @@ func (p *GroqProvider) Models() []Model {
{ {
ID: "whisper-large-v3", ID: "whisper-large-v3",
Name: "Whisper Large v3", Name: "Whisper Large v3",
Description: "Full Whisper v3 model, best accuracy", Description: "Best accuracy; generous free tier makes this great default",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -44,7 +48,7 @@ func (p *GroqProvider) Models() []Model {
{ {
ID: "whisper-large-v3-turbo", ID: "whisper-large-v3-turbo",
Name: "Whisper Large v3 Turbo", Name: "Whisper Large v3 Turbo",
Description: "Faster Whisper v3 with slightly lower accuracy", Description: "Faster with slight accuracy tradeoff; still very good",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -58,7 +62,7 @@ func (p *GroqProvider) Models() []Model {
{ {
ID: "llama-3.3-70b-versatile", ID: "llama-3.3-70b-versatile",
Name: "Llama 3.3 70B Versatile", Name: "Llama 3.3 70B Versatile",
Description: "Most capable Llama model", Description: "Best quality cleanup; smart rewrites, free tier available",
Type: LLM, Type: LLM,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -69,18 +73,7 @@ func (p *GroqProvider) Models() []Model {
{ {
ID: "llama-3.1-8b-instant", ID: "llama-3.1-8b-instant",
Name: "Llama 3.1 8B Instant", Name: "Llama 3.1 8B Instant",
Description: "Fast and efficient", Description: "Very fast; good for simple cleanup tasks",
Type: LLM,
SupportsBatch: true,
SupportsStreaming: false,
Local: false,
AdapterType: AdapterOpenAI,
Endpoint: &EndpointConfig{BaseURL: "https://api.groq.com/openai", Path: "/v1/chat/completions"},
},
{
ID: "mixtral-8x7b-32768",
Name: "Mixtral 8x7B",
Description: "Mixture of experts model",
Type: LLM, Type: LLM,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
+2
View File
@@ -34,6 +34,8 @@ var deepgramNova2Languages = []string{
"ru", "sk", "es", "es-419", "sv", "sv-SE", "th", "th-TH", "tr", "uk", "vi", "ru", "sk", "es", "es-419", "sv", "sv-SE", "th", "th-TH", "tr", "uk", "vi",
} }
var deepgramFluxLanguages = []string{"en"}
var elevenLabsTranscriptionLanguages = []string{ var elevenLabsTranscriptionLanguages = []string{
"bel", "bos", "bul", "cat", "hrv", "ces", "dan", "nld", "eng", "est", "fin", "fra", "bel", "bos", "bul", "cat", "hrv", "ces", "dan", "nld", "eng", "est", "fin", "fra",
"glg", "deu", "ell", "hun", "isl", "ind", "ita", "jpn", "kan", "lav", "mkd", "msa", "glg", "deu", "ell", "hun", "isl", "ind", "ita", "jpn", "kan", "lav", "mkd", "msa",
+6 -17
View File
@@ -16,6 +16,10 @@ func (p *MistralProvider) ValidateAPIKey(key string) bool {
return len(key) > 0 return len(key) > 0
} }
func (p *MistralProvider) APIKeyURL() string {
return "https://admin.mistral.ai/organization/api-keys"
}
func (p *MistralProvider) IsLocal() bool { func (p *MistralProvider) IsLocal() bool {
return false return false
} }
@@ -29,27 +33,12 @@ func (p *MistralProvider) Models() []Model {
{ {
ID: "voxtral-mini-latest", ID: "voxtral-mini-latest",
Name: "Voxtral Mini Latest", Name: "Voxtral Mini Latest",
Description: "Latest Voxtral model, best for most uses", Description: "EU-hosted; good for data residency or Mistral ecosystem",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: true, SupportsStreaming: false,
Local: false, Local: false,
AdapterType: AdapterOpenAI, AdapterType: AdapterOpenAI,
StreamingAdapter: "mistral-streaming", // not yet implemented
SupportedLanguages: allLangs,
Endpoint: &EndpointConfig{BaseURL: "https://api.mistral.ai", Path: "/v1/audio/transcriptions"},
DocsURL: docsURL,
},
{
ID: "voxtral-mini-2507",
Name: "Voxtral Mini 2507",
Description: "Stable Voxtral version from July 2025",
Type: Transcription,
SupportsBatch: true,
SupportsStreaming: true,
Local: false,
AdapterType: AdapterOpenAI,
StreamingAdapter: "mistral-streaming", // not yet implemented
SupportedLanguages: allLangs, SupportedLanguages: allLangs,
Endpoint: &EndpointConfig{BaseURL: "https://api.mistral.ai", Path: "/v1/audio/transcriptions"}, Endpoint: &EndpointConfig{BaseURL: "https://api.mistral.ai", Path: "/v1/audio/transcriptions"},
DocsURL: docsURL, DocsURL: docsURL,
+3 -3
View File
@@ -57,7 +57,7 @@ func TestModel_IsStreaming(t *testing.T) {
}{ }{
{ {
name: "streaming-only model", name: "streaming-only model",
model: Model{ID: "scribe_v2_realtime", SupportsBatch: false, SupportsStreaming: true}, model: Model{ID: "flux-general-en", SupportsBatch: false, SupportsStreaming: true},
expected: true, expected: true,
}, },
{ {
@@ -89,7 +89,7 @@ func TestModel_SupportsBothModes(t *testing.T) {
}{ }{
{ {
name: "streaming-only model", name: "streaming-only model",
model: Model{ID: "scribe_v2_realtime", SupportsBatch: false, SupportsStreaming: true}, model: Model{ID: "flux-general-en", SupportsBatch: false, SupportsStreaming: true},
expected: false, expected: false,
}, },
{ {
@@ -325,7 +325,7 @@ func TestAllTranscriptionModels_HaveDocsURL(t *testing.T) {
"mistral": "https://docs.mistral.ai/capabilities/audio/", "mistral": "https://docs.mistral.ai/capabilities/audio/",
"elevenlabs": "https://elevenlabs.io/speech-to-text", "elevenlabs": "https://elevenlabs.io/speech-to-text",
"deepgram": "https://developers.deepgram.com/docs/language", "deepgram": "https://developers.deepgram.com/docs/language",
"whisper-cpp": "https://github.com/openai/whisper#available-models-and-languages", "whisper-cpp": "https://github.com/ggml-org/whisper.cpp#models",
} }
for _, pName := range providers { for _, pName := range providers {
+10 -6
View File
@@ -17,6 +17,10 @@ func (p *OpenAIProvider) ValidateAPIKey(key string) bool {
return strings.HasPrefix(key, "sk-") return strings.HasPrefix(key, "sk-")
} }
func (p *OpenAIProvider) APIKeyURL() string {
return "https://platform.openai.com/api-keys"
}
func (p *OpenAIProvider) IsLocal() bool { func (p *OpenAIProvider) IsLocal() bool {
return false return false
} }
@@ -32,7 +36,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "whisper-1", ID: "whisper-1",
Name: "Whisper 1", Name: "Whisper 1",
Description: "OpenAI's production speech-to-text model", Description: "Reliable and cost-effective; good default for most use cases",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -45,7 +49,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "gpt-4o-transcribe", ID: "gpt-4o-transcribe",
Name: "GPT-4o Transcribe", Name: "GPT-4o Transcribe",
Description: "High quality transcription with GPT-4o", Description: "Top accuracy; slower and pricier but best quality",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -58,7 +62,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "gpt-4o-mini-transcribe", ID: "gpt-4o-mini-transcribe",
Name: "GPT-4o Mini Transcribe", Name: "GPT-4o Mini Transcribe",
Description: "Fast transcription with GPT-4o Mini", Description: "Good balance of speed, cost, and quality",
Type: Transcription, Type: Transcription,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -71,7 +75,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "gpt-4o-realtime-preview", ID: "gpt-4o-realtime-preview",
Name: "GPT-4o Realtime Preview", Name: "GPT-4o Realtime Preview",
Description: "Real-time streaming transcription with GPT-4o", Description: "Instant words as you speak; fastest but most expensive",
Type: Transcription, Type: Transcription,
SupportsBatch: false, SupportsBatch: false,
SupportsStreaming: true, SupportsStreaming: true,
@@ -85,7 +89,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "gpt-4o-mini", ID: "gpt-4o-mini",
Name: "GPT-4o Mini", Name: "GPT-4o Mini",
Description: "Fast and affordable GPT-4 variant", Description: "Fast and cheap; good default for text cleanup",
Type: LLM, Type: LLM,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
@@ -96,7 +100,7 @@ func (p *OpenAIProvider) Models() []Model {
{ {
ID: "gpt-4o", ID: "gpt-4o",
Name: "GPT-4o", Name: "GPT-4o",
Description: "Most capable GPT-4 model", Description: "Best quality cleanup; pricier but smarter rewrites",
Type: LLM, Type: LLM,
SupportsBatch: true, SupportsBatch: true,
SupportsStreaming: false, SupportsStreaming: false,
+1
View File
@@ -11,6 +11,7 @@ type Provider interface {
Name() string Name() string
RequiresAPIKey() bool RequiresAPIKey() bool
ValidateAPIKey(key string) bool ValidateAPIKey(key string) bool
APIKeyURL() string
IsLocal() bool IsLocal() bool
Models() []Model Models() []Model
DefaultModel(t ModelType) string DefaultModel(t ModelType) string
+13 -5
View File
@@ -279,7 +279,7 @@ func TestValidateModelLanguage_ErrorFormat(t *testing.T) {
} }
// should contain docs URL // should contain docs URL
if !strings.Contains(errMsg, "https://github.com/openai/whisper") { if !strings.Contains(errMsg, "https://github.com/ggml-org/whisper.cpp") {
t.Errorf("error should contain docs URL, got: %s", errMsg) t.Errorf("error should contain docs URL, got: %s", errMsg)
} }
@@ -343,7 +343,7 @@ func TestElevenLabsProvider(t *testing.T) {
t.Errorf("ElevenLabsProvider.Models() = %d models, want 3", len(models)) t.Errorf("ElevenLabsProvider.Models() = %d models, want 3", len(models))
} }
// Check batch-only models // Check batch + streaming models
scribeV1, err := GetModel("elevenlabs", "scribe_v1") scribeV1, err := GetModel("elevenlabs", "scribe_v1")
if err != nil { if err != nil {
t.Fatalf("GetModel('elevenlabs', 'scribe_v1') error: %v", err) t.Fatalf("GetModel('elevenlabs', 'scribe_v1') error: %v", err)
@@ -357,6 +357,9 @@ func TestElevenLabsProvider(t *testing.T) {
if scribeV1.AdapterType != "elevenlabs" { if scribeV1.AdapterType != "elevenlabs" {
t.Errorf("scribe_v1 AdapterType=%q, want 'elevenlabs'", scribeV1.AdapterType) t.Errorf("scribe_v1 AdapterType=%q, want 'elevenlabs'", scribeV1.AdapterType)
} }
if scribeV1.StreamingAdapter != "elevenlabs-streaming" {
t.Errorf("scribe_v1 StreamingAdapter=%q, want 'elevenlabs-streaming'", scribeV1.StreamingAdapter)
}
scribeV2, err := GetModel("elevenlabs", "scribe_v2") scribeV2, err := GetModel("elevenlabs", "scribe_v2")
if err != nil { if err != nil {
@@ -371,8 +374,10 @@ func TestElevenLabsProvider(t *testing.T) {
if scribeV2.AdapterType != "elevenlabs" { if scribeV2.AdapterType != "elevenlabs" {
t.Errorf("scribe_v2 AdapterType=%q, want 'elevenlabs'", scribeV2.AdapterType) t.Errorf("scribe_v2 AdapterType=%q, want 'elevenlabs'", scribeV2.AdapterType)
} }
if scribeV2.StreamingAdapter != "elevenlabs-streaming" {
t.Errorf("scribe_v2 StreamingAdapter=%q, want 'elevenlabs-streaming'", scribeV2.StreamingAdapter)
}
// Check streaming-only model
scribeV2Realtime, err := GetModel("elevenlabs", "scribe_v2_realtime") scribeV2Realtime, err := GetModel("elevenlabs", "scribe_v2_realtime")
if err != nil { if err != nil {
t.Fatalf("GetModel('elevenlabs', 'scribe_v2_realtime') error: %v", err) t.Fatalf("GetModel('elevenlabs', 'scribe_v2_realtime') error: %v", err)
@@ -383,8 +388,11 @@ func TestElevenLabsProvider(t *testing.T) {
if !scribeV2Realtime.SupportsStreaming { if !scribeV2Realtime.SupportsStreaming {
t.Error("scribe_v2_realtime should have SupportsStreaming=true") t.Error("scribe_v2_realtime should have SupportsStreaming=true")
} }
if scribeV2Realtime.AdapterType != "elevenlabs-streaming" { if scribeV2Realtime.AdapterType != "elevenlabs" {
t.Errorf("scribe_v2_realtime AdapterType=%q, want 'elevenlabs-streaming'", scribeV2Realtime.AdapterType) t.Errorf("scribe_v2_realtime AdapterType=%q, want 'elevenlabs'", scribeV2Realtime.AdapterType)
}
if scribeV2Realtime.StreamingAdapter != "elevenlabs-streaming" {
t.Errorf("scribe_v2_realtime StreamingAdapter=%q, want 'elevenlabs-streaming'", scribeV2Realtime.StreamingAdapter)
} }
// All models should share the same supported language list // All models should share the same supported language list
+36 -6
View File
@@ -17,16 +17,20 @@ func (p *WhisperCppProvider) ValidateAPIKey(key string) bool {
return true // no API key needed return true // no API key needed
} }
func (p *WhisperCppProvider) APIKeyURL() string {
return ""
}
func (p *WhisperCppProvider) IsLocal() bool { func (p *WhisperCppProvider) IsLocal() bool {
return true return true
} }
func (p *WhisperCppProvider) Models() []Model { func (p *WhisperCppProvider) Models() []Model {
// https://github.com/openai/whisper#available-models-and-languages // https://github.com/ggml-org/whisper.cpp#models
allLangs := whisperTranscriptionLanguages allLangs := whisperTranscriptionLanguages
// https://github.com/openai/whisper#available-models-and-languages // https://github.com/ggml-org/whisper.cpp#models
englishOnly := whisperEnglishOnlyLanguages englishOnly := whisperEnglishOnlyLanguages
docsURL := "https://github.com/openai/whisper#available-models-and-languages" docsURL := "https://github.com/ggml-org/whisper.cpp#models"
whisperModels := whisper.ListModels() whisperModels := whisper.ListModels()
result := make([]Model, 0, len(whisperModels)) result := make([]Model, 0, len(whisperModels))
@@ -63,10 +67,36 @@ func (p *WhisperCppProvider) Models() []Model {
} }
func modelDescription(m whisper.ModelInfo) string { func modelDescription(m whisper.ModelInfo) string {
if m.Multilingual { switch m.ID {
return "Multilingual local transcription" case "tiny.en":
return "Free/offline; fastest but low accuracy, good for weak hardware"
case "base.en":
return "Free/offline; balanced speed and accuracy, recommended start"
case "small.en":
return "Free/offline; better accuracy, needs decent CPU"
case "medium.en":
return "Free/offline; best .en accuracy, needs good CPU/RAM"
case "tiny":
return "Free/offline multilingual; fastest but low accuracy"
case "base":
return "Free/offline multilingual; balanced, recommended start"
case "small":
return "Free/offline multilingual; better accuracy, needs decent CPU"
case "medium":
return "Free/offline multilingual; great accuracy, needs good CPU/RAM"
case "large-v1":
return "Free/offline; high accuracy, needs strong CPU/GPU"
case "large-v2":
return "Free/offline; high accuracy, needs strong CPU/GPU"
case "large-v3":
return "Free/offline; best accuracy available, needs strong hardware"
case "large-v3-turbo":
return "Free/offline; near-best accuracy with better speed"
} }
return "English-only local transcription (faster)" if m.Multilingual {
return "Free/offline multilingual model"
}
return "Free/offline English model"
} }
func (p *WhisperCppProvider) DefaultModel(t ModelType) string { func (p *WhisperCppProvider) DefaultModel(t ModelType) string {
+6 -3
View File
@@ -16,9 +16,9 @@ func TestWhisperCppProvider_Models(t *testing.T) {
p := &WhisperCppProvider{} p := &WhisperCppProvider{}
models := p.Models() models := p.Models()
// verify we have 9 models // verify we have 12 models
if len(models) != 9 { if len(models) != 12 {
t.Errorf("expected 9 models, got %d", len(models)) t.Errorf("expected 12 models, got %d", len(models))
} }
// verify all models have required fields // verify all models have required fields
@@ -81,7 +81,10 @@ func TestWhisperCppProvider_MultilingualModels(t *testing.T) {
"base": true, "base": true,
"small": true, "small": true,
"medium": true, "medium": true,
"large-v1": true,
"large-v2": true,
"large-v3": true, "large-v3": true,
"large-v3-turbo": true,
} }
for _, m := range models { for _, m := range models {
+25 -1
View File
@@ -36,6 +36,7 @@ type DeepgramAdapter struct {
// finalization signaling // finalization signaling
finalizeDone chan struct{} finalizeDone chan struct{}
finalizing bool // true when Finalize() has been called
} }
// deepgramCloseStream message to signal end of audio // deepgramCloseStream message to signal end of audio
@@ -235,7 +236,8 @@ func (a *DeepgramAdapter) buildURL() (string, error) {
q.Set("language", lang) q.Set("language", lang)
} }
if len(a.keywords) > 0 { // nova-3 uses "keyterm" (singular), others use "keywords" (plural)
if len(a.keywords) > 0 && !strings.HasPrefix(a.model, "nova-3") && !strings.HasPrefix(a.model, "flux") {
q.Set("keywords", strings.Join(a.keywords, ",")) q.Set("keywords", strings.Join(a.keywords, ","))
} }
@@ -277,6 +279,20 @@ func (a *DeepgramAdapter) readLoop() {
default: default:
} }
// check if we're finalizing - normal close after finalize is expected
a.mu.Lock()
finalizing := a.finalizing
a.mu.Unlock()
if finalizing {
// expected close after finalization, signal done and exit gracefully
select {
case a.finalizeDone <- struct{}{}:
default:
}
return
}
// attempt reconnection // attempt reconnection
log.Printf("deepgram: read error: %v, attempting reconnection", err) log.Printf("deepgram: read error: %v, attempting reconnection", err)
if !a.reconnect() { if !a.reconnect() {
@@ -411,6 +427,11 @@ func (a *DeepgramAdapter) Finalize(ctx context.Context) error {
default: default:
} }
// mark as finalizing to prevent reconnection attempts on normal close
a.mu.Lock()
a.finalizing = true
a.mu.Unlock()
// send CloseStream message // send CloseStream message
msg := deepgramCloseStream{Type: "CloseStream"} msg := deepgramCloseStream{Type: "CloseStream"}
@@ -447,6 +468,9 @@ func (a *DeepgramAdapter) Close() error {
return nil return nil
} }
// mark as finalizing to prevent reconnection attempts
a.finalizing = true
// cancel context first to signal reader to stop // cancel context first to signal reader to stop
if a.cancel != nil { if a.cancel != nil {
a.cancel() a.cancel()
+15 -4
View File
@@ -49,21 +49,31 @@ func NewDeepgramBatchAdapter(endpoint *provider.EndpointConfig, apiKey, model, l
// Transcribe sends audio data to Deepgram's pre-recorded API // Transcribe sends audio data to Deepgram's pre-recorded API
func (a *DeepgramBatchAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) { func (a *DeepgramBatchAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) {
if len(audioData) == 0 {
return "", nil
}
// convert raw PCM to WAV format
wavData, err := convertToWAV(audioData)
if err != nil {
return "", fmt.Errorf("convert to WAV: %w", err)
}
// build URL with query parameters // build URL with query parameters
apiURL, err := a.buildURL() apiURL, err := a.buildURL()
if err != nil { if err != nil {
return "", fmt.Errorf("build url: %w", err) return "", fmt.Errorf("build url: %w", err)
} }
// create request with audio data as body // create request with WAV data as body
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewReader(audioData)) req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewReader(wavData))
if err != nil { if err != nil {
return "", fmt.Errorf("create request: %w", err) return "", fmt.Errorf("create request: %w", err)
} }
// set headers // set headers
req.Header.Set("Authorization", "Token "+a.apiKey) req.Header.Set("Authorization", "Token "+a.apiKey)
req.Header.Set("Content-Type", "audio/wav") // we send raw PCM wrapped as WAV req.Header.Set("Content-Type", "audio/wav")
// send request // send request
resp, err := http.DefaultClient.Do(req) resp, err := http.DefaultClient.Do(req)
@@ -123,7 +133,8 @@ func (a *DeepgramBatchAdapter) buildURL() (string, error) {
q.Set("language", lang) q.Set("language", lang)
} }
if len(a.keywords) > 0 { // nova-3 uses "keyterm" (singular), others use "keywords" (plural)
if len(a.keywords) > 0 && !strings.HasPrefix(a.model, "nova-3") && !strings.HasPrefix(a.model, "flux") {
q.Set("keywords", strings.Join(a.keywords, ",")) q.Set("keywords", strings.Join(a.keywords, ","))
} }
+5 -6
View File
@@ -82,15 +82,14 @@ func (a *ElevenLabsAdapter) Transcribe(ctx context.Context, audioData []byte) (s
} }
} }
if len(a.keywords) > 0 { // keyterms only supported on scribe_v2, not scribe_v1
keytermsJSON, err := json.Marshal(a.keywords) if a.model != "scribe_v1" {
if err != nil { for _, keyword := range a.keywords {
return "", fmt.Errorf("marshal keyterms: %w", err) if err := writer.WriteField("keyterms", keyword); err != nil {
}
if err := writer.WriteField("keyterms", string(keytermsJSON)); err != nil {
return "", fmt.Errorf("write keyterms: %w", err) return "", fmt.Errorf("write keyterms: %w", err)
} }
} }
}
if err := writer.Close(); err != nil { if err := writer.Close(); err != nil {
return "", fmt.Errorf("close writer: %w", err) return "", fmt.Errorf("close writer: %w", err)
@@ -4,6 +4,7 @@ import (
"context" "context"
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
@@ -242,6 +243,9 @@ func (a *ElevenLabsStreamingAdapter) readLoop() {
_, message, err := conn.ReadMessage() _, message, err := conn.ReadMessage()
if err != nil { if err != nil {
if a.handleFatalClose(err) {
return
}
// check if context was cancelled (normal shutdown) // check if context was cancelled (normal shutdown)
select { select {
case <-a.ctx.Done(): case <-a.ctx.Done():
@@ -291,21 +295,92 @@ func (a *ElevenLabsStreamingAdapter) readLoop() {
case "error", "auth_error", "quota_exceeded", "rate_limited", case "error", "auth_error", "quota_exceeded", "rate_limited",
"queue_overflow", "resource_exhausted", "session_time_limit_exceeded", "queue_overflow", "resource_exhausted", "session_time_limit_exceeded",
"input_error", "chunk_size_exceeded", "insufficient_audio_activity", "input_error", "chunk_size_exceeded", "insufficient_audio_activity",
"transcriber_error", "commit_throttled", "unaccepted_terms": "transcriber_error", "commit_throttled", "unaccepted_terms", "invalid_request":
// error message // error message
errMsg := msg.Error errMsg := msg.Error
if errMsg == "" { if errMsg == "" {
errMsg = msg.MessageType errMsg = msg.MessageType
} }
log.Printf("elevenlabs-streaming: error: %s", errMsg) log.Printf("elevenlabs-streaming: error: %s", errMsg)
a.resultsCh <- TranscriptionResult{Error: fmt.Errorf("elevenlabs: %s", errMsg)} err := fmt.Errorf("elevenlabs: %s", errMsg)
if isElevenLabsFatalMessageType(msg.MessageType) {
a.handleFatalError(err)
return
}
a.emitResultError(err)
default: default:
log.Printf("elevenlabs-streaming: unknown message type: %s", msg.MessageType) log.Printf("elevenlabs-streaming: unknown message type: %s payload=%s", msg.MessageType, strings.TrimSpace(string(message)))
} }
} }
} }
func (a *ElevenLabsStreamingAdapter) emitResultError(err error) {
select {
case a.resultsCh <- TranscriptionResult{Error: err}:
default:
}
}
func (a *ElevenLabsStreamingAdapter) handleFatalError(err error) {
fatalErr := NewFatalTranscriptionError(err)
log.Printf("elevenlabs-streaming: fatal error: %v", err)
a.emitResultError(fatalErr)
a.closeConn()
if a.cancel != nil {
a.cancel()
}
}
func (a *ElevenLabsStreamingAdapter) handleFatalClose(err error) bool {
var closeErr *websocket.CloseError
if !errors.As(err, &closeErr) {
return false
}
if !isElevenLabsFatalCloseCode(closeErr.Code) {
return false
}
reason := strings.TrimSpace(closeErr.Text)
if reason == "" {
reason = "no reason provided"
}
a.handleFatalError(fmt.Errorf("elevenlabs websocket closed (%d): %s", closeErr.Code, reason))
return true
}
func (a *ElevenLabsStreamingAdapter) closeConn() {
a.mu.Lock()
conn := a.conn
a.conn = nil
a.mu.Unlock()
if conn != nil {
_ = conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
_ = conn.Close()
}
}
func isElevenLabsFatalCloseCode(code int) bool {
switch code {
case websocket.ClosePolicyViolation,
websocket.CloseUnsupportedData,
websocket.CloseInvalidFramePayloadData,
websocket.CloseMessageTooBig,
websocket.CloseProtocolError:
return true
default:
return false
}
}
func isElevenLabsFatalMessageType(messageType string) bool {
switch messageType {
case "auth_error", "unaccepted_terms", "invalid_request", "input_error", "chunk_size_exceeded":
return true
default:
return false
}
}
// SendChunk sends audio data to the WebSocket // SendChunk sends audio data to the WebSocket
func (a *ElevenLabsStreamingAdapter) SendChunk(audio []byte) error { func (a *ElevenLabsStreamingAdapter) SendChunk(audio []byte) error {
a.mu.Lock() a.mu.Lock()
+34
View File
@@ -0,0 +1,34 @@
package transcriber
import "errors"
// FatalTranscriptionError marks an error as non-recoverable for the current session.
type FatalTranscriptionError struct {
Err error
}
func (e *FatalTranscriptionError) Error() string {
if e == nil || e.Err == nil {
return "fatal transcription error"
}
return e.Err.Error()
}
func (e *FatalTranscriptionError) Unwrap() error {
if e == nil {
return nil
}
return e.Err
}
func NewFatalTranscriptionError(err error) error {
if err == nil {
return nil
}
return &FatalTranscriptionError{Err: err}
}
func IsFatalTranscriptionError(err error) bool {
var fatal *FatalTranscriptionError
return errors.As(err, &fatal)
}
+60 -1
View File
@@ -2,6 +2,7 @@ package transcriber
import ( import (
"context" "context"
"errors"
"log" "log"
"strings" "strings"
"sync" "sync"
@@ -19,6 +20,7 @@ type StreamingTranscriber struct {
// accumulated final text // accumulated final text
finalText strings.Builder finalText strings.Builder
mu sync.Mutex mu sync.Mutex
fatalErr error
// coordination // coordination
ctx context.Context ctx context.Context
@@ -66,6 +68,24 @@ func (t *StreamingTranscriber) sendAudio(frameCh <-chan recording.AudioFrame, er
return return
} }
if err := t.adapter.SendChunk(frame.Data); err != nil { if err := t.adapter.SendChunk(frame.Data); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
if t.ctx.Err() == nil && t.cancel != nil {
t.cancel()
}
return
}
if IsFatalTranscriptionError(err) {
if t.setFatalErr(err) {
select {
case errCh <- err:
default:
}
}
if t.cancel != nil {
t.cancel()
}
return
}
select { select {
case errCh <- err: case errCh <- err:
default: default:
@@ -98,6 +118,19 @@ func (t *StreamingTranscriber) receiveResults(errCh chan<- error) {
func (t *StreamingTranscriber) processResult(result TranscriptionResult, errCh chan<- error) { func (t *StreamingTranscriber) processResult(result TranscriptionResult, errCh chan<- error) {
if result.Error != nil { if result.Error != nil {
if IsFatalTranscriptionError(result.Error) {
if t.setFatalErr(result.Error) {
select {
case errCh <- result.Error:
default:
}
}
log.Printf("streaming transcriber: result error: %v", result.Error)
if t.cancel != nil {
t.cancel()
}
return
}
select { select {
case errCh <- result.Error: case errCh <- result.Error:
default: default:
@@ -154,11 +187,37 @@ func (t *StreamingTranscriber) Stop(ctx context.Context) error {
t.wg.Wait() t.wg.Wait()
// close the adapter // close the adapter
return t.adapter.Close() closeErr := t.adapter.Close()
if fatalErr := t.getFatalErr(); fatalErr != nil {
return fatalErr
}
return closeErr
} }
func (t *StreamingTranscriber) GetFinalTranscription() (string, error) { func (t *StreamingTranscriber) GetFinalTranscription() (string, error) {
t.mu.Lock() t.mu.Lock()
defer t.mu.Unlock() defer t.mu.Unlock()
if t.fatalErr != nil {
return "", t.fatalErr
}
return t.finalText.String(), nil return t.finalText.String(), nil
} }
func (t *StreamingTranscriber) setFatalErr(err error) bool {
if err == nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
if t.fatalErr != nil {
return false
}
t.fatalErr = err
return true
}
func (t *StreamingTranscriber) getFatalErr() error {
t.mu.Lock()
defer t.mu.Unlock()
return t.fatalErr
}
+7 -5
View File
@@ -83,14 +83,16 @@ func NewTranscriber(config Config) (Transcriber, error) {
config.Language = "" config.Language = ""
} }
// determine if we should use streaming mode // validate streaming/batch mode compatibility
useStreaming := config.Streaming && model.SupportsStreaming if config.Streaming && !model.SupportsStreaming {
return nil, fmt.Errorf("model %s does not support streaming mode", model.ID)
// fail if streaming-only model is used without streaming enabled }
if !useStreaming && !model.SupportsBatch { if !config.Streaming && !model.SupportsBatch {
return nil, fmt.Errorf("model %s requires streaming mode (set streaming = true in config)", model.ID) return nil, fmt.Errorf("model %s requires streaming mode (set streaming = true in config)", model.ID)
} }
useStreaming := config.Streaming
// streaming mode: use StreamingTranscriber // streaming mode: use StreamingTranscriber
if useStreaming { if useStreaming {
// pick the right adapter type for streaming // pick the right adapter type for streaming
+11
View File
@@ -145,6 +145,17 @@ func TestNewTranscriber(t *testing.T) {
}, },
wantErr: false, wantErr: false,
}, },
{
name: "elevenlabs batch model with streaming enabled fails",
config: Config{
Provider: "elevenlabs",
APIKey: "test-key",
Language: "en",
Model: "scribe_v2",
Streaming: true,
},
wantErr: true,
},
{ {
name: "deepgram streaming model creates StreamingTranscriber", name: "deepgram streaming model creates StreamingTranscriber",
config: Config{ config: Config{
+4 -8
View File
@@ -8,7 +8,7 @@ import (
) )
func TestGetTranscriptionModelOptions_ShowsCapabilities(t *testing.T) { func TestGetTranscriptionModelOptions_ShowsCapabilities(t *testing.T) {
// test elevenlabs - has batch-only and streaming-only models // test elevenlabs - includes batch+streaming and streaming-only models
options := getTranscriptionModelOptions("elevenlabs") options := getTranscriptionModelOptions("elevenlabs")
// should have 3 models: scribe_v1, scribe_v2, scribe_v2_realtime // should have 3 models: scribe_v1, scribe_v2, scribe_v2_realtime
@@ -23,12 +23,7 @@ func TestGetTranscriptionModelOptions_ShowsCapabilities(t *testing.T) {
continue continue
} }
if model.SupportsStreaming && !model.SupportsBatch { if model.SupportsBothModes() {
// streaming-only should mention streaming
if !strings.Contains(opt.Desc, "streaming") {
t.Errorf("streaming-only model %s should mention streaming in desc: %s", opt.ID, opt.Desc)
}
} else if model.SupportsBothModes() {
// both modes should mention batch+streaming // both modes should mention batch+streaming
if !strings.Contains(opt.Desc, "batch+streaming") { if !strings.Contains(opt.Desc, "batch+streaming") {
t.Errorf("both-modes model %s should mention batch+streaming in desc: %s", opt.ID, opt.Desc) t.Errorf("both-modes model %s should mention batch+streaming in desc: %s", opt.ID, opt.Desc)
@@ -75,11 +70,12 @@ func TestGetTranscriptionModelOptions_OpenAI_ShowsCapabilities(t *testing.T) {
func TestGetTranscriptionModelOptions_Deepgram_ShowsBothModes(t *testing.T) { func TestGetTranscriptionModelOptions_Deepgram_ShowsBothModes(t *testing.T) {
options := getTranscriptionModelOptions("deepgram") options := getTranscriptionModelOptions("deepgram")
// Deepgram has 2 models: nova-3, nova-2 - both support batch+streaming // Deepgram has 2 models: nova-3, nova-2
if len(options) != 2 { if len(options) != 2 {
t.Errorf("expected 2 options for deepgram, got %d", len(options)) t.Errorf("expected 2 options for deepgram, got %d", len(options))
} }
// all deepgram models support both modes
for _, opt := range options { for _, opt := range options {
if !strings.Contains(opt.Desc, "batch+streaming") { if !strings.Contains(opt.Desc, "batch+streaming") {
t.Errorf("deepgram model %s should mention batch+streaming: %s", opt.ID, opt.Desc) t.Errorf("deepgram model %s should mention batch+streaming: %s", opt.ID, opt.Desc)
+10 -4
View File
@@ -66,13 +66,13 @@ func onboardingSummaryScreen(state *wizardState, onBack func() screen) screen {
func newMenuScreen(state *wizardState) screen { func newMenuScreen(state *wizardState) screen {
items := []optionItem{ items := []optionItem{
{title: formatProvidersLabel(state.cfg), desc: "Manage API keys for cloud providers.", value: menuProviders}, {title: "Save & Exit", desc: "Write config changes to disk.", value: menuSave},
{title: formatVoiceModelLabel(state.cfg), desc: "Pick the transcription provider, model, and language.", value: menuVoiceModel}, {title: formatVoiceModelLabel(state.cfg), desc: "Pick the transcription provider, model, and language.", value: menuVoiceModel},
{title: formatLLMLabel(state.cfg), desc: "Configure post-processing and custom prompts.", value: menuLLM}, {title: formatLLMLabel(state.cfg), desc: "Configure post-processing and custom prompts.", value: menuLLM},
{title: formatKeywordsLabel(state.cfg), desc: "Words to preserve spelling and phrasing.", value: menuKeywords}, {title: formatKeywordsLabel(state.cfg), desc: "Words to preserve spelling and phrasing.", value: menuKeywords},
{title: formatProvidersLabel(state.cfg), desc: "Manage API keys for cloud providers.", value: menuProviders},
{title: formatNotificationsLabel(state.cfg), desc: "Notification type and message text.", value: menuNotifications}, {title: formatNotificationsLabel(state.cfg), desc: "Notification type and message text.", value: menuNotifications},
{title: "Advanced Settings", desc: "Recording, injection, and timeout settings.", value: menuAdvanced}, {title: "Advanced Settings", desc: "Recording, injection, and timeout settings.", value: menuAdvanced},
{title: "Save & Exit", desc: "Write config changes to disk.", value: menuSave},
{title: "Discard & Exit", desc: "Exit without saving changes.", value: menuDiscard}, {title: "Discard & Exit", desc: "Exit without saving changes.", value: menuDiscard},
} }
@@ -178,6 +178,9 @@ func newAPIKeyInputScreen(state *wizardState, providerName string, onContinue fu
} }
} }
desc := []string{fmt.Sprintf("Enter your %s API key", displayName)} desc := []string{fmt.Sprintf("Enter your %s API key", displayName)}
if url := getProviderKeyURL(providerName); url != "" {
desc = append(desc, fmt.Sprintf("Get key: %s", url))
}
validate := func(s string) error { validate := func(s string) error {
if s == "" { if s == "" {
return fmt.Errorf("API key is required") return fmt.Errorf("API key is required")
@@ -354,8 +357,11 @@ func newLanguageScreen(state *wizardState, model *provider.Model, onBack func()
func applyStreamingSelection(state *wizardState, model *provider.Model, onBack func() screen, next func() screen) screen { func applyStreamingSelection(state *wizardState, model *provider.Model, onBack func() screen, next func() screen) screen {
if model.SupportsBothModes() { if model.SupportsBothModes() {
desc := []string{"This model supports both batch and streaming modes."} desc := []string{
return newConfirmScreen(state, "Enable Streaming Mode?", desc, "Yes, streaming", "Lower latency, higher resource use.", "No, batch", "Wait for full transcription.", func() screen { "This model supports both batch and streaming modes.",
"Streaming is quicker but more expensive.",
}
return newConfirmScreen(state, "Enable Streaming Mode?", desc, "Yes, streaming", "Quicker response, higher cost.", "No, batch", "Wait for full transcription (cheaper).", func() screen {
state.cfg.Transcription.Streaming = true state.cfg.Transcription.Streaming = true
return next() return next()
}, func() screen { }, func() screen {
+8
View File
@@ -33,6 +33,14 @@ func getProviderDisplayName(providerName string) string {
return providerName return providerName
} }
func getProviderKeyURL(providerName string) string {
p := provider.GetProvider(providerName)
if p == nil {
return ""
}
return p.APIKeyURL()
}
func maskAPIKey(key string) string { func maskAPIKey(key string) string {
if len(key) <= 8 { if len(key) <= 8 {
return "***" return "***"
+4
View File
@@ -15,6 +15,10 @@ func TestWizardMenuTransitionAppliesSize(t *testing.T) {
updated, _ := model.Update(tea.WindowSizeMsg{Width: 120, Height: 40}) updated, _ := model.Update(tea.WindowSizeMsg{Width: 120, Height: 40})
model = updated.(wizardModel) model = updated.(wizardModel)
// move down to "Voice Model" item (index 1) which leads to a listScreen
updated, _ = model.Update(tea.KeyMsg{Type: tea.KeyDown})
model = updated.(wizardModel)
updated, _ = model.Update(tea.KeyMsg{Type: tea.KeyEnter}) updated, _ = model.Update(tea.KeyMsg{Type: tea.KeyEnter})
model = updated.(wizardModel) model = updated.(wizardModel)