diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 638f26b..95f5a88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24" + go-version-file: go.mod - name: Install dependencies run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fa1b24a..c05a178 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -3,15 +3,10 @@ name: E2E Tests on: workflow_dispatch: - inputs: - timeout: - description: 'Per-model timeout (e.g. 60s)' - required: false - default: '60s' jobs: - test-models: - name: Test All Models + integration: + name: Integration Tests runs-on: ubuntu-latest steps: - name: Checkout code @@ -20,7 +15,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24" + go-version-file: go.mod - name: Install dependencies run: | @@ -35,27 +30,11 @@ jobs: - name: Download dependencies run: go mod download - - name: Build binary - env: - CGO_ENABLED: 1 - run: go build -o hyprvoice ./cmd/hyprvoice - - - name: Run test-models + - name: Run integration tests env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - run: | - ./hyprvoice test-models \ - --timeout=${{ inputs.timeout }} \ - --output=test-models-report.json - - - name: Upload report - uses: actions/upload-artifact@v4 - if: always() - with: - name: test-models-report - path: test-models-report.json - retention-days: 30 + run: go test -tags=integration -v ./cmd/hyprvoice -timeout 15m diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ee3308..4a416e0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24" + go-version-file: go.mod - name: Install dependencies run: | diff --git a/cmd/hyprvoice/integration_test.go b/cmd/hyprvoice/integration_test.go new file mode 100644 index 0000000..f421f08 --- /dev/null +++ b/cmd/hyprvoice/integration_test.go @@ -0,0 +1,541 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/leonardotrapani/hyprvoice/internal/config" + "github.com/leonardotrapani/hyprvoice/internal/llm" + "github.com/leonardotrapani/hyprvoice/internal/models/whisper" + "github.com/leonardotrapani/hyprvoice/internal/provider" + "github.com/leonardotrapani/hyprvoice/internal/recording" + "github.com/leonardotrapani/hyprvoice/internal/transcriber" +) + +const ( + testSampleRate = 16000 + testChannels = 1 + testBitsPerSample = 16 + testTimeout = 45 * time.Second +) + +var testKeywords = []string{"Hyprvoice", "transcription", "dictation"} + +func TestTranscriptionModels(t *testing.T) { + audio, err := loadTestAudio(t) + if err != nil { + t.Fatalf("failed to load test audio: %v", err) + } + + cfg := loadTestConfig(t) + + providerNames := provider.ListProvidersWithTranscription() + sort.Strings(providerNames) + + smallestLocalModel := selectSmallestLocalModel() + + for _, providerName := range providerNames { + p := provider.GetProvider(providerName) + if p == nil { + continue + } + + models := provider.ModelsOfType(p, provider.Transcription) + sort.Slice(models, func(i, j int) bool { + return models[i].ID < models[j].ID + }) + + for _, model := range models { + if model.Local && providerName == provider.ProviderWhisperCpp && model.ID != smallestLocalModel { + continue + } + + modes := getModesForModel(model) + languages := []string{"en", ""} + keywordOptions := []bool{true, false} + + for _, mode := range modes { + for _, lang := range languages { + for _, useKeywords := range keywordOptions { + testName := fmt.Sprintf("%s/%s/%s/lang=%s/keywords=%v", + providerName, model.ID, mode, langDisplay(lang), useKeywords) + + model := model + mode := mode + lang := lang + useKeywords := useKeywords + providerName := providerName + + t.Run(testName, func(t *testing.T) { + t.Parallel() + runTranscriptionTest(t, cfg, providerName, model, mode, lang, useKeywords, audio) + }) + } + } + } + } + } +} + +func TestLLMModels(t *testing.T) { + cfg := loadTestConfig(t) + + providerNames := provider.ListProvidersWithLLM() + sort.Strings(providerNames) + + for _, providerName := range providerNames { + p := provider.GetProvider(providerName) + if p == nil { + continue + } + + models := provider.ModelsOfType(p, provider.LLM) + sort.Slice(models, func(i, j int) bool { + return models[i].ID < models[j].ID + }) + + for _, model := range models { + for _, useKeywords := range []bool{true, false} { + testName := fmt.Sprintf("%s/%s/keywords=%v", providerName, model.ID, useKeywords) + + model := model + useKeywords := useKeywords + providerName := providerName + + t.Run(testName, func(t *testing.T) { + t.Parallel() + runLLMTest(t, cfg, providerName, model, useKeywords) + }) + } + } + } +} + +func runTranscriptionTest(t *testing.T, cfg *config.Config, providerName string, model provider.Model, mode, lang string, useKeywords bool, audio []byte) { + if model.Local { + if _, err := exec.LookPath("whisper-cli"); err != nil { + t.Skip("whisper-cli not found") + } + if !whisper.IsInstalled(model.ID) { + t.Skipf("local model %s not installed", model.ID) + } + } + + apiKey := resolveTestAPIKey(cfg, providerName) + if testProviderRequiresKey(providerName) && apiKey == "" { + t.Skipf("missing api key for %s", providerName) + } + + var keywords []string + if useKeywords { + keywords = testKeywords + } + + streaming := mode == "streaming" + transcribeCfg := transcriber.Config{ + Provider: providerName, + APIKey: apiKey, + Language: lang, + Model: model.ID, + Keywords: keywords, + Threads: 0, + Streaming: streaming, + } + + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + text, err := runTestTranscriber(ctx, transcribeCfg, audio) + if err != nil { + t.Errorf("transcription failed: %v", err) + return + } + + text = strings.TrimSpace(text) + if text == "" { + t.Error("transcription returned empty text") + return + } + + t.Logf("output (%d chars): %q", len(text), truncateTestString(text, 100)) +} + +func runLLMTest(t *testing.T, cfg *config.Config, providerName string, model provider.Model, useKeywords bool) { + apiKey := resolveTestAPIKey(cfg, providerName) + if testProviderRequiresKey(providerName) && apiKey == "" { + t.Skipf("missing api key for %s", providerName) + } + + var keywords []string + if useKeywords { + keywords = testKeywords + } + + llmCfg := llm.Config{ + Provider: providerName, + APIKey: apiKey, + Model: model.ID, + RemoveStutters: true, + AddPunctuation: true, + FixGrammar: true, + RemoveFillerWords: true, + CustomPrompt: "", + Keywords: keywords, + } + + adapter, err := llm.NewAdapter(llmCfg) + if err != nil { + t.Errorf("failed to create adapter: %v", err) + return + } + + input := "uh i i i want to test hyprvoice you know this is just a cleanup check" + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + output, err := adapter.Process(ctx, input) + if err != nil { + t.Errorf("llm processing failed: %v", err) + return + } + + output = strings.TrimSpace(output) + if output == "" { + t.Error("llm returned empty output") + return + } + + t.Logf("output (%d chars): %q", len(output), truncateTestString(output, 100)) +} + +func runTestTranscriber(ctx context.Context, cfg transcriber.Config, audio []byte) (string, error) { + tr, err := transcriber.NewTranscriber(cfg) + if err != nil { + return "", err + } + + frameCh := make(chan recording.AudioFrame, 8) + errCh, err := tr.Start(ctx, frameCh) + if err != nil { + return "", err + } + + sendErr := sendTestAudioFrames(ctx, frameCh, audio) + close(frameCh) + + stopErr := tr.Stop(ctx) + errChErr := readTestErrorChannel(errCh) + + if sendErr != nil { + return "", sendErr + } + if stopErr != nil { + return "", stopErr + } + if errChErr != nil { + return "", errChErr + } + + return tr.GetFinalTranscription() +} + +func sendTestAudioFrames(ctx context.Context, frameCh chan<- recording.AudioFrame, audio []byte) error { + const chunkBytes = 3200 + bytesPerSecond := testSampleRate * (testBitsPerSample / 8) * testChannels + chunkDuration := time.Duration(float64(chunkBytes) / float64(bytesPerSecond) * float64(time.Second)) + + for offset := 0; offset < len(audio); offset += chunkBytes { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + end := offset + chunkBytes + if end > len(audio) { + end = len(audio) + } + + frame := recording.AudioFrame{Data: audio[offset:end], Timestamp: time.Now()} + select { + case frameCh <- frame: + case <-ctx.Done(): + return ctx.Err() + } + + time.Sleep(chunkDuration) + } + + return nil +} + +func readTestErrorChannel(errCh <-chan error) error { + if errCh == nil { + return nil + } + + var firstErr error + idleTimer := time.NewTimer(150 * time.Millisecond) + defer idleTimer.Stop() + + for { + select { + case err, ok := <-errCh: + if !ok { + return firstErr + } + if err != nil && firstErr == nil { + firstErr = err + } + if !idleTimer.Stop() { + <-idleTimer.C + } + idleTimer.Reset(150 * time.Millisecond) + case <-idleTimer.C: + return firstErr + } + } +} + +func loadTestAudio(t *testing.T) ([]byte, error) { + _, currentFile, _, ok := runtime.Caller(0) + if !ok { + return nil, fmt.Errorf("could not determine current file path") + } + + projectRoot := filepath.Dir(filepath.Dir(filepath.Dir(currentFile))) + samplePath := filepath.Join(projectRoot, "testdata", "sample.wav") + + data, err := os.ReadFile(samplePath) + if err != nil { + return nil, fmt.Errorf("could not read sample audio: %w", err) + } + + return parseTestWAV(data) +} + +func parseTestWAV(data []byte) ([]byte, error) { + if len(data) < 12 { + return nil, fmt.Errorf("invalid wav: too short") + } + if string(data[0:4]) != "RIFF" || string(data[8:12]) != "WAVE" { + return nil, fmt.Errorf("invalid wav: missing riff/wave header") + } + + offset := 12 + var fmtFound, dataFound bool + var sampleRate, channels, bitsPerSample int + var audioData []byte + + for offset+8 <= len(data) { + chunkID := string(data[offset : offset+4]) + chunkSize := int(binary.LittleEndian.Uint32(data[offset+4 : offset+8])) + offset += 8 + if offset+chunkSize > len(data) { + return nil, fmt.Errorf("invalid wav: chunk overflows file") + } + + switch chunkID { + case "fmt ": + if chunkSize < 16 { + return nil, fmt.Errorf("invalid wav: fmt chunk too short") + } + audioFormat := binary.LittleEndian.Uint16(data[offset : offset+2]) + if audioFormat != 1 { + return nil, fmt.Errorf("unsupported wav format: %d", audioFormat) + } + channels = int(binary.LittleEndian.Uint16(data[offset+2 : offset+4])) + sampleRate = int(binary.LittleEndian.Uint32(data[offset+4 : offset+8])) + bitsPerSample = int(binary.LittleEndian.Uint16(data[offset+14 : offset+16])) + fmtFound = true + case "data": + audioData = data[offset : offset+chunkSize] + dataFound = true + } + + offset += chunkSize + if chunkSize%2 == 1 { + offset++ + } + } + + if !fmtFound || !dataFound { + return nil, fmt.Errorf("invalid wav: missing fmt or data chunk") + } + if bitsPerSample != testBitsPerSample { + return nil, fmt.Errorf("unsupported wav bits per sample: %d", bitsPerSample) + } + + monoData, err := downmixTestToMono(audioData, channels) + if err != nil { + return nil, err + } + resampled := resampleTestPCM16(monoData, sampleRate, testSampleRate) + if len(resampled) == 0 { + return nil, fmt.Errorf("invalid wav: empty audio data") + } + + return resampled, nil +} + +func downmixTestToMono(data []byte, channels int) ([]byte, error) { + if channels == 1 { + return data, nil + } + if channels <= 0 { + return nil, fmt.Errorf("invalid channel count: %d", channels) + } + frameSize := 2 * channels + if len(data)%frameSize != 0 { + return nil, fmt.Errorf("invalid pcm data length") + } + + frames := len(data) / frameSize + out := make([]byte, frames*2) + for i := 0; i < frames; i++ { + var sum int32 + for c := 0; c < channels; c++ { + idx := (i*channels + c) * 2 + sample := int16(binary.LittleEndian.Uint16(data[idx : idx+2])) + sum += int32(sample) + } + mono := int16(sum / int32(channels)) + out[i*2] = byte(mono) + out[i*2+1] = byte(mono >> 8) + } + + return out, nil +} + +func resampleTestPCM16(data []byte, inRate, outRate int) []byte { + if inRate <= 0 || outRate <= 0 || inRate == outRate { + return data + } + if len(data) < 2 { + return data + } + + numInSamples := len(data) / 2 + numOutSamples := int(math.Round(float64(numInSamples) * float64(outRate) / float64(inRate))) + if numOutSamples <= 0 { + return nil + } + + out := make([]byte, numOutSamples*2) + for i := 0; i < numOutSamples; i++ { + srcPos := float64(i) * float64(inRate) / float64(outRate) + srcIdx := int(srcPos) + frac := srcPos - float64(srcIdx) + + sample1 := sampleTestAtPCM16(data, srcIdx) + sample2 := sampleTestAtPCM16(data, srcIdx+1) + outSample := int16(float64(sample1)*(1-frac) + float64(sample2)*frac) + + out[i*2] = byte(outSample) + out[i*2+1] = byte(outSample >> 8) + } + + return out +} + +func sampleTestAtPCM16(data []byte, idx int) int16 { + if idx <= 0 { + return int16(binary.LittleEndian.Uint16(data[0:2])) + } + pos := idx * 2 + if pos+1 >= len(data) { + last := len(data) - 2 + if last < 0 { + return 0 + } + return int16(binary.LittleEndian.Uint16(data[last : last+2])) + } + return int16(binary.LittleEndian.Uint16(data[pos : pos+2])) +} + +func loadTestConfig(t *testing.T) *config.Config { + cfg, err := config.Load() + if err != nil { + if errors.Is(err, config.ErrConfigNotFound) { + return config.DefaultConfig() + } + t.Logf("warning: could not load config: %v", err) + return config.DefaultConfig() + } + if cfg.Providers == nil { + cfg.Providers = make(map[string]config.ProviderConfig) + } + return cfg +} + +func resolveTestAPIKey(cfg *config.Config, providerName string) string { + base := provider.BaseProviderName(providerName) + if cfg != nil && cfg.Providers != nil { + if pc, ok := cfg.Providers[base]; ok && pc.APIKey != "" { + return pc.APIKey + } + } + if envVar := provider.EnvVarForProvider(providerName); envVar != "" { + return os.Getenv(envVar) + } + return "" +} + +func testProviderRequiresKey(providerName string) bool { + p := provider.GetProvider(provider.BaseProviderName(providerName)) + if p == nil { + return false + } + return p.RequiresAPIKey() +} + +func selectSmallestLocalModel() string { + models := whisper.ListModels() + if len(models) == 0 { + return "" + } + sort.Slice(models, func(i, j int) bool { + if models[i].SizeBytes == models[j].SizeBytes { + return !models[i].Multilingual && models[j].Multilingual + } + return models[i].SizeBytes < models[j].SizeBytes + }) + return models[0].ID +} + +func getModesForModel(model provider.Model) []string { + if model.SupportsBothModes() { + return []string{"batch", "streaming"} + } + if model.SupportsStreaming && !model.SupportsBatch { + return []string{"streaming"} + } + return []string{"batch"} +} + +func langDisplay(lang string) string { + if lang == "" { + return "auto" + } + return lang +} + +func truncateTestString(s string, max int) string { + if len(s) <= max { + return s + } + return s[:max] + "..." +} diff --git a/cmd/hyprvoice/main.go b/cmd/hyprvoice/main.go index 0582535..f4741ac 100644 --- a/cmd/hyprvoice/main.go +++ b/cmd/hyprvoice/main.go @@ -40,7 +40,6 @@ func init() { onboardingCmd(), configureCmd(), modelCmd(), - testModelsCmd(), ) } diff --git a/cmd/hyprvoice/test_models.go b/cmd/hyprvoice/test_models.go deleted file mode 100644 index 68b70f7..0000000 --- a/cmd/hyprvoice/test_models.go +++ /dev/null @@ -1,865 +0,0 @@ -package main - -import ( - "context" - "encoding/binary" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "net/http" - "os" - "os/exec" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/leonardotrapani/hyprvoice/internal/config" - "github.com/leonardotrapani/hyprvoice/internal/llm" - "github.com/leonardotrapani/hyprvoice/internal/models/whisper" - "github.com/leonardotrapani/hyprvoice/internal/provider" - "github.com/leonardotrapani/hyprvoice/internal/recording" - "github.com/leonardotrapani/hyprvoice/internal/transcriber" - "github.com/spf13/cobra" -) - -const ( - mockSampleRate = 16000 - mockChannels = 1 - mockBitsPerSample = 16 - defaultSampleURL = "https://raw.githubusercontent.com/mozilla/DeepSpeech/master/data/smoke_test/LDC93S1.wav" - defaultSampleName = "testaudio.wav" -) - -var ( - defaultTestKeywords = []string{"Hyprvoice", "transcription", "dictation"} - defaultTestLanguage = "en" -) - -type testModelsOptions struct { - audioPath string - recordFor time.Duration - timeout time.Duration - outputPath string - realtime bool - bothModes bool - localModel string - downloadLocal bool - language string - keywords []string - noKeywords bool - noLanguage bool -} - -type modelTest struct { - provider string - model provider.Model - mode string -} - -type modelTestResult struct { - Provider string `json:"provider"` - Model string `json:"model"` - Type string `json:"type"` - Mode string `json:"mode"` - Local bool `json:"local"` - Status string `json:"status"` - DurationMS int64 `json:"duration_ms"` - Output string `json:"output,omitempty"` - OutputChars int `json:"output_chars,omitempty"` - Error string `json:"error,omitempty"` -} - -type testReport struct { - StartedAt time.Time `json:"started_at"` - AudioSrc string `json:"audio_src"` - Results []modelTestResult `json:"results"` - PassCount int `json:"pass_count"` - FailCount int `json:"fail_count"` - SkipCount int `json:"skip_count"` - TotalCount int `json:"total_count"` -} - -func testModelsCmd() *cobra.Command { - var opts testModelsOptions - - cmd := &cobra.Command{ - Use: "test-models", - Short: "Run E2E tests for all providers/models", - RunE: func(cmd *cobra.Command, args []string) error { - return runTestModels(cmd.Context(), opts) - }, - } - - cmd.Flags().StringVar(&opts.audioPath, "audio", "", "WAV file to use (defaults to downloaded sample)") - cmd.Flags().DurationVar(&opts.recordFor, "record-seconds", 0, "Record mic audio (e.g. 5s)") - cmd.Flags().DurationVar(&opts.timeout, "timeout", 45*time.Second, "Per-model timeout") - cmd.Flags().BoolVar(&opts.realtime, "realtime", true, "Pace streaming chunks in real time") - cmd.Flags().BoolVar(&opts.bothModes, "both-modes", true, "Test batch+streaming models in both modes") - cmd.Flags().StringVar(&opts.outputPath, "output", "", "Write JSON report to file") - cmd.Flags().StringVar(&opts.localModel, "local-model", "", "whisper-cpp model ID to test") - cmd.Flags().BoolVar(&opts.downloadLocal, "download-local", false, "Download local whisper model if missing") - cmd.Flags().StringVar(&opts.language, "language", defaultTestLanguage, "Language code to test") - cmd.Flags().StringSliceVar(&opts.keywords, "keywords", defaultTestKeywords, "Keywords to test") - cmd.Flags().BoolVar(&opts.noKeywords, "no-keywords", false, "Skip keyword testing") - cmd.Flags().BoolVar(&opts.noLanguage, "no-language", false, "Skip language (use auto-detect)") - - return cmd -} - -func runTestModels(ctx context.Context, opts testModelsOptions) error { - if opts.audioPath != "" && opts.recordFor > 0 { - return fmt.Errorf("use either --audio or --record-seconds, not both") - } - if opts.timeout <= 0 { - return fmt.Errorf("timeout must be positive") - } - startedAt := time.Now().UTC() - - audio, audioSrc, err := loadTestAudio(ctx, opts) - if err != nil { - return err - } - - cfg, err := loadConfigForTests() - if err != nil { - return err - } - - transcriptionTests, err := buildTranscriptionTests(opts) - if err != nil { - return err - } - - llmTests := buildLLMTests() - - var results []modelTestResult - - for _, test := range transcriptionTests { - result := runTranscriptionTest(ctx, cfg, test, audio, opts) - results = append(results, result) - } - - for _, test := range llmTests { - result := runLLMTest(ctx, cfg, test, opts) - results = append(results, result) - } - - report := summarizeReport(startedAt, audioSrc, results) - printReport(report) - - if opts.outputPath != "" { - if err := writeReport(opts.outputPath, report); err != nil { - return err - } - } - - if report.FailCount > 0 || report.SkipCount > 0 { - return fmt.Errorf("%d failed, %d skipped", report.FailCount, report.SkipCount) - } - - return nil -} - -func loadConfigForTests() (*config.Config, error) { - cfg, err := config.Load() - if err != nil { - if errors.Is(err, config.ErrConfigNotFound) { - return config.DefaultConfig(), nil - } - return nil, err - } - if cfg.Providers == nil { - cfg.Providers = make(map[string]config.ProviderConfig) - } - return cfg, nil -} - -func buildTranscriptionTests(opts testModelsOptions) ([]modelTest, error) { - providerNames := provider.ListProvidersWithTranscription() - sort.Strings(providerNames) - - localModel := opts.localModel - if localModel == "" { - localModel = selectSmallestWhisperModel() - } - - var tests []modelTest - for _, providerName := range providerNames { - p := provider.GetProvider(providerName) - if p == nil { - continue - } - - models := provider.ModelsOfType(p, provider.Transcription) - sort.Slice(models, func(i, j int) bool { - return models[i].ID < models[j].ID - }) - - for _, model := range models { - if model.Local { - if providerName == provider.ProviderWhisperCpp && model.ID != localModel { - // test only the smallest local model; if it works the rest should too - continue - } - } - - if opts.bothModes && model.SupportsBothModes() { - tests = append(tests, modelTest{provider: providerName, model: model, mode: "batch"}) - tests = append(tests, modelTest{provider: providerName, model: model, mode: "streaming"}) - continue - } - - mode := "batch" - if model.SupportsStreaming && !model.SupportsBatch { - mode = "streaming" - } - tests = append(tests, modelTest{provider: providerName, model: model, mode: mode}) - } - } - - return tests, nil -} - -func buildLLMTests() []modelTest { - providerNames := provider.ListProvidersWithLLM() - sort.Strings(providerNames) - - var tests []modelTest - for _, providerName := range providerNames { - p := provider.GetProvider(providerName) - if p == nil { - continue - } - models := provider.ModelsOfType(p, provider.LLM) - sort.Slice(models, func(i, j int) bool { - return models[i].ID < models[j].ID - }) - for _, model := range models { - tests = append(tests, modelTest{provider: providerName, model: model, mode: "batch"}) - } - } - - return tests -} - -func runTranscriptionTest(ctx context.Context, cfg *config.Config, test modelTest, audio []byte, opts testModelsOptions) modelTestResult { - result := modelTestResult{ - Provider: test.provider, - Model: test.model.ID, - Type: "transcription", - Mode: test.mode, - Local: test.model.Local, - Status: "fail", - } - - if test.model.Local { - if _, err := exec.LookPath("whisper-cli"); err != nil { - result.Status = "skip" - result.Error = "whisper-cli not found" - return result - } - if !whisper.IsInstalled(test.model.ID) { - if opts.downloadLocal { - dlCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) - defer cancel() - if err := downloadLocalModel(dlCtx, test.model.ID); err != nil { - result.Status = "fail" - result.Error = err.Error() - return result - } - } else { - result.Status = "skip" - result.Error = "local model not installed" - return result - } - } - } - - apiKey := resolveAPIKey(cfg, test.provider) - if providerRequiresKey(test.provider) && apiKey == "" { - result.Status = "skip" - result.Error = "missing api key" - return result - } - - language := opts.language - if opts.noLanguage { - language = "" - } - keywords := opts.keywords - if opts.noKeywords { - keywords = nil - } - - streaming := test.mode == "streaming" - transcribeCfg := transcriber.Config{ - Provider: test.provider, - APIKey: apiKey, - Language: language, - Model: test.model.ID, - Keywords: keywords, - Threads: 0, - Streaming: streaming, - } - - testCtx, cancel := context.WithTimeout(ctx, opts.timeout) - defer cancel() - start := time.Now() - text, err := runTranscriber(testCtx, transcribeCfg, audio, opts.realtime) - result.DurationMS = time.Since(start).Milliseconds() - if err != nil { - result.Error = err.Error() - return result - } - - result.Status = "pass" - result.Output = strings.TrimSpace(text) - result.OutputChars = len(result.Output) - return result -} - -func runLLMTest(ctx context.Context, cfg *config.Config, test modelTest, opts testModelsOptions) modelTestResult { - result := modelTestResult{ - Provider: test.provider, - Model: test.model.ID, - Type: "llm", - Mode: "batch", - Local: test.model.Local, - Status: "fail", - } - - apiKey := resolveAPIKey(cfg, test.provider) - if providerRequiresKey(test.provider) && apiKey == "" { - result.Status = "skip" - result.Error = "missing api key" - return result - } - - keywords := opts.keywords - if opts.noKeywords { - keywords = nil - } - - llmCfg := llm.Config{ - Provider: test.provider, - APIKey: apiKey, - Model: test.model.ID, - RemoveStutters: true, - AddPunctuation: true, - FixGrammar: true, - RemoveFillerWords: true, - CustomPrompt: "", - Keywords: keywords, - } - - adapter, err := llm.NewAdapter(llmCfg) - if err != nil { - result.Error = err.Error() - return result - } - - input := "uh i i i want to test hyprvoice you know this is just a cleanup check" - testCtx, cancel := context.WithTimeout(ctx, opts.timeout) - defer cancel() - start := time.Now() - output, err := adapter.Process(testCtx, input) - result.DurationMS = time.Since(start).Milliseconds() - if err != nil { - result.Error = err.Error() - return result - } - - result.Status = "pass" - result.Output = strings.TrimSpace(output) - result.OutputChars = len(result.Output) - return result -} - -func runTranscriber(ctx context.Context, cfg transcriber.Config, audio []byte, realtime bool) (string, error) { - t, err := transcriber.NewTranscriber(cfg) - if err != nil { - return "", err - } - - frameCh := make(chan recording.AudioFrame, 8) - errCh, err := t.Start(ctx, frameCh) - if err != nil { - return "", err - } - - sendErr := sendAudioFrames(ctx, frameCh, audio, realtime) - close(frameCh) - - stopErr := t.Stop(ctx) - errChErr := readErrorChannel(errCh) - - if sendErr != nil { - return "", sendErr - } - if stopErr != nil { - return "", stopErr - } - if errChErr != nil { - return "", errChErr - } - - return t.GetFinalTranscription() -} - -func sendAudioFrames(ctx context.Context, frameCh chan<- recording.AudioFrame, audio []byte, realtime bool) error { - const chunkBytes = 3200 - bytesPerSecond := mockSampleRate * (mockBitsPerSample / 8) * mockChannels - chunkDuration := time.Duration(float64(chunkBytes) / float64(bytesPerSecond) * float64(time.Second)) - - for offset := 0; offset < len(audio); offset += chunkBytes { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - end := offset + chunkBytes - if end > len(audio) { - end = len(audio) - } - - frame := recording.AudioFrame{Data: audio[offset:end], Timestamp: time.Now()} - select { - case frameCh <- frame: - case <-ctx.Done(): - return ctx.Err() - } - - if realtime { - time.Sleep(chunkDuration) - } - } - - return nil -} - -func readErrorChannel(errCh <-chan error) error { - var firstErr error - if errCh == nil { - return nil - } - - idleTimer := time.NewTimer(150 * time.Millisecond) - defer idleTimer.Stop() - - for { - select { - case err, ok := <-errCh: - if !ok { - return firstErr - } - if err != nil && firstErr == nil { - firstErr = err - } - if !idleTimer.Stop() { - <-idleTimer.C - } - idleTimer.Reset(150 * time.Millisecond) - case <-idleTimer.C: - return firstErr - } - } -} - -func loadTestAudio(ctx context.Context, opts testModelsOptions) ([]byte, string, error) { - if opts.audioPath != "" { - wav, err := readWAVFile(opts.audioPath) - if err != nil { - return nil, "", err - } - return wav.data, opts.audioPath, nil - } - - if opts.recordFor > 0 { - audio, err := recordAudio(ctx, opts.recordFor) - if err != nil { - return nil, "", err - } - return audio, fmt.Sprintf("recording:%s", opts.recordFor), nil - } - - path, err := ensureDefaultSample(ctx) - if err != nil { - return nil, "", err - } - wav, err := readWAVFile(path) - if err != nil { - return nil, "", err - } - return wav.data, path, nil -} - -type wavData struct { - data []byte - sampleRate int - channels int - bitsPerSample int -} - -func readWAVFile(path string) (*wavData, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - return parseWAV(data) -} - -func parseWAV(data []byte) (*wavData, error) { - if len(data) < 12 { - return nil, fmt.Errorf("invalid wav: too short") - } - if string(data[0:4]) != "RIFF" || string(data[8:12]) != "WAVE" { - return nil, fmt.Errorf("invalid wav: missing riff/wave header") - } - - offset := 12 - var fmtFound bool - var dataFound bool - var info wavData - - for offset+8 <= len(data) { - chunkID := string(data[offset : offset+4]) - chunkSize := int(binary.LittleEndian.Uint32(data[offset+4 : offset+8])) - offset += 8 - if offset+chunkSize > len(data) { - return nil, fmt.Errorf("invalid wav: chunk overflows file") - } - - switch chunkID { - case "fmt ": - if chunkSize < 16 { - return nil, fmt.Errorf("invalid wav: fmt chunk too short") - } - audioFormat := binary.LittleEndian.Uint16(data[offset : offset+2]) - if audioFormat != 1 { - return nil, fmt.Errorf("unsupported wav format: %d", audioFormat) - } - info.channels = int(binary.LittleEndian.Uint16(data[offset+2 : offset+4])) - info.sampleRate = int(binary.LittleEndian.Uint32(data[offset+4 : offset+8])) - info.bitsPerSample = int(binary.LittleEndian.Uint16(data[offset+14 : offset+16])) - fmtFound = true - case "data": - info.data = data[offset : offset+chunkSize] - dataFound = true - } - - offset += chunkSize - if chunkSize%2 == 1 { - offset++ - } - } - - if !fmtFound || !dataFound { - return nil, fmt.Errorf("invalid wav: missing fmt or data chunk") - } - if info.bitsPerSample != mockBitsPerSample { - return nil, fmt.Errorf("unsupported wav bits per sample: %d", info.bitsPerSample) - } - if info.sampleRate <= 0 { - return nil, fmt.Errorf("invalid wav sample rate: %d", info.sampleRate) - } - if info.channels <= 0 { - return nil, fmt.Errorf("invalid wav: channels=%d", info.channels) - } - if len(info.data)%2 != 0 { - return nil, fmt.Errorf("invalid wav: pcm data not aligned") - } - - monoData, err := downmixToMono(info.data, info.channels) - if err != nil { - return nil, err - } - resampled := resamplePCM16(monoData, info.sampleRate, mockSampleRate) - if len(resampled) == 0 { - return nil, fmt.Errorf("invalid wav: empty audio data") - } - info.data = resampled - info.sampleRate = mockSampleRate - info.channels = mockChannels - info.bitsPerSample = mockBitsPerSample - return &info, nil -} - -func downmixToMono(data []byte, channels int) ([]byte, error) { - if channels == 1 { - return data, nil - } - if channels <= 0 { - return nil, fmt.Errorf("invalid channel count: %d", channels) - } - frameSize := 2 * channels - if len(data)%frameSize != 0 { - return nil, fmt.Errorf("invalid pcm data length") - } - - frames := len(data) / frameSize - out := make([]byte, frames*2) - for i := 0; i < frames; i++ { - var sum int32 - for c := 0; c < channels; c++ { - idx := (i*channels + c) * 2 - sample := int16(binary.LittleEndian.Uint16(data[idx : idx+2])) - sum += int32(sample) - } - mono := int16(sum / int32(channels)) - out[i*2] = byte(mono) - out[i*2+1] = byte(mono >> 8) - } - - return out, nil -} - -func resamplePCM16(data []byte, inRate, outRate int) []byte { - if inRate <= 0 || outRate <= 0 { - return data - } - if inRate == outRate { - return data - } - if len(data) < 2 { - return data - } - - numInSamples := len(data) / 2 - numOutSamples := int(math.Round(float64(numInSamples) * float64(outRate) / float64(inRate))) - if numOutSamples <= 0 { - return nil - } - - out := make([]byte, numOutSamples*2) - for i := 0; i < numOutSamples; i++ { - srcPos := float64(i) * float64(inRate) / float64(outRate) - srcIdx := int(srcPos) - frac := srcPos - float64(srcIdx) - - sample1 := sampleAtPCM16(data, srcIdx) - sample2 := sampleAtPCM16(data, srcIdx+1) - outSample := int16(float64(sample1)*(1-frac) + float64(sample2)*frac) - - out[i*2] = byte(outSample) - out[i*2+1] = byte(outSample >> 8) - } - - return out -} - -func sampleAtPCM16(data []byte, idx int) int16 { - if idx <= 0 { - return int16(binary.LittleEndian.Uint16(data[0:2])) - } - pos := idx * 2 - if pos+1 >= len(data) { - last := len(data) - 2 - if last < 0 { - return 0 - } - return int16(binary.LittleEndian.Uint16(data[last : last+2])) - } - return int16(binary.LittleEndian.Uint16(data[pos : pos+2])) -} - -func recordAudio(ctx context.Context, duration time.Duration) ([]byte, error) { - recorder := recording.NewRecorder(recording.Config{ - SampleRate: mockSampleRate, - Channels: mockChannels, - Format: "s16", - BufferSize: 8192, - Device: "", - ChannelBufferSize: 30, - Timeout: duration + 2*time.Second, - }) - - frameCh, errCh, err := recorder.Start(ctx) - if err != nil { - return nil, err - } - - var audio []byte - stopCh := make(chan struct{}) - go func() { - for frame := range frameCh { - audio = append(audio, frame.Data...) - } - close(stopCh) - }() - - select { - case <-time.After(duration): - recorder.Stop() - case <-ctx.Done(): - recorder.Stop() - } - - <-stopCh - if err := readErrorChannel(errCh); err != nil { - return nil, err - } - - return audio, nil -} - -func ensureDefaultSample(ctx context.Context) (string, error) { - cacheDir, err := os.UserCacheDir() - if err != nil { - return "", err - } - path := filepath.Join(cacheDir, "hyprvoice", defaultSampleName) - if info, err := os.Stat(path); err == nil && info.Size() > 0 { - return path, nil - } - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - return "", err - } - - fmt.Printf("test-models: downloading sample audio...\n") - if err := downloadSample(ctx, defaultSampleURL, path); err != nil { - return "", fmt.Errorf("download sample: %w (use --audio or --record-seconds to skip download)", err) - } - return path, nil -} - -func downloadSample(ctx context.Context, url, path string) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download failed: %s", resp.Status) - } - - tmpPath := path + ".downloading" - out, err := os.Create(tmpPath) - if err != nil { - return err - } - defer func() { - out.Close() - _ = os.Remove(tmpPath) - }() - - if _, err := io.Copy(out, resp.Body); err != nil { - return err - } - if err := out.Close(); err != nil { - return err - } - return os.Rename(tmpPath, path) -} - -func resolveAPIKey(cfg *config.Config, providerName string) string { - base := provider.BaseProviderName(providerName) - if cfg != nil && cfg.Providers != nil { - if pc, ok := cfg.Providers[base]; ok && pc.APIKey != "" { - return pc.APIKey - } - } - if envVar := provider.EnvVarForProvider(providerName); envVar != "" { - return os.Getenv(envVar) - } - return "" -} - -func providerRequiresKey(providerName string) bool { - p := provider.GetProvider(provider.BaseProviderName(providerName)) - if p == nil { - return false - } - return p.RequiresAPIKey() -} - -func selectSmallestWhisperModel() string { - models := whisper.ListModels() - if len(models) == 0 { - return "" - } - sort.Slice(models, func(i, j int) bool { - if models[i].SizeBytes == models[j].SizeBytes { - return !models[i].Multilingual && models[j].Multilingual - } - return models[i].SizeBytes < models[j].SizeBytes - }) - return models[0].ID -} - -func downloadLocalModel(ctx context.Context, modelID string) error { - var lastPercent int64 - return whisper.Download(ctx, modelID, func(downloaded, total int64) { - if total <= 0 { - return - } - percent := downloaded * 100 / total - if percent >= lastPercent+10 { - fmt.Printf("downloading %s... %d%%\n", modelID, percent) - lastPercent = percent - } - }) -} - -func summarizeReport(startedAt time.Time, audioSrc string, results []modelTestResult) testReport { - report := testReport{ - StartedAt: startedAt, - AudioSrc: audioSrc, - Results: results, - } - for _, r := range results { - report.TotalCount++ - switch r.Status { - case "pass": - report.PassCount++ - case "fail": - report.FailCount++ - case "skip": - report.SkipCount++ - } - } - return report -} - -func printReport(report testReport) { - fmt.Printf("test-models: total=%d pass=%d fail=%d skip=%d\n", report.TotalCount, report.PassCount, report.FailCount, report.SkipCount) - fmt.Printf("audio: %s\n", report.AudioSrc) - for _, r := range report.Results { - line := fmt.Sprintf("%s %s/%s %s", r.Status, r.Provider, r.Model, r.Mode) - if r.Type == "llm" { - line = fmt.Sprintf("%s %s/%s llm", r.Status, r.Provider, r.Model) - } - if r.DurationMS > 0 { - line += fmt.Sprintf(" %dms", r.DurationMS) - } - if r.Error != "" { - line += fmt.Sprintf(" error=%s", truncateString(r.Error, 160)) - } - if r.Output != "" { - line += fmt.Sprintf(" output=%q", truncateString(r.Output, 120)) - } - fmt.Println(line) - } -} - -func writeReport(path string, report testReport) error { - data, err := json.MarshalIndent(report, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, data, 0600) -} - -func truncateString(s string, max int) string { - if len(s) <= max { - return s - } - return s[:max] + "..." -} diff --git a/testdata/sample.wav b/testdata/sample.wav new file mode 100644 index 0000000..62b65f8 Binary files /dev/null and b/testdata/sample.wav differ