From 20516e8dc7909ae51f739a9c360dd00d1ccb8543 Mon Sep 17 00:00:00 2001 From: LeonardoTrapani Date: Sat, 16 Aug 2025 18:45:30 +0200 Subject: [PATCH] add simple transcription with adapters --- README.md | 44 ++- internal/pipeline/pipeline.go | 6 +- internal/transcriber/adapter_openai.go | 94 +++++++ internal/transcriber/adapter_whispercpp.go | 31 +++ internal/transcriber/openai.go | 304 --------------------- internal/transcriber/simple_transcriber.go | 122 +++++++++ internal/transcriber/transcriber.go | 60 ++-- 7 files changed, 323 insertions(+), 338 deletions(-) create mode 100644 internal/transcriber/adapter_openai.go create mode 100644 internal/transcriber/adapter_whispercpp.go delete mode 100644 internal/transcriber/openai.go create mode 100644 internal/transcriber/simple_transcriber.go diff --git a/README.md b/README.md index a303634..141a767 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,9 @@ hyprvoice stop | Recording control | ✅ | `hyprvoice toggle` | | Desktop notifications | ✅ | `notify-send` (logs fallback) | | Audio capture | ✅ | PipeWire (`pw-record`) frames + bounded channels | -| ASR backends | ⏳ | Not implemented yet (cloud/local planned) | +| Simple transcriber | ✅ | Collect audio and transcribe when complete | +| OpenAI adapter | ✅ | HTTP API calls with clean audio buffering | +| whisper.cpp adapter | ⏳ | Local inference ready for implementation | | Text injection | ⏳ | Not implemented; will use clipboard + `wtype`/`ydotool` | | Service management | 🔄 | `systemd --user` unit example provided | @@ -134,13 +136,31 @@ stateDiagram-v2 injecting --> idle: abort ``` +### Transcription Strategy + +Hyprvoice uses a **simple collect-and-transcribe** approach for reliable transcription: + +- **Collect all audio** during recording session +- **Single transcription** when recording stops +- **Clean, predictable results** with full context +- **Provider-agnostic adapter** pattern for different backends + +**Architecture:** + +``` +Audio Frames → Audio Buffer → Backend Adapter → Transcription + ↓ + [OpenAI API, whisper.cpp, etc.] +``` + ### Data flow 1. `toggle` (daemon) → create pipeline → recording 2. First frame arrives → transcribing (daemon may notify `Transcribing` later) -3. Second `toggle` during transcribing → send `inject` action → injecting (simulated) -4. Complete → idle; pipeline stops; daemon clears reference -5. Notifications at key transitions +3. Audio frames → audio buffer (collect all audio during session) +4. Second `toggle` during transcribing → send `inject` action → transcribe collected audio → injecting (simulated) +5. Complete → idle; pipeline stops; daemon clears reference +6. Notifications at key transitions --- @@ -215,7 +235,9 @@ hyprvoice/ │ ├── bus/ # IPC (Unix socket) + PID management │ ├── daemon/ # Control plane (IPC server, lifecycle; no state) │ ├── notify/ # Desktop notifications -│ └── pipeline/ # Pipeline + state machine (record/transcribe/inject) +│ ├── pipeline/ # Pipeline + state machine (record/transcribe/inject) +│ ├── recording/ # Audio capture via PipeWire +│ └── transcriber/ # Simple transcriber + adapters (OpenAI, whisper.cpp) ├── go.mod # Go module definition └── README.md ``` @@ -250,12 +272,12 @@ go run ./cmd/hyprvoice status ## Direction / Roadmap -- ASR integration: start with a cloud streaming backend; add a local backend later. -- Proper injection: clipboard save/restore + Ctrl+V, with `wtype`/`ydotool` fallbacks. -- VAD / endpointing to auto‑stop on silence (in addition to manual toggle). -- Configuration for devices, sample rate, and buffer sizing. -- Tests for pipeline state transitions and IPC. -- Direction is flexible; we can adjust based on UX feedback and perf. +- **ASR integration**: OpenAI adapter complete; whisper.cpp adapter ready for implementation. +- **Proper injection**: clipboard save/restore + Ctrl+V, with `wtype`/`ydotool` fallbacks. +- **Configuration options**: devices, sample rate, transcription providers. +- **Enhanced features**: VAD for auto-stop, improved chunking strategies if needed. +- **Tests**: comprehensive testing for pipeline state transitions and transcription. +- Direction is flexible; we can adjust based on UX feedback and performance needs. --- diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 364214a..e771d27 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -181,7 +181,7 @@ func (p *pipeline) run(ctx context.Context) { } defer func() { - if stopErr := t.Stop(); stopErr != nil { + if stopErr := t.Stop(ctx); stopErr != nil { log.Printf("Pipeline: Error stopping transcriber: %v", stopErr) p.sendError("Transcription Error", "Failed to stop transcriber cleanly", stopErr) } @@ -231,15 +231,13 @@ func (p *pipeline) run(ctx context.Context) { // Wait for the recorder to fully stop and frameCh to be closed // The transcriber will process any remaining frames when frameCh closes - log.Printf("Pipeline: Waiting for recording channel to close and final transcription to complete") - // Drain the frameCh to ensure it's fully closed for range frameCh { // Continue draining until channel is closed } // Stop the transcriber to ensure all buffered audio is processed - if err := t.Stop(); err != nil { + if err := t.Stop(ctx); err != nil { log.Printf("Pipeline: Error stopping transcriber: %v", err) p.sendError("Transcription Error", "Failed to stop transcriber during injection", err) } diff --git a/internal/transcriber/adapter_openai.go b/internal/transcriber/adapter_openai.go new file mode 100644 index 0000000..4259860 --- /dev/null +++ b/internal/transcriber/adapter_openai.go @@ -0,0 +1,94 @@ +package transcriber + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "log" + "time" + + "github.com/sashabaranov/go-openai" +) + +// OpenAIAdapter implements TranscriptionAdapter for OpenAI Whisper API +type OpenAIAdapter struct { + client *openai.Client + config Config +} + +func NewOpenAIAdapter(config Config) *OpenAIAdapter { + client := openai.NewClient(config.APIKey) + return &OpenAIAdapter{ + client: client, + config: config, + } +} + +func (a *OpenAIAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) { + if len(audioData) == 0 { + return "", nil + } + + // Convert raw PCM to WAV format + wavData, err := a.convertToWAV(audioData) + if err != nil { + return "", fmt.Errorf("convert to WAV: %w", err) + } + + // Create transcription request + req := openai.AudioRequest{ + Model: a.config.Model, + Reader: bytes.NewReader(wavData), + FilePath: "audio.wav", + Language: a.config.Language, + } + + start := time.Now() + resp, err := a.client.CreateTranscription(ctx, req) + duration := time.Since(start) + + if err != nil { + log.Printf("openai-adapter: API call failed after %v: %v", duration, err) + return "", fmt.Errorf("openai transcription: %w", err) + } + + log.Printf("openai-adapter: transcribed %d bytes in %v: %q", len(audioData), duration, resp.Text) + return resp.Text, nil +} + +// convertToWAV converts raw 16-bit PCM audio to WAV format +func (a *OpenAIAdapter) convertToWAV(rawAudio []byte) ([]byte, error) { + var buf bytes.Buffer + + const sampleRate = 16000 + const channels = 1 + const bitsPerSample = 16 + const byteRate = sampleRate * channels * bitsPerSample / 8 + const blockAlign = channels * bitsPerSample / 8 + + dataSize := len(rawAudio) + fileSize := 36 + dataSize + + // WAV header + buf.WriteString("RIFF") + binary.Write(&buf, binary.LittleEndian, uint32(fileSize)) + buf.WriteString("WAVE") + + // fmt chunk + buf.WriteString("fmt ") + binary.Write(&buf, binary.LittleEndian, uint32(16)) // fmt chunk size + binary.Write(&buf, binary.LittleEndian, uint16(1)) // PCM format + binary.Write(&buf, binary.LittleEndian, uint16(channels)) // number of channels + binary.Write(&buf, binary.LittleEndian, uint32(sampleRate)) // sample rate + binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) // byte rate + binary.Write(&buf, binary.LittleEndian, uint16(blockAlign)) // block align + binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample)) // bits per sample + + // data chunk + buf.WriteString("data") + binary.Write(&buf, binary.LittleEndian, uint32(dataSize)) + buf.Write(rawAudio) + + return buf.Bytes(), nil +} diff --git a/internal/transcriber/adapter_whispercpp.go b/internal/transcriber/adapter_whispercpp.go new file mode 100644 index 0000000..8e32dd2 --- /dev/null +++ b/internal/transcriber/adapter_whispercpp.go @@ -0,0 +1,31 @@ +package transcriber + +import ( + "context" + "fmt" +) + +// WhisperCppAdapter implements TranscriptionAdapter for local whisper.cpp +type WhisperCppAdapter struct { + config Config + modelPath string +} + +func NewWhisperCppAdapter(config Config) *WhisperCppAdapter { + return &WhisperCppAdapter{ + config: config, + // TODO: Configure model path based on config + modelPath: "./models/ggml-base.en.bin", + } +} + +func (a *WhisperCppAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) { + // TODO: Implement whisper.cpp transcription + // This will involve: + // 1. Writing audio data to a temporary file or pipe + // 2. Calling whisper.cpp binary with appropriate flags + // 3. Reading the transcription result + // 4. Cleaning up temporary files + + return "", fmt.Errorf("whisper.cpp adapter not implemented yet") +} diff --git a/internal/transcriber/openai.go b/internal/transcriber/openai.go deleted file mode 100644 index 5cae942..0000000 --- a/internal/transcriber/openai.go +++ /dev/null @@ -1,304 +0,0 @@ -package transcriber - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - "log" - "strings" - "sync" - "time" - - "github.com/leonardotrapani/hyprvoice/internal/recording" - "github.com/sashabaranov/go-openai" -) - -type OpenAITranscriber struct { - client *openai.Client - config Config - buffer *audioBuffer - cancel context.CancelFunc - wg sync.WaitGroup - mu sync.Mutex - transcribing bool - - transcriptionMu sync.RWMutex - transcriptionText strings.Builder -} - -type audioBuffer struct { - data []byte - mu sync.Mutex - lastAdd time.Time - maxSize int -} - -func NewOpenAITranscriber(config Config) *OpenAITranscriber { - client := openai.NewClient(config.APIKey) - - buffer := &audioBuffer{ - data: make([]byte, 0, config.ChunkSize*2), - maxSize: config.ChunkSize, - } - - return &OpenAITranscriber{ - client: client, - config: config, - buffer: buffer, - } -} - -func (t *OpenAITranscriber) Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) { - t.mu.Lock() - if t.transcribing { - t.mu.Unlock() - return nil, fmt.Errorf("transcriber: already transcribing") - } - t.transcribing = true - t.mu.Unlock() - - transcribeCtx, cancel := context.WithCancel(ctx) - t.cancel = cancel - - errCh := make(chan error, 1) - - t.wg.Add(1) - go t.processFrames(transcribeCtx, frameCh, errCh) - - return errCh, nil -} - -func (t *OpenAITranscriber) Stop() error { - t.mu.Lock() - if !t.transcribing { - t.mu.Unlock() - return nil - } - cancel := t.cancel - t.mu.Unlock() - - if cancel != nil { - cancel() - } - t.wg.Wait() - - // After stopping, ensure any remaining buffered audio is transcribed - if t.buffer.hasData() { - log.Printf("transcriber: processing remaining buffered audio on stop") - ctx := context.Background() - errCh := make(chan error, 1) - t.transcribeBuffer(ctx, errCh) - close(errCh) - } - - return nil -} - -func (t *OpenAITranscriber) GetTranscription() (string, error) { - t.transcriptionMu.RLock() - defer t.transcriptionMu.RUnlock() - - if t.transcriptionText.Len() == 0 { - return "", nil - } - - return t.transcriptionText.String(), nil -} - -func (t *OpenAITranscriber) processFrames(ctx context.Context, frameCh <-chan recording.AudioFrame, errCh chan<- error) { - defer func() { - // Always process remaining buffer when shutting down - if t.buffer.hasData() { - log.Printf("transcriber: processing final buffered audio on shutdown") - // Use background context for final transcription to avoid timeout - finalCtx := context.Background() - t.transcribeBuffer(finalCtx, errCh) - } - close(errCh) - t.mu.Lock() - t.transcribing = false - t.cancel = nil - t.mu.Unlock() - t.wg.Done() - }() - - ticker := time.NewTicker(t.config.BufferTime) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Printf("transcriber: context cancelled, processing remaining frames") - // Continue processing remaining frames from channel before stopping - for { - select { - case frame, ok := <-frameCh: - if !ok { - log.Printf("transcriber: recording channel closed") - return - } - t.buffer.addFrame(frame) - default: - // No more frames available, exit - return - } - } - - case frame, ok := <-frameCh: - if !ok { - log.Printf("transcriber: recording channel closed, finishing with remaining buffer") - return - } - t.buffer.addFrame(frame) - - case <-ticker.C: - if t.buffer.shouldFlush(t.config.BufferTime) { - t.transcribeBuffer(ctx, errCh) - } - } - } -} - -func (t *OpenAITranscriber) transcribeBuffer(ctx context.Context, errCh chan<- error) { - audioData := t.buffer.flush() - if len(audioData) == 0 { - return - } - - log.Printf("transcriber: sending %d bytes to OpenAI API", len(audioData)) - - wavData, err := t.convertToWAV(audioData) - if err != nil { - log.Printf("transcriber: failed to convert audio to WAV: %v", err) - select { - case errCh <- fmt.Errorf("transcriber: convert to WAV: %w", err): - default: - } - return - } - - req := openai.AudioRequest{ - Model: t.config.Model, - Reader: bytes.NewReader(wavData), - FilePath: "audio.wav", - Language: t.config.Language, - } - - start := time.Now() - resp, err := t.client.CreateTranscription(ctx, req) - duration := time.Since(start) - - if err != nil { - log.Printf("transcriber: API call failed after %v: %v", duration, err) - select { - case errCh <- fmt.Errorf("transcriber: transcription failed: %w", err): - default: - } - return - } - - if resp.Text != "" { - log.Printf("transcriber: received result in %v: %q", duration, resp.Text) - t.transcriptionMu.Lock() - if t.transcriptionText.Len() > 0 { - t.transcriptionText.WriteString(" ") - } - t.transcriptionText.WriteString(strings.TrimSpace(resp.Text)) - t.transcriptionMu.Unlock() - } else { - log.Printf("transcriber: received empty result after %v", duration) - } -} - -func (t *OpenAITranscriber) convertToWAV(rawAudio []byte) ([]byte, error) { - var buf bytes.Buffer - - const sampleRate = 16000 - const channels = 1 - const bitsPerSample = 16 - const byteRate = sampleRate * channels * bitsPerSample / 8 - const blockAlign = channels * bitsPerSample / 8 - - dataSize := len(rawAudio) - fileSize := 36 + dataSize - - buf.WriteString("RIFF") - binary.Write(&buf, binary.LittleEndian, uint32(fileSize)) - buf.WriteString("WAVE") - - buf.WriteString("fmt ") - binary.Write(&buf, binary.LittleEndian, uint32(16)) - binary.Write(&buf, binary.LittleEndian, uint16(1)) - binary.Write(&buf, binary.LittleEndian, uint16(channels)) - binary.Write(&buf, binary.LittleEndian, uint32(sampleRate)) - binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) - binary.Write(&buf, binary.LittleEndian, uint16(blockAlign)) - binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample)) - - buf.WriteString("data") - binary.Write(&buf, binary.LittleEndian, uint32(dataSize)) - buf.Write(rawAudio) - - return buf.Bytes(), nil -} - -func (b *audioBuffer) addFrame(frame recording.AudioFrame) { - b.mu.Lock() - defer b.mu.Unlock() - - b.data = append(b.data, frame.Data...) - b.lastAdd = frame.Timestamp - - if len(b.data) > b.maxSize*2 { - b.data = b.data[len(b.data)-b.maxSize:] - } -} - -func (b *audioBuffer) flush() []byte { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.data) == 0 { - return nil - } - - result := make([]byte, len(b.data)) - copy(result, b.data) - b.data = b.data[:0] - - return result -} - -func (b *audioBuffer) hasData() bool { - b.mu.Lock() - defer b.mu.Unlock() - return len(b.data) > 0 -} - -func (b *audioBuffer) shouldFlush(bufferTime time.Duration) bool { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.data) == 0 { - return false - } - - if len(b.data) >= b.maxSize { - return true - } - - return time.Since(b.lastAdd) >= bufferTime -} - -func NewTranscriber(config Config) (Transcriber, error) { - switch config.Provider { - case "openai": - if config.APIKey == "" { - return nil, fmt.Errorf("OpenAI API key required") - } - return NewOpenAITranscriber(config), nil - default: - return nil, fmt.Errorf("unsupported provider: %s", config.Provider) - } -} diff --git a/internal/transcriber/simple_transcriber.go b/internal/transcriber/simple_transcriber.go new file mode 100644 index 0000000..99110ae --- /dev/null +++ b/internal/transcriber/simple_transcriber.go @@ -0,0 +1,122 @@ +package transcriber + +import ( + "context" + "fmt" + "log" + "sync" + + "github.com/leonardotrapani/hyprvoice/internal/recording" +) + +// SimpleTranscriber collects all audio and transcribes when stopped +type SimpleTranscriber struct { + adapter TranscriptionAdapter + config Config + + // Audio collection + audioBuffer []byte + bufferMu sync.Mutex + + // Control + running bool + wg sync.WaitGroup + + // Transcription result + transcriptionMu sync.RWMutex + transcriptionText string +} + +func NewSimpleTranscriber(config Config, adapter TranscriptionAdapter) *SimpleTranscriber { + return &SimpleTranscriber{ + adapter: adapter, + config: config, + } +} + +func (t *SimpleTranscriber) Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) { + if t.running { + return nil, fmt.Errorf("transcriber already running") + } + + t.running = true + + errCh := make(chan error, 1) + + t.wg.Add(1) + go t.collectAudio(ctx, frameCh, errCh) + + return errCh, nil +} + +func (t *SimpleTranscriber) Stop(ctx context.Context) error { + if !t.running { + return nil + } + + t.wg.Wait() + t.running = false + + // Transcribe all collected audio using the passed context + return t.transcribeAll(ctx) +} + +func (t *SimpleTranscriber) GetTranscription() (string, error) { + t.transcriptionMu.RLock() + defer t.transcriptionMu.RUnlock() + return t.transcriptionText, nil +} + +func (t *SimpleTranscriber) collectAudio(ctx context.Context, frameCh <-chan recording.AudioFrame, errCh chan<- error) { + defer func() { + close(errCh) + t.wg.Done() + }() + + for { + select { + case <-ctx.Done(): + log.Printf("transcriber: stopping audio collection") + return + + case frame, ok := <-frameCh: + if !ok { + log.Printf("transcriber: audio channel closed") + return + } + + t.bufferMu.Lock() + t.audioBuffer = append(t.audioBuffer, frame.Data...) + t.bufferMu.Unlock() + } + } +} + +func (t *SimpleTranscriber) transcribeAll(ctx context.Context) error { + t.bufferMu.Lock() + audioData := make([]byte, len(t.audioBuffer)) + copy(audioData, t.audioBuffer) + t.bufferMu.Unlock() + + if len(audioData) == 0 { + log.Printf("transcriber: no audio data to transcribe") + return nil + } + + log.Printf("transcriber: transcribing %d bytes of audio", len(audioData)) + + // Use the context passed from the pipeline for proper cancellation chain + text, err := t.adapter.Transcribe(ctx, audioData) + if err != nil { + log.Printf("transcriber: transcription failed: %v", err) + return fmt.Errorf("transcription failed: %w", err) + } + + log.Printf("transcriber: transcription completed: %q", text) + + t.transcriptionMu.Lock() + t.transcriptionText = text + t.transcriptionMu.Unlock() + + return nil +} diff --git a/internal/transcriber/transcriber.go b/internal/transcriber/transcriber.go index 4246c56..01a1c26 100644 --- a/internal/transcriber/transcriber.go +++ b/internal/transcriber/transcriber.go @@ -2,38 +2,60 @@ package transcriber import ( "context" - "time" + "fmt" "github.com/leonardotrapani/hyprvoice/internal/recording" ) -type TranscriptionResult struct { - Text string - Timestamp time.Time - IsFinal bool -} - +// Main transcriber interface type Transcriber interface { Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) - Stop() error + Stop(ctx context.Context) error GetTranscription() (string, error) } +// Adapter interface for different transcription backends +type TranscriptionAdapter interface { + Transcribe(ctx context.Context, audioData []byte) (string, error) +} + +// Configuration for the transcriber type Config struct { - Provider string - APIKey string - Language string - ChunkSize int - BufferTime time.Duration - Model string + Provider string + APIKey string + Language string + Model string } func DefaultConfig() Config { return Config{ - Provider: "openai", - Language: "it", - ChunkSize: 16384, - BufferTime: 2 * time.Second, - Model: "whisper-1", + Provider: "openai", + Language: "it", + Model: "whisper-1", } } + +// NewTranscriber creates a new simple transcriber +func NewTranscriber(config Config) (Transcriber, error) { + // Create the appropriate adapter + var adapter TranscriptionAdapter + + switch config.Provider { + case "openai": + if config.APIKey == "" { + return nil, fmt.Errorf("OpenAI API key required") + } + adapter = NewOpenAIAdapter(config) + + case "whisper.cpp": + adapter = NewWhisperCppAdapter(config) + + default: + return nil, fmt.Errorf("unsupported provider: %s", config.Provider) + } + + // Create simple transcriber that collects all audio + transcriber := NewSimpleTranscriber(config, adapter) + + return transcriber, nil +}