thread/hyprvoice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add simple transcription with adapters

Browse Source
This commit is contained in:
LeonardoTrapani
2025-08-16 18:45:30 +02:00
parent 0b64a71afd
commit 20516e8dc7
7 changed files with 323 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+33 -11
View File
@@ -82,7 +82,9 @@ hyprvoice stop
| Recording control | ✅ | `hyprvoice toggle` | | Recording control | ✅ | `hyprvoice toggle` |
| Desktop notifications | ✅ | `notify-send` (logs fallback) | | Desktop notifications | ✅ | `notify-send` (logs fallback) |
| Audio capture | ✅ | PipeWire (`pw-record`) frames + bounded channels | | Audio capture | ✅ | PipeWire (`pw-record`) frames + bounded channels |
| ASR backends | ⏳ | Not implemented yet (cloud/local planned) | | Simple transcriber | ✅ | Collect audio and transcribe when complete |
| OpenAI adapter | ✅ | HTTP API calls with clean audio buffering |
| whisper.cpp adapter | ⏳ | Local inference ready for implementation |
| Text injection | ⏳ | Not implemented; will use clipboard + `wtype`/`ydotool` | | Text injection | ⏳ | Not implemented; will use clipboard + `wtype`/`ydotool` |
| Service management | 🔄 | `systemd --user` unit example provided | | Service management | 🔄 | `systemd --user` unit example provided |
@@ -134,13 +136,31 @@ stateDiagram-v2
injecting --> idle: abort injecting --> idle: abort
``` ```
### Transcription Strategy
Hyprvoice uses a **simple collect-and-transcribe** approach for reliable transcription:
- **Collect all audio** during recording session
- **Single transcription** when recording stops
- **Clean, predictable results** with full context
- **Provider-agnostic adapter** pattern for different backends
**Architecture:**
```
Audio Frames → Audio Buffer → Backend Adapter → Transcription
↓
[OpenAI API, whisper.cpp, etc.]
```
### Data flow ### Data flow
1. `toggle` (daemon) → create pipeline → recording 1. `toggle` (daemon) → create pipeline → recording
2. First frame arrives → transcribing (daemon may notify `Transcribing` later) 2. First frame arrives → transcribing (daemon may notify `Transcribing` later)
3. Second `toggle` during transcribing → send `inject` action → injecting (simulated) 3. Audio frames → audio buffer (collect all audio during session)
4. Complete → idle; pipeline stops; daemon clears reference 4. Second `toggle` during transcribing → send `inject` action → transcribe collected audio → injecting (simulated)
5. Notifications at key transitions 5. Complete → idle; pipeline stops; daemon clears reference
6. Notifications at key transitions
--- ---
@@ -215,7 +235,9 @@ hyprvoice/
│ ├── bus/ # IPC (Unix socket) + PID management │ ├── bus/ # IPC (Unix socket) + PID management
│ ├── daemon/ # Control plane (IPC server, lifecycle; no state) │ ├── daemon/ # Control plane (IPC server, lifecycle; no state)
│ ├── notify/ # Desktop notifications │ ├── notify/ # Desktop notifications
│ └── pipeline/ # Pipeline + state machine (record/transcribe/inject) │ ├── pipeline/ # Pipeline + state machine (record/transcribe/inject)
│ ├── recording/ # Audio capture via PipeWire
│ └── transcriber/ # Simple transcriber + adapters (OpenAI, whisper.cpp)
├── go.mod # Go module definition ├── go.mod # Go module definition
└── README.md └── README.md
``` ```
@@ -250,12 +272,12 @@ go run ./cmd/hyprvoice status
## Direction / Roadmap ## Direction / Roadmap
- ASR integration: start with a cloud streaming backend; add a local backend later. - **ASR integration**: OpenAI adapter complete; whisper.cpp adapter ready for implementation.
- Proper injection: clipboard save/restore + Ctrl+V, with `wtype`/`ydotool` fallbacks. - **Proper injection**: clipboard save/restore + Ctrl+V, with `wtype`/`ydotool` fallbacks.
- VAD / endpointing to auto‑stop on silence (in addition to manual toggle). - **Configuration options**: devices, sample rate, transcription providers.
- Configuration for devices, sample rate, and buffer sizing. - **Enhanced features**: VAD for auto-stop, improved chunking strategies if needed.
- Tests for pipeline state transitions and IPC. - **Tests**: comprehensive testing for pipeline state transitions and transcription.
- Direction is flexible; we can adjust based on UX feedback and perf. - Direction is flexible; we can adjust based on UX feedback and performance needs.
--- ---
internal/pipeline/pipeline.go
+2 -4
View File
@@ -181,7 +181,7 @@ func (p *pipeline) run(ctx context.Context) {
} }
defer func() { defer func() {
if stopErr := t.Stop(); stopErr != nil { if stopErr := t.Stop(ctx); stopErr != nil {
log.Printf("Pipeline: Error stopping transcriber: %v", stopErr) log.Printf("Pipeline: Error stopping transcriber: %v", stopErr)
p.sendError("Transcription Error", "Failed to stop transcriber cleanly", stopErr) p.sendError("Transcription Error", "Failed to stop transcriber cleanly", stopErr)
} }
@@ -231,15 +231,13 @@ func (p *pipeline) run(ctx context.Context) {
// Wait for the recorder to fully stop and frameCh to be closed // Wait for the recorder to fully stop and frameCh to be closed
// The transcriber will process any remaining frames when frameCh closes // The transcriber will process any remaining frames when frameCh closes
log.Printf("Pipeline: Waiting for recording channel to close and final transcription to complete")
// Drain the frameCh to ensure it's fully closed // Drain the frameCh to ensure it's fully closed
for range frameCh { for range frameCh {
// Continue draining until channel is closed // Continue draining until channel is closed
} }
// Stop the transcriber to ensure all buffered audio is processed // Stop the transcriber to ensure all buffered audio is processed
if err := t.Stop(); err != nil { if err := t.Stop(ctx); err != nil {
log.Printf("Pipeline: Error stopping transcriber: %v", err) log.Printf("Pipeline: Error stopping transcriber: %v", err)
p.sendError("Transcription Error", "Failed to stop transcriber during injection", err) p.sendError("Transcription Error", "Failed to stop transcriber during injection", err)
} }
internal/transcriber/adapter_openai.go
+94
View File
@@ -0,0 +1,94 @@
package transcriber
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"log"
"time"
"github.com/sashabaranov/go-openai"
)
// OpenAIAdapter implements TranscriptionAdapter for OpenAI Whisper API
type OpenAIAdapter struct {
client *openai.Client
config Config
}
func NewOpenAIAdapter(config Config) *OpenAIAdapter {
client := openai.NewClient(config.APIKey)
return &OpenAIAdapter{
client: client,
config: config,
}
}
func (a *OpenAIAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) {
if len(audioData) == 0 {
return "", nil
}
// Convert raw PCM to WAV format
wavData, err := a.convertToWAV(audioData)
if err != nil {
return "", fmt.Errorf("convert to WAV: %w", err)
}
// Create transcription request
req := openai.AudioRequest{
Model: a.config.Model,
Reader: bytes.NewReader(wavData),
FilePath: "audio.wav",
Language: a.config.Language,
}
start := time.Now()
resp, err := a.client.CreateTranscription(ctx, req)
duration := time.Since(start)
if err != nil {
log.Printf("openai-adapter: API call failed after %v: %v", duration, err)
return "", fmt.Errorf("openai transcription: %w", err)
}
log.Printf("openai-adapter: transcribed %d bytes in %v: %q", len(audioData), duration, resp.Text)
return resp.Text, nil
}
// convertToWAV converts raw 16-bit PCM audio to WAV format
func (a *OpenAIAdapter) convertToWAV(rawAudio []byte) ([]byte, error) {
var buf bytes.Buffer
const sampleRate = 16000
const channels = 1
const bitsPerSample = 16
const byteRate = sampleRate * channels * bitsPerSample / 8
const blockAlign = channels * bitsPerSample / 8
dataSize := len(rawAudio)
fileSize := 36 + dataSize
// WAV header
buf.WriteString("RIFF")
binary.Write(&buf, binary.LittleEndian, uint32(fileSize))
buf.WriteString("WAVE")
// fmt chunk
buf.WriteString("fmt ")
binary.Write(&buf, binary.LittleEndian, uint32(16)) // fmt chunk size
binary.Write(&buf, binary.LittleEndian, uint16(1)) // PCM format
binary.Write(&buf, binary.LittleEndian, uint16(channels)) // number of channels
binary.Write(&buf, binary.LittleEndian, uint32(sampleRate)) // sample rate
binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) // byte rate
binary.Write(&buf, binary.LittleEndian, uint16(blockAlign)) // block align
binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample)) // bits per sample
// data chunk
buf.WriteString("data")
binary.Write(&buf, binary.LittleEndian, uint32(dataSize))
buf.Write(rawAudio)
return buf.Bytes(), nil
}
internal/transcriber/adapter_whispercpp.go
+31
View File
@@ -0,0 +1,31 @@
package transcriber
import (
"context"
"fmt"
)
// WhisperCppAdapter implements TranscriptionAdapter for local whisper.cpp
type WhisperCppAdapter struct {
config Config
modelPath string
}
func NewWhisperCppAdapter(config Config) *WhisperCppAdapter {
return &WhisperCppAdapter{
config: config,
// TODO: Configure model path based on config
modelPath: "./models/ggml-base.en.bin",
}
}
func (a *WhisperCppAdapter) Transcribe(ctx context.Context, audioData []byte) (string, error) {
// TODO: Implement whisper.cpp transcription
// This will involve:
// 1. Writing audio data to a temporary file or pipe
// 2. Calling whisper.cpp binary with appropriate flags
// 3. Reading the transcription result
// 4. Cleaning up temporary files
return "", fmt.Errorf("whisper.cpp adapter not implemented yet")
}
internal/transcriber/openai.go
-304
View File
@@ -1,304 +0,0 @@
package transcriber
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"log"
"strings"
"sync"
"time"
"github.com/leonardotrapani/hyprvoice/internal/recording"
"github.com/sashabaranov/go-openai"
)
type OpenAITranscriber struct {
client *openai.Client
config Config
buffer *audioBuffer
cancel context.CancelFunc
wg sync.WaitGroup
mu sync.Mutex
transcribing bool
transcriptionMu sync.RWMutex
transcriptionText strings.Builder
}
type audioBuffer struct {
data []byte
mu sync.Mutex
lastAdd time.Time
maxSize int
}
func NewOpenAITranscriber(config Config) *OpenAITranscriber {
client := openai.NewClient(config.APIKey)
buffer := &audioBuffer{
data: make([]byte, 0, config.ChunkSize*2),
maxSize: config.ChunkSize,
}
return &OpenAITranscriber{
client: client,
config: config,
buffer: buffer,
}
}
func (t *OpenAITranscriber) Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) {
t.mu.Lock()
if t.transcribing {
t.mu.Unlock()
return nil, fmt.Errorf("transcriber: already transcribing")
}
t.transcribing = true
t.mu.Unlock()
transcribeCtx, cancel := context.WithCancel(ctx)
t.cancel = cancel
errCh := make(chan error, 1)
t.wg.Add(1)
go t.processFrames(transcribeCtx, frameCh, errCh)
return errCh, nil
}
func (t *OpenAITranscriber) Stop() error {
t.mu.Lock()
if !t.transcribing {
t.mu.Unlock()
return nil
}
cancel := t.cancel
t.mu.Unlock()
if cancel != nil {
cancel()
}
t.wg.Wait()
// After stopping, ensure any remaining buffered audio is transcribed
if t.buffer.hasData() {
log.Printf("transcriber: processing remaining buffered audio on stop")
ctx := context.Background()
errCh := make(chan error, 1)
t.transcribeBuffer(ctx, errCh)
close(errCh)
}
return nil
}
func (t *OpenAITranscriber) GetTranscription() (string, error) {
t.transcriptionMu.RLock()
defer t.transcriptionMu.RUnlock()
if t.transcriptionText.Len() == 0 {
return "", nil
}
return t.transcriptionText.String(), nil
}
func (t *OpenAITranscriber) processFrames(ctx context.Context, frameCh <-chan recording.AudioFrame, errCh chan<- error) {
defer func() {
// Always process remaining buffer when shutting down
if t.buffer.hasData() {
log.Printf("transcriber: processing final buffered audio on shutdown")
// Use background context for final transcription to avoid timeout
finalCtx := context.Background()
t.transcribeBuffer(finalCtx, errCh)
}
close(errCh)
t.mu.Lock()
t.transcribing = false
t.cancel = nil
t.mu.Unlock()
t.wg.Done()
}()
ticker := time.NewTicker(t.config.BufferTime)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
log.Printf("transcriber: context cancelled, processing remaining frames")
// Continue processing remaining frames from channel before stopping
for {
select {
case frame, ok := <-frameCh:
if !ok {
log.Printf("transcriber: recording channel closed")
return
}
t.buffer.addFrame(frame)
default:
// No more frames available, exit
return
}
}
case frame, ok := <-frameCh:
if !ok {
log.Printf("transcriber: recording channel closed, finishing with remaining buffer")
return
}
t.buffer.addFrame(frame)
case <-ticker.C:
if t.buffer.shouldFlush(t.config.BufferTime) {
t.transcribeBuffer(ctx, errCh)
}
}
}
}
func (t *OpenAITranscriber) transcribeBuffer(ctx context.Context, errCh chan<- error) {
audioData := t.buffer.flush()
if len(audioData) == 0 {
return
}
log.Printf("transcriber: sending %d bytes to OpenAI API", len(audioData))
wavData, err := t.convertToWAV(audioData)
if err != nil {
log.Printf("transcriber: failed to convert audio to WAV: %v", err)
select {
case errCh <- fmt.Errorf("transcriber: convert to WAV: %w", err):
default:
}
return
}
req := openai.AudioRequest{
Model: t.config.Model,
Reader: bytes.NewReader(wavData),
FilePath: "audio.wav",
Language: t.config.Language,
}
start := time.Now()
resp, err := t.client.CreateTranscription(ctx, req)
duration := time.Since(start)
if err != nil {
log.Printf("transcriber: API call failed after %v: %v", duration, err)
select {
case errCh <- fmt.Errorf("transcriber: transcription failed: %w", err):
default:
}
return
}
if resp.Text != "" {
log.Printf("transcriber: received result in %v: %q", duration, resp.Text)
t.transcriptionMu.Lock()
if t.transcriptionText.Len() > 0 {
t.transcriptionText.WriteString(" ")
}
t.transcriptionText.WriteString(strings.TrimSpace(resp.Text))
t.transcriptionMu.Unlock()
} else {
log.Printf("transcriber: received empty result after %v", duration)
}
}
func (t *OpenAITranscriber) convertToWAV(rawAudio []byte) ([]byte, error) {
var buf bytes.Buffer
const sampleRate = 16000
const channels = 1
const bitsPerSample = 16
const byteRate = sampleRate * channels * bitsPerSample / 8
const blockAlign = channels * bitsPerSample / 8
dataSize := len(rawAudio)
fileSize := 36 + dataSize
buf.WriteString("RIFF")
binary.Write(&buf, binary.LittleEndian, uint32(fileSize))
buf.WriteString("WAVE")
buf.WriteString("fmt ")
binary.Write(&buf, binary.LittleEndian, uint32(16))
binary.Write(&buf, binary.LittleEndian, uint16(1))
binary.Write(&buf, binary.LittleEndian, uint16(channels))
binary.Write(&buf, binary.LittleEndian, uint32(sampleRate))
binary.Write(&buf, binary.LittleEndian, uint32(byteRate))
binary.Write(&buf, binary.LittleEndian, uint16(blockAlign))
binary.Write(&buf, binary.LittleEndian, uint16(bitsPerSample))
buf.WriteString("data")
binary.Write(&buf, binary.LittleEndian, uint32(dataSize))
buf.Write(rawAudio)
return buf.Bytes(), nil
}
func (b *audioBuffer) addFrame(frame recording.AudioFrame) {
b.mu.Lock()
defer b.mu.Unlock()
b.data = append(b.data, frame.Data...)
b.lastAdd = frame.Timestamp
if len(b.data) > b.maxSize*2 {
b.data = b.data[len(b.data)-b.maxSize:]
}
}
func (b *audioBuffer) flush() []byte {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.data) == 0 {
return nil
}
result := make([]byte, len(b.data))
copy(result, b.data)
b.data = b.data[:0]
return result
}
func (b *audioBuffer) hasData() bool {
b.mu.Lock()
defer b.mu.Unlock()
return len(b.data) > 0
}
func (b *audioBuffer) shouldFlush(bufferTime time.Duration) bool {
b.mu.Lock()
defer b.mu.Unlock()
if len(b.data) == 0 {
return false
}
if len(b.data) >= b.maxSize {
return true
}
return time.Since(b.lastAdd) >= bufferTime
}
func NewTranscriber(config Config) (Transcriber, error) {
switch config.Provider {
case "openai":
if config.APIKey == "" {
return nil, fmt.Errorf("OpenAI API key required")
}
return NewOpenAITranscriber(config), nil
default:
return nil, fmt.Errorf("unsupported provider: %s", config.Provider)
}
}
internal/transcriber/simple_transcriber.go
+122
View File
@@ -0,0 +1,122 @@
package transcriber
import (
"context"
"fmt"
"log"
"sync"
"github.com/leonardotrapani/hyprvoice/internal/recording"
)
// SimpleTranscriber collects all audio and transcribes when stopped
type SimpleTranscriber struct {
adapter TranscriptionAdapter
config Config
// Audio collection
audioBuffer []byte
bufferMu sync.Mutex
// Control
running bool
wg sync.WaitGroup
// Transcription result
transcriptionMu sync.RWMutex
transcriptionText string
}
func NewSimpleTranscriber(config Config, adapter TranscriptionAdapter) *SimpleTranscriber {
return &SimpleTranscriber{
adapter: adapter,
config: config,
}
}
func (t *SimpleTranscriber) Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) {
if t.running {
return nil, fmt.Errorf("transcriber already running")
}
t.running = true
errCh := make(chan error, 1)
t.wg.Add(1)
go t.collectAudio(ctx, frameCh, errCh)
return errCh, nil
}
func (t *SimpleTranscriber) Stop(ctx context.Context) error {
if !t.running {
return nil
}
t.wg.Wait()
t.running = false
// Transcribe all collected audio using the passed context
return t.transcribeAll(ctx)
}
func (t *SimpleTranscriber) GetTranscription() (string, error) {
t.transcriptionMu.RLock()
defer t.transcriptionMu.RUnlock()
return t.transcriptionText, nil
}
func (t *SimpleTranscriber) collectAudio(ctx context.Context, frameCh <-chan recording.AudioFrame, errCh chan<- error) {
defer func() {
close(errCh)
t.wg.Done()
}()
for {
select {
case <-ctx.Done():
log.Printf("transcriber: stopping audio collection")
return
case frame, ok := <-frameCh:
if !ok {
log.Printf("transcriber: audio channel closed")
return
}
t.bufferMu.Lock()
t.audioBuffer = append(t.audioBuffer, frame.Data...)
t.bufferMu.Unlock()
}
}
}
func (t *SimpleTranscriber) transcribeAll(ctx context.Context) error {
t.bufferMu.Lock()
audioData := make([]byte, len(t.audioBuffer))
copy(audioData, t.audioBuffer)
t.bufferMu.Unlock()
if len(audioData) == 0 {
log.Printf("transcriber: no audio data to transcribe")
return nil
}
log.Printf("transcriber: transcribing %d bytes of audio", len(audioData))
// Use the context passed from the pipeline for proper cancellation chain
text, err := t.adapter.Transcribe(ctx, audioData)
if err != nil {
log.Printf("transcriber: transcription failed: %v", err)
return fmt.Errorf("transcription failed: %w", err)
}
log.Printf("transcriber: transcription completed: %q", text)
t.transcriptionMu.Lock()
t.transcriptionText = text
t.transcriptionMu.Unlock()
return nil
}
internal/transcriber/transcriber.go
+34 -12
View File
@@ -2,29 +2,28 @@ package transcriber
import ( import (
"context" "context"
"time" "fmt"
"github.com/leonardotrapani/hyprvoice/internal/recording" "github.com/leonardotrapani/hyprvoice/internal/recording"
) )
type TranscriptionResult struct { // Main transcriber interface
Text string
Timestamp time.Time
IsFinal bool
}
type Transcriber interface { type Transcriber interface {
Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error) Start(ctx context.Context, frameCh <-chan recording.AudioFrame) (<-chan error, error)
Stop() error Stop(ctx context.Context) error
GetTranscription() (string, error) GetTranscription() (string, error)
} }
// Adapter interface for different transcription backends
type TranscriptionAdapter interface {
Transcribe(ctx context.Context, audioData []byte) (string, error)
}
// Configuration for the transcriber
type Config struct { type Config struct {
Provider string Provider string
APIKey string APIKey string
Language string Language string
ChunkSize int
BufferTime time.Duration
Model string Model string
} }
@@ -32,8 +31,31 @@ func DefaultConfig() Config {
return Config{ return Config{
Provider: "openai", Provider: "openai",
Language: "it", Language: "it",
ChunkSize: 16384,
BufferTime: 2 * time.Second,
Model: "whisper-1", Model: "whisper-1",
} }
} }
// NewTranscriber creates a new simple transcriber
func NewTranscriber(config Config) (Transcriber, error) {
// Create the appropriate adapter
var adapter TranscriptionAdapter
switch config.Provider {
case "openai":
if config.APIKey == "" {
return nil, fmt.Errorf("OpenAI API key required")
}
adapter = NewOpenAIAdapter(config)
case "whisper.cpp":
adapter = NewWhisperCppAdapter(config)
default:
return nil, fmt.Errorf("unsupported provider: %s", config.Provider)
}
// Create simple transcriber that collects all audio
transcriber := NewSimpleTranscriber(config, adapter)
return transcriber, nil
}