diff --git a/README.md b/README.md index 4ed0a46..5bb0da6 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Press a toggle key, speak, and get instant text input. Built natively for Waylan ## Features - **Toggle workflow**: Press once to start recording, press again to stop and inject text +- **LLM post-processing**: Automatically cleans up transcriptions - removes stutters, fixes grammar, adds punctuation (enabled by default) - **Wayland native**: Purpose-built for Wayland compositors - no legacy X11 dependencies or hacky workarounds - **Real-time feedback**: Desktop notifications for recording states and transcription status - **Multiple transcription backends**: OpenAI Whisper, Groq, Mistral Voxtral, and Eleven Labs Scribe (99 languages, excellent accuracy) @@ -210,14 +211,39 @@ hyprvoice configure This will guide you through setting up: -- OpenAI API key for transcription -- Language preferences (auto-detect or specific language) +- Provider API keys (OpenAI, Groq, Mistral, ElevenLabs) +- Transcription provider and model +- LLM post-processing options (enabled by default) +- Keywords for domain-specific terms - Text injection method (clipboard/typing/fallback) - Notification settings -- Recording timeout Configuration is stored in `~/.config/hyprvoice/config.toml` and can also be edited manually. Changes are applied immediately without restarting the daemon. +### Unified Provider System + +Hyprvoice uses a unified provider system where API keys are configured once and shared between transcription and LLM features: + +```toml +# Configure API keys for providers you want to use +[providers.openai] + api_key = "sk-..." # Or set OPENAI_API_KEY env var + +[providers.groq] + api_key = "gsk_..." # Or set GROQ_API_KEY env var + +[providers.mistral] + api_key = "..." # Or set MISTRAL_API_KEY env var + +[providers.elevenlabs] + api_key = "..." # Or set ELEVENLABS_API_KEY env var +``` + +**API key resolution order:** +1. `[providers.X]` section in config +2. Legacy `transcription.api_key` (backward compatible) +3. Environment variable (`OPENAI_API_KEY`, `GROQ_API_KEY`, etc.) + ### Transcription Providers Hyprvoice supports multiple transcription backends: @@ -229,7 +255,6 @@ Cloud-based transcription using OpenAI's Whisper API: ```toml [transcription] provider = "openai" -api_key = "sk-..." # Or set OPENAI_API_KEY environment variable language = "" # Empty for auto-detect, or "en", "es", "fr", etc. model = "whisper-1" ``` @@ -246,7 +271,6 @@ Fast cloud-based transcription using Groq's Whisper API: ```toml [transcription] provider = "groq-transcription" -api_key = "gsk_..." # Or set GROQ_API_KEY environment variable language = "" # Empty for auto-detect, or "en", "es", "fr", etc. model = "whisper-large-v3" # Or "whisper-large-v3-turbo" for faster processing ``` @@ -264,7 +288,6 @@ Fast translation of audio to English using Groq's Whisper API: ```toml [transcription] provider = "groq-translation" -api_key = "gsk_..." # Or set GROQ_API_KEY environment variable language = "es" # Optional: hint source language for better accuracy model = "whisper-large-v3-turbo" ``` @@ -275,45 +298,175 @@ model = "whisper-large-v3-turbo" - Language field hints at source language (improves accuracy) - Always outputs English regardless of input language -#### Generated Configuration Example +### LLM Post-Processing -The daemon automatically creates `~/.config/hyprvoice/config.toml` with helpful comments: +LLM post-processing is **enabled by default** and significantly improves transcription quality. After transcription, the text is processed by an LLM to: + +- Remove stutters and repeated words ("I I I want" → "I want") +- Add proper punctuation +- Fix grammar errors +- Remove filler words ("um", "uh", "like", "you know", etc.) + +#### Basic Configuration ```toml -# Hyprvoice Configuration -# This file is automatically generated with defaults. -# Edit values as needed - changes are applied immediately without daemon restart. - -# Audio Recording Configuration -[recording] - sample_rate = 16000 # Audio sample rate in Hz (16000 recommended for speech) - channels = 1 # Number of audio channels (1 = mono, 2 = stereo) - format = "s16" # Audio format (s16 = 16-bit signed integers) - buffer_size = 8192 # Internal buffer size in bytes (larger = less CPU, more latency) - device = "" # PipeWire audio device (empty = use default microphone) - channel_buffer_size = 30 # Audio frame buffer size (frames to buffer) - timeout = "5m" # Maximum recording duration (e.g., "30s", "2m", "5m") - -# Speech Transcription Configuration -[transcription] - provider = "openai" # Transcription service: "openai", "groq-transcription", or "groq-translation" - api_key = "" # API key (or set OPENAI_API_KEY/GROQ_API_KEY environment variable) - language = "" # Language code (empty for auto-detect, "en", "it", "es", "fr", etc.) - model = "whisper-1" # Model: OpenAI="whisper-1", Groq="whisper-large-v3" or "whisper-large-v3-turbo" - -# Text Injection Configuration -[injection] - backends = ["ydotool", "wtype", "clipboard"] # Ordered fallback chain - ydotool_timeout = "5s" # Timeout for ydotool commands - wtype_timeout = "5s" # Timeout for wtype commands - clipboard_timeout = "3s" # Timeout for clipboard operations - -# Desktop Notification Configuration -[notifications] - enabled = true # Enable desktop notifications - type = "desktop" # Notification type ("desktop", "log", "none") -- always keep "desktop" unless debugging +[llm] + enabled = true # Disable with false if you want raw transcriptions + provider = "openai" # "openai" or "groq" + model = "gpt-4o-mini" # OpenAI: "gpt-4o-mini", Groq: "llama-3.3-70b-versatile" ``` +#### Post-Processing Options + +All options are enabled by default. Disable specific ones as needed: + +```toml +[llm.post_processing] + remove_stutters = true # "I I I want" → "I want" + add_punctuation = true # Adds periods, commas, etc. + fix_grammar = true # Fixes grammatical errors + remove_filler_words = true # Removes "um", "uh", "like", "you know" +``` + +#### Custom Prompts + +Add custom instructions for specific use cases: + +```toml +[llm.custom_prompt] + enabled = true + prompt = "Format as bullet points" +``` + +**Use cases for custom prompts:** +- "Format as bullet points" - for note-taking +- "Keep technical terms exactly as spoken" - for programming dictation +- "Use formal language" - for professional documents +- "Translate to Spanish" - for translation workflows + +#### LLM Provider Recommendations + +| Provider | Model | Best For | +| -------- | ----- | -------- | +| OpenAI | gpt-4o-mini | Best quality/cost balance (default) | +| Groq | llama-3.3-70b-versatile | Fastest processing, free tier | + +Both providers use the same API key as transcription if you're using OpenAI or Groq for transcription. + +### Keywords + +Keywords help both transcription and LLM understand domain-specific terms, names, and technical vocabulary: + +```toml +keywords = ["Hyprland", "Wayland", "PipeWire", "Claude", "TypeScript"] +``` + +**How keywords work:** +- **Transcription**: Passed as initial_prompt to Whisper, improving recognition of these terms +- **LLM**: Included in the system prompt to ensure correct spelling + +**When to use keywords:** +- Names of people, companies, or products +- Technical terminology specific to your field +- Acronyms or abbreviations +- Words commonly misheard by speech-to-text + +### Example Configurations + +#### Fast Transcription Only (No LLM) + +```toml +[providers.groq] + api_key = "gsk_..." + +[transcription] + provider = "groq-transcription" + model = "whisper-large-v3-turbo" + +[llm] + enabled = false +``` + +#### High Quality with OpenAI (Default) + +```toml +[providers.openai] + api_key = "sk-..." + +[transcription] + provider = "openai" + model = "whisper-1" + +[llm] + enabled = true + provider = "openai" + model = "gpt-4o-mini" +``` + +#### Budget-Friendly with Groq + +```toml +[providers.groq] + api_key = "gsk_..." + +[transcription] + provider = "groq-transcription" + model = "whisper-large-v3-turbo" + +[llm] + enabled = true + provider = "groq" + model = "llama-3.3-70b-versatile" +``` + +#### Mixed Providers (Groq Transcription + OpenAI LLM) + +```toml +[providers.openai] + api_key = "sk-..." + +[providers.groq] + api_key = "gsk_..." + +[transcription] + provider = "groq-transcription" + model = "whisper-large-v3-turbo" + +[llm] + enabled = true + provider = "openai" + model = "gpt-4o-mini" +``` + +### Migration from Old Config Format + +If you're upgrading from an older version with `transcription.api_key`: + +**Old format (still works):** +```toml +[transcription] + provider = "openai" + api_key = "sk-..." # Legacy location + model = "whisper-1" +``` + +**New format (recommended):** +```toml +[providers.openai] + api_key = "sk-..." # Unified location + +[transcription] + provider = "openai" + model = "whisper-1" + +[llm] + enabled = true + provider = "openai" + model = "gpt-4o-mini" +``` + +Run `hyprvoice configure` to interactively update your config to the new format. + #### whisper.cpp Local (Planned) -> Not yet implemented Private, offline transcription using local models: @@ -436,6 +589,9 @@ You can customize notification text via the `[notifications.messages]` section. [notifications.messages.transcribing] title = "Hyprvoice" body = "Recording Ended... Transcribing" + [notifications.messages.llm_processing] + title = "Hyprvoice" + body = "Processing..." [notifications.messages.config_reloaded] title = "Hyprvoice" body = "Config Reloaded" @@ -454,7 +610,7 @@ The daemon automatically watches the config file for changes and applies them im - **Notification settings**: Applied instantly - **Injection settings**: Applied to current and future operations -- **Recording/Transcription settings**: Applied to new recording sessions +- **Recording/Transcription/LLM settings**: Applied to new recording sessions - **Invalid configs**: Rejected with error notification, daemon continues with previous config ### Service Management @@ -493,9 +649,12 @@ journalctl --user -u hyprvoice.service -f | Desktop notifications | ✅ | Status feedback via notify-send | | OpenAI transcription | ✅ | HTTP API integration | | Groq transcription | ✅ | Fast Whisper API with transcription and translation | -| Text injection | ✅ | Clipboard + wtype with fallback | +| Mistral transcription | ✅ | Voxtral API for European languages | +| ElevenLabs transcription| ✅ | Scribe API with 99 language support | +| LLM post-processing | ✅ | OpenAI/Groq text cleanup (enabled by default) | +| Text injection | ✅ | Clipboard + wtype/ydotool with fallback | | Configuration system | ✅ | TOML-based user settings with hot-reload | -| Interactive setup | ✅ | `hyprvoice configure` wizard for easy setup | +| Interactive TUI setup | ✅ | `hyprvoice configure` wizard with section editing | | Unit test coverage | ✅ | Comprehensive test suite (100% pass) | | CI/CD Pipeline | ✅ | Automated builds and releases via GitHub Actions | | Installation (AUR etc) | ✅ | AUR package with automated dependency installation | @@ -509,8 +668,8 @@ journalctl --user -u hyprvoice.service -f Hyprvoice uses a **daemon + pipeline** architecture for efficient resource management: - **Control Daemon**: Lightweight IPC server managing lifecycle -- **Pipeline**: Stateful audio processing (recording → transcribing → injecting) -- **State Machine**: `idle → recording → transcribing → injecting → idle` +- **Pipeline**: Stateful audio processing (recording → transcribing → processing → injecting) +- **State Machine**: `idle → recording → transcribing → processing → injecting → idle` ### System Architecture @@ -544,7 +703,9 @@ stateDiagram-v2 [*] --> idle idle --> recording: toggle recording --> transcribing: first_frame - transcribing --> injecting: inject_action + transcribing --> processing: llm_enabled + transcribing --> injecting: llm_disabled + processing --> injecting: inject_action injecting --> idle: done recording --> idle: abort injecting --> idle: abort @@ -555,17 +716,20 @@ stateDiagram-v2 1. **Toggle recording** → Pipeline starts, audio capture begins 2. **Audio streaming** → PipeWire frames buffered for transcription 3. **Toggle stop** → Recording ends, transcription starts -4. **Text injection** → Result typed or copied to clipboard -5. **Return to idle** → Pipeline cleaned up, ready for next session +4. **LLM processing** → Text cleaned up (if enabled, which is the default) +5. **Text injection** → Result typed or copied to clipboard +6. **Return to idle** → Pipeline cleaned up, ready for next session ### Data Flow 1. `toggle` (daemon) → create pipeline → recording 2. First frame arrives → transcribing (daemon may notify `Transcribing` later) 3. Audio frames → audio buffer (collect all audio during session) -4. Second `toggle` during transcribing → send `inject` action → transcribe collected audio → injecting (simulated) -5. Complete → idle; pipeline stops; daemon clears reference -6. Notifications at key transitions +4. Second `toggle` during transcribing → transcribe collected audio +5. If LLM enabled → processing → clean up text with LLM +6. injecting → type or paste text +7. Complete → idle; pipeline stops; daemon clears reference +8. Notifications at key transitions ## Troubleshooting @@ -717,12 +881,16 @@ hyprvoice/ ├── cmd/hyprvoice/ # CLI application entry point ├── internal/ │ ├── bus/ # IPC (Unix socket) + PID management +│ ├── config/ # Configuration loading and validation │ ├── daemon/ # Control daemon (lifecycle management) -│ ├── injection/ # Text injection (clipboard + wtype) +│ ├── injection/ # Text injection (clipboard + wtype + ydotool) +│ ├── llm/ # LLM post-processing adapters (OpenAI, Groq) │ ├── notify/ # Desktop notification integration │ ├── pipeline/ # Audio processing pipeline + state machine +│ ├── provider/ # Provider registry and capability detection │ ├── recording/ # PipeWire audio capture -│ └── transcriber/ # Transcription adapters (OpenAI, whisper.cpp) +│ ├── transcriber/ # Transcription adapters (OpenAI, Groq, Mistral, ElevenLabs) +│ └── tui/ # Interactive configuration wizard ├── go.mod # Go module definition └── README.md ``` diff --git a/progress.txt b/progress.txt index 105bbc4..99fa62a 100644 --- a/progress.txt +++ b/progress.txt @@ -205,3 +205,26 @@ Key decisions: - Notification channel approach (vs direct notifier access) keeps pipeline decoupled - Notification sent at same time status changes to Processing - Configurable like all other notifications via `[notifications.messages.llm_processing]` + +## Task 12: Update README documentation - COMPLETE + +Updated README.md with comprehensive LLM post-processing documentation: +- Added LLM feature to Features list at top +- Added "Unified Provider System" section with API key configuration examples +- Added "LLM Post-Processing" section with full configuration guide +- Added post-processing options documentation (remove_stutters, add_punctuation, etc.) +- Added custom prompt documentation with use cases +- Added "Keywords" section explaining how they help transcription + LLM +- Added 4 example configurations: fast transcription only, high quality, budget-friendly, mixed providers +- Added "Migration from Old Config Format" section with before/after examples +- Updated Development Status table: added Mistral, ElevenLabs, LLM post-processing, TUI setup +- Updated architecture diagrams to show processing state +- Updated state machine description: idle → recording → transcribing → processing → injecting +- Updated project structure to include new packages (config, llm, provider, tui) +- Added llm_processing to custom notification messages example + +Key decisions: +- Put Unified Provider System before Transcription Providers (sets context) +- LLM section after transcription providers (logical flow) +- Example configs ordered by use case (fast → quality → budget → mixed) +- Migration section shows both old and new format side by side diff --git a/tasks/prd.jsonc b/tasks/prd.jsonc index 8a80751..af3498a 100644 --- a/tasks/prd.jsonc +++ b/tasks/prd.jsonc @@ -236,7 +236,7 @@ "Migration documented", "Keywords explained" ], - "passes": false + "passes": true }, { "title": "End-to-end testing",