commit 5d680928b9d4975a378559b5951c59f42228a839 Author: Bryan Gilliom Date: Mon Jul 27 00:33:25 2026 +0800 feat: add CW-026 Local Voice plugin diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..aefe164 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,17 @@ +{ + "name": "local-voice", + "version": "0.1.0", + "description": "Generate validated local speech with reusable CosyVoice profiles.", + "author": { + "name": "Bryan Gilliom / Message Point Media" + }, + "repository": "https://git.mpm.to/mpm/local-voice", + "license": "Apache-2.0", + "keywords": [ + "tts", + "voice", + "cosyvoice", + "briefing", + "local-ai" + ] +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..6e18dfd --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,26 @@ +{ + "name": "local-voice", + "version": "0.1.0", + "description": "Generate validated local speech with reusable CosyVoice profiles.", + "author": { + "name": "Bryan Gilliom / Message Point Media" + }, + "skills": "./skills/", + "interface": { + "displayName": "Local Voice", + "shortDescription": "Generate local briefing and notification audio.", + "longDescription": "Local Voice plans, renders, validates, and assembles reusable CosyVoice profiles on local hardware.", + "developerName": "Message Point Media", + "category": "Productivity", + "capabilities": [ + "local-text-to-speech", + "voice-profiles", + "audio-validation" + ], + "defaultPrompt": [ + "Generate this script with Local Voice and validate the finished audio.", + "List the locally installed voice profiles.", + "Diagnose my Local Voice runtime." + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..39690f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +.DS_Store +__pycache__/ +*.pyc +.venv/ +runtime/ +generated/ +outputs/ +models/ +voice-assets/ +*.wav +*.mp3 +*.m4a +*.flac +*.onnx +*.safetensors +*.plugin +*.zip diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..28175da --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +## 0.1.0 — 2026-07-27 + +- Added a shared Claude CoWork and Codex Local Voice skill. +- Added unified planning, Metal/CPU/CUDA rendering, Whisper alignment, cadence + correction, resumable assembly, fixed assets, and transcript QA. +- Added Donna, Chris Engineer, Grandpa Bomber, Ryan Pitch Meeting, and Val + Holiday profiles. +- Added verified macOS installation and recovery instructions. +- Added provisional Windows/NVIDIA deployment guidance. +- Added Donna dependency contract and fresh acceptance cases. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e7992cd --- /dev/null +++ b/LICENSE @@ -0,0 +1,17 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +Copyright 2026 Message Point Media + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3a2c85b --- /dev/null +++ b/README.md @@ -0,0 +1,120 @@ +# Local Voice + +Local, validated CosyVoice3 speech generation for reusable MPM briefing and +character profiles. + +**Version:** 0.1.0 +**Author:** Bryan Gilliom / Message Point Media +**Repository:** https://git.mpm.to/mpm/local-voice +**CoWork Project:** CW-026 — Local Voice + +## Overview + +Local Voice is a shared audio dependency for Claude CoWork, Codex, and other +local automation. It converts a canonical script or structured two-role +dialogue into WAV or MP3, retaining raw generations and Whisper alignment data +so failed seams can be repaired without regenerating successful speech. Voice +reference audio is installed separately from the public plugin and remains in +the private CW-026 recovery package. + +The verified production configuration is Apple Silicon Metal with the +CosyVoice3 Candle model. A provisional NVIDIA/CUDA path is documented for the +planned Windows host but is not yet certified. + +## Skill + +| Skill | What it does | +|---|---| +| `local-voice` | Plans, renders, resumes, aligns, validates, and assembles local speech when a user asks for a voice brief, local TTS message, spoken notification, character dialogue, CosyVoice render, or MP3 briefing. | + +## Commands + +| Command | Purpose | +|---|---| +| `list` | List bundled voice profiles and their roles. | +| `doctor` | Verify Python, audio tools, model files, and authorized assets. | +| `plan` | Convert simple Markdown or Ryan-labelled dialogue into a render plan. | +| `validate` | Validate a hand-authored render plan without generating audio. | +| `render` | Generate, align, process, assemble, and quality-check audio. | + +`render --resume` reuses completed raw generations. `render --assemble-only` +repairs trimming or seams without rerunning the model. + +## Supported profiles + +- Donna +- Chris Engineer +- Grandpa Bomber +- Ryan Pitch Meeting: Producer and Writer roles +- Val Holiday + +See `skills/local-voice/references/voice-catalog.md` for the production rules +that distinguish these profiles. + +## Setup + +1. Follow `docs/INSTALL_MACOS.md` on the verified Apple Silicon path. +2. Install the private `local-voice-authorized-assets` archive from CW-026. +3. Download the exact Candle model into the runtime `models` directory. +4. Run `scripts/verify_install.py`. +5. Install the Claude or Codex plugin package. +6. Start a new task so the host discovers the skill. + +The private recovery package and public plugin are intentionally separate: +the public repository contains redistributable software, while the private +Drive folder contains the authorized voice configuration. + +## Calling Local Voice + +```bash +python3 scripts/local_voice.py plan \ + --voice donna \ + --script /absolute/path/brief.md \ + --output-plan /absolute/path/brief-plan.json \ + --output-audio /absolute/path/brief.mp3 + +python3 scripts/local_voice.py render \ + /absolute/path/brief-plan.json \ + --resume +``` + +Set `LOCAL_VOICE_RUNTIME` when the runtime is not installed at the +platform-default location. + +## Requirements + +- Python 3.11 recommended +- CosyVoice3 0.1.0 Candle build +- CosyVoice3-0.5B-Candle model +- ffmpeg +- Whisper CLI with word timestamps +- NumPy and SoundFile +- Authorized reference audio and transcripts + +## Operational rules + +- Keep canonical `text` separate from pronunciation-safe `tts_text`. +- Generate the largest safe complete passages; paragraph breaks are candidates, + not mandatory cuts. +- Do not normalize tempo on short greetings, closings, or reactions. +- Preserve raw WAV, Whisper JSON, processed WAV, and QA reports. +- Use fixed assets for exact Ryan signatures and very short reactions. +- Do not silently fall back to a cloud provider. + +## Troubleshooting + +| Symptom | Resolution | +|---|---| +| A word is clipped | Retain more natural tail or add neighboring sacrificial context, then rerun with `--assemble-only` when possible. | +| A seam clicks | Move the join to a quiet boundary or add a natural pause; do not regenerate speech that already passed. | +| A short reaction sounds generic | Use an authorized fixed asset from the Ryan signature library. | +| Opening or closing sounds drunk or rushed | Remove tempo adjustment; short frames stay native. | +| Literal alignment fails | Inspect retained Whisper JSON; normalized fuzzy alignment is expected to tolerate ordinary ASR variation. | +| `lead` uses the wrong pronunciation | Keep canonical text and use a local `tts_text` override such as `led` for ammunition. | + +## Security and rights + +Local Voice does not require a service credential. Reference audio must only be +installed or used when the operator has authorization. The public repository +must never contain private voice assets, source recordings, generated briefings, +or model binaries. diff --git a/docs/ACCEPTANCE.md b/docs/ACCEPTANCE.md new file mode 100644 index 0000000..7c95250 --- /dev/null +++ b/docs/ACCEPTANCE.md @@ -0,0 +1,46 @@ +# Acceptance results + +Fresh acceptance audio was generated on 2026-07-27 using Apple Silicon Metal, +the production Candle model, the private authorized assets, and the unified +renderer in Local Voice 0.1.0. + +| Profile | Duration | Transcript coverage | Transcript precision | Click-risk seams | +|---|---:|---:|---:|---:| +| Donna | 17.49 s | 0.978 | 0.957 | 0 | +| Chris Engineer | 14.73 s | 0.960 | 0.923 | 0 | +| Grandpa Bomber | 21.79 s | 0.934 | 0.934 | 0 | +| Val Holiday | 23.45 s | 0.964 | 0.946 | 0 | +| Ryan Pitch Meeting | 16.62 s | 0.963 | 0.981 | 0 | + +## What the suite verifies + +- Every profile and both Ryan roles load from the private runtime. +- Generated targets contain their expected beginning and final-word marker. +- Final transcript coverage is at least 0.60. +- Final transcript precision is at least 0.75, preventing sacrificial context + from silently leaking into the finished audio. +- Every join has a boundary step below the click-risk threshold. +- Short frames remain at native tempo. +- Val normalizes only sufficiently long substantive passages toward 170 WPM. +- Chris uses 30-word safe blocks after a 42-word block proved vulnerable to + model truncation. +- Ryan uses fixed `Yes sir, I do`, `is tight`, and + `super easy, barely an inconvenience / Oh really` assets in the acceptance + dialogue. + +## Human review + +Objective acceptance cannot judge character similarity or comic timing. +The packaged acceptance MP3s are retained for listening review. Future changes +to reference audio, cadence thresholds, trimming, or asset selection should be +compared against these files before release. + +## Latest jobs + +| Profile | Job ID | +|---|---| +| Donna | `acceptance-v2-donna` | +| Chris Engineer | `acceptance-v6-chris-engineer` | +| Grandpa Bomber | `acceptance-v7-grandpa-bomber` | +| Val Holiday | `acceptance-v7-val-holiday` | +| Ryan Pitch Meeting | `acceptance-v9-ryan-pitch-meeting` | diff --git a/docs/DONNA_DEPENDENCY.md b/docs/DONNA_DEPENDENCY.md new file mode 100644 index 0000000..dceea7e --- /dev/null +++ b/docs/DONNA_DEPENDENCY.md @@ -0,0 +1,62 @@ +# Donna integration contract + +Donna remains responsible for collecting organizational information, deciding +priority, selecting a character, and writing the final canonical script. Local +Voice is an optional audio dependency responsible for rendering and validating +that script. + +## Invocation + +Donna supplies: + +- voice profile ID; +- canonical Markdown or Ryan-labelled dialogue; +- absolute output path; +- pronunciation overrides when needed; +- optional delivery, role, signature-asset, and pause metadata. + +Donna then asks Local Voice to: + +1. run `doctor` if runtime health is unknown; +2. build or validate the render plan; +3. render with `--resume`; +4. return the final audio and QA-report paths. + +## Required behavior + +- Do not send Markdown headings, numbered section titles, speaker labels, or + stage directions as spoken text. +- Keep greetings and closings as short standalone paragraphs so Local Voice can + preserve native tempo. +- Keep substantive paragraphs to complete sentence groups of roughly 20–42 + words unless a profile specifies otherwise. +- Use canonical `text` for the written brief and `tts_text` only for local + pronunciation corrections. +- Do not silently fall back to ElevenLabs. If Local Voice fails, Donna should + report the failure and apply the caller's configured fallback policy. +- Attach or link the final audio alongside the full written organizational + brief; the audio is a quick summary, not the sole record. + +## Character mapping + +| Donna character | Local Voice profile | +|---|---| +| Production Donna | `donna` | +| Chris Engineer | `chris-engineer` | +| Grandpa Bomber | `grandpa-bomber` | +| Pitch Meeting | `ryan-pitch-meeting` | +| Val Holiday | `val-holiday` | + +## Example handoff + +```json +{ + "voice": "val-holiday", + "script": "/absolute/path/val-brief.md", + "output": "/absolute/path/val-brief.mp3", + "fallback": "report-and-return-text" +} +``` + +Local Voice returns the final audio path, QA report path, duration, and +segment-level completeness and cadence results. diff --git a/docs/INSTALL_MACOS.md b/docs/INSTALL_MACOS.md new file mode 100644 index 0000000..fe69427 --- /dev/null +++ b/docs/INSTALL_MACOS.md @@ -0,0 +1,132 @@ +# Verified macOS installation + +This procedure recreates the production configuration verified on an Apple +Silicon MacBook Pro. It installs the software and model separately from the +private authorized voice-asset archive. + +## Verified configuration + +- Apple Silicon macOS +- Python 3.11 +- CosyVoice3 `0.1.0+metal` +- 24 kHz Candle model: `spensercai/CosyVoice3-0.5B-Candle` +- ffmpeg and Whisper available on `PATH` +- Metal inference with full-precision weights + +## 1. Install system prerequisites + +Install Homebrew if it is not already present, then install: + +```bash +brew install python@3.11 ffmpeg +``` + +Install the Whisper CLI into an isolated environment or with `pipx`. Confirm: + +```bash +ffmpeg -version +whisper --help +python3.11 --version +``` + +## 2. Create the runtime + +The default runtime is: + +```text +~/Library/Application Support/MPM Local Voice/runtime +``` + +Create its `models`, `voices`, `jobs`, and `wheels` directories. Create a Python +3.11 virtual environment beside or inside the runtime and activate it. + +For a guided installation after downloading the model and private packages: + +```bash +zsh scripts/install_macos.sh \ + --wheel /path/to/cosyvoice3-0.1.0+metal-cp310-abi3-macosx_11_0_arm64.whl \ + --assets /path/to/local-voice-authorized-assets-v0.1.0.tar.gz \ + --model-dir /path/to/CosyVoice3-0.5B-Candle +``` + +## 3. Install the tested Metal wheel + +Copy this file from the private CW-026 recovery package: + +```text +cosyvoice3-0.1.0+metal-cp310-abi3-macosx_11_0_arm64.whl +``` + +Expected SHA-256: + +```text +c9c04352fe0e559b7b43129baae7035b25b48922e2296498efc53c39874d4e39 +``` + +Install the wheel and runtime Python dependencies: + +```bash +python -m pip install ./wheels/cosyvoice3-0.1.0+metal-cp310-abi3-macosx_11_0_arm64.whl +python -m pip install numpy==2.4.6 soundfile==0.14.0 +``` + +## 4. Install the model + +Download `spensercai/CosyVoice3-0.5B-Candle` from Hugging Face into: + +```text +runtime/models/CosyVoice3-0.5B-Candle +``` + +The private recovery manifest contains hashes for the production model files. +At minimum the directory must contain: + +- `llm.safetensors` +- `flow.safetensors` +- `hift.safetensors` +- `campplus.onnx` +- `speech_tokenizer_v3.onnx` +- `config.json` + +The model is approximately 4.7 GB and is not bundled with the public plugin. + +## 5. Install authorized voice assets + +Extract the private archive so the runtime contains: + +```text +runtime/voices/donna +runtime/voices/chris-engineer +runtime/voices/grandpa-bomber +runtime/voices/ryan-pitch-meeting +runtime/voices/val-holiday +``` + +Do not publish or redistribute this archive. + +## 6. Verify + +From the plugin root: + +```bash +python scripts/local_voice.py doctor --device metal +python scripts/verify_install.py --device metal +``` + +Every dependency, model, and voice profile must report `PASS`. + +## 7. Install the plugin + +For Claude CoWork, install `local-voice-v0.1.0.plugin` from the private CW-026 +folder. For Codex, install from the MPM marketplace or use the same repository +source containing `.codex-plugin/plugin.json`. + +Start a new task after installation so the host loads the skill. + +## Restore policy + +The public repository is authoritative for code and documentation. The private +CW-026 folder is authoritative for the tested wheel, authorized voice assets, +checksums, plugin packages, and recovery notes. The model is reproducible from +its exact source and hashes; an optional offline model archive may be added +later. diff --git a/docs/RECOVERY.md b/docs/RECOVERY.md new file mode 100644 index 0000000..2b926f6 --- /dev/null +++ b/docs/RECOVERY.md @@ -0,0 +1,31 @@ +# Recovery and private asset policy + +CW-026 uses a split distribution: + +- **Public Gitea:** redistributable source, profiles, schemas, tests, and docs. +- **Private Google Drive:** authorized voice assets, the tested Metal wheel, + installable plugin packages, checksums, and recovery manifests. +- **External model source:** exact Candle model repository and verified hashes. + +## Private recovery folder + +`CW-026 — Local Voice` + +https://drive.google.com/drive/folders/1bcgkABj-JGavyFHRqSj2gwZqhnPHBAhk + +The private folder is visible only to authorized MPM staff. It is the recovery +source of truth for the machine-specific configuration. + +## Restore sequence + +1. Restore the plugin source or installable package. +2. Recreate the Python 3.11 environment. +3. Install the tested platform wheel. +4. Download and verify the exact model. +5. Extract the authorized voice-assets archive into the runtime. +6. Run `doctor`. +7. Run the acceptance suite. +8. Configure calling skills, such as Donna, to use the Local Voice dependency. + +Never delete the working runtime during recovery preparation. Copy assets into +the private archive and verify the archive before treating it as a backup. diff --git a/docs/WINDOWS_NVIDIA_NOTES.md b/docs/WINDOWS_NVIDIA_NOTES.md new file mode 100644 index 0000000..9bbc5bb --- /dev/null +++ b/docs/WINDOWS_NVIDIA_NOTES.md @@ -0,0 +1,52 @@ +# Windows and NVIDIA deployment notes + +This path is a recommendation for the planned dedicated Windows/NVIDIA host. +It has not yet passed the Local Voice acceptance suite and must be treated as +provisional. + +## Recommended host + +- Windows 11 +- Current NVIDIA Studio or production driver +- Python 3.11 x64 +- ffmpeg and Whisper available on `PATH` +- Sufficient SSD space for the 4.7 GB model, jobs, and retained WAV files +- NVIDIA GPU with supported CUDA capability and practical VRAM headroom + +## Preferred deployment order + +1. Start with native Windows and a CosyVoice3 wheel built for the installed CUDA + runtime. +2. If native dependency resolution is unreliable, use WSL2 with NVIDIA CUDA + passthrough and the Linux CUDA build. +3. Use `LOCAL_VOICE_RUNTIME` to point at a dedicated data directory, for example + `D:\MPM-Local-Voice\runtime`. +4. Copy the same private `voices` archive and model directory used on macOS. +5. Run `doctor --device cuda`. +6. Run the full acceptance suite before scheduling production briefings. + +## Important differences + +- The verified Metal wheel cannot run on Windows. +- CUDA, driver, and wheel versions must agree. +- Do not assume that a CUDA build exists merely because the NVIDIA driver is + installed. +- Keep ffmpeg path quoting and Windows long-path behavior in mind. +- Compare transcript coverage, ending confidence, seam derivatives, duration, + and subjective voice similarity against the macOS acceptance outputs. + +## Fallback + +CPU generation remains functionally possible but may be slower. It is suitable +for overnight batches if CUDA setup is delayed, provided the acceptance suite +passes on that host. + +## Certification checklist + +- [ ] `cosyvoice3`, NumPy, and SoundFile import +- [ ] Candle model hashes match the recovery manifest +- [ ] All authorized voice assets resolve +- [ ] CUDA device loads successfully +- [ ] Every acceptance output passes transcript QA +- [ ] Ryan fixed assets and dialogue assembly are seamless +- [ ] Five production briefs can complete inside the overnight window diff --git a/evals/evals.json b/evals/evals.json new file mode 100644 index 0000000..0e03058 --- /dev/null +++ b/evals/evals.json @@ -0,0 +1,44 @@ +{ + "skill_name": "local-voice", + "evals": [ + { + "id": 1, + "prompt": "Generate this three-paragraph operational update with Donna using local TTS, preserve the short closing at native tempo, and return the MP3 and QA report.", + "expected_output": "A complete Donna MP3 with passing transcript QA, retained raw and alignment artifacts, and no cloud fallback.", + "files": [ + "tests/acceptance/donna.md" + ], + "assertions": [ + "The final transcript QA passes.", + "The closing segment uses native tempo.", + "The response returns both audio and QA report paths." + ] + }, + { + "id": 2, + "prompt": "Render this fast Producer and Writer exchange as a seamless Pitch Meeting dialogue, keeping speaker labels out of the audio.", + "expected_output": "A complete two-role MP3 with both roles, no spoken labels, and passing transcript QA.", + "files": [ + "tests/acceptance/ryan-pitch-meeting.md" + ], + "assertions": [ + "Producer and Writer use their configured role references.", + "Speaker labels are not synthesized.", + "The final transcript QA passes." + ] + }, + { + "id": 3, + "prompt": "Render this Val Holiday brief locally. Keep the greeting and sign-off native, normalize only substantive passages, and do not clip paragraph endings.", + "expected_output": "A complete Val Holiday MP3 with native short frames, 170 WPM long passages, natural tails, and passing QA.", + "files": [ + "tests/acceptance/val-holiday.md" + ], + "assertions": [ + "Short greeting and closing remain at native tempo.", + "Long passages target 170 WPM.", + "Every target ending passes completeness checks." + ] + } + ] +} diff --git a/profiles/chris-engineer.json b/profiles/chris-engineer.json new file mode 100644 index 0000000..bef2b4f --- /dev/null +++ b/profiles/chris-engineer.json @@ -0,0 +1,60 @@ +{ + "schema_version": "1.0", + "id": "chris-engineer", + "display_name": "Chris Engineer", + "adapter": "single", + "roles": { + "default": { + "reference_audio": "voices/chris-engineer/reference.wav", + "reference_transcript": "voices/chris-engineer/reference.txt" + }, + "high-energy": { + "reference_audio": "voices/chris-engineer/reference-high-energy.wav", + "reference_transcript": "voices/chris-engineer/reference-high-energy.txt" + } + }, + "deliveries": { + "narrative": { + "role": "default", + "tempo_multiplier": 1.12 + }, + "technical": { + "role": "default", + "tempo_multiplier": 1.12 + }, + "high-energy": { + "role": "high-energy", + "tempo_multiplier": 1.2 + }, + "joke": { + "role": "high-energy", + "tempo_multiplier": 1.2 + }, + "punchline": { + "role": "high-energy", + "tempo_multiplier": 1.2 + } + }, + "generation": { + "minimum_words": 4, + "maximum_words": 30, + "short_native_max_words": 10, + "default_context": "Wanna see something cool!!!" + }, + "cadence": { + "default_mode": "multiplier", + "default_tempo_multiplier": 1.12, + "minimum_words_for_normalization": 10, + "minimum_tempo": 0.8, + "maximum_tempo": 1.25 + }, + "assembly": { + "crossfade_ms": 60, + "default_pause_ms": 120, + "minimum_natural_tail_ms": 200 + }, + "signature_assets": { + "opening": "voices/chris-engineer/signature-opening.wav" + }, + "persona_guide": "voices/chris-engineer/persona.md" +} diff --git a/profiles/donna.json b/profiles/donna.json new file mode 100644 index 0000000..43d9fa6 --- /dev/null +++ b/profiles/donna.json @@ -0,0 +1,29 @@ +{ + "schema_version": "1.0", + "id": "donna", + "display_name": "Donna", + "adapter": "single", + "roles": { + "default": { + "reference_audio": "voices/donna/reference-neutral.wav", + "reference_transcript": "voices/donna/reference-neutral.txt" + } + }, + "generation": { + "minimum_words": 4, + "maximum_words": 42, + "short_native_max_words": 18, + "default_context": "" + }, + "cadence": { + "default_mode": "native", + "minimum_words_for_normalization": 20, + "minimum_tempo": 0.8, + "maximum_tempo": 1.25 + }, + "assembly": { + "crossfade_ms": 60, + "default_pause_ms": 100, + "minimum_natural_tail_ms": 200 + } +} diff --git a/profiles/grandpa-bomber.json b/profiles/grandpa-bomber.json new file mode 100644 index 0000000..6aa9c40 --- /dev/null +++ b/profiles/grandpa-bomber.json @@ -0,0 +1,30 @@ +{ + "schema_version": "1.0", + "id": "grandpa-bomber", + "display_name": "Grandpa Bomber", + "adapter": "single", + "roles": { + "default": { + "reference_audio": "voices/grandpa-bomber/reference.wav", + "reference_transcript": "voices/grandpa-bomber/reference.txt" + } + }, + "generation": { + "minimum_words": 4, + "maximum_words": 45, + "short_native_max_words": 18, + "default_context": "" + }, + "cadence": { + "default_mode": "native", + "minimum_words_for_normalization": 20, + "minimum_tempo": 0.8, + "maximum_tempo": 1.25 + }, + "assembly": { + "crossfade_ms": 60, + "default_pause_ms": 120, + "minimum_natural_tail_ms": 200 + }, + "persona_guide": "voices/grandpa-bomber/persona.md" +} diff --git a/profiles/ryan-pitch-meeting.json b/profiles/ryan-pitch-meeting.json new file mode 100644 index 0000000..70419d2 --- /dev/null +++ b/profiles/ryan-pitch-meeting.json @@ -0,0 +1,38 @@ +{ + "schema_version": "1.0", + "id": "ryan-pitch-meeting", + "display_name": "Ryan Pitch Meeting", + "adapter": "dialogue", + "roles": { + "producer": { + "reference_audio": "voices/ryan-pitch-meeting/producer-reference-v3.wav", + "reference_transcript": "voices/ryan-pitch-meeting/producer-reference-v3.txt", + "default_context": "I think that's like a slogan.", + "tempo_multiplier": 1.15 + }, + "writer": { + "reference_audio": "voices/ryan-pitch-meeting/writer-reference-v3.wav", + "reference_transcript": "voices/ryan-pitch-meeting/writer-reference-v3.txt", + "default_context": "No, no, no, that would never work.", + "tempo_multiplier": 1.3 + } + }, + "generation": { + "minimum_words": 4, + "maximum_words": 42, + "short_native_max_words": 6, + "default_context": "" + }, + "cadence": { + "default_mode": "multiplier", + "minimum_words_for_normalization": 7, + "minimum_tempo": 0.8, + "maximum_tempo": 1.35 + }, + "assembly": { + "crossfade_ms": 35, + "default_pause_ms": 95, + "minimum_natural_tail_ms": 180 + }, + "signature_library": "voices/ryan-pitch-meeting/signature-exchanges/manifest.json" +} diff --git a/profiles/val-holiday.json b/profiles/val-holiday.json new file mode 100644 index 0000000..972d63a --- /dev/null +++ b/profiles/val-holiday.json @@ -0,0 +1,49 @@ +{ + "schema_version": "1.0", + "id": "val-holiday", + "display_name": "Val Holiday", + "adapter": "single", + "roles": { + "default": { + "reference_audio": "voices/val-holiday/reference-v3.wav", + "reference_transcript": "voices/val-holiday/reference-v3.txt" + } + }, + "deliveries": { + "narrative": { + "target_wpm": 170 + }, + "rollup": { + "target_wpm": 170 + } + }, + "generation": { + "minimum_words": 4, + "maximum_words": 42, + "short_native_max_words": 18, + "default_context": "" + }, + "cadence": { + "default_mode": "wpm", + "default_wpm": 170, + "minimum_words_for_normalization": 20, + "minimum_tempo": 0.75, + "maximum_tempo": 1.35 + }, + "assembly": { + "crossfade_ms": 60, + "default_pause_ms": 100, + "minimum_natural_tail_ms": 200 + }, + "pronunciation": { + "use_explicit_tts_text_for_homographs": true, + "examples": [ + { + "written": "lead", + "tts": "led", + "meaning": "bullets or ammunition" + } + ] + }, + "persona_guide": "voices/val-holiday/Val Holiday Pulse Brief Voice Guide.md" +} diff --git a/scripts/install_macos.sh b/scripts/install_macos.sh new file mode 100755 index 0000000..7bc6b8b --- /dev/null +++ b/scripts/install_macos.sh @@ -0,0 +1,79 @@ +#!/bin/zsh +set -euo pipefail + +usage() { + echo "Usage: install_macos.sh --wheel PATH --assets PATH --model-dir PATH [--runtime PATH]" +} + +runtime_root="$HOME/Library/Application Support/MPM Local Voice/runtime" +wheel_path="" +assets_path="" +model_source="" + +while (( $# )); do + case "$1" in + --runtime) + runtime_root="$2" + shift 2 + ;; + --wheel) + wheel_path="$2" + shift 2 + ;; + --assets) + assets_path="$2" + shift 2 + ;; + --model-dir) + model_source="$2" + shift 2 + ;; + *) + usage + exit 2 + ;; + esac +done + +if [[ -z "$wheel_path" || -z "$assets_path" || -z "$model_source" ]]; then + usage + exit 2 +fi + +for required in "$wheel_path" "$assets_path" "$model_source"; do + if [[ ! -e "$required" ]]; then + echo "Missing required input: $required" >&2 + exit 2 + fi +done + +if ! command -v python3.11 >/dev/null 2>&1; then + echo "Python 3.11 is required. Install it with Homebrew first." >&2 + exit 2 +fi +if ! command -v ffmpeg >/dev/null 2>&1; then + echo "ffmpeg is required and must be available on PATH." >&2 + exit 2 +fi +if ! command -v whisper >/dev/null 2>&1; then + echo "Whisper is required and must be available on PATH." >&2 + exit 2 +fi + +mkdir -p "$runtime_root/models" "$runtime_root/jobs" "$runtime_root/wheels" + +python3.11 -m venv "$runtime_root/.venv" +"$runtime_root/.venv/bin/python" -m pip install --upgrade pip +cp -p "$wheel_path" "$runtime_root/wheels/" +"$runtime_root/.venv/bin/python" -m pip install "$wheel_path" +"$runtime_root/.venv/bin/python" -m pip install numpy==2.4.6 soundfile==0.14.0 + +tar -xzf "$assets_path" -C "$runtime_root" + +model_target="$runtime_root/models/CosyVoice3-0.5B-Candle" +mkdir -p "$model_target" +rsync -a "$model_source/" "$model_target/" + +echo "Runtime installed at: $runtime_root" +echo "Run scripts/verify_install.py with:" +echo "\"$runtime_root/.venv/bin/python\" scripts/verify_install.py --runtime-root \"$runtime_root\" --device metal" diff --git a/scripts/local_voice.py b/scripts/local_voice.py new file mode 100755 index 0000000..482b53d --- /dev/null +++ b/scripts/local_voice.py @@ -0,0 +1,1320 @@ +#!/usr/bin/env python3 +"""Unified local CosyVoice production renderer for Claude and Codex plugins.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import re +import shutil +import subprocess +import sys +import time +from difflib import SequenceMatcher +from pathlib import Path +from typing import Any + + +PLUGIN_ROOT = Path(__file__).resolve().parent.parent +PROFILE_DIR = PLUGIN_ROOT / "profiles" +PROMPT_PREFIX = "You are a helpful assistant.<|endofprompt|>" +MODEL_RELATIVE = Path("models") / "CosyVoice3-0.5B-Candle" +ID_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]*$") + + +class VoiceError(RuntimeError): + """A production failure with an actionable message.""" + + +def platform_default_runtime() -> Path: + configured = os.environ.get("LOCAL_VOICE_RUNTIME") + if configured: + return Path(configured).expanduser() + if platform.system() == "Windows": + base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) + return base / "MPM Local Voice" / "runtime" + if platform.system() == "Darwin": + return ( + Path.home() + / "Library" + / "Application Support" + / "MPM Local Voice" + / "runtime" + ) + return Path.home() / ".local" / "share" / "mpm-local-voice" / "runtime" + + +def json_read(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise VoiceError(f"Missing file: {path}") from exc + except json.JSONDecodeError as exc: + raise VoiceError(f"Invalid JSON in {path}: {exc}") from exc + + +def json_write(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def normalized_tokens(text: str) -> list[str]: + return re.findall(r"[a-z0-9]+", text.casefold()) + + +def word_count(text: str) -> int: + return len(normalized_tokens(text)) + + +def clean_markdown(text: str) -> str: + text = re.sub(r"^\s{0,3}#{1,6}\s+.*$", "", text, flags=re.MULTILINE) + text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) + text = re.sub(r"^\s*\d+[.)]\s+", "", text, flags=re.MULTILINE) + text = re.sub(r"^\s*>\s?", "", text, flags=re.MULTILINE) + text = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", text) + text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) + text = re.sub(r"[*_~`]", "", text) + return text + + +def paragraphs_from_markdown(path: Path) -> list[str]: + cleaned = clean_markdown(path.read_text(encoding="utf-8")) + paragraphs = [ + re.sub(r"\s+", " ", block).strip() + for block in re.split(r"\n\s*\n", cleaned) + ] + return [block for block in paragraphs if block] + + +def last_sentence(text: str, maximum_words: int = 10) -> str: + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + candidate = parts[-1] if parts else text.strip() + words = candidate.split() + return " ".join(words[-maximum_words:]) + + +def first_sentence(text: str, maximum_words: int = 10) -> str: + parts = re.split(r"(?<=[.!?])\s+", text.strip()) + selected: list[str] = [] + for part in parts or [text.strip()]: + selected.extend(part.split()) + if len(selected) >= 6: + break + return " ".join(selected[:maximum_words]) + + +def slug(value: str) -> str: + value = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").casefold() + return value or "segment" + + +def profile_path(voice: str) -> Path: + return PROFILE_DIR / f"{voice}.json" + + +def load_profile(voice: str) -> dict[str, Any]: + profile = json_read(profile_path(voice)) + if profile.get("id") != voice: + raise VoiceError(f"Profile ID mismatch in {profile_path(voice)}") + return profile + + +def available_profiles() -> list[dict[str, Any]]: + profiles = [] + for path in sorted(PROFILE_DIR.glob("*.json")): + profile = json_read(path) + profiles.append( + { + "id": profile["id"], + "display_name": profile["display_name"], + "adapter": profile["adapter"], + "roles": sorted(profile["roles"]), + } + ) + return profiles + + +def validate_plan_data(plan: dict[str, Any]) -> dict[str, Any]: + if plan.get("schema_version") != "1.0": + raise VoiceError("Render plan schema_version must be '1.0'") + job_id = str(plan.get("job_id", "")) + if not ID_PATTERN.fullmatch(job_id): + raise VoiceError("job_id must be filesystem-safe") + voice = str(plan.get("voice", "")) + profile = load_profile(voice) + output = Path(str(plan.get("output", ""))).expanduser() + if not output.is_absolute(): + raise VoiceError("Render plan output must be an absolute path") + if output.suffix.casefold() not in {".wav", ".mp3"}: + raise VoiceError("Render plan output must end in .wav or .mp3") + segments = plan.get("segments") + if not isinstance(segments, list) or not segments: + raise VoiceError("Render plan must contain at least one segment") + seen: set[str] = set() + roles = set(profile["roles"]) + deliveries = set(profile.get("deliveries", {})) + for index, segment in enumerate(segments, start=1): + if not isinstance(segment, dict): + raise VoiceError(f"Segment {index} must be an object") + segment_id = str(segment.get("id", "")) + if not ID_PATTERN.fullmatch(segment_id): + raise VoiceError(f"Segment {index} has invalid id {segment_id!r}") + if segment_id in seen: + raise VoiceError(f"Duplicate segment id: {segment_id}") + seen.add(segment_id) + text = str(segment.get("text", "")).strip() + if not text: + raise VoiceError(f"{segment_id}: canonical text is required") + role = segment.get("role") + if role is not None and role not in roles: + raise VoiceError(f"{segment_id}: unknown role {role!r}") + if profile["adapter"] == "dialogue" and role is None: + raise VoiceError(f"{segment_id}: dialogue segments require a role") + delivery = segment.get("delivery") + if delivery is not None and delivery not in deliveries: + raise VoiceError(f"{segment_id}: unknown delivery {delivery!r}") + fixed = segment.get("fixed_asset") + pool = segment.get("asset_pool") + if fixed and pool: + raise VoiceError(f"{segment_id}: choose fixed_asset or asset_pool") + if pool is not None and ( + not isinstance(pool, list) or not all(isinstance(x, str) for x in pool) + ): + raise VoiceError(f"{segment_id}: asset_pool must be a list of paths") + if segment.get("target_wpm") and segment.get("tempo_multiplier"): + raise VoiceError( + f"{segment_id}: target_wpm and tempo_multiplier are mutually exclusive" + ) + return profile + + +def build_plan(args: argparse.Namespace) -> dict[str, Any]: + profile = load_profile(args.voice) + max_words = int(profile["generation"]["maximum_words"]) + short_max = int(profile["generation"]["short_native_max_words"]) + raw_blocks = paragraphs_from_markdown(args.script) + if not raw_blocks: + raise VoiceError("Script contains no speakable text") + + segments: list[dict[str, Any]] = [] + if profile["adapter"] == "dialogue": + role_pattern = re.compile( + r"^(?:\*\*)?(PRODUCER(?:\s+GUY)?|EXEC(?:\s+GUY)?|" + r"WRITER(?:\s+GUY)?)(?:\*\*)?\s*:\s*(.+)$", + re.IGNORECASE, + ) + for block in raw_blocks: + match = role_pattern.match(block) + if not match: + raise VoiceError( + "Ryan scripts require Producer/Writer labels on every paragraph" + ) + label, spoken = match.groups() + role = ( + "writer" + if label.casefold().startswith("writer") + else "producer" + ) + segments.append( + { + "id": f"{len(segments) + 1:02d}-{role}", + "role": role, + "text": spoken.strip(), + } + ) + else: + packed: list[str] = [] + for index, block in enumerate(raw_blocks): + is_short_frame = ( + word_count(block) <= short_max + and index in {0, len(raw_blocks) - 1} + ) + if is_short_frame: + if packed: + segments.append( + { + "id": f"{len(segments) + 1:02d}-passage", + "text": " ".join(packed), + "delivery": "narrative" + if "narrative" in profile.get("deliveries", {}) + else None, + } + ) + packed = [] + segments.append( + { + "id": f"{len(segments) + 1:02d}-frame", + "text": block, + "native_tempo": True, + } + ) + continue + proposed = " ".join([*packed, block]) + if packed and word_count(proposed) > max_words: + segments.append( + { + "id": f"{len(segments) + 1:02d}-passage", + "text": " ".join(packed), + "delivery": "narrative" + if "narrative" in profile.get("deliveries", {}) + else None, + } + ) + packed = [block] + else: + packed.append(block) + if packed: + segments.append( + { + "id": f"{len(segments) + 1:02d}-passage", + "text": " ".join(packed), + "delivery": "narrative" + if "narrative" in profile.get("deliveries", {}) + else None, + } + ) + for segment in segments: + if segment.get("delivery") is None: + segment.pop("delivery", None) + + for index, segment in enumerate(segments): + if index: + segment["context_before"] = last_sentence( + segments[index - 1]["text"] + ) + if index + 1 < len(segments): + segment["context_after"] = first_sentence( + segments[index + 1]["text"] + ) + segment.setdefault( + "pause_after_ms", + int(profile["assembly"]["default_pause_ms"]), + ) + plan = { + "schema_version": "1.0", + "job_id": args.job_id or slug(args.script.stem), + "voice": args.voice, + "output": str(args.output_audio.resolve()), + "final_transcript_qa": True, + "segments": segments, + } + validate_plan_data(plan) + json_write(args.output_plan, plan) + return plan + + +def resolve_runtime(path: str | None) -> Path: + return ( + Path(path).expanduser().resolve() + if path + else platform_default_runtime().resolve() + ) + + +def executable(name: str, preferred: str | None = None) -> str: + if preferred: + candidate = Path(preferred).expanduser() + if candidate.exists(): + return str(candidate) + found = shutil.which(name) + if not found: + raise VoiceError(f"Required executable is unavailable: {name}") + return found + + +def resolve_asset(relative: str, runtime: Path) -> Path: + candidate = Path(relative).expanduser() + if candidate.is_absolute() and candidate.exists(): + return candidate + for base in (runtime, PLUGIN_ROOT): + resolved = base / candidate + if resolved.exists(): + return resolved + raise VoiceError(f"Missing authorized asset: {relative}") + + +def select_asset(segment: dict[str, Any], plan: dict[str, Any]) -> str | None: + if segment.get("fixed_asset"): + return str(segment["fixed_asset"]) + pool = segment.get("asset_pool") + if not pool: + return None + digest = hashlib.sha256( + f"{plan['job_id']}:{segment['id']}".encode() + ).digest() + return str(pool[int.from_bytes(digest[:4], "big") % len(pool)]) + + +def role_for_segment( + segment: dict[str, Any], + profile: dict[str, Any], +) -> tuple[str, dict[str, Any], dict[str, Any]]: + delivery = profile.get("deliveries", {}).get(segment.get("delivery"), {}) + role_name = segment.get("role") or delivery.get("role") or "default" + role = profile["roles"][role_name] + return role_name, role, delivery + + +def job_paths(plan: dict[str, Any], runtime: Path) -> dict[str, Path]: + root = runtime / "jobs" / plan["job_id"] + return { + "root": root, + "raw": root / "raw", + "alignment": root / "alignment", + "processed": root / "processed", + "qa": root / "qa", + } + + +def import_audio_stack() -> tuple[Any, Any]: + try: + import numpy as np + import soundfile as sf + except ImportError as exc: + raise VoiceError( + "numpy and soundfile must be installed in the Local Voice environment" + ) from exc + return np, sf + + +def import_model() -> tuple[Any, Any]: + try: + from cosyvoice3 import CosyVoice3, PyDevice + except ImportError as exc: + raise VoiceError( + "cosyvoice3 is not installed in this Python environment" + ) from exc + return CosyVoice3, PyDevice + + +def validate_generated(samples: Any, segment_id: str, np: Any) -> None: + samples = np.asarray(samples).squeeze() + if samples.ndim != 1 or samples.size == 0: + raise VoiceError(f"{segment_id}: unexpected audio shape {samples.shape}") + if not np.isfinite(samples).all(): + raise VoiceError(f"{segment_id}: generated non-finite audio") + + +def generate_segments( + plan: dict[str, Any], + profile: dict[str, Any], + runtime: Path, + paths: dict[str, Path], + *, + device_name: str, + resume: bool, +) -> None: + np, sf = import_audio_stack() + CosyVoice3, PyDevice = import_model() + model_dir = runtime / MODEL_RELATIVE + if not model_dir.exists(): + raise VoiceError(f"Missing Candle model directory: {model_dir}") + generated = [ + segment + for segment in plan["segments"] + if not select_asset(segment, plan) + ] + pending = [ + segment + for segment in generated + if not (resume and (paths["raw"] / f"{segment['id']}.wav").exists()) + ] + if not pending: + print("All raw speech segments already exist.") + return + print(f"Loading CosyVoice3 on {device_name}...") + model = CosyVoice3( + str(model_dir), + device=PyDevice(device_name), + use_f16=False, + ) + for index, segment in enumerate(pending, start=1): + _, role, _ = role_for_segment(segment, profile) + reference_audio = resolve_asset(role["reference_audio"], runtime) + reference_text = resolve_asset(role["reference_transcript"], runtime) + context = segment.get("context_before") + if context is None: + context = role.get( + "default_context", + profile["generation"].get("default_context", ""), + ) + target = str(segment.get("tts_text", segment["text"])).strip() + spoken = target + if context: + spoken = f"{str(context).strip()} ... ... ... {spoken}" + if segment.get("context_after"): + spoken += f" ... ... ... {str(segment['context_after']).strip()}" + prompt_text = PROMPT_PREFIX + reference_text.read_text( + encoding="utf-8" + ).strip() + started = time.perf_counter() + audio = model.inference_zero_shot( + text=spoken, + prompt_text=prompt_text, + prompt_wav=str(reference_audio), + ) + elapsed = time.perf_counter() - started + samples = np.asarray(audio, dtype=np.float32).squeeze() + validate_generated(samples, segment["id"], np) + destination = paths["raw"] / f"{segment['id']}.wav" + sf.write(destination, samples, model.sample_rate, subtype="PCM_16") + duration = samples.size / model.sample_rate + print( + f"[{index}/{len(pending)}] {segment['id']}: " + f"{duration:.2f}s in {elapsed:.2f}s" + ) + + +def run_whisper( + source: Path, + output_dir: Path, + *, + whisper_path: str | None, +) -> Path: + whisper = executable("whisper", whisper_path) + output_dir.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + whisper, + str(source), + "--model", + "base", + "--device", + "cpu", + "--fp16", + "False", + "--language", + "en", + "--word_timestamps", + "True", + "--output_format", + "json", + "--output_dir", + str(output_dir), + "--verbose", + "False", + ], + check=True, + ) + result = output_dir / f"{source.stem}.json" + if not result.exists(): + raise VoiceError(f"Whisper did not create {result}") + return result + + +def align_segments( + plan: dict[str, Any], + runtime: Path, + paths: dict[str, Path], + *, + whisper_path: str | None, +) -> None: + for segment in plan["segments"]: + if select_asset(segment, plan): + continue + raw = paths["raw"] / f"{segment['id']}.wav" + if not raw.exists(): + raise VoiceError(f"{segment['id']}: missing raw WAV") + alignment = paths["alignment"] / f"{segment['id']}.json" + if not alignment.exists() or raw.stat().st_mtime > alignment.stat().st_mtime: + run_whisper( + raw, + paths["alignment"], + whisper_path=whisper_path, + ) + + +def aligned_words(path: Path) -> list[dict[str, Any]]: + data = json_read(path) + return [ + word + for segment in data.get("segments", []) + for word in segment.get("words", []) + if normalized_tokens(str(word.get("word", ""))) + ] + + +def locate_phrase( + phrase: str, + words: list[dict[str, Any]], + *, + start_at: int = 0, + minimum_score: float = 0.55, +) -> tuple[int, int, float]: + expected = normalized_tokens(phrase)[:10] + if len(expected) < 2: + raise VoiceError(f"Phrase too short for alignment: {phrase!r}") + heard = [normalized_tokens(str(word["word"]))[0] for word in words] + best = (-1.0, -1, -1) + for start in range(start_at, len(heard)): + for width in range(max(2, len(expected) - 3), len(expected) + 4): + candidate = heard[start:start + width] + if len(candidate) < 2: + continue + score = SequenceMatcher( + None, + "".join(expected), + "".join(candidate), + ).ratio() + if score > best[0]: + best = (score, start, width) + if best[0] < minimum_score: + raise VoiceError( + f"Could not locate {phrase!r}; confidence={best[0]:.3f}" + ) + return best[1], best[2], best[0] + + +def dbfs(value: float) -> float: + return 20 * math.log10(max(value, 1e-10)) + + +def quiet_cut( + audio: Any, + sample_rate: int, + *, + lower: float, + upper: float, + target: float, + ceiling: float, + np: Any, +) -> tuple[int, float]: + lo = max(0, round(lower * sample_rate)) + hi = min(len(audio), round(upper * sample_rate)) + window = max(1, round(0.010 * sample_rate)) + if hi - lo < window: + fallback_lo = max(0, round((upper - 0.14) * sample_rate)) + fallback_hi = min( + len(audio), + max(fallback_lo + 1, round((upper - 0.035) * sample_rate)), + ) + section = np.abs(audio[fallback_lo:fallback_hi]) + chosen = fallback_lo + int(np.argmin(section)) + local = audio[ + max(0, chosen - window // 2):min(len(audio), chosen + window // 2) + ] + return chosen, dbfs(float(np.sqrt(np.mean(local ** 2)))) + squared = audio[lo:hi] ** 2 + rms = np.sqrt( + np.convolve(squared, np.ones(window) / window, mode="valid") + ) + centers = np.arange(len(rms)) + window // 2 + candidates = np.flatnonzero(rms <= 10 ** (ceiling / 20)) + if not candidates.size: + chosen = int(np.argmin(rms)) + return lo + int(centers[chosen]), dbfs(float(rms[chosen])) + desired = round(target * sample_rate) - lo + chosen = candidates[int(np.argmin(np.abs(centers[candidates] - desired)))] + return lo + int(centers[chosen]), dbfs(float(rms[chosen])) + + +def transcode_asset( + source: Path, + destination: Path, + *, + ffmpeg_path: str | None, +) -> None: + ffmpeg = executable("ffmpeg", ffmpeg_path) + destination.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(source), + "-filter:a", + ( + "loudnorm=I=-18:LRA=7:TP=-1.5," + "afade=t=in:st=0:d=0.008," + "areverse,afade=t=in:st=0:d=0.008,areverse" + ), + "-ar", + "24000", + "-ac", + "1", + "-codec:a", + "pcm_s16le", + str(destination), + ], + check=True, + ) + + +def tempo_for_segment( + segment: dict[str, Any], + profile: dict[str, Any], + role: dict[str, Any], + delivery: dict[str, Any], + *, + body_duration: float, + target_words: int, +) -> tuple[float, float | None, float]: + raw_wpm = target_words / max(body_duration, 0.001) * 60 + short_limit = int(profile["generation"]["short_native_max_words"]) + cadence = profile["cadence"] + minimum_words = int(cadence["minimum_words_for_normalization"]) + if ( + segment.get("native_tempo") + or target_words <= short_limit + or target_words < minimum_words + ): + return 1.0, None, raw_wpm + multiplier = segment.get("tempo_multiplier") + target_wpm = segment.get("target_wpm") + if multiplier is None and target_wpm is None: + multiplier = delivery.get("tempo_multiplier") + target_wpm = delivery.get("target_wpm") + if multiplier is None and target_wpm is None: + multiplier = role.get("tempo_multiplier") + if multiplier is None and target_wpm is None: + mode = cadence.get("default_mode", "native") + if mode == "multiplier": + multiplier = cadence.get("default_tempo_multiplier", 1.0) + elif mode == "wpm": + target_wpm = cadence.get("default_wpm") + if target_wpm is not None: + tempo = float(target_wpm) / raw_wpm + else: + tempo = float(multiplier if multiplier is not None else 1.0) + minimum = float(cadence["minimum_tempo"]) + maximum = float(cadence["maximum_tempo"]) + if not minimum <= tempo <= maximum: + raise VoiceError( + f"{segment['id']}: cadence correction {tempo:.3f} " + f"outside validated range {minimum:.2f}-{maximum:.2f}" + ) + return tempo, float(target_wpm) if target_wpm is not None else None, raw_wpm + + +def apply_processing( + source: Path, + destination: Path, + *, + tempo: float, + ffmpeg_path: str | None, +) -> None: + ffmpeg = executable("ffmpeg", ffmpeg_path) + subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(source), + "-filter:a", + ( + f"atempo={tempo:.6f}," + "loudnorm=I=-18:LRA=7:TP=-1.5," + "afade=t=in:st=0:d=0.008," + "areverse,afade=t=in:st=0:d=0.008,areverse" + ), + "-ar", + "24000", + "-ac", + "1", + "-codec:a", + "pcm_s16le", + str(destination), + ], + check=True, + ) + + +def append_suffix( + body: Any, + suffix: Any, + *, + gap_ms: int, + sample_rate: int, + np: Any, +) -> Any: + gap = np.zeros(round(gap_ms / 1000 * sample_rate), dtype=np.float32) + return np.concatenate([body, gap, suffix]) + + +def process_generated_segment( + segment: dict[str, Any], + profile: dict[str, Any], + runtime: Path, + paths: dict[str, Path], + *, + ffmpeg_path: str | None, +) -> tuple[Any, dict[str, Any]]: + np, sf = import_audio_stack() + raw = paths["raw"] / f"{segment['id']}.wav" + alignment = paths["alignment"] / f"{segment['id']}.json" + words = aligned_words(alignment) + target_text = str(segment.get("tts_text", segment["text"])) + context = segment.get("context_before") + role_name, role, delivery = role_for_segment(segment, profile) + if context is None: + context = role.get( + "default_context", + profile["generation"].get("default_context", ""), + ) + start_index, _, match_confidence = locate_phrase( + target_text, + words, + start_at=2 if context else 0, + ) + expected_tokens = normalized_tokens(target_text) + ending_marker = " ".join(expected_tokens[-min(7, len(expected_tokens)):]) + ending_start, ending_width, marker_confidence = locate_phrase( + ending_marker, + words, + start_at=max(start_index, start_index + len(expected_tokens) - 12), + minimum_score=0.45, + ) + after_index = min(len(words), ending_start + ending_width) + after_confidence: float | None = None + if segment.get("context_after"): + try: + context_start, _, after_confidence = locate_phrase( + str(segment["context_after"]), + words, + start_at=after_index, + minimum_score=0.45, + ) + after_index = context_start + except VoiceError: + after_confidence = None + target_words = words[start_index:after_index] + if not target_words: + raise VoiceError(f"{segment['id']}: alignment selected no target words") + body_start = float(target_words[0]["start"]) + body_end = float(target_words[-1]["end"]) + audio, sample_rate = sf.read(raw, dtype="float32") + audio = np.asarray(audio).reshape(-1) + if start_index == 0: + start_cut = 0 + start_level = dbfs( + float(np.sqrt(np.mean(audio[:max(1, sample_rate // 100)] ** 2))) + ) + else: + prior_end = float(words[start_index - 1]["end"]) + start_cut, start_level = quiet_cut( + audio, + sample_rate, + lower=prior_end, + upper=body_start, + target=body_start - 0.10, + ceiling=-30.0, + np=np, + ) + start_cut = max(start_cut, round(prior_end * sample_rate)) + if after_index == len(words): + end_cut = len(audio) + end_level = dbfs( + float(np.sqrt(np.mean(audio[-max(1, sample_rate // 100):] ** 2))) + ) + else: + after_start = float(words[after_index]["start"]) + end_cut, end_level = quiet_cut( + audio, + sample_rate, + lower=body_end + 0.030, + upper=after_start, + target=body_end + + int(profile["assembly"]["minimum_natural_tail_ms"]) / 1000, + ceiling=-38.0, + np=np, + ) + minimum_end = min( + len(audio), + round( + ( + body_end + + min( + 0.08, + int(profile["assembly"]["minimum_natural_tail_ms"]) + / 1000, + ) + ) + * sample_rate + ), + ) + end_cut = max(end_cut, minimum_end) + selected = audio[start_cut:end_cut] + expected = normalized_tokens(target_text) + heard = [ + normalized_tokens(str(word["word"]))[0] + for word in target_words + if normalized_tokens(str(word["word"])) + ] + matcher = SequenceMatcher(None, expected, heard, autojunk=False) + matched = sum(block.size for block in matcher.get_matching_blocks()) + coverage = matched / max(1, len(expected)) + ending_expected = expected[-min(7, len(expected)):] + ending_heard = heard[-min(9, len(heard)):] + ending = SequenceMatcher( + None, + ending_expected, + ending_heard, + autojunk=False, + ).ratio() + if coverage < 0.65 or ending < 0.50: + raise VoiceError( + f"{segment['id']}: incomplete target; " + f"coverage={coverage:.3f}, ending={ending:.3f}" + ) + tempo, target_wpm, raw_wpm = tempo_for_segment( + segment, + profile, + role, + delivery, + body_duration=body_end - body_start, + target_words=len(expected), + ) + trimmed = paths["processed"] / f"{segment['id']}-trimmed.wav" + ready = paths["processed"] / f"{segment['id']}.wav" + sf.write(trimmed, selected, sample_rate, subtype="PCM_16") + apply_processing( + trimmed, + ready, + tempo=tempo, + ffmpeg_path=ffmpeg_path, + ) + ready_audio, ready_rate = sf.read(ready, dtype="float32") + if ready_rate != 24000: + raise VoiceError(f"{segment['id']}: processed sample rate is not 24 kHz") + result = np.asarray(ready_audio).reshape(-1) + if segment.get("suffix_asset"): + suffix_source = resolve_asset(str(segment["suffix_asset"]), runtime) + suffix_ready = paths["processed"] / f"{segment['id']}-suffix.wav" + transcode_asset( + suffix_source, + suffix_ready, + ffmpeg_path=ffmpeg_path, + ) + suffix, suffix_rate = sf.read(suffix_ready, dtype="float32") + if suffix_rate != 24000: + raise VoiceError(f"{segment['id']}: suffix sample rate mismatch") + result = append_suffix( + result, + np.asarray(suffix).reshape(-1), + gap_ms=int(segment.get("suffix_gap_ms", 90)), + sample_rate=24000, + np=np, + ) + sf.write(ready, result, 24000, subtype="PCM_16") + return result, { + "id": segment["id"], + "source": "generated", + "role": role_name, + "delivery": segment.get("delivery"), + "target_match_confidence": round(match_confidence, 3), + "after_match_confidence": ( + round(after_confidence, 3) + if after_confidence is not None + else None + ), + "transcript_coverage": round(coverage, 3), + "ending_confidence": round(ending, 3), + "ending_marker_confidence": round(marker_confidence, 3), + "retained_pre_word_ms": round( + (body_start - start_cut / sample_rate) * 1000, + 1, + ), + "retained_post_word_ms": round( + (end_cut / sample_rate - body_end) * 1000, + 1, + ), + "start_cut_dbfs": round(start_level, 1), + "end_cut_dbfs": round(end_level, 1), + "raw_wpm": round(raw_wpm, 1), + "target_wpm": round(target_wpm, 1) if target_wpm else None, + "applied_tempo": round(tempo, 4), + "native_tempo": tempo == 1.0, + "suffix_asset": segment.get("suffix_asset"), + } + + +def process_asset_segment( + segment: dict[str, Any], + plan: dict[str, Any], + runtime: Path, + paths: dict[str, Path], + *, + ffmpeg_path: str | None, +) -> tuple[Any, dict[str, Any]]: + np, sf = import_audio_stack() + selected = select_asset(segment, plan) + if not selected: + raise VoiceError(f"{segment['id']}: asset selection failed") + source = resolve_asset(selected, runtime) + ready = paths["processed"] / f"{segment['id']}.wav" + transcode_asset(source, ready, ffmpeg_path=ffmpeg_path) + audio, sample_rate = sf.read(ready, dtype="float32") + if sample_rate != 24000: + raise VoiceError(f"{segment['id']}: fixed asset is not 24 kHz") + return np.asarray(audio).reshape(-1), { + "id": segment["id"], + "source": "fixed_asset", + "asset": str(source), + "native_tempo": True, + "duration_seconds": round(len(audio) / sample_rate, 3), + } + + +def assemble( + plan: dict[str, Any], + profile: dict[str, Any], + runtime: Path, + paths: dict[str, Path], + *, + ffmpeg_path: str | None, + whisper_path: str | None, + final_qa: bool, +) -> dict[str, Any]: + np, sf = import_audio_stack() + processed: list[Any] = [] + reports: list[dict[str, Any]] = [] + for segment in plan["segments"]: + if select_asset(segment, plan): + audio, report = process_asset_segment( + segment, + plan, + runtime, + paths, + ffmpeg_path=ffmpeg_path, + ) + else: + audio, report = process_generated_segment( + segment, + profile, + runtime, + paths, + ffmpeg_path=ffmpeg_path, + ) + processed.append(audio) + reports.append(report) + + crossfade = round( + int(profile["assembly"]["crossfade_ms"]) / 1000 * 24000 + ) + joined = processed[0] + seam_times: list[float] = [] + for index, right in enumerate(processed[1:], start=1): + pause_ms = int( + plan["segments"][index - 1].get( + "pause_after_ms", + profile["assembly"]["default_pause_ms"], + ) + ) + if pause_ms > 0: + joined = np.concatenate( + [ + joined, + np.zeros(round(pause_ms / 1000 * 24000), dtype=np.float32), + right, + ] + ) + seam_times.append((len(joined) - len(right)) / 24000) + else: + overlap = min(crossfade, len(joined), len(right)) + if overlap: + theta = np.linspace( + 0, + np.pi / 2, + overlap, + dtype=np.float32, + ) + seam_times.append((len(joined) - overlap) / 24000) + joined = np.concatenate( + [ + joined[:-overlap], + joined[-overlap:] * np.cos(theta) + + right[:overlap] * np.sin(theta), + right[overlap:], + ] + ) + else: + seam_times.append(len(joined) / 24000) + joined = np.concatenate([joined, right]) + + output = Path(plan["output"]) + output.parent.mkdir(parents=True, exist_ok=True) + wav_output = output if output.suffix.casefold() == ".wav" else output.with_suffix( + ".wav" + ) + sf.write(wav_output, joined, 24000, subtype="PCM_16") + if output.suffix.casefold() == ".mp3": + ffmpeg = executable("ffmpeg", ffmpeg_path) + subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(wav_output), + "-codec:a", + "libmp3lame", + "-q:a", + "2", + str(output), + ], + check=True, + ) + + derivative = np.abs(np.diff(joined)) + seam_report = [] + for seam in seam_times: + center = round(seam * 24000) + lo = max(0, center - round(0.08 * 24000)) + hi = min(len(derivative), center + round(0.14 * 24000)) + boundary_index = min(max(0, center - 1), len(derivative) - 1) + boundary_step = ( + float(derivative[boundary_index]) if len(derivative) else 0.0 + ) + seam_report.append( + { + "time_seconds": round(seam, 3), + "boundary_step": round(boundary_step, 6), + "click_risk": boundary_step > 0.08, + "maximum_derivative": round( + float(derivative[lo:hi].max()) if hi > lo else 0.0, + 6, + ), + } + ) + + report: dict[str, Any] = { + "schema_version": "1.0", + "job_id": plan["job_id"], + "voice": plan["voice"], + "output": str(output), + "wav_output": str(wav_output), + "duration_seconds": round(len(joined) / 24000, 3), + "segments": reports, + "seams": seam_report, + "global_maximum_derivative": round(float(derivative.max()), 6), + } + if final_qa: + final_alignment = run_whisper( + output, + paths["qa"], + whisper_path=whisper_path, + ) + data = json_read(final_alignment) + heard_text = " ".join( + str(segment.get("text", "")).strip() + for segment in data.get("segments", []) + ).strip() + expected = normalized_tokens( + " ".join(segment["text"] for segment in plan["segments"]) + ) + heard = normalized_tokens(heard_text) + matcher = SequenceMatcher(None, expected, heard, autojunk=False) + matched = sum(block.size for block in matcher.get_matching_blocks()) + coverage = matched / max(1, len(expected)) + precision = matched / max(1, len(heard)) + passed = coverage >= 0.60 and precision >= 0.75 + report["final_transcript_qa"] = { + "coverage": round(coverage, 3), + "precision": round(precision, 3), + "heard_text": heard_text, + "passed": passed, + } + if not passed: + json_write(paths["root"] / "qa-report.json", report) + raise VoiceError( + "Final transcript QA failed with " + f"coverage={coverage:.3f}, precision={precision:.3f}" + ) + report_path = paths["root"] / "qa-report.json" + json_write(report_path, report) + report["report_path"] = str(report_path) + return report + + +def doctor(runtime: Path, device_name: str) -> int: + checks: list[tuple[str, bool, str]] = [] + checks.append( + ( + "Python", + sys.version_info >= (3, 10), + platform.python_version(), + ) + ) + for name in ("ffmpeg", "whisper"): + path = shutil.which(name) + checks.append((name, path is not None, path or "not found")) + for module in ("numpy", "soundfile", "cosyvoice3"): + try: + __import__(module) + checks.append((module, True, "importable")) + except ImportError: + checks.append((module, False, "not importable")) + model = runtime / MODEL_RELATIVE + required_model_files = [ + "llm.safetensors", + "flow.safetensors", + "hift.safetensors", + "campplus.onnx", + "speech_tokenizer_v3.onnx", + "config.json", + ] + missing_model = [name for name in required_model_files if not (model / name).exists()] + checks.append( + ( + "Candle model", + not missing_model, + str(model) if not missing_model else "missing " + ", ".join(missing_model), + ) + ) + for summary in available_profiles(): + try: + profile = load_profile(summary["id"]) + missing = [] + for role in profile["roles"].values(): + for field in ("reference_audio", "reference_transcript"): + try: + resolve_asset(role[field], runtime) + except VoiceError: + missing.append(role[field]) + checks.append( + ( + summary["display_name"], + not missing, + "ready" if not missing else "missing assets: " + ", ".join(missing), + ) + ) + except VoiceError as exc: + checks.append((summary["display_name"], False, str(exc))) + print(f"Runtime: {runtime}") + print(f"Requested device: {device_name}") + for name, passed, detail in checks: + print(f"{'PASS' if passed else 'FAIL'} {name}: {detail}") + return 0 if all(item[1] for item in checks) else 1 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(prog="local_voice") + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("list", help="List installed voice profiles") + + validate_parser = subparsers.add_parser( + "validate", + help="Validate a render plan without generating audio", + ) + validate_parser.add_argument("plan", type=Path) + + plan_parser = subparsers.add_parser( + "plan", + help="Create a basic render plan from Markdown", + ) + plan_parser.add_argument("--voice", required=True) + plan_parser.add_argument("--script", required=True, type=Path) + plan_parser.add_argument("--output-plan", required=True, type=Path) + plan_parser.add_argument("--output-audio", required=True, type=Path) + plan_parser.add_argument("--job-id") + + doctor_parser = subparsers.add_parser( + "doctor", + help="Check the installed runtime and authorized voice assets", + ) + doctor_parser.add_argument("--runtime-root") + doctor_parser.add_argument("--device", default="metal") + + render_parser = subparsers.add_parser( + "render", + help="Generate, align, process, and assemble a render plan", + ) + render_parser.add_argument("plan", type=Path) + render_parser.add_argument("--runtime-root") + render_parser.add_argument( + "--device", + choices=("metal", "cpu", "cuda"), + default="metal" if platform.system() == "Darwin" else "cpu", + ) + render_parser.add_argument("--resume", action="store_true") + render_parser.add_argument("--assemble-only", action="store_true") + render_parser.add_argument("--skip-final-qa", action="store_true") + render_parser.add_argument("--ffmpeg-path") + render_parser.add_argument("--whisper-path") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.command == "list": + print(json.dumps(available_profiles(), indent=2)) + return 0 + if args.command == "validate": + plan = json_read(args.plan) + profile = validate_plan_data(plan) + print( + json.dumps( + { + "valid": True, + "voice": profile["id"], + "segments": len(plan["segments"]), + "output": plan["output"], + }, + indent=2, + ) + ) + return 0 + if args.command == "plan": + plan = build_plan(args) + print(json.dumps(plan, indent=2)) + return 0 + if args.command == "doctor": + return doctor(resolve_runtime(args.runtime_root), args.device) + if args.command == "render": + plan = json_read(args.plan) + profile = validate_plan_data(plan) + runtime = resolve_runtime(args.runtime_root) + paths = job_paths(plan, runtime) + for path in paths.values(): + path.mkdir(parents=True, exist_ok=True) + if not args.assemble_only: + generate_segments( + plan, + profile, + runtime, + paths, + device_name=args.device, + resume=args.resume, + ) + align_segments( + plan, + runtime, + paths, + whisper_path=args.whisper_path, + ) + report = assemble( + plan, + profile, + runtime, + paths, + ffmpeg_path=args.ffmpeg_path, + whisper_path=args.whisper_path, + final_qa=bool( + plan.get("final_transcript_qa", True) + and not args.skip_final_qa + ), + ) + print(json.dumps(report, indent=2)) + return 0 + raise VoiceError(f"Unsupported command: {args.command}") + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except VoiceError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(2) diff --git a/scripts/verify_install.py b/scripts/verify_install.py new file mode 100755 index 0000000..f80657f --- /dev/null +++ b/scripts/verify_install.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Run a deterministic Local Voice installation preflight.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +RENDERER = ROOT / "scripts" / "local_voice.py" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runtime-root") + parser.add_argument("--device", choices=("metal", "cpu", "cuda"), default="metal") + args = parser.parse_args() + command = [ + sys.executable, + str(RENDERER), + "doctor", + "--device", + args.device, + ] + if args.runtime_root: + command.extend(["--runtime-root", args.runtime_root]) + completed = subprocess.run(command, check=False) + if completed.returncode: + return completed.returncode + completed = subprocess.run( + [sys.executable, str(RENDERER), "list"], + check=False, + ) + if completed.returncode: + return completed.returncode + print("Local Voice installation preflight passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/local-voice/SKILL.md b/skills/local-voice/SKILL.md new file mode 100644 index 0000000..12fd3b8 --- /dev/null +++ b/skills/local-voice/SKILL.md @@ -0,0 +1,135 @@ +--- +name: local-voice +description: > + Generate, validate, resume, and assemble local text-to-speech audio using the + approved Donna, Chris Engineer, Grandpa Bomber, Ryan Pitch Meeting, and Val + Holiday profiles. Use this skill whenever a user asks to make a voice brief, + local TTS message, spoken notification, character dialogue, CosyVoice render, + MP3 briefing, or asks another skill such as Donna to produce audio locally. + Also use it to list voices, diagnose the local voice stack, prepare a render + plan, or correct pronunciation and audio seams. +metadata: + version: "0.1.0" +--- + +# Local Voice + +Use the shared renderer as an audio dependency. Keep briefing intelligence and +persona writing in the calling skill; keep chunking, pronunciation, generation, +alignment, timing, and assembly here. + +The skill requires the separately installed Local Voice runtime, CosyVoice3 +Candle model, ffmpeg, Whisper, and authorized reference audio. Apple Silicon +Metal is the verified production path. + +## Before rendering + +1. Read `references/voice-catalog.md` for the selected voice. +2. Read `references/render-plan-schema.md`. +3. Read the selected voice's persona guide from the configured runtime when the + request includes script writing, not just rendering. +4. Preserve two forms of every passage: + - `text`: canonical readable wording. + - `tts_text`: optional pronunciation-safe wording sent only to the engine. +5. Never include Markdown headings, speaker labels, stage directions, or + renderer metadata in spoken text. + +## Core production rules + +- Generate the largest safe group of complete sentences or paragraphs. +- Treat paragraph boundaries as candidate splits, not mandatory audio cuts. +- Preserve short openings, closings, reactions, and signature clips at native + tempo. WPM normalization is unreliable on short clips. +- Apply speed adjustment only to sufficiently long passages and only within the + profile's validated correction range. +- Create drawl, emphasis, and emotional weight through punctuation and wording, + not global slowdown. +- Use actual neighboring script text as sacrificial context when a cold start + needs trimming. Do not use unrelated generic carriers. +- Retain natural word-release tails and join at quiet waveform boundaries. +- Keep raw WAV and alignment files so `--assemble-only` can repair a brief + without regenerating model output. +- Validate transcript completeness before changing cadence or assembling. +- Prefer an approved fixed asset for Ryan's very short reactions and exact + signature exchanges. +- Never publish or redistribute reference audio unless the user has rights to + that asset. + +## Workflow + +### 1. Check the runtime + +Run: + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/local_voice.py" doctor +``` + +In Codex, resolve the plugin root from this skill's location if +`${CLAUDE_PLUGIN_ROOT}` is unavailable. + +### 2. Build or validate a render plan + +For simple single-voice prose: + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/local_voice.py" plan \ + --voice val-holiday \ + --script /absolute/path/brief.md \ + --output-plan /absolute/path/render-plan.json \ + --output-audio /absolute/path/brief.mp3 +``` + +For Chris, Ryan, signature assets, pronunciation overrides, or deliberate +delivery changes, write the JSON plan directly and validate it: + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/local_voice.py" validate \ + /absolute/path/render-plan.json +``` + +### 3. Render + +```bash +python3 "${CLAUDE_PLUGIN_ROOT}/scripts/local_voice.py" render \ + /absolute/path/render-plan.json \ + --resume +``` + +Use `--assemble-only` when all required raw WAV files already exist. + +### 4. Check the report + +Do not report success until the command produces: + +- the requested WAV or MP3; +- a QA JSON file; +- complete target text for every generated segment; +- no rejected cadence correction; +- no missing fixed or suffix asset. + +If a segment truncates, split it at a complete sentence boundary and regenerate +only that segment. If a seam clicks, retain more tail or move the cut to a quiet +boundary; do not keep regenerating speech that already passed. + +## Dependency contract + +Calling skills provide: + +- selected `voice`; +- canonical script or structured dialogue; +- desired output path; +- optional role, delivery class, fixed asset, pronunciation override, and + pause metadata. + +Local Voice returns: + +- final audio path; +- QA report path; +- duration; +- segment-level completeness and cadence results; +- a clear failure with the segment ID when human review or regeneration is + required. + +Do not silently fall back to cloud TTS. Let the calling skill decide whether to +use ElevenLabs, text-only delivery, or another fallback. diff --git a/skills/local-voice/references/render-plan-schema.md b/skills/local-voice/references/render-plan-schema.md new file mode 100644 index 0000000..65be1d7 --- /dev/null +++ b/skills/local-voice/references/render-plan-schema.md @@ -0,0 +1,80 @@ +# Render plan schema + +Use JSON with this structure: + +```json +{ + "schema_version": "1.0", + "job_id": "morning-brief-2026-07-27", + "voice": "val-holiday", + "output": "/absolute/path/brief.mp3", + "segments": [ + { + "id": "01-greeting", + "text": "Morning, Tracy. Let us get straight to the point.", + "native_tempo": true + }, + { + "id": "02-update", + "text": "The team closed the issue and documented the root cause.", + "tts_text": "The team closed the issue and documented the root cause.", + "delivery": "narrative", + "context_before": "Let us get straight to the point.", + "pause_after_ms": 100 + } + ] +} +``` + +## Plan fields + +### Root + +| Field | Required | Meaning | +|---|---:|---| +| `schema_version` | Yes | Currently `1.0`. | +| `job_id` | Yes | Stable filesystem-safe identifier. | +| `voice` | Yes | Profile ID from the voice catalog. | +| `output` | Yes | Absolute `.wav` or `.mp3` destination. | +| `segments` | Yes | Ordered list of speech or fixed assets. | +| `final_transcript_qa` | No | Defaults to `true`. | + +### Segment + +| Field | Required | Meaning | +|---|---:|---| +| `id` | Yes | Unique filesystem-safe ID. | +| `text` | Yes | Canonical readable transcript. | +| `tts_text` | No | Pronunciation-safe engine text. | +| `role` | No | Role within a multi-role voice, such as `producer`. | +| `delivery` | No | Profile delivery class, such as `high-energy`. | +| `native_tempo` | No | Disable all tempo processing for this segment. | +| `target_wpm` | No | Override the profile WPM for a long passage. | +| `tempo_multiplier` | No | Explicit validated multiplier for a complete passage. | +| `context_before` | No | Sacrificial neighboring text trimmed before target. | +| `context_after` | No | Sacrificial neighboring text trimmed after target. | +| `fixed_asset` | No | Authorized asset path relative to runtime root. | +| `asset_pool` | No | Authorized asset paths; one is selected deterministically. | +| `suffix_asset` | No | Asset appended to the generated passage. | +| `suffix_gap_ms` | No | Silence before suffix asset. | +| `pause_after_ms` | No | Silence after the assembled segment. | + +Do not combine `fixed_asset` with generated `tts_text`. A fixed segment still +needs canonical `text` for the final transcript and report. + +## Pronunciation examples + +Keep canonical text: + +```json +"text": "Nobody else wastes lead on it." +``` + +Send the engine: + +```json +"tts_text": "Nobody else wastes led on it." +``` + +Use explicit overrides for homographs. Do not globally replace every instance of +`lead`, because leadership and ammunition require different pronunciations. diff --git a/skills/local-voice/references/voice-catalog.md b/skills/local-voice/references/voice-catalog.md new file mode 100644 index 0000000..f17e238 --- /dev/null +++ b/skills/local-voice/references/voice-catalog.md @@ -0,0 +1,45 @@ +# Voice catalog + +## Donna + +- ID: `donna` +- Type: single voice +- Reference: neutral designed-voice recording +- Default: native timing, punctuation-led expression +- Use for Bryan's primary Donna brief and lower-priority notifications +- Do not feed natural-language shaping instructions into zero-shot production + +## Chris Engineer + +- ID: `chris-engineer` +- Type: single voice with narrative and high-energy references +- Use the authentic fixed opening for `Wanna see something cool!!!` +- Narrative and technical explanation use the narrative reference +- Jokes, punchlines, sharp warnings, and the closing tag use high energy +- Change delivery only at a complete sentence boundary + +## Grandpa Bomber + +- ID: `grandpa-bomber` +- Type: single voice +- Default: native timing +- Shape sleepy setup and deadpan escalation through punctuation +- Keep the edge PG and use `freaking`, not `effing` or `fudge` + +## Ryan Pitch Meeting + +- ID: `ryan-pitch-meeting` +- Type: two roles from the same performer +- Roles: `producer`, `writer` +- Writer is faster and more eager; Producer is calmer and skeptical +- Use fixed assets for exact short signatures and reactions when available +- Preserve role labels in the plan but never synthesize them + +## Val Holiday + +- ID: `val-holiday` +- Type: single voice +- Long substantive passages target 170 WPM +- Short greeting, closing, and reaction passages stay at native timing +- Do not create drawl by slowing playback +- Use punctuation and sentence construction for weighted delivery diff --git a/tests/acceptance/chris-engineer.md b/tests/acceptance/chris-engineer.md new file mode 100644 index 0000000..ef62e39 --- /dev/null +++ b/tests/acceptance/chris-engineer.md @@ -0,0 +1,5 @@ +Three warning lights all trace back to one loose connector. That is not three failures; it is one design flaw wearing three different hats. + +Replace the connector, document the root cause, and stop blaming the operator for what the hardware did. + +Fix the system once... and that is pretty cool. diff --git a/tests/acceptance/donna.md b/tests/acceptance/donna.md new file mode 100644 index 0000000..cbc050e --- /dev/null +++ b/tests/acceptance/donna.md @@ -0,0 +1,5 @@ +Good morning. Overnight, the service team closed the kiosk alert and confirmed the replacement controller is stable. + +One decision remains: approve the revised field schedule before noon so the Monday installation stays on track. + +That is the whole story. Handle the schedule, then enjoy the quiet. diff --git a/tests/acceptance/grandpa-bomber.md b/tests/acceptance/grandpa-bomber.md new file mode 100644 index 0000000..57ed879 --- /dev/null +++ b/tests/acceptance/grandpa-bomber.md @@ -0,0 +1,5 @@ +Morning. We have one little weather problem parked on the runway. + +The supplier missed the shipment, the field crew is waiting, and somebody scheduled the backup truck for the wrong county. Find the truck before I have to drop the sun on the dispatch board. + +That is all. Now get out of my briefing room before something starts leaking hydraulic fluid. diff --git a/tests/acceptance/ryan-pitch-meeting.md b/tests/acceptance/ryan-pitch-meeting.md new file mode 100644 index 0000000..96772e0 --- /dev/null +++ b/tests/acceptance/ryan-pitch-meeting.md @@ -0,0 +1,9 @@ +**PRODUCER GUY:** So, you have a Monday briefing for me? + +**WRITER GUY:** Yes sir, I do! The Phoenix hardware shipped, but the final quantity approval is still missing! + +**PRODUCER GUY:** Oh, paperwork blocking shipped hardware is tight! + +**PRODUCER GUY:** Is it going to be hard to clear that before noon? + +**WRITER GUY:** Actually, it's gonna be super easy, barely an inconvenience! Oh, really? diff --git a/tests/acceptance/val-holiday.md b/tests/acceptance/val-holiday.md new file mode 100644 index 0000000..41d6b5e --- /dev/null +++ b/tests/acceptance/val-holiday.md @@ -0,0 +1,7 @@ +Morning, Tracy. + +The Phoenix installation is ready, but the final quantity approval remains unsigned. The hardware is already moving, so paperwork is the only gentleman still blocking the road. + +The engineering team settled yesterday's controller affair and documented the root cause. A clean ending, for once. + +Sign the quantity approval before noon. Say when. diff --git a/tests/run_acceptance.py b/tests/run_acceptance.py new file mode 100755 index 0000000..6d9eb8e --- /dev/null +++ b/tests/run_acceptance.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Create and optionally render fresh Local Voice acceptance plans.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +RENDERER = ROOT / "scripts" / "local_voice.py" +CASES = ( + "donna", + "chris-engineer", + "grandpa-bomber", + "val-holiday", + "ryan-pitch-meeting", +) + + +def run(command: list[str]) -> None: + subprocess.run(command, check=True) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runtime-root", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--device", choices=("metal", "cpu", "cuda"), default="metal") + parser.add_argument("--render", action="store_true") + parser.add_argument( + "--voices", + help="Comma-separated subset of profile IDs; defaults to every case.", + ) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + selected_cases = ( + tuple(item.strip() for item in args.voices.split(",") if item.strip()) + if args.voices + else CASES + ) + unknown = sorted(set(selected_cases) - set(CASES)) + if unknown: + parser.error("Unknown acceptance voice(s): " + ", ".join(unknown)) + results = [] + for voice in selected_cases: + script = ROOT / "tests" / "acceptance" / f"{voice}.md" + plan = args.output_dir / f"{voice}.json" + audio = args.output_dir / f"{voice}.mp3" + job_id = f"acceptance-v9-{voice}" + run( + [ + sys.executable, + str(RENDERER), + "plan", + "--voice", + voice, + "--script", + str(script), + "--output-plan", + str(plan), + "--output-audio", + str(audio), + "--job-id", + job_id, + ] + ) + if voice == "ryan-pitch-meeting": + data = json.loads(plan.read_text(encoding="utf-8")) + data["segments"] = [ + { + "id": "01-producer", + "role": "producer", + "text": "So, you have a Monday briefing for me?", + "context_after": "Yes sir, I do!", + "pause_after_ms": 95, + }, + { + "id": "02-yes-sir", + "role": "writer", + "text": "Yes sir, I do!", + "fixed_asset": ( + "voices/ryan-pitch-meeting/signature-exchanges/" + "yes-sir-i-do/lotr_two_towers.wav" + ), + "pause_after_ms": 55, + }, + { + "id": "03-writer", + "role": "writer", + "text": ( + "The Phoenix hardware shipped, but the final quantity " + "approval is still missing!" + ), + "context_before": "Yes sir, I do!", + "context_after": ( + "Oh, paperwork blocking shipped hardware is tight!" + ), + "pause_after_ms": 95, + }, + { + "id": "04-tight", + "role": "producer", + "text": "Oh, paperwork blocking shipped hardware is tight!", + "tts_text": "Oh, paperwork blocking shipped hardware", + "context_before": ( + "The final quantity approval is still missing!" + ), + "context_after": "It sure is!", + "suffix_asset": ( + "voices/ryan-pitch-meeting/signature-exchanges/" + "tight-endings/is/disclosure.wav" + ), + "suffix_gap_ms": 130, + "pause_after_ms": 95, + }, + { + "id": "05-producer", + "role": "producer", + "text": ( + "Is it going to be hard to clear that before noon?" + ), + "context_before": ( + "Oh, paperwork blocking shipped hardware is tight!" + ), + "context_after": ( + "Actually, it's gonna be super easy, barely an " + "inconvenience! Oh, really?" + ), + "pause_after_ms": 95, + }, + { + "id": "06-super-easy", + "role": "writer", + "text": ( + "Actually, it's gonna be super easy, barely an " + "inconvenience! Oh, really?" + ), + "fixed_asset": ( + "voices/ryan-pitch-meeting/signature-exchanges/" + "super_easy_oh_really.wav" + ), + "pause_after_ms": 95, + }, + ] + plan.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8") + run([sys.executable, str(RENDERER), "validate", str(plan)]) + if args.render: + run( + [ + sys.executable, + str(RENDERER), + "render", + str(plan), + "--runtime-root", + str(args.runtime_root), + "--device", + args.device, + "--resume", + ] + ) + results.append({"voice": voice, "plan": str(plan), "audio": str(audio)}) + summary = args.output_dir / "acceptance-summary.json" + existing = [] + if summary.exists(): + existing = json.loads(summary.read_text(encoding="utf-8")) + merged = {item["voice"]: item for item in existing} + merged.update({item["voice"]: item for item in results}) + summary.write_text( + json.dumps([merged[key] for key in sorted(merged)], indent=2) + "\n", + encoding="utf-8", + ) + print(f"Acceptance summary: {summary}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())