From 272b8ffbf660b0cfd07e20fdef538f335a4f92ca Mon Sep 17 00:00:00 2001 From: zengguoyang Date: Tue, 16 Sep 2025 11:46:47 +0800 Subject: [PATCH] init --- LICENSE | 201 ++++++ README.md | 268 ++++++++ app.py | 279 ++++++++ assets/modelbest_logo.png | Bin 0 -> 55126 bytes assets/thuhcsi_logo.png | Bin 0 -> 10187 bytes assets/voxcpm_logo.png | Bin 0 -> 24487 bytes assets/voxcpm_model.png | Bin 0 -> 144982 bytes examples/example.wav | Bin 0 -> 1439096 bytes pyproject.toml | 96 +++ requirements.txt | 16 + src/voxcpm/__init__.py | 5 + src/voxcpm/cli.py | 292 +++++++++ src/voxcpm/core.py | 181 ++++++ src/voxcpm/model/__init__.py | 3 + src/voxcpm/model/utils.py | 122 ++++ src/voxcpm/model/voxcpm.py | 605 ++++++++++++++++++ src/voxcpm/modules/__init__.py | 0 src/voxcpm/modules/audiovae/__init__.py | 1 + src/voxcpm/modules/audiovae/audio_vae.py | 359 +++++++++++ src/voxcpm/modules/layers/__init__.py | 1 + .../layers/scalar_quantization_layer.py | 26 + src/voxcpm/modules/locdit/__init__.py | 2 + src/voxcpm/modules/locdit/local_dit.py | 114 ++++ src/voxcpm/modules/locdit/unified_cfm.py | 137 ++++ src/voxcpm/modules/locenc/__init__.py | 1 + src/voxcpm/modules/locenc/local_encoder.py | 30 + src/voxcpm/modules/minicpm4/__init__.py | 3 + src/voxcpm/modules/minicpm4/cache.py | 47 ++ src/voxcpm/modules/minicpm4/config.py | 29 + src/voxcpm/modules/minicpm4/model.py | 411 ++++++++++++ src/voxcpm/utils/text_normalize.py | 244 +++++++ 31 files changed, 3473 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 app.py create mode 100644 assets/modelbest_logo.png create mode 100644 assets/thuhcsi_logo.png create mode 100644 assets/voxcpm_logo.png create mode 100644 assets/voxcpm_model.png create mode 100644 examples/example.wav create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 src/voxcpm/__init__.py create mode 100644 src/voxcpm/cli.py create mode 100644 src/voxcpm/core.py create mode 100644 src/voxcpm/model/__init__.py create mode 100644 src/voxcpm/model/utils.py create mode 100644 src/voxcpm/model/voxcpm.py create mode 100644 src/voxcpm/modules/__init__.py create mode 100644 src/voxcpm/modules/audiovae/__init__.py create mode 100644 src/voxcpm/modules/audiovae/audio_vae.py create mode 100644 src/voxcpm/modules/layers/__init__.py create mode 100644 src/voxcpm/modules/layers/scalar_quantization_layer.py create mode 100644 src/voxcpm/modules/locdit/__init__.py create mode 100644 src/voxcpm/modules/locdit/local_dit.py create mode 100644 src/voxcpm/modules/locdit/unified_cfm.py create mode 100644 src/voxcpm/modules/locenc/__init__.py create mode 100644 src/voxcpm/modules/locenc/local_encoder.py create mode 100644 src/voxcpm/modules/minicpm4/__init__.py create mode 100644 src/voxcpm/modules/minicpm4/cache.py create mode 100644 src/voxcpm/modules/minicpm4/config.py create mode 100644 src/voxcpm/modules/minicpm4/model.py create mode 100644 src/voxcpm/utils/text_normalize.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..abd7fd6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright OpenBMB + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b4cb85f --- /dev/null +++ b/README.md @@ -0,0 +1,268 @@ +## 🎙️ VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation and True-to-Life Voice Cloning + + +[![Project Page](https://img.shields.io/badge/Project%20Page-GitHub-blue)](https://github.com/OpenBMB/VoxCPM/) [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-OpenBMB-yellow)](hhttps://huggingface.co/openbmb/VoxCPM-0.5B) [![Live Playground](https://img.shields.io/badge/Live%20PlayGround-Demo-orange)](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) [![Samples](https://img.shields.io/badge/Page-Samples-red)](https://thuhcsi.github.io/VoxCPM/) + + +
+ VoxCPM Logo +
+ +## News +* [2025.09.16] 🔥 🔥 🔥 We Open Source the VoxCPM-0.5B weights! +* [2025.09.16] 🎉 🎉 🎉 We Provide the [Gradio PlayGround](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) for VoxCPM-0.5B, try it now! + +## Overview + +VoxCPM is a novel tokenizer-free Text-to-Speech (TTS) system that redefines realism in speech synthesis. By modeling speech in a continuous space, it overcomes the limitations of discrete tokenization and enables two flagship capabilities: context-aware speech generation and true-to-life zero-shot voice cloning. + +Unlike mainstream approaches that convert speech to discrete tokens, VoxCPM uses an end-to-end diffusion autoregressive architecture that directly generates continuous speech representations from text. Built on [MiniCPM-4](https://huggingface.co/openbmb/MiniCPM4-0.5B), it achieves implicit semantic-acoustic decoupling through hierachical language modeling and FSQ constraints, greatly enhancing both expressiveness and generation stability. + +
+ VoxCPM Model Architecture +
+ + +### 🚀 Key Features +- **Context-Aware, Expressive Speech Generation** - VoxCPM comprehends text to infer and generate appropriate prosody, delivering speech with remarkable expressiveness and natural flow. It spontaneously adapts speaking style based on content, producing highly fitting vocal expression trained on a massive 1.8 million-hour bilingual corpus. +- **True-to-Life Voice Cloning** - With only a short reference audio clip, VoxCPM performs accurate zero-shot voice cloning, capturing not only the speaker’s timbre but also fine-grained characteristics such as accent, emotional tone, rhythm, and pacing to create a faithful and natural replica. +- **High-Efficiency Synthesis** - VoxCPM supports streaming synthesis with a Real-Time Factor (RTF) as low as 0.17 on a consumer-grade NVIDIA RTX 4090 GPU, making it possible for real-time applications. + + + +## Quick Start + +### 🔧 Install from PyPI +``` sh +pip install voxcpm +``` +### 1. Model Download (Optional) +By default, when you first run the script, the model will be downloaded automatically, but you can also download the model in advance. +- Download VoxCPM-0.5B + ``` + from huggingface_hub import snapshot_download + snapshot_download("openbmb/VoxCPM-0.5B",local_files_only=local_files_only) + ``` +- Download ZipEnhancer and SenseVoice-Small. We use ZipEnhancer to enhance speech prompts and SenseVoice-Small for speech prompt ASR in the web demo. + ``` + from modelscope import snapshot_download + snapshot_download('iic/speech_zipenhancer_ans_multiloss_16k_base') + snapshot_download('iic/SenseVoiceSmall') + ``` + +### 2. Basic Usage +```python +import soundfile as sf +from voxcpm import VoxCPM + +model = VoxCPM.from_pretrained("openbmb/VoxCPM-0.5B") + +wav = model.generate( + text="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly expressive speech.", + prompt_wav_path=None, # optional: path to a prompt speech for voice cloning + prompt_text=None, # optional: reference text + cfg_value=2.0, + inference_timesteps=10, + normalize=True, + denoise=True, + retry_badcase=True, # optional: enable retrying mode + retry_badcase_max_times=3, + retry_badcase_ratio_threshold=6.0, +) + +sf.write("output.wav", wav, 16000) +print("saved: output.wav") +``` + +### 3. CLI Usage + +After installation, the entry point is `voxcpm` (or use `python -m voxcpm.cli`). + +```bash +# 1) Direct synthesis (single text) +voxcpm --text "Hello VoxCPM" --output out.wav + +# 2) Voice cloning (reference audio + transcript) +voxcpm --text "Hello" \ + --prompt-audio path/to/voice.wav \ + --prompt-text "reference transcript" \ + --output out.wav \ + --denoise + +# 3) Batch processing (one text per line) +voxcpm --input examples/input.txt --output-dir outs +# (optional) Batch + cloning +voxcpm --input examples/input.txt --output-dir outs \ + --prompt-audio path/to/voice.wav \ + --prompt-text "reference transcript" \ + --denoise + +# 4) Inference parameters (quality/speed) +voxcpm --text "..." --output out.wav \ + --cfg-value 2.0 --inference-timesteps 10 --normalize + +# 5) Model loading +# Prefer local path +voxcpm --text "..." --output out.wav --model-path /path/to/VoxCPM_model_dir +# Or from Hugging Face (auto download/cache) +voxcpm --text "..." --output out.wav \ + --hf-model-id openbmb/VoxCPM-0.5B --cache-dir ~/.cache/huggingface --local-files-only + +# 6) Denoiser control +voxcpm --text "..." --output out.wav \ + --no-denoiser --zipenhancer-path iic/speech_zipenhancer_ans_multiloss_16k_base + +# 7) Help +voxcpm --help +python -m voxcpm.cli --help +``` + +### 4. Start web demo + +You can start the UI interface by running `python app.py`, which allows you to perform Voice Cloning and Voice Creation. + + + +## 👩‍🍳 A Voice Chef's Guide +Welcome to the VoxCPM kitchen! Follow this recipe to cook up perfect generated speech. Let’s begin. + +--- +### 🥚 Step 1: Prepare Your Base Ingredients (Content) + +First, choose how you’d like to input your text:. +1. Regular Text (Classic Mode) +- ✅ Keep "Text Normalization" ON. Type naturally (e.g., "Hello, world! 123"). The system will automatically process numbers, abbreviations, and punctuation using WeTextProcessing library. +2. Phoneme Input (Native Mode) +- ❌ Turn "Text Normalization" OFF. Enter phoneme text like {HH AH0 L OW1} (EN) or {ni3}{hao3} (ZH) for precise pronunciation control. In this mode, VoxCPM also supports native understanding of other complex non-normalized text—try it out! + +--- +### 🍳 Step 2: Choose Your Flavor Profile (Voice Style) + +This is the secret sauce that gives your audio its unique sound. +1. Cooking with a Prompt Speech (Following a Famous Recipe) + - A prompt speech provides the desired acoustic characteristics for VoxCPM. The speaker's timbre, speaking style, and even the background sounds and ambiance will be replicated. + - For a Clean, Studio-Quality Voice: + - ✅ Enable "Prompt Speech Enhancement". This acts like a noise filter, removing background hiss and rumble to give you a pure, clean voice clone. +2. Cooking au Naturel (Letting the Model Improvise) + - If no reference is provided, VoxCPM becomes a creative chef! It will infer a fitting speaking style based on the text itself, thanks to the text-smartness of its foundation model, MiniCPM-4. + - Pro Tip: Challenge VoxCPM with any text—poetry, song lyrics, dramatic monologues—it may deliver some interesting results! + +--- +### 🧂 Step 3: The Final Seasoning (Fine-Tuning Your Results) +You're ready to serve! But for master chefs who want to tweak the flavor, here are two key spices. +- CFG Value (How Closely to Follow the Recipe) + - Default: A great starting point. + - Voice sounds strained or weird? Lower this value. It tells the model to be more relaxed and improvisational, great for expressive prompts. + - Need maximum clarity and adherence to the text? Raise it slightly to keep the model on a tighter leash. +- Inference Timesteps (Simmering Time: Quality vs. Speed) + - Need a quick snack? Use a lower number. Perfect for fast drafts and experiments. + - Cooking a gourmet meal? Use a higher number. This lets the model "simmer" longer, refining the audio for superior detail and naturalness. + +--- +Happy creating! 🎉 Start with the default settings and tweak from there to suit your project. The kitchen is yours! + + +--- + + + +## 📊 Performance Highlights + +VoxCPM achieves competitive results on public zero-shot TTS benchmarks: + +### Seed-TTS-eval Benchmark + +| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | | +|------|------|------|:------------:|:--:|:------------:|:--:|:-------------:|:--:| +| | | | WER/%⬇ | SIM/%⬆| CER/%⬇| SIM/%⬆ | CER/%⬇ | SIM/%⬆ | +| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 | +| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | 6.83 | 72.4 | +| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 | +| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - | +| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 | +| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - | +| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | 74.7 | +| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | - | - | +| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | - | - | +| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - | +| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | - | - | +| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 | +| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 | +| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - | +| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - | +| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 | +| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - | +| **VoxCPM** | **0.5B** | **✅** | **1.85** | **72.9** | **0.93** | **77.2** | 8.87 | 73.0 | + + +### CV3-eval Benchmark + +| Model | zh | en | hard-zh | | | hard-en | | | | +|-------|:--:|:--:|:-------:|:--:|:--:|:-------:|:--:|:--:|:--:| +| | CER/%⬇ | WER/%⬇ | CER/%⬇ | SIM/%⬆ | DNSMOS⬆ | WER/%⬇ | SIM/%⬆ | DNSMOS⬆ | | +| F5-TTS | 5.47 | 8.90 | - | - | - | - | - | - | | +| SparkTTS | 5.15 | 11.0 | - | - | - | - | - | - | | +| GPT-SoVits | 7.34 | 12.5 | - | - | - | - | - | - | | +| CosyVoice2 | 4.08 | 6.32 | 12.58 | 72.6 | 3.81 | 11.96 | 66.7 | 3.95 | | +| OpenAudio-s1-mini | 4.00 | 5.54 | 18.1 | 58.2 | 3.77 | 12.4 | 55.7 | 3.89 | | +| IndexTTS2 | 3.58 | 4.45 | 12.8 | 74.6 | 3.65 | fail | fail | fail | | +| HiggsAudio-v2 | 9.54 | 7.89 | 41.0 | 60.2 | 3.39 | 10.3 | 61.8 | 3.68 | | +| CosyVoice3-0.5B | 3.89 | 5.24 | 14.15 | 78.6 | 3.75 | 9.04 | 75.9 | 3.92 | | +| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 78.5 | 3.79 | 10.55 | 76.1 | 3.95 | | +| **VoxCPM** | **3.40** | **4.04** | 12.9 | 66.1 | 3.59 | **7.89** | 64.3 | 3.74 | | + + + + + + + + + + + + +## ⚠️ Risks and limitations +- General Model Behavior: While VoxCPM has been trained on a large-scale dataset, it may still produce outputs that are unexpected, biased, or contain artifacts. +- Potential for Misuse of Voice Cloning: VoxCPM's powerful zero-shot voice cloning capability can generate highly realistic synthetic speech. This technology could be misused for creating convincing deepfakes for purposes of impersonation, fraud, or spreading disinformation. Users of this model must not use it to create content that infringes upon the rights of individuals. It is strictly forbidden to use VoxCPM for any illegal or unethical purposes. We strongly recommend that any publicly shared content generated with this model be clearly marked as AI-generated. +- Current Technical Limitations: Although generally stable, the model may occasionally exhibit instability, especially with very long or expressive inputs. Furthermore, the current version offers limited direct control over specific speech attributes like emotion or speaking style. +- Bilingual Model: VoxCPM is trained primarily on Chinese and English data. Performance on other languages is not guaranteed and may result in unpredictable or low-quality audio. +- This model is released for research and development purposes only. We do not recommend its use in production or commercial applications without rigorous testing and safety evaluations. Please use VoxCPM responsibly. + + + +## 📄 License +The VoxCPM model weights and code are open-sourced under the [Apache-2.0](LICENSE) license. + +## 🙏 Acknowledgments + +We extend our sincere gratitude to the following works and resources for their inspiration and contributions: + +- [DiTAR](https://arxiv.org/abs/2502.03930) for the diffusion autoregressive backbone used in speech generation +- [MiniCPM-4](https://github.com/OpenBMB/MiniCPM) for serving as the language model foundation +- [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) for the implementation of Flow Matching-based LocDiT +- [DAC](https://github.com/descriptinc/descript-audio-codec) for providing the Audio VAE backbone + +## Institutions + +This project is developed by the following institutions: +- [ModelBest](https://modelbest.cn/) + +- [THUHCSI](https://github.com/thuhcsi) + + + + +## 📚 Citation + +If you find our model helpful, please consider citing our projects 📝 and staring us ⭐️！ + +```bib +@misc{voxcpm2025, + author = {{Yixuan Zhou, Guoyang Zeng, Xin Liu, Xiang Li, Renjie Yu, Ziyang Wang, Runchuan Ye, Weiyue Sun, Jiancheng Gui, Kehan Li, Zhiyong Wu, Zhiyuan Liu}}, + title = {{VoxCPM}}, + year = {2025}, + publish = {\url{https://github.com/OpenBMB/VoxCPM}}, + note = {GitHub repository} +} +``` diff --git a/app.py b/app.py new file mode 100644 index 0000000..f07b8fd --- /dev/null +++ b/app.py @@ -0,0 +1,279 @@ +import os +import numpy as np +import torch +import gradio as gr +import spaces +from typing import Optional, Tuple +from funasr import AutoModel +from pathlib import Path +os.environ["TOKENIZERS_PARALLELISM"] = "false" +if os.environ.get("HF_REPO_ID", "").strip() == "": + os.environ["HF_REPO_ID"] = "openbmb/VoxCPM-0.5B" + +import voxcpm + + +class VoxCPMDemo: + def __init__(self) -> None: + self.device = "cuda" if torch.cuda.is_available() else "cpu" + print(f"🚀 Running on device: {self.device}") + + # ASR model for prompt text recognition + self.asr_model_id = "iic/SenseVoiceSmall" + self.asr_model: Optional[AutoModel] = AutoModel( + model=self.asr_model_id, + disable_update=True, + log_level='DEBUG', + device="cuda:0" if self.device == "cuda" else "cpu", + ) + + # TTS model (lazy init) + self.voxcpm_model: Optional[voxcpm.VoxCPM] = None + self.default_local_model_dir = "./models/VoxCPM-0.5B" + + # ---------- Model helpers ---------- + def _resolve_model_dir(self) -> str: + """ + Resolve model directory: + 1) Use local checkpoint directory if exists + 2) If HF_REPO_ID env is set, download into models/{repo} + 3) Fallback to 'models' + """ + if os.path.isdir(self.default_local_model_dir): + return self.default_local_model_dir + + repo_id = os.environ.get("HF_REPO_ID", "").strip() + if len(repo_id) > 0: + target_dir = os.path.join("models", repo_id.replace("/", "__")) + if not os.path.isdir(target_dir): + try: + from huggingface_hub import snapshot_download # type: ignore + os.makedirs(target_dir, exist_ok=True) + print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...") + snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False) + except Exception as e: + print(f"Warning: HF download failed: {e}. Falling back to 'data'.") + return "models" + return target_dir + return "models" + + def get_or_load_voxcpm(self) -> voxcpm.VoxCPM: + if self.voxcpm_model is not None: + return self.voxcpm_model + print("Model not loaded, initializing...") + model_dir = self._resolve_model_dir() + print(f"Using model dir: {model_dir}") + self.voxcpm_model = voxcpm.VoxCPM(voxcpm_model_path=model_dir) + print("Model loaded successfully.") + return self.voxcpm_model + + # ---------- Functional endpoints ---------- + def prompt_wav_recognition(self, prompt_wav: Optional[str]) -> str: + if prompt_wav is None: + return "" + res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True) + text = res[0]["text"].split('|>')[-1] + return text + + def generate_tts_audio( + self, + text_input: str, + prompt_wav_path_input: Optional[str] = None, + prompt_text_input: Optional[str] = None, + cfg_value_input: float = 2.0, + inference_timesteps_input: int = 10, + do_normalize: bool = True, + denoise: bool = True, + ) -> Tuple[int, np.ndarray]: + """ + Generate speech from text using VoxCPM; optional reference audio for voice style guidance. + Returns (sample_rate, waveform_numpy) + """ + current_model = self.get_or_load_voxcpm() + + text = (text_input or "").strip() + if len(text) == 0: + raise ValueError("Please input text to synthesize.") + + prompt_wav_path = prompt_wav_path_input if prompt_wav_path_input else None + prompt_text = prompt_text_input if prompt_text_input else None + + print(f"Generating audio for text: '{text[:60]}...'") + wav = current_model.generate( + text=text, + prompt_text=prompt_text, + prompt_wav_path=prompt_wav_path, + cfg_value=float(cfg_value_input), + inference_timesteps=int(inference_timesteps_input), + normalize=do_normalize, + denoise=denoise, + ) + return (16000, wav) + + +# ---------- UI Builders ---------- + +def create_demo_interface(demo: VoxCPMDemo): + """Build the Gradio UI for VoxCPM demo.""" + # static assets (logo path) + gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"]) + + with gr.Blocks( + theme=gr.themes.Soft( + primary_hue="blue", + secondary_hue="gray", + neutral_hue="slate", + font=[gr.themes.GoogleFont("Inter"), "Arial", "sans-serif"] + ), + css=""" + .logo-container { + text-align: center; + margin: 0.5rem 0 1rem 0; + } + .logo-container img { + height: 80px; + width: auto; + max-width: 200px; + display: inline-block; + } + /* Bold accordion labels */ + #acc_quick details > summary, + #acc_tips details > summary { + font-weight: 600 !important; + font-size: 1.1em !important; + } + /* Bold labels for specific checkboxes */ + #chk_denoise label, + #chk_denoise span, + #chk_normalize label, + #chk_normalize span { + font-weight: 600; + } + """ + ) as interface: + # Header logo + gr.HTML('
VoxCPM Logo
') + + # Quick Start + with gr.Accordion("📋 Quick Start Guide ｜快速入门", open=False, elem_id="acc_quick"): + gr.Markdown(""" + ### How to Use ｜使用说明 + 1. **(Optional) Provide a Voice Prompt** - Upload or record an audio clip to provide the desired voice characteristics for synthesis. + **（可选）提供参考声音** - 上传或录制一段音频，为声音合成提供音色、语调和情感等个性化特征 + 2. **(Optional) Enter prompt text** - If you provided a voice prompt, enter the corresponding transcript here (auto-recognition available). + **（可选项）输入参考文本** - 如果提供了参考语音，请输入其对应的文本内容（支持自动识别）。 + 3. **Enter target text** - Type the text you want the model to speak. + **输入目标文本** - 输入您希望模型朗读的文字内容。 + 4. **Generate Speech** - Click the "Generate" button to create your audio. + **生成语音** - 点击"生成"按钮，即可为您创造出音频。 + """) + + # Pro Tips + with gr.Accordion("💡 Pro Tips ｜使用建议", open=False, elem_id="acc_tips"): + gr.Markdown(f""" + ### Prompt Speech Enhancement｜参考语音降噪 + - **Enable** to remove background noise for a clean, studio-like voice, with an external ZipEnhancer component. + **启用**：通过 ZipEnhancer 组件消除背景噪音，获得更好的音质。 + - **Disable** to preserve the original audio's background atmosphere. + **禁用**：保留原始音频的背景环境声，如果想复刻相应声学环境。 + + ### Text Normalization｜文本正则化 + - **Enable** to process general text with an external WeTextProcessing component. + **启用**：使用 WeTextProcessing 组件，可处理常见文本。 + - **Disable** to use VoxCPM's native text understanding ability. For example, it supports phonemes input ({HH AH0 L OW1}), try it! + **禁用**：将使用 VoxCPM 内置的文本理解能力。如，支持音素输入（如 {da4}{jia1}好）和公式符号合成，尝试一下！ + + ### CFG Value｜CFG 值 + - **Lower CFG** if the voice prompt sounds strained or expressive. + **调低**：如果提示语音听起来不自然或过于夸张。 + - **Higher CFG** for better adherence to the prompt speech style or input text. + **调高**：为更好地贴合提示音频的风格或输入文本。 + + ### Inference Timesteps｜推理时间步 + - **Lower** for faster synthesis speed. + **调低**：合成速度更快。 + - **Higher** for better synthesis quality. + **调高**：合成质量更佳。 + + ### Long Text (e.g., >5 min speech)｜长文本 (如 >5分钟的合成语音) + While VoxCPM can handle long texts directly, we recommend using empty lines to break very long content into paragraphs; the model will then synthesize each paragraph individually. + 虽然 VoxCPM 支持直接生成长文本，但如果目标文本过长，我们建议使用换行符将内容分段；模型将对每个段落分别合成。 + """) + + # Main controls + with gr.Row(): + with gr.Column(): + prompt_wav = gr.Audio( + sources=["upload", 'microphone'], + type="filepath", + label="Prompt Speech", + value="./examples/example.wav", + ) + DoDenoisePromptAudio = gr.Checkbox( + value=False, + label="Prompt Speech Enhancement", + elem_id="chk_denoise", + info="We use ZipEnhancer model to denoise the prompt audio." + ) + with gr.Row(): + prompt_text = gr.Textbox( + value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.", + label="Prompt Text", + placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..." + ) + run_btn = gr.Button("Generate Speech", variant="primary") + + with gr.Column(): + cfg_value = gr.Slider( + minimum=1.0, + maximum=3.0, + value=2.0, + step=0.1, + label="CFG Value (Guidance Scale)", + info="Higher values increase adherence to prompt, lower values allow more creativity" + ) + inference_timesteps = gr.Slider( + minimum=4, + maximum=30, + value=10, + step=1, + label="Inference Timesteps", + info="Number of inference timesteps for generation (higher values may improve quality but slower)" + ) + with gr.Row(): + text = gr.Textbox( + value="VoxCPM is an innovative end-to-end TTS model from ModelBest, designed to generate highly realistic speech.", + label="Target Text", + info="Default processing splits text on \\n into paragraphs; each is synthesized as a chunk and then concatenated into the final audio." + ) + with gr.Row(): + DoNormalizeText = gr.Checkbox( + value=False, + label="Text Normalization", + elem_id="chk_normalize", + info="We use WeTextPorcessing library to normalize the input text." + ) + audio_output = gr.Audio(label="Output Audio") + + # Wiring + run_btn.click( + fn=demo.generate_tts_audio, + inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio], + outputs=[audio_output], + show_progress=True, + api_name="generate", + ) + prompt_wav.change(fn=demo.prompt_wav_recognition, inputs=[prompt_wav], outputs=[prompt_text]) + + return interface + + +def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error: bool = True): + demo = VoxCPMDemo() + interface = create_demo_interface(demo) + # Recommended to enable queue on Spaces for better throughput + interface.queue(max_size=10).launch(server_name=server_name, server_port=server_port, show_error=show_error) + + +if __name__ == "__main__": + run_demo() \ No newline at end of file diff --git a/assets/modelbest_logo.png b/assets/modelbest_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..afbf01b5ff16cdb57988f0806424abaa71f6be00 GIT binary patch literal 55126 zcmeFZh7t37u>wmH4w-n!I&hbnAhbo0mpzb; zS@aagQPg!=Se&XX-o0|s>o2O`--#X}7MDfew%a%#IZBYiiQ;nG7v5}6N8b1lw*Atk zkGXC=a1!Zp9j8H#fp3ODcD_@!-2TCj6FU&~hfEf92Gs8?pTw|HzkhlNMXk{<`~UtL zwH^PrE&mq|APoL5Oa2cX{tp%Z|4V}(B({A062jT{{@S}8EPRzNzY9@lx1RX;@tJfv zy5Ab8P59<@IT|gNc8v6DWmzUH@$n9chZBLJ9`sOqVJ}7yZK zdah0D?bkWM_!07Q@JWl&kyjtm*a*{vMcYbB+UqCK{ES-N!krMQvG4P1D#uJt-a%x) zh++SLxHf&;ykj8M{xdL7G|0ZOJt<~AUEp{+A;cQJRRF>f(P@+>#Bk(KCvBDNT3N8T z%L{u2F_odmfU2~&$A`cq32C7zP~We^8NW%c&z@7_#gjB6HY>$S??EDdH>hoOG(0JM z;<|*a%8k_{hB%`>NRRp;9h?=nyNCBYre~#sO*<;OrOI+MU9I^s7CmRN#_qV_#&$mO zQ2Il)H*XxavLP(~;62?K7~=JXbFfY$KZn~5S`78B9HHK@j-@U8V-*jCDK^&9)77<) zS4*(<7-=9Ufi32XAr2A1C8#;uq2w*_wY6195wWvx;`Q3pJpZlT^Y7l&m4ge!uvy36 zku5QWm}v2*tyygZIyM9l2ECW!r7d({Xs2KBJ$~HFZZMKY)a+F3xtI;FTVy-$C&z%8 z@QatmJZ3tpXW#Kcq7g9oLFl3Ahuk5ZSG}F1cdojnWv-cV`5XmGpQg(%M0-0P9Ctl< zf{c4djS%oDTlavQv$RR{4lN0@8 ze6~-74voVY@F0HXV7HYXBurQJcx?9Ygn444VL<$$V&PJOI&{1D=jxOd>c^M$Fg1($ zd=&nWracIcCso&S%7yw-y8MMUzPERjz6#pt;VF!c8H1dFU>QLtB6@Xm^;76fHe!4W zyd0gEXh#~6kjvzE_(y@9DRcEQ1%cG~b);0}|w0>yf;?Htqm`*%-@;)1c1VeAX zjfdg!Mls7pj{fKu70&GCnf)eau5lV6yUFhzG0g^8P!*clkY}k0e}*iN;E7HjHZlBu zfIz#5pQf1m;ll}D{T?X?F5k{?PG>jQjfjRG5x7O%s0~^sM!3}O1ou_ahcH*qaMWS^ z&-ZyTvQ6?;7}y?!WJ`#{k9kZ=5)(5&+l*bmr z-Qj+%-(g-KkoJ@<*)glLHXW`9g6loyIAZrO#Kly@aBAD{{Gr_;S}$K^Gj}z!kWs4j zy!yME@n8+C0BPxK)tl|{B*|WMAkr$@H>B@n+9b0~H`ZM%LAMwjG%q}VsEX2?Y`fkc z^;id54{`ZEMON8qXSAA1wLAFuc5l@{po;*LA1lj_ovQtAv%pR>Fkrg=dFZYa^~lEJ zD_t8GWRwUStwwiJgvrJvnB^SzDq+DsNs#-p@9Bf$De}oF5XxdUKf9;Ngj2lira1K4 z?vd1zk5UE2BB4;6lVR*P>wWg@WNOXmWi|l zc65A!1!28dd_J`4#C56^(0V5jk%l%-fz7K(&BbB=cMHSU#|82b07u{DKb65%W!lAb zC%KvS6uolCHB&SFMb6(_K@h0)`wFvxtw0U=@$8&CM*_bN>c#}qmfoHH@cF*l!*q47 z1A&r&JyDeiaJw$c&SE@{Eyax?w$rBO22nVMdcu!y34v>-4Opk+X33T@aG$3w2ly*9 zSHe`>RzgWoKeE^jTALFp2t`hA^-4k;|LUo=UfFw$yH9uwyO~*f$M(L2j;QQ<9%#Wx z_vFxYb%(LCHH8wW=*UOD<7N10o?d12-(1{nS|T0&-sx5+r91kIh}y>UIt^FUhY`|| z#efkjt?{zL;O~{jw(3^^W`}y5oUd%+^9NIRaN!E(G_?_YH@#EWSzFsNdSvVRIOrF2 z=nQ^H-=gukoxkVHx&Y=70hQ9##2I z7Q5h6h7V-+g@(~kAp_&Fh*~A9g6mAx3p4ogHIXzshTqX&*ZvsB)v0I7-0vTJcpa8; z$H@I6i1w1US=9RM#FUF6^YIW!%Z%u;U<1Te^;EG-!xGA))?d>(=0a7b z9_@)wmMb*vDg_u2-t4@{cOsm+I>I~iTq&_?UmO#ov^_`AJGQ5iOe|ueFK1zi z-A8y9Y#fbx%m``-X}BMM9@5j^m!G9XN@CL+X6FQGK37Z;ULbv7`{|`ROyIo*_V8K2 zfy}ey1ly{lKZI!i0n^~>sOw+NYE}h#g2nHTa_06{z($DC&wJ$@U#1QmEEo^K6Caaa zU*l(Wj?T-pHtk2eh1=pLK~=CU0EmVjsAu)$=zW_m6mMD>yD5_`*SbE;&;YpWrw{h> ze&=ewvMaAB``zxzzcD^S{p4ikGy*JTmq_nC!TL@!hH~V$`PHL2DoA2{g9rkW7*)>t zK78-Z{AD zpLkb_{rfWAXfmq4)ra)lJ{@S)5y=`x_2Z3y8meF2n0(g9_Ty_=UFk0i7a5TeVCo)UvNS~?4E*mH7KSmR=w zLB(gRtnxjoT63x7nT>tOA-PX2T6J_e>wK+5Y0+VsAY-k=pseC|>J}*E`SK9<;`c6L zFWJ$* zV}2n)H@;XA{@PTvK*Pqp3JyH(;s!rU^rLRVQ<`0^ygFeWbPj5tBDCs>tod0YR>c9gQ$A!5(tLEZA334wq>jSlBeos$|H+X`= z9Q4W=kPRzEm@`QdGz!#iKT!{%cSHI$MIHC{BpfRjru6QcpNK=e9x)2LUvESZb^ZxT z1dI!b)$v|b^gwS?#B_Z2B$0xPf&LCU1AX%}MaoRD{#8+jqK@+9B&#Z^G>8p@xXC&~ z8C3e1|>DKaTkjU@wCt?I6)9Rf>0NH*un_eQ9k3#m$n5@12UN5JIPL&3~#>Ef9 zoLaj9S?KXJF@2d$Q}EtWoX1q!28?}Dz-NiQNKInvzT_+HP(Y%v%d=KeN+u3Q6H62+ zpZwaG71=qm-Y%f;2xbvu+%3LISjw|_&linq0kJ;j7+~GDr#}MaeKN`ji3Npt!&onnP=Sj z_2vpIvJ9CCO-(X?7q<7A{@n)aHLxw$x}Ya# z57Ed9Uo!+$Yg7-U0>}?7=lIf}kO;@Yi7(zwvn~hbjmmt!oR_61ORpsUJLLl}|B-V`ZZxW=tz6Qkb~vgUdT-9-R`u`0d5##!Ev3Lus+ou)XtO4&6FI z=0q;sUeIk}AE1N6vN0zjU{pd`7YP%ZO~2`4RS%kSCg}g===L}Br_=Kh*A=B*(BOq{ z=D&1KiLv7IX$y&Zo&#Z1h>O$`Gp1F>L#(!X8&~p~<>sLdTR96z}&eg>xMgWC| zu3By%{?5gDxVdrL+l?rDNXd(gr}$@aLD(Y1uzNG5k}5!wFU%=6dRKpEgTH9})=h+E)c8dtb8Btr znlGXv-7_PhjkOs-bm}v=Yk7$Xux>K91G>!-RK_9f~-N=+(`2p6?C(b^MlnY?7=Fn??F_LB+y#`H^5Po^)PH+9 zAVfhwYZm?nQ01z_%)76Hg0wdW<@6g_G^Q3J9*&vy6~sINAr?qzEQaaR);qm6CqrG_ z@i|&Eh%Ef<=ivqTs0M8vwzl5P%y&1L4e#>;5z)NY=;{BnN3^^U%8NRNlOsDiZ}U`- z+sGCvEbb7r$6IQcNK5bjDlXe$J%M@hd7+kbp_&?sua(_NDK1m+C1Zt;N?gnfU|5{Oy4>8! zXTV?r_-C4FKu=!|;OQG`&I{CMx!Gfhv6LP_M*nkI2Hr;rxT)UdNwtTweB!Ba6 zR>k08lhCO}+3H>CPfsy-)YrpoDTPxm>msd_UDqu4w(;!L-|yLpXc~Jzz+1or-FO!tThj`d8_Xi9gfM1J66uW z&?N`O(l5N?U5!s<@(pS9%+@F<@+XS6Jb``~s1Ko!L$9#E-zKqaV-W`zm*qm=2>=F= zOYikxht(?DjP*pmW_n&NTS)7#vKc%gEpc0Udg-J5M|!|A0bs@`)9JTq*A|UfPdtuL zOcfR6X9~=>OB}MR3qF`j@=(-*{UsaEXIR5$@Cyg+G1rJ^fkI5+G%ACKk#C2J4guy= z@tN00E2RC(B9SR@ndF%Q6px7Lw{)rMn?FeIz?Bui^vGX738QB%s!9L9W&Ai4+nN)(nuo`RDW`a&Y88$9chlv=W