Merge pull request #8 from fakerybakery/main

Handle edge case, improve model caching, enhance API, add auto GPU support, add WebUI, add CLI

README.md

@@ -23,18 +23,84 @@ Some other features include:
 - The Chinese speaker supports `mixed Chinese and English`.
 - Fast enough for `CPU real-time inference`.
 
-## Install on Linux
+## Install on Linux or macOS
+
 ```bash
-git clone git@github.com:myshell-ai/MeloTTS.git
+git clone git+https://github.com/myshell-ai/MeloTTS.git
 cd MeloTTS
 pip install -e .
 python -m unidic download
 ```
-We welcome the open-source community to make this repo `Mac` and `Windows` compatible. If you find this repo useful, please consider contributing to the repo.
+
+We welcome the open-source community to make this repo `Windows` compatible. If you find this repo useful, please consider contributing to the repo.
 
 ## Usage
 
-### English with Multi Accents
+An unofficial [live demo](https://huggingface.co/spaces/mrfakename/MeloTTS) is hosted on Hugging Face Spaces.
+
+### WebUI
+
+The WebUI supports muliple languages and voices. First, follow the installation steps. Then, simply run:
+
+```bash
+melo-ui
+# Or: python melo/app.py
+```
+
+### CLI
+
+You may use the MeloTTS CLI to interact with MeloTTS. The CLI may be invoked using either `melotts` or `melo`. Here are some examples:
+
+**Read English text:**
+
+```bash
+melo "Text to read" output.wav
+```
+
+**Specify a language:**
+
+```bash
+melo "Text to read" output.wav --language EN
+```
+
+**Specify a speaker:**
+
+```bash
+melo "Text to read" output.wav --language EN --speaker EN-US
+melo "Text to read" output.wav --language EN --speaker EN-AU
+```
+
+The available speakers are: `EN-Default`, `EN-US`, `EN-BR`, `EN-INDIA` `EN-AU`.
+
+**Specify a speed:**
+
+```bash
+melo "Text to read" output.wav --language EN --speaker EN-US --speed 1.5
+melo "Text to read" output.wav --speed 1.5
+```
+
+**Use a different language:**
+
+```bash
+melo "语音合成领域近年来发展迅速" zh.wav -l ZH
+```
+
+**Load from a file:**
+
+```bash
+melo file.txt out.wav --file
+```
+
+The full API documentation may be found using:
+
+```bash
+melo --help
+```
+
+### Python API
+
+#### English with Multiple Accents
+
 ```python
 from melo.api import TTS
 
@@ -42,8 +108,8 @@ from melo.api import TTS
 speed = 1.0
 
 # CPU is sufficient for real-time inference.
-# You can also change to cuda:0
+# You can set it manually to 'cpu' or 'cuda' or 'cuda:0' or 'mps'
-device = 'cpu'
+device = 'auto' # Will automatically use GPU if available
 
 # English 
 text = "Did you ever hear a folk tale about a giant turtle?"
@@ -91,7 +157,8 @@ output_path = 'es.wav'
 model.tts_to_file(text, speaker_ids['ES'], output_path, speed=speed)
 ```
 
-### French
+#### French
+
 ```python
 from melo.api import TTS
 
@@ -107,7 +174,8 @@ output_path = 'fr.wav'
 model.tts_to_file(text, speaker_ids['FR'], output_path, speed=speed)
 ```
 
-### Chinese
+#### Chinese
+
 ```python
 from melo.api import TTS
 
@@ -123,7 +191,8 @@ output_path = 'zh.wav'
 model.tts_to_file(text, speaker_ids['ZH'], output_path, speed=speed)
 ```
 
-### Japanese
+#### Japanese
+
 ```python
 from melo.api import TTS
 
@@ -139,7 +208,8 @@ output_path = 'jp.wav'
 model.tts_to_file(text, speaker_ids['JP'], output_path, speed=speed)
 ```
 
-### Korean
+#### Korean
+
 ```python
 from melo.api import TTS
 
@@ -156,7 +226,9 @@ model.tts_to_file(text, speaker_ids['KR'], output_path, speed=speed)
 ```
 
 ## License
-This library is under MIT License. Free for both commercial and non-commercial use.
 
-## Acknowledgement
+This library is under MIT License, which means it is free for both commercial and non-commercial use.
+
+## Acknowledgements
+
 This implementation is based on several excellent projects, [TTS](https://github.com/coqui-ai/TTS), [VITS](https://github.com/jaywalnut310/vits), [VITS2](https://github.com/daniilrobnikov/vits2) and [Bert-VITS2](https://github.com/fishaudio/Bert-VITS2). We appreciate their awesome work!

melo/api.py

@@ -7,6 +7,8 @@ import soundfile
 import torchaudio
 import numpy as np
 import torch.nn as nn
+from tqdm import tqdm
+import torch
 
 from . import utils
 from . import commons
@@ -18,8 +20,12 @@ from .download_utils import load_or_download_config, load_or_download_model
 class TTS(nn.Module):
     def __init__(self, 
                 language,
-                device='cuda:0'):
+                device='auto'):
         super().__init__()
+        if device == 'auto':
+            device = 'cpu'
+            if torch.cuda.is_available(): device = 'cuda'
+            if torch.backends.mps.is_available(): device = 'mps'
         if 'cuda' in device:
             assert torch.cuda.is_available()
 
@@ -63,18 +69,28 @@ class TTS(nn.Module):
         return audio_segments
 
     @staticmethod
-    def split_sentences_into_pieces(text, language):
+    def split_sentences_into_pieces(text, language, quiet=False):
         texts = split_sentence(text, language_str=language)
-        print(" > Text splitted to sentences.")
+        if not quiet:
-        print('\n'.join(texts))
+            print(" > Text split to sentences.")
-        print(" > ===========================")
+            print('\n'.join(texts))
+            print(" > ===========================")
         return texts
 
-    def tts_to_file(self, text, speaker_id, output_path=None, sdp_ratio=0.2, noise_scale=0.6, noise_scale_w=0.8, speed=1.0):
+    def tts_to_file(self, text, speaker_id, output_path=None, sdp_ratio=0.2, noise_scale=0.6, noise_scale_w=0.8, speed=1.0, pbar=None, format=None, position=None, quiet=False,):
         language = self.language
-        texts = self.split_sentences_into_pieces(text, language)
+        texts = self.split_sentences_into_pieces(text, language, quiet)
         audio_list = []
-        for t in texts:
+        if pbar:
+            tx = pbar(texts)
+        else:
+            if position:
+                tx = tqdm(texts, position=position)
+            elif quiet:
+                tx = texts
+            else:
+                tx = tqdm(texts)
+        for t in tx:
             if language in ['EN', 'ZH_MIX_EN']:
                 t = re.sub(r'([a-z])([A-Z])', r'\1 \2', t)
             device = self.device
@@ -110,4 +126,7 @@ class TTS(nn.Module):
         if output_path is None:
             return audio
         else:
-            soundfile.write(output_path, audio, self.hps.data.sampling_rate)
+            if format:
+                soundfile.write(output_path, audio, self.hps.data.sampling_rate, format=format)
+            else:
+                soundfile.write(output_path, audio, self.hps.data.sampling_rate)

melo/app.py

@@ -0,0 +1,47 @@
+# WebUI by mrfakename <X @realmrfakename / HF @mrfakename>
+# Demo also available on HF Spaces: https://huggingface.co/spaces/mrfakename/MeloTTS
+import gradio as gr
+import os, torch, io
+# os.system('python -m unidic download')
+print("Make sure you've downloaded unidic (python -m unidic download) for this WebUI to work.")
+from melo.api import TTS
+speed = 1.0
+import tempfile
+import click
+device = 'auto'
+models = {
+    'EN': TTS(language='EN', device=device),
+    'ES': TTS(language='ES', device=device),
+    'FR': TTS(language='FR', device=device),
+    'ZH': TTS(language='ZH', device=device),
+    'JP': TTS(language='JP', device=device),
+    'KR': TTS(language='KR', device=device),
+}
+speaker_ids = models['EN'].hps.data.spk2id
+def synthesize(speaker, text, speed, language, progress=gr.Progress()):
+    bio = io.BytesIO()
+    models[language].tts_to_file(text, models[language].hps.data.spk2id[speaker], bio, speed=speed, pbar=progress.tqdm, format='wav')
+    return bio.getvalue()
+def load_speakers(language):
+    return gr.update(value=list(models[language].hps.data.spk2id.keys())[0], choices=list(models[language].hps.data.spk2id.keys()))
+with gr.Blocks() as demo:
+    gr.Markdown('# MeloTTS WebUI\n\nA WebUI for MeloTTS.')
+    with gr.Group():
+        speaker = gr.Dropdown(speaker_ids.keys(), interactive=True, value='EN-Default', label='Speaker')
+        language = gr.Radio(['EN', 'ES', 'FR', 'ZH', 'JP', 'KR'], label='Language', value='EN')
+        language.input(load_speakers, inputs=language, outputs=speaker)
+        speed = gr.Slider(label='Speed', minimum=0.1, maximum=10.0, value=1.0, interactive=True, step=0.1)
+        text = gr.Textbox(label="Text to speak", value='The field of text to speech has seen rapid development recently')
+    btn = gr.Button('Synthesize', variant='primary')
+    aud = gr.Audio(interactive=False)
+    btn.click(synthesize, inputs=[speaker, text, speed, language], outputs=[aud])
+    gr.Markdown('WebUI by [mrfakename](https://twitter.com/realmrfakename).')
+@click.command()
+@click.option('--share', '-s', is_flag=True, show_default=True, default=False, help="Expose a publicly-accessible shared Gradio link usable by anyone with the link. Only share the link with people you trust.")
+@click.option('--host', '-h', default=None)
+@click.option('--port', '-p', default=None)
+def main(share, host, port):
+    demo.queue(api_open=False, default_concurrency_limit=10).launch(show_api=False, share=share, server_name=host, server_port=port)
+
+if __name__ == "__main__":
+    main()

melo/download_utils.py

@@ -1,7 +1,7 @@
 import torch
 import os
 from . import utils
-
+from cached_path import cached_path
 DOWNLOAD_CKPT_URLS = {
     'EN': 'https://myshell-public-repo-hosting.s3.amazonaws.com/openvoice/basespeakers/EN/checkpoint.pth',
     'EN_V2': 'https://myshell-public-repo-hosting.s3.amazonaws.com/openvoice/basespeakers/EN_V2/checkpoint.pth',
@@ -25,23 +25,11 @@ DOWNLOAD_CONFIG_URLS = {
 def load_or_download_config(locale):
     language = locale.split('-')[0].upper()
     assert language in DOWNLOAD_CONFIG_URLS
-    config_path = os.path.expanduser(f'~/.local/share/openvoice/basespeakers/{language}/config.json')
+    config_path = cached_path(DOWNLOAD_CONFIG_URLS[language])
-    try:
-        return utils.get_hparams_from_file(config_path)
-    except:
-        # download
-        os.makedirs(os.path.dirname(config_path), exist_ok=True)
-        os.system(f'wget {DOWNLOAD_CONFIG_URLS[language]} -O {config_path}')
     return utils.get_hparams_from_file(config_path)
 
 def load_or_download_model(locale, device):
     language = locale.split('-')[0].upper()
     assert language in DOWNLOAD_CKPT_URLS
-    ckpt_path = os.path.expanduser(f'~/.local/share/openvoice/basespeakers/{language}/checkpoint.pth')
+    ckpt_path = cached_path(DOWNLOAD_CKPT_URLS[language])
-    try:
+    return torch.load(ckpt_path, map_location=device)
-        return torch.load(ckpt_path, map_location=device)
-    except:
-        # download
-        os.makedirs(os.path.dirname(ckpt_path), exist_ok=True)
-        os.system(f'wget {DOWNLOAD_CKPT_URLS[language]} -O {ckpt_path}')
-    return torch.load(ckpt_path, map_location=device)

melo/main.py

@@ -0,0 +1,38 @@
+import click
+import warnings
+import os
+
+@click.command
+@click.argument('text')
+@click.argument('output_path')
+@click.option("--file", '-f', is_flag=True, show_default=True, default=False, help="Text is a file")
+@click.option('--language', '-l', default='EN', help='Language, defaults to English', type=click.Choice(['EN', 'ES', 'FR', 'ZH', 'JP', 'KR'], case_sensitive=False))
+@click.option('--speaker', '-spk', default='EN-Default', help='Speaker ID, only for English, leave empty for default, ignored if not English. If English, defaults to "EN-Default"', type=click.Choice(['EN-Default', 'EN-US', 'EN-BR', 'EN-INDIA', 'EN-AU']))
+@click.option('--speed', '-s', default=1.0, help='Speed, defaults to 1.0', type=float)
+@click.option('--device', '-d', default='auto', help='Device, defaults to auto')
+def main(text, file, output_path, language, speaker, speed, device):
+    if file:
+        if not os.path.exists(text):
+            raise FileNotFoundError(f'Trying to load text from file due to --file/-f flag, but file not found. Remove the --file/-f flag to pass a string.')
+        else:
+            with open(text) as f:
+                text = f.read().strip()
+    if text == '':
+        raise ValueError('You entered empty text or the file you passed was empty.')
+    language = language.upper()
+    if language == '': language = 'EN'
+    if speaker == '': speaker = None
+    if (not language == 'EN') and speaker:
+        warnings.warn('You specified a speaker but the language is English.')
+    from melo.api import TTS
+    model = TTS(language=language, device=device)
+    speaker_ids = model.hps.data.spk2id
+    if language == 'EN':
+        if not speaker: speaker = 'EN-Default'
+        spkr = speaker_ids[speaker]
+    else:
+        spkr = speaker_ids[list(speaker_ids.keys())[0]]
+    model.tts_to_file(text, spkr, output_path, speed=speed)
+
+if __name__ == "__main__":
+    main()

melo/split_utils.py

@@ -4,7 +4,7 @@ import glob
 import numpy as np
 import soundfile as sf
 import torchaudio
-
+from txtsplit import txtsplit
 def split_sentence(text, min_len=10, language_str='EN'):
     if language_str in ['EN', 'FR', 'ES', 'SP', 'DE', 'RU']:
         sentences = split_sentences_latin(text, min_len=min_len)
@@ -18,26 +18,27 @@ def split_sentences_latin(text, min_len=10):
     text = re.sub('[“”]', '"', text)
     text = re.sub('[‘’]', "'", text)
     text = re.sub(r"[\<\>\(\)\[\]\"\«\»]+", "", text)
+    return [item.strip() for item in txtsplit(text, 512, 512) if item.strip()]
     # 将文本中的换行符、空格和制表符替换为空格
-    text = re.sub('[\n\t ]+', ' ', text)
+    # text = re.sub('[\n\t ]+', ' ', text)
-    # 在标点符号后添加一个空格
+    # # 在标点符号后添加一个空格
-    text = re.sub('([,.!?;])', r'\1 $#!', text)
+    # text = re.sub('([,.!?;])', r'\1 $#!', text)
-    # 分隔句子并去除前后空格
+    # # 分隔句子并去除前后空格
-    sentences = [s.strip() for s in text.split('$#!')]
+    # sentences = [s.strip() for s in text.split('$#!')]
-    if len(sentences[-1]) == 0: del sentences[-1]
+    # if len(sentences[-1]) == 0: del sentences[-1]
 
-    new_sentences = []
+    # new_sentences = []
-    new_sent = []
+    # new_sent = []
-    count_len = 0
+    # count_len = 0
-    for ind, sent in enumerate(sentences):
+    # for ind, sent in enumerate(sentences):
-        # print(sent)
+    #     # print(sent)
-        new_sent.append(sent)
+    #     new_sent.append(sent)
-        count_len += len(sent.split(" "))
+    #     count_len += len(sent.split(" "))
-        if count_len > min_len or ind == len(sentences) - 1:
+    #     if count_len > min_len or ind == len(sentences) - 1:
-            count_len = 0
+    #         count_len = 0
-            new_sentences.append(' '.join(new_sent))
+    #         new_sentences.append(' '.join(new_sent))
-            new_sent = []
+    #         new_sent = []
-    return merge_short_sentences_en(new_sentences)
+    # return merge_short_sentences_en(new_sentences)
 
 def split_sentences_zh(text, min_len=10):
     text = re.sub('[。！？；]', '.', text)
@@ -127,4 +128,4 @@ if __name__ == '__main__':
     print(split_sentence(sp_text, language_str='SP'))
     print(split_sentence(fr_text, language_str='FR'))
     print(split_sentence(de_text, language_str='DE'))
-    print(split_sentence(ru_text, language_str='RU'))
+    print(split_sentence(ru_text, language_str='RU'))

requirements.txt

@@ -1,5 +1,7 @@
+txtsplit
 torch<2.0
 torchaudio
+cached_path
 transformers==4.27.4
 mecab-python3==1.0.5
 num2words==0.5.12
@@ -21,5 +23,6 @@ unidecode==1.3.7
 pypinyin==0.50.0
 cn2an==0.5.22
 jieba==0.42.1
-gradio==3.48.0
+gradio
-langid==1.1.6
+langid==1.1.6
+tqdm

setup.py

@@ -21,11 +21,18 @@ class PostDevelopCommand(develop):
 
 setup(
     name='melo',
-    version='0.1.0',
+    version='0.1.1',
     packages=find_packages(),
     include_package_data=True,
     install_requires=requirements,
     package_data={
         '': ['*.txt', 'cmudict_*'],
     },
+    entry_points={
+        "console_scripts": [
+            "melotts = melo.main:main",
+            "melo = melo.main:main",
+            "melo-ui = melo.app:main",
+        ],
+    },
 )