diff --git a/.gitignore b/.gitignore index 22ec6e410a..c2fd44cb4d 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,5 @@ wandb depot/* coqui_recipes/* local_scripts/* -coqui_demos/* \ No newline at end of file +coqui_demos/* +recipes/ljspeech/xtts_v2/run/* \ No newline at end of file diff --git a/TTS/tts/datasets/formatters.py b/TTS/tts/datasets/formatters.py index 053444b0c1..bdccc8386f 100644 --- a/TTS/tts/datasets/formatters.py +++ b/TTS/tts/datasets/formatters.py @@ -653,3 +653,18 @@ def bel_tts_formatter(root_path, meta_file, **kwargs): # pylint: disable=unused text = cols[1] items.append({"text": text, "audio_file": wav_file, "speaker_name": speaker_name, "root_path": root_path}) return items + +def afrotts(root_path, meta_file, **kwargs): + csv_path = os.path.join(root_path, meta_file) + csv_file = pd.read_csv(csv_path) + csv_file["char_count"] = csv_file.transcript.apply(lambda x: len(list(x))) + csv_file = csv_file[csv_file.char_count < 400].copy() + csv_file["audio_paths"] = csv_file["audio_paths"].apply( + lambda x: x.replace("/AfriSpeech-TTS-D/", root_path) + ) + csv_file = csv_file.rename(columns={"transcript":"text", "audio_paths":"audio_file", "user_ids":"speaker_name"}) + csv_file = csv_file[["audio_file", "text", "speaker_name"]] + csv_file['root_path'] = root_path + items = csv_file.to_dict('records') + return items + diff --git a/TTS/tts/layers/xtts/tokenizer.py b/TTS/tts/layers/xtts/tokenizer.py index 1a3cc47aaf..f66ab93c85 100644 --- a/TTS/tts/layers/xtts/tokenizer.py +++ b/TTS/tts/layers/xtts/tokenizer.py @@ -32,7 +32,7 @@ def get_spacy_lang(lang): return English() -def split_sentence(text, lang, text_split_length=250): +def split_sentence(text, lang, text_split_length=400): """Preprocess the input text""" text_splits = [] if text_split_length is not None and len(text) >= text_split_length: @@ -595,7 +595,7 @@ def __init__(self, vocab_file=None): if vocab_file is not None: self.tokenizer = Tokenizer.from_file(vocab_file) self.char_limits = { - "en": 250, + "en": 400, "de": 253, "fr": 273, "es": 239, @@ -621,7 +621,7 @@ def katsu(self): def check_input_length(self, txt, lang): lang = lang.split("-")[0] # remove the region - limit = self.char_limits.get(lang, 250) + limit = self.char_limits.get(lang, 400) if len(txt) > limit: print( f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio." diff --git a/recipes/ljspeech/xtts_v2/README.md b/recipes/ljspeech/xtts_v2/README.md new file mode 100644 index 0000000000..15b6aa1a75 --- /dev/null +++ b/recipes/ljspeech/xtts_v2/README.md @@ -0,0 +1,67 @@ + +# Coqui AI TTS + +This repository contains the code for training a text-to-speech (TTS) model using Coqui AI's TTS framework. + +## Installation and Configuration + +1. **Clone the repository:** + + ```bash + git clone https://github.com/owos/coqui-ai-TTS.git + ``` + +2. **Navigate to the repository root:** + + ```bash + cd coqui-ai-TTS + ``` +3. **Create a virtual environment with python version 3.10** + + ```bash + conda create -n xtts python==3.10 + conda activate xtts + ``` + +4. **Install system dependencies and the code:** + + ```bash + make system-deps # Intended to be used on Ubuntu (Debian). Let us know if you have a different OS. + make install + ``` + +5. **Open the following file and redefine the specified variables:** + + File: `recipes/ljspeech/xtts_v2/train_gpt_xtts.py` + + ```python + # Line 30 + path = 'the root path to the audio dirs on your machine' + + # Line 31 + meta_file_train = "the root path to the train CSV on your machine" + + # Line 32 + meta_file_val = "the root path to the train CSV on your machine" + + # Line 75 + SPEAKER_REFERENCE = "a list with a single path to a test audio from the afro tts data" + ``` + +## Running the Code + +From the repository root, run the following command: + +```bash python +python3 recipes/ljspeech/xtts_v2/train_gpt_xtts.py +``` + +You are now ready to train your TTS model using Coqui AI's framework. Enjoy! + +## Optional: Resuming from a checkpoint + +File: `recipes/ljspeech/xtts_v2/train_gpt_xtts_resume.py` + +Update the parameters in the file for the models + + diff --git a/recipes/ljspeech/xtts_v2/train_gpt_xtts.py b/recipes/ljspeech/xtts_v2/train_gpt_xtts.py index 626917381a..a456cbe9bd 100644 --- a/recipes/ljspeech/xtts_v2/train_gpt_xtts.py +++ b/recipes/ljspeech/xtts_v2/train_gpt_xtts.py @@ -1,4 +1,6 @@ import os +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"] = "0" from trainer import Trainer, TrainerArgs @@ -8,7 +10,7 @@ from TTS.utils.manage import ModelManager # Logging parameters -RUN_NAME = "GPT_XTTS_v2.0_LJSpeech_FT" +RUN_NAME = "GPT_XTTS_v2.0_AfroTTS_FT" PROJECT_NAME = "XTTS_trainer" DASHBOARD_LOGGER = "tensorboard" LOGGER_URI = None @@ -18,17 +20,18 @@ # Training Parameters OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False -START_WITH_EVAL = True # if True it will star with evaluation -BATCH_SIZE = 3 # set here the batch size -GRAD_ACUMM_STEPS = 84 # set here the grad accumulation steps +START_WITH_EVAL = False # if True it will star with evaluation +BATCH_SIZE = 16 # set here the batch size +GRAD_ACUMM_STEPS = 4 # set here the grad accumulation steps # Note: we recommend that BATCH_SIZE * GRAD_ACUMM_STEPS need to be at least 252 for more efficient training. You can increase/decrease BATCH_SIZE but then set GRAD_ACUMM_STEPS accordingly. # Define here the dataset that you want to use for the fine-tuning on. config_dataset = BaseDatasetConfig( - formatter="ljspeech", - dataset_name="ljspeech", - path="/raid/datasets/LJSpeech-1.1_24khz/", - meta_file_train="/raid/datasets/LJSpeech-1.1_24khz/metadata.csv", + formatter="afrotts", + dataset_name="afrotts", + path="/data4/data/AfriSpeech-TTS-D/", + meta_file_train="/data4/abraham/tts/AfriSpeech-TTS/data/afritts-train-clean.csv", + meta_file_val="/data4/abraham/tts/AfriSpeech-TTS/data/afritts-dev-clean.csv", language="en", ) @@ -72,7 +75,7 @@ # Training sentences generations SPEAKER_REFERENCE = [ - "./tests/data/ljspeech/wavs/LJ001-0002.wav" # speaker reference to be used in training test sentences + "/data4/data/AfriSpeech-TTS-D/train/1dddeb9f-18ec-4498-b74b-84ac59f2fcf1/e9af9831281555e8685e511f7becdf32_P2L385Vp.wav" # speaker reference to be used in training test sentences ] LANGUAGE = config_dataset.language @@ -83,8 +86,8 @@ def main(): max_conditioning_length=132300, # 6 secs min_conditioning_length=66150, # 3 secs debug_loading_failures=False, - max_wav_length=255995, # ~11.6 seconds - max_text_length=200, + max_wav_length=255995, # ~11.6 seconds 661500, #~ 30 seconds # + max_text_length=300, mel_norm_file=MEL_NORM_FILE, dvae_checkpoint=DVAE_CHECKPOINT, xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune @@ -110,18 +113,18 @@ def main(): logger_uri=LOGGER_URI, audio=audio_config, batch_size=BATCH_SIZE, - batch_group_size=48, + batch_group_size=64, eval_batch_size=BATCH_SIZE, num_loader_workers=8, eval_split_max_size=256, print_step=50, plot_step=100, - log_model_step=1000, - save_step=10000, - save_n_checkpoints=1, + log_model_step=100, + save_step=1000, + save_n_checkpoints=3, save_checkpoints=True, # target_loss="loss", - print_eval=False, + print_eval=True, # Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters. optimizer="AdamW", optimizer_wd_only_on_weights=OPTIMIZER_WD_ONLY_ON_WEIGHTS, @@ -154,7 +157,6 @@ def main(): eval_split_max_size=config.eval_split_max_size, eval_split_size=config.eval_split_size, ) - # init the trainer and 🚀 trainer = Trainer( TrainerArgs( @@ -174,3 +176,4 @@ def main(): if __name__ == "__main__": main() +DATASETS_CONFIG_LIST \ No newline at end of file diff --git a/recipes/ljspeech/xtts_v2/train_gpt_xtts_resume_ft.py b/recipes/ljspeech/xtts_v2/train_gpt_xtts_resume_ft.py new file mode 100644 index 0000000000..3324bc2ec8 --- /dev/null +++ b/recipes/ljspeech/xtts_v2/train_gpt_xtts_resume_ft.py @@ -0,0 +1,171 @@ +import os +os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" +os.environ["CUDA_VISIBLE_DEVICES"] = "0" + +from trainer import Trainer, TrainerArgs + +from TTS.config.shared_configs import BaseDatasetConfig +from TTS.tts.datasets import load_tts_samples +from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig +from TTS.utils.manage import ModelManager + +# Logging parameters +RUN_NAME = "GPT_XTTS_v2.0_AfroTTS_FT" +PROJECT_NAME = "XTTS_trainer" +DASHBOARD_LOGGER = "tensorboard" +LOGGER_URI = None + +# Set here the path that the checkpoints will be saved. Default: ./run/training/ +OUT_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "run", "training") + +# Training Parameters +OPTIMIZER_WD_ONLY_ON_WEIGHTS = True # for multi-gpu training please make it False +START_WITH_EVAL = False # if True it will star with evaluation +BATCH_SIZE = 2 # set here the batch size +GRAD_ACUMM_STEPS = 126 # set here the grad accumulation steps +# Note: we recommend that BATCH_SIZE * GRAD_ACUMM_STEPS need to be at least 252 for more efficient training. You can increase/decrease BATCH_SIZE but then set GRAD_ACUMM_STEPS accordingly. + +afrotts_dir = "AfriSpeech-TTS-D" # add path to afrotts data here + +# Define here the dataset that you want to use for the fine-tuning on. +config_dataset = BaseDatasetConfig( + formatter="afrotts", + dataset_name="afrotts", + path=afrotts_dir, + meta_file_train=os.path.join(afrotts_dir, "data/afritts-train-clean-upsamp.csv") #afritts-train-clean-upsamp.csv + meta_file_val=os.path.join(afrotts_dir,"data/afritts-dev-clean.csv"), + language="en", +) + +# Add here the configs of the datasets +DATASETS_CONFIG_LIST = [config_dataset] + +# Define the path where XTTS v2.0.1 files will be downloaded +CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, "XTTS_v2.0_original_model_files/") +os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True) + +# Set the path to the downloaded files +DVAE_CHECKPOINT = "coqui-ai-TTS/recipes/ljspeech/xtts_v2/run/training/XTTS_v2.0_original_model_files/dvae.pth" +MEL_NORM_FILE = "coqui-ai-TTS/recipes/ljspeech/xtts_v2/run/training/XTTS_v2.0_original_model_files/mel_stats.pth" + +# download DVAE files if needed +if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE): + print(" > Downloading DVAE files!") + ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True) + + +# XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning. +TOKENIZER_FILE = "coqui-ai-TTS/recipes/ljspeech/xtts_v2/run/training/XTTS_v2.0_original_model_files/vocab.json" +XTTS_CHECKPOINT = "coqui-ai-TTS/recipes/ljspeech/xtts_v2/run/training/GPT_XTTS_v2.0_AfroTTS_FT-March-06-2024_06+36AM-581cf506/checkpoint_135000.pth" + + +# download XTTS v2.0 files if needed +if not os.path.isfile(TOKENIZER_FILE) or not os.path.isfile(XTTS_CHECKPOINT): + print(" > Downloading XTTS v2.0 files!") + ModelManager._download_model_files( + [TOKENIZER_FILE_LINK, XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True + ) + + +# Training sentences generations +SPEAKER_REFERENCE = [ + "/AfriSpeech-TTS/train/defc5e03-926c-4e0b-a639-c821e5e7db89/14f64f13c57f9a64a2a1521253934a0b_KYA8MaKS.wav" # speaker reference to be used in training test sentences +] +LANGUAGE = config_dataset.language + +def main(): + # init args and config + model_args = GPTArgs( + max_conditioning_length=132300, # 6 secs + min_conditioning_length=66150, # 3 secs + debug_loading_failures=False, + max_wav_length=255995, # ~11.6 seconds 661500, #~ 30 seconds # + max_text_length=300, + mel_norm_file=MEL_NORM_FILE, + dvae_checkpoint=DVAE_CHECKPOINT, + xtts_checkpoint=XTTS_CHECKPOINT, # checkpoint path of the model that you want to fine-tune + tokenizer_file=TOKENIZER_FILE, + gpt_num_audio_tokens=1026, + gpt_start_audio_token=1024, + gpt_stop_audio_token=1025, + gpt_use_masking_gt_prompt_approach=True, + gpt_use_perceiver_resampler=True, + ) + # define audio config + audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000) + # training parameters config + config = GPTTrainerConfig( + output_path=OUT_PATH, + model_args=model_args, + run_name=RUN_NAME, + project_name=PROJECT_NAME, + run_description=""" + GPT XTTS training + """, + dashboard_logger=DASHBOARD_LOGGER, + logger_uri=LOGGER_URI, + audio=audio_config, + batch_size=BATCH_SIZE, + batch_group_size=64, + eval_batch_size=BATCH_SIZE, + num_loader_workers=8, + eval_split_max_size=256, + print_step=1000, + plot_step=1000, + log_model_step=1000, + save_step=1000, + save_n_checkpoints=3, + save_checkpoints=True, + # target_loss="loss", + print_eval=True, + # Optimizer values like tortoise, pytorch implementation with modifications to not apply WD to non-weight parameters. + optimizer="AdamW", + optimizer_wd_only_on_weights=OPTIMIZER_WD_ONLY_ON_WEIGHTS, + optimizer_params={"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": 1e-2}, + lr=5e-06, # learning rate + lr_scheduler="MultiStepLR", + # it was adjusted accordly for the new step scheme + lr_scheduler_params={"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}, + test_sentences=[ + { + "text": "It took me quite a long time to develop a voice, and now that I have it I'm not going to be silent.", + "speaker_wav": SPEAKER_REFERENCE, + "language": LANGUAGE, + }, + { + "text": "This cake is great. It's so delicious and moist.", + "speaker_wav": SPEAKER_REFERENCE, + "language": LANGUAGE, + }, + ], + ) + + # init the model from config + model = GPTTrainer.init_from_config(config) + + # load training samples + train_samples, eval_samples = load_tts_samples( + DATASETS_CONFIG_LIST, + eval_split=True, + eval_split_max_size=config.eval_split_max_size, + eval_split_size=config.eval_split_size, + ) + # init the trainer and 🚀 + trainer = Trainer( + TrainerArgs( + restore_path=None, # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter + skip_train_epoch=False, + start_with_eval=START_WITH_EVAL, + grad_accum_steps=GRAD_ACUMM_STEPS, + ), + config, + output_path=OUT_PATH, + model=model, + train_samples=train_samples, + eval_samples=eval_samples, + ) + trainer.fit() + + +if __name__ == "__main__": + main()