frequency noise

[Shenkailai]

dali.zip (279000 steps)

This is a synthesized 16kHz Chinese audio, where noise is consistently present at specific frequencies.

[Shenkailai]

config:
{
"train": {
"log_interval": 200,
"eval_interval": 1000,
"seed": 2024,
"epochs": 20000,
"learning_rate": 2e-4,
"betas": [0.8, 0.99],
"eps": 1e-9,
"batch_size": 32,
"fp16_run": false,
"lr_decay": 0.999875,
"segment_size": 16384 ,
"init_lr_ratio": 1,
"warmup_epochs": 0,
"c_mel": 45,
"c_kl": 1.0,
"fft_sizes": [768, 1366, 342],
"hop_sizes": [60, 120, 20],
"win_lengths": [300, 600, 120],
"window": "hann_window"
},
"data": {
"use_mel_posterior_encoder": true,
"training_files":"filelists/all/train.txt.cleaned3",
"validation_files":"filelists/all/val.txt.cleaned3",
"max_wav_value": 32768.0,
"sampling_rate": 16000,
"filter_length": 2048,
"hop_length": 512,
"win_length": 2048,
"n_mel_channels": 80,
"mel_fmin": 0.0,
"mel_fmax": null,
"add_blank": true,
"n_speakers": 1304,
"cleaned_text": true
},
"model": {
"use_mel_posterior_encoder": true,
"use_transformer_flows": true,
"transformer_flow_type": "pre_conv2",
"use_spk_conditioned_encoder": true,
"use_noise_scaled_mas": true,
"use_duration_discriminator": true,
"duration_discriminator_type": "dur_disc_2",
"ms_istft_vits": false,
"mb_istft_vits": true,
"istft_vits": false,
"subbands": 4,
"gen_istft_n_fft": 16,
"gen_istft_hop_size": 4,
"inter_channels": 192,
"hidden_channels": 96,
"filter_channels": 768,
"n_heads": 2,
"n_layers": 3,
"kernel_size": 3,
"p_dropout": 0.1,
"resblock": "1",
"resblock_kernel_sizes": [3,7,11],
"resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
"upsample_rates": [4,4,2],
"upsample_initial_channel": 256,
"upsample_kernel_sizes": [16,16,8],
"n_layers_q": 3,
"use_spectral_norm": false,
"use_sdp": false,
"gin_channels": 256
}
}

[Shenkailai]

In addition, I have also adopted the frontend processing method for both Chinese and English from GPT-SoVITS: https://github.com/RVC-Boss/GPT-SoVITS/tree/main/GPT_SoVITS/text.

[Shenkailai]

However, it seems that the original MB-iSTFT-VITS2 (without any modifications) also exhibits this issue on LJSpeech.