Cool_Tools/Retrieval-based-Voice-Conversion-WebUI
1
0
Fork 0
mirror of synced 2024-11-24 07:30:16 +01:00
Code Releases Wiki Activity

Format code (#727)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
github-actions[bot] 2023-07-13 14:35:24 +08:00 committed by GitHub
parent 6c13f1fe52
commit 9739f3085d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 418 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
gui_v1.py
View File

@ -1,7 +1,10 @@
import os, sys import os, sys
now_dir = os.getcwd() now_dir = os.getcwd()
sys.path.append(now_dir) sys.path.append(now_dir)
import multiprocessing import multiprocessing
class Harvest(multiprocessing.Process): class Harvest(multiprocessing.Process):
def __init__(self, inp_q, opt_q): def __init__(self, inp_q, opt_q):
multiprocessing.Process.__init__(self) multiprocessing.Process.__init__(self)
@ -10,7 +13,8 @@ class Harvest(multiprocessing.Process):
def run(self): def run(self):
import numpy as np, pyworld import numpy as np, pyworld
while(1):
while 1:
idx, x, res_f0, n_cpu, ts = self.inp_q.get() idx, x, res_f0, n_cpu, ts = self.inp_q.get()
f0, t = pyworld.harvest( f0, t = pyworld.harvest(
x.astype(np.double), x.astype(np.double),
@ -20,10 +24,11 @@ class Harvest(multiprocessing.Process):
frame_period=10, frame_period=10,
) )
res_f0[idx] = f0 res_f0[idx] = f0
if(len(res_f0.keys())>=n_cpu): if len(res_f0.keys()) >= n_cpu:
self.opt_q.put(ts) self.opt_q.put(ts)
if __name__ == '__main__':
if __name__ == "__main__":
from multiprocessing import Queue from multiprocessing import Queue
from queue import Empty from queue import Empty
import numpy as np import numpy as np
@ -48,6 +53,7 @@ if __name__ == '__main__':
for _ in range(n_cpu): for _ in range(n_cpu):
Harvest(inp_q, opt_q).start() Harvest(inp_q, opt_q).start()
from rvc_for_realtime import RVC from rvc_for_realtime import RVC
class GUIConfig: class GUIConfig:
def __init__(self) -> None: def __init__(self) -> None:
self.pth_path: str = "" self.pth_path: str = ""
@ -65,7 +71,6 @@ if __name__ == '__main__':
self.n_cpu = min(n_cpu, 8) self.n_cpu = min(n_cpu, 8)
self.f0method = "harvest" self.f0method = "harvest"
class GUI: class GUI:
def __init__(self) -> None: def __init__(self) -> None:
self.config = GUIConfig() self.config = GUIConfig()
@ -191,10 +196,30 @@ if __name__ == '__main__':
], ],
[ [
sg.Text(i18n("音高算法")), sg.Text(i18n("音高算法")),
sg.Radio("pm","f0method",key="pm",default=data.get("pm","")==True), sg.Radio(
sg.Radio("harvest","f0method",key="harvest",default=data.get("harvest","")==True), "pm",
sg.Radio("crepe","f0method",key="crepe",default=data.get("crepe","")==True), "f0method",
sg.Radio("rmvpe","f0method",key="rmvpe",default=data.get("rmvpe","")==True), key="pm",
default=data.get("pm", "") == True,
),
sg.Radio(
"harvest",
"f0method",
key="harvest",
default=data.get("harvest", "") == True,
),
sg.Radio(
"crepe",
"f0method",
key="crepe",
default=data.get("crepe", "") == True,
),
sg.Radio(
"rmvpe",
"f0method",
key="rmvpe",
default=data.get("rmvpe", "") == True,
),
], ],
], ],
title=i18n("常规设置"), title=i18n("常规设置"),
@ -218,7 +243,9 @@ if __name__ == '__main__':
key="n_cpu", key="n_cpu",
resolution=1, resolution=1,
orientation="h", orientation="h",
default_value=data.get("n_cpu", min(self.config.n_cpu,n_cpu)), default_value=data.get(
"n_cpu", min(self.config.n_cpu, n_cpu)
),
), ),
], ],
[ [
@ -281,7 +308,14 @@ if __name__ == '__main__':
"crossfade_length": values["crossfade_length"], "crossfade_length": values["crossfade_length"],
"extra_time": values["extra_time"], "extra_time": values["extra_time"],
"n_cpu": values["n_cpu"], "n_cpu": values["n_cpu"],
"f0method": ["pm","harvest","crepe","rmvpe"][[values["pm"],values["harvest"],values["crepe"],values["rmvpe"]].index(True)], "f0method": ["pm", "harvest", "crepe", "rmvpe"][
[
values["pm"],
values["harvest"],
values["crepe"],
values["rmvpe"],
].index(True)
],
} }
with open("values1.json", "w") as j: with open("values1.json", "w") as j:
json.dump(settings, j) json.dump(settings, j)
@ -314,7 +348,14 @@ if __name__ == '__main__':
self.config.O_noise_reduce = values["O_noise_reduce"] self.config.O_noise_reduce = values["O_noise_reduce"]
self.config.index_rate = values["index_rate"] self.config.index_rate = values["index_rate"]
self.config.n_cpu = values["n_cpu"] self.config.n_cpu = values["n_cpu"]
self.config.f0method = ["pm","harvest","crepe","rmvpe"][[values["pm"],values["harvest"],values["crepe"],values["rmvpe"]].index(True)] self.config.f0method = ["pm", "harvest", "crepe", "rmvpe"][
[
values["pm"],
values["harvest"],
values["crepe"],
values["rmvpe"],
].index(True)
]
return True return True
def start_vc(self): def start_vc(self):
@ -325,20 +366,64 @@ if __name__ == '__main__':
self.config.pth_path, self.config.pth_path,
self.config.index_path, self.config.index_path,
self.config.index_rate, self.config.index_rate,
self.config.n_cpu,inp_q,opt_q,device self.config.n_cpu,
inp_q,
opt_q,
device,
) )
self.config.samplerate = self.rvc.tgt_sr self.config.samplerate = self.rvc.tgt_sr
self.config.crossfade_time=min(self.config.crossfade_time,self.config.block_time) self.config.crossfade_time = min(
self.config.crossfade_time, self.config.block_time
)
self.block_frame = int(self.config.block_time * self.config.samplerate) self.block_frame = int(self.config.block_time * self.config.samplerate)
self.crossfade_frame = int(self.config.crossfade_time * self.config.samplerate) self.crossfade_frame = int(
self.config.crossfade_time * self.config.samplerate
)
self.sola_search_frame = int(0.01 * self.config.samplerate) self.sola_search_frame = int(0.01 * self.config.samplerate)
self.extra_frame = int(self.config.extra_time * self.config.samplerate) self.extra_frame = int(self.config.extra_time * self.config.samplerate)
self.zc = self.rvc.tgt_sr // 100 self.zc = self.rvc.tgt_sr // 100
self.input_wav: np.ndarray = np.zeros(int(np.ceil((self.extra_frame+ self.crossfade_frame+ self.sola_search_frame+ self.block_frame)/self.zc)*self.zc),dtype="float32",) self.input_wav: np.ndarray = np.zeros(
self.output_wav_cache: torch.Tensor = torch.zeros(int(np.ceil((self.extra_frame+ self.crossfade_frame+ self.sola_search_frame+ self.block_frame)/self.zc)*self.zc), device=device,dtype=torch.float32) int(
self.pitch: np.ndarray = np.zeros(self.input_wav.shape[0]//self.zc,dtype="int32",) np.ceil(
self.pitchf: np.ndarray = np.zeros(self.input_wav.shape[0]//self.zc,dtype="float64",) (
self.output_wav: torch.Tensor = torch.zeros(self.block_frame, device=device, dtype=torch.float32) self.extra_frame
+ self.crossfade_frame
+ self.sola_search_frame
+ self.block_frame
)
/ self.zc
)
* self.zc
),
dtype="float32",
)
self.output_wav_cache: torch.Tensor = torch.zeros(
int(
np.ceil(
(
self.extra_frame
+ self.crossfade_frame
+ self.sola_search_frame
+ self.block_frame
)
/ self.zc
)
* self.zc
),
device=device,
dtype=torch.float32,
)
self.pitch: np.ndarray = np.zeros(
self.input_wav.shape[0] // self.zc,
dtype="int32",
)
self.pitchf: np.ndarray = np.zeros(
self.input_wav.shape[0] // self.zc,
dtype="float64",
)
self.output_wav: torch.Tensor = torch.zeros(
self.block_frame, device=device, dtype=torch.float32
)
self.sola_buffer: torch.Tensor = torch.zeros( self.sola_buffer: torch.Tensor = torch.zeros(
self.crossfade_frame, device=device, dtype=torch.float32 self.crossfade_frame, device=device, dtype=torch.float32
) )
@ -384,8 +469,10 @@ if __name__ == '__main__':
rms = librosa.feature.rms( rms = librosa.feature.rms(
y=indata, frame_length=frame_length, hop_length=hop_length y=indata, frame_length=frame_length, hop_length=hop_length
) )
if(self.config.threhold>-60): if self.config.threhold > -60:
db_threhold = librosa.amplitude_to_db(rms, ref=1.0)[0] < self.config.threhold db_threhold = (
librosa.amplitude_to_db(rms, ref=1.0)[0] < self.config.threhold
)
for i in range(db_threhold.shape[0]): for i in range(db_threhold.shape[0]):
if db_threhold[i]: if db_threhold[i]:
indata[i * hop_length : (i + 1) * hop_length] = 0 indata[i * hop_length : (i + 1) * hop_length] = 0
@ -395,11 +482,33 @@ if __name__ == '__main__':
##0 ##0
res1 = self.resampler(inp) res1 = self.resampler(inp)
###55% ###55%
rate1=self.block_frame/(self.extra_frame+ self.crossfade_frame+ self.sola_search_frame+ self.block_frame) rate1 = self.block_frame / (
rate2=(self.crossfade_frame + self.sola_search_frame + self.block_frame)/(self.extra_frame+ self.crossfade_frame+ self.sola_search_frame+ self.block_frame) self.extra_frame
res2=self.rvc.infer(res1,res1[-self.block_frame:].cpu().numpy(),rate1,rate2,self.pitch,self.pitchf,self.config.f0method) + self.crossfade_frame
+ self.sola_search_frame
+ self.block_frame
)
rate2 = (
self.crossfade_frame + self.sola_search_frame + self.block_frame
) / (
self.extra_frame
+ self.crossfade_frame
+ self.sola_search_frame
+ self.block_frame
)
res2 = self.rvc.infer(
res1,
res1[-self.block_frame :].cpu().numpy(),
rate1,
rate2,
self.pitch,
self.pitchf,
self.config.f0method,
)
self.output_wav_cache[-res2.shape[0] :] = res2 self.output_wav_cache[-res2.shape[0] :] = res2
infer_wav = self.output_wav_cache[-self.crossfade_frame - self.sola_search_frame - self.block_frame :] infer_wav = self.output_wav_cache[
-self.crossfade_frame - self.sola_search_frame - self.block_frame :
]
# SOLA algorithm from https://github.com/yxlllc/DDSP-SVC # SOLA algorithm from https://github.com/yxlllc/DDSP-SVC
cor_nom = F.conv1d( cor_nom = F.conv1d(
infer_wav[None, None, : self.crossfade_frame + self.sola_search_frame], infer_wav[None, None, : self.crossfade_frame + self.sola_search_frame],
@ -407,7 +516,9 @@ if __name__ == '__main__':
) )
cor_den = torch.sqrt( cor_den = torch.sqrt(
F.conv1d( F.conv1d(
infer_wav[None, None, : self.crossfade_frame + self.sola_search_frame] infer_wav[
None, None, : self.crossfade_frame + self.sola_search_frame
]
** 2, ** 2,
torch.ones(1, 1, self.crossfade_frame, device=device), torch.ones(1, 1, self.crossfade_frame, device=device),
) )
@ -491,12 +602,15 @@ if __name__ == '__main__':
input_device_indices, input_device_indices,
output_device_indices, output_device_indices,
) = self.get_devices() ) = self.get_devices()
sd.default.device[0] = input_device_indices[input_devices.index(input_device)] sd.default.device[0] = input_device_indices[
input_devices.index(input_device)
]
sd.default.device[1] = output_device_indices[ sd.default.device[1] = output_device_indices[
output_devices.index(output_device) output_devices.index(output_device)
] ]
print("input device:" + str(sd.default.device[0]) + ":" + str(input_device)) print("input device:" + str(sd.default.device[0]) + ":" + str(input_device))
print("output device:" + str(sd.default.device[1]) + ":" + str(output_device)) print(
"output device:" + str(sd.default.device[1]) + ":" + str(output_device)
)
gui = GUI() gui = GUI()

8
lib/infer_pack/models.py
View File

@ -635,7 +635,7 @@ class SynthesizerTrnMs256NSFsid(nn.Module):
g = self.emb_g(sid).unsqueeze(-1) g = self.emb_g(sid).unsqueeze(-1)
m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
if(rate): if rate:
head = int(z_p.shape[2] * rate) head = int(z_p.shape[2] * rate)
z_p = z_p[:, :, -head:] z_p = z_p[:, :, -head:]
x_mask = x_mask[:, :, -head:] x_mask = x_mask[:, :, -head:]
@ -751,7 +751,7 @@ class SynthesizerTrnMs768NSFsid(nn.Module):
g = self.emb_g(sid).unsqueeze(-1) g = self.emb_g(sid).unsqueeze(-1)
m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths) m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
if(rate): if rate:
head = int(z_p.shape[2] * rate) head = int(z_p.shape[2] * rate)
z_p = z_p[:, :, -head:] z_p = z_p[:, :, -head:]
x_mask = x_mask[:, :, -head:] x_mask = x_mask[:, :, -head:]
@ -858,7 +858,7 @@ class SynthesizerTrnMs256NSFsid_nono(nn.Module):
g = self.emb_g(sid).unsqueeze(-1) g = self.emb_g(sid).unsqueeze(-1)
m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
if(rate): if rate:
head = int(z_p.shape[2] * rate) head = int(z_p.shape[2] * rate)
z_p = z_p[:, :, -head:] z_p = z_p[:, :, -head:]
x_mask = x_mask[:, :, -head:] x_mask = x_mask[:, :, -head:]
@ -964,7 +964,7 @@ class SynthesizerTrnMs768NSFsid_nono(nn.Module):
g = self.emb_g(sid).unsqueeze(-1) g = self.emb_g(sid).unsqueeze(-1)
m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths) m_p, logs_p, x_mask = self.enc_p(phone, None, phone_lengths)
z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66666) * x_mask
if(rate): if rate:
head = int(z_p.shape[2] * rate) head = int(z_p.shape[2] * rate)
z_p = z_p[:, :, -head:] z_p = z_p[:, :, -head:]
x_mask = x_mask[:, :, -head:] x_mask = x_mask[:, :, -head:]

168
rmvpe.py
View File

@ -3,32 +3,44 @@ import torch.nn as nn
from time import time as ttime from time import time as ttime
import torch.nn.functional as F import torch.nn.functional as F
class BiGRU(nn.Module): class BiGRU(nn.Module):
def __init__(self, input_features, hidden_features, num_layers): def __init__(self, input_features, hidden_features, num_layers):
super(BiGRU, self).__init__() super(BiGRU, self).__init__()
self.gru = nn.GRU(input_features, hidden_features, num_layers=num_layers, batch_first=True, bidirectional=True) self.gru = nn.GRU(
input_features,
hidden_features,
num_layers=num_layers,
batch_first=True,
bidirectional=True,
)
def forward(self, x): def forward(self, x):
return self.gru(x)[0] return self.gru(x)[0]
class ConvBlockRes(nn.Module): class ConvBlockRes(nn.Module):
def __init__(self, in_channels, out_channels, momentum=0.01): def __init__(self, in_channels, out_channels, momentum=0.01):
super(ConvBlockRes, self).__init__() super(ConvBlockRes, self).__init__()
self.conv = nn.Sequential( self.conv = nn.Sequential(
nn.Conv2d(in_channels=in_channels, nn.Conv2d(
in_channels=in_channels,
out_channels=out_channels, out_channels=out_channels,
kernel_size=(3, 3), kernel_size=(3, 3),
stride=(1, 1), stride=(1, 1),
padding=(1, 1), padding=(1, 1),
bias=False), bias=False,
),
nn.BatchNorm2d(out_channels, momentum=momentum), nn.BatchNorm2d(out_channels, momentum=momentum),
nn.ReLU(), nn.ReLU(),
nn.Conv2d(
nn.Conv2d(in_channels=out_channels, in_channels=out_channels,
out_channels=out_channels, out_channels=out_channels,
kernel_size=(3, 3), kernel_size=(3, 3),
stride=(1, 1), stride=(1, 1),
padding=(1, 1), padding=(1, 1),
bias=False), bias=False,
),
nn.BatchNorm2d(out_channels, momentum=momentum), nn.BatchNorm2d(out_channels, momentum=momentum),
nn.ReLU(), nn.ReLU(),
) )
@ -44,15 +56,29 @@ class ConvBlockRes(nn.Module):
else: else:
return self.conv(x) + x return self.conv(x) + x
class Encoder(nn.Module): class Encoder(nn.Module):
def __init__(self, in_channels, in_size, n_encoders, kernel_size, n_blocks, out_channels=16, momentum=0.01): def __init__(
self,
in_channels,
in_size,
n_encoders,
kernel_size,
n_blocks,
out_channels=16,
momentum=0.01,
):
super(Encoder, self).__init__() super(Encoder, self).__init__()
self.n_encoders = n_encoders self.n_encoders = n_encoders
self.bn = nn.BatchNorm2d(in_channels, momentum=momentum) self.bn = nn.BatchNorm2d(in_channels, momentum=momentum)
self.layers = nn.ModuleList() self.layers = nn.ModuleList()
self.latent_channels = [] self.latent_channels = []
for i in range(self.n_encoders): for i in range(self.n_encoders):
self.layers.append(ResEncoderBlock(in_channels, out_channels, kernel_size, n_blocks, momentum=momentum)) self.layers.append(
ResEncoderBlock(
in_channels, out_channels, kernel_size, n_blocks, momentum=momentum
)
)
self.latent_channels.append([out_channels, in_size]) self.latent_channels.append([out_channels, in_size])
in_channels = out_channels in_channels = out_channels
out_channels *= 2 out_channels *= 2
@ -67,8 +93,12 @@ class Encoder(nn.Module):
_, x = self.layers[i](x) _, x = self.layers[i](x)
concat_tensors.append(_) concat_tensors.append(_)
return x, concat_tensors return x, concat_tensors
class ResEncoderBlock(nn.Module): class ResEncoderBlock(nn.Module):
def __init__(self, in_channels, out_channels, kernel_size, n_blocks=1, momentum=0.01): def __init__(
self, in_channels, out_channels, kernel_size, n_blocks=1, momentum=0.01
):
super(ResEncoderBlock, self).__init__() super(ResEncoderBlock, self).__init__()
self.n_blocks = n_blocks self.n_blocks = n_blocks
self.conv = nn.ModuleList() self.conv = nn.ModuleList()
@ -86,32 +116,42 @@ class ResEncoderBlock(nn.Module):
return x, self.pool(x) return x, self.pool(x)
else: else:
return x return x
class Intermediate(nn.Module): # class Intermediate(nn.Module): #
def __init__(self, in_channels, out_channels, n_inters, n_blocks, momentum=0.01): def __init__(self, in_channels, out_channels, n_inters, n_blocks, momentum=0.01):
super(Intermediate, self).__init__() super(Intermediate, self).__init__()
self.n_inters = n_inters self.n_inters = n_inters
self.layers = nn.ModuleList() self.layers = nn.ModuleList()
self.layers.append(ResEncoderBlock(in_channels, out_channels, None, n_blocks, momentum)) self.layers.append(
ResEncoderBlock(in_channels, out_channels, None, n_blocks, momentum)
)
for i in range(self.n_inters - 1): for i in range(self.n_inters - 1):
self.layers.append(ResEncoderBlock(out_channels, out_channels, None, n_blocks, momentum)) self.layers.append(
ResEncoderBlock(out_channels, out_channels, None, n_blocks, momentum)
)
def forward(self, x): def forward(self, x):
for i in range(self.n_inters): for i in range(self.n_inters):
x = self.layers[i](x) x = self.layers[i](x)
return x return x
class ResDecoderBlock(nn.Module): class ResDecoderBlock(nn.Module):
def __init__(self, in_channels, out_channels, stride, n_blocks=1, momentum=0.01): def __init__(self, in_channels, out_channels, stride, n_blocks=1, momentum=0.01):
super(ResDecoderBlock, self).__init__() super(ResDecoderBlock, self).__init__()
out_padding = (0, 1) if stride == (1, 2) else (1, 1) out_padding = (0, 1) if stride == (1, 2) else (1, 1)
self.n_blocks = n_blocks self.n_blocks = n_blocks
self.conv1 = nn.Sequential( self.conv1 = nn.Sequential(
nn.ConvTranspose2d(in_channels=in_channels, nn.ConvTranspose2d(
in_channels=in_channels,
out_channels=out_channels, out_channels=out_channels,
kernel_size=(3, 3), kernel_size=(3, 3),
stride=stride, stride=stride,
padding=(1, 1), padding=(1, 1),
output_padding=out_padding, output_padding=out_padding,
bias=False), bias=False,
),
nn.BatchNorm2d(out_channels, momentum=momentum), nn.BatchNorm2d(out_channels, momentum=momentum),
nn.ReLU(), nn.ReLU(),
) )
@ -126,6 +166,8 @@ class ResDecoderBlock(nn.Module):
for i in range(self.n_blocks): for i in range(self.n_blocks):
x = self.conv2[i](x) x = self.conv2[i](x)
return x return x
class Decoder(nn.Module): class Decoder(nn.Module):
def __init__(self, in_channels, n_decoders, stride, n_blocks, momentum=0.01): def __init__(self, in_channels, n_decoders, stride, n_blocks, momentum=0.01):
super(Decoder, self).__init__() super(Decoder, self).__init__()
@ -133,7 +175,9 @@ class Decoder(nn.Module):
self.n_decoders = n_decoders self.n_decoders = n_decoders
for i in range(self.n_decoders): for i in range(self.n_decoders):
out_channels = in_channels // 2 out_channels = in_channels // 2
self.layers.append(ResDecoderBlock(in_channels, out_channels, stride, n_blocks, momentum)) self.layers.append(
ResDecoderBlock(in_channels, out_channels, stride, n_blocks, momentum)
)
in_channels = out_channels in_channels = out_channels
def forward(self, x, concat_tensors): def forward(self, x, concat_tensors):
@ -141,12 +185,30 @@ class Decoder(nn.Module):
x = self.layers[i](x, concat_tensors[-1 - i]) x = self.layers[i](x, concat_tensors[-1 - i])
return x return x
class DeepUnet(nn.Module): class DeepUnet(nn.Module):
def __init__(self, kernel_size, n_blocks, en_de_layers=5, inter_layers=4, in_channels=1, en_out_channels=16): def __init__(
self,
kernel_size,
n_blocks,
en_de_layers=5,
inter_layers=4,
in_channels=1,
en_out_channels=16,
):
super(DeepUnet, self).__init__() super(DeepUnet, self).__init__()
self.encoder = Encoder(in_channels, 128, en_de_layers, kernel_size, n_blocks, en_out_channels) self.encoder = Encoder(
self.intermediate = Intermediate(self.encoder.out_channel // 2, self.encoder.out_channel, inter_layers, n_blocks) in_channels, 128, en_de_layers, kernel_size, n_blocks, en_out_channels
self.decoder = Decoder(self.encoder.out_channel, en_de_layers, kernel_size, n_blocks) )
self.intermediate = Intermediate(
self.encoder.out_channel // 2,
self.encoder.out_channel,
inter_layers,
n_blocks,
)
self.decoder = Decoder(
self.encoder.out_channel, en_de_layers, kernel_size, n_blocks
)
def forward(self, x): def forward(self, x):
x, concat_tensors = self.encoder(x) x, concat_tensors = self.encoder(x)
@ -154,24 +216,38 @@ class DeepUnet(nn.Module):
x = self.decoder(x, concat_tensors) x = self.decoder(x, concat_tensors)
return x return x
class E2E(nn.Module): class E2E(nn.Module):
def __init__(self, n_blocks, n_gru, kernel_size, en_de_layers=5, inter_layers=4, in_channels=1, def __init__(
en_out_channels=16): self,
n_blocks,
n_gru,
kernel_size,
en_de_layers=5,
inter_layers=4,
in_channels=1,
en_out_channels=16,
):
super(E2E, self).__init__() super(E2E, self).__init__()
self.unet = DeepUnet(kernel_size, n_blocks, en_de_layers, inter_layers, in_channels, en_out_channels) self.unet = DeepUnet(
kernel_size,
n_blocks,
en_de_layers,
inter_layers,
in_channels,
en_out_channels,
)
self.cnn = nn.Conv2d(en_out_channels, 3, (3, 3), padding=(1, 1)) self.cnn = nn.Conv2d(en_out_channels, 3, (3, 3), padding=(1, 1))
if n_gru: if n_gru:
self.fc = nn.Sequential( self.fc = nn.Sequential(
BiGRU(3 * 128, 256, n_gru), BiGRU(3 * 128, 256, n_gru),
nn.Linear(512, 360), nn.Linear(512, 360),
nn.Dropout(0.25), nn.Dropout(0.25),
nn.Sigmoid() nn.Sigmoid(),
) )
else: else:
self.fc = nn.Sequential( self.fc = nn.Sequential(
nn.Linear(3 * N_MELS, N_CLASS), nn.Linear(3 * N_MELS, N_CLASS), nn.Dropout(0.25), nn.Sigmoid()
nn.Dropout(0.25),
nn.Sigmoid()
) )
def forward(self, mel): def forward(self, mel):
@ -179,7 +255,11 @@ class E2E(nn.Module):
x = self.cnn(self.unet(mel)).transpose(1, 2).flatten(-2) x = self.cnn(self.unet(mel)).transpose(1, 2).flatten(-2)
x = self.fc(x) x = self.fc(x)
return x return x
from librosa.filters import mel from librosa.filters import mel
class MelSpectrogram(torch.nn.Module): class MelSpectrogram(torch.nn.Module):
def __init__( def __init__(
self, self,
@ -191,7 +271,7 @@ class MelSpectrogram(torch.nn.Module):
n_fft=None, n_fft=None,
mel_fmin=0, mel_fmin=0,
mel_fmax=None, mel_fmax=None,
clamp=1e-5 clamp=1e-5,
): ):
super().__init__() super().__init__()
n_fft = win_length if n_fft is None else n_fft n_fft = win_length if n_fft is None else n_fft
@ -202,7 +282,8 @@ class MelSpectrogram(torch.nn.Module):
n_mels=n_mel_channels, n_mels=n_mel_channels,
fmin=mel_fmin, fmin=mel_fmin,
fmax=mel_fmax, fmax=mel_fmax,
htk=True) htk=True,
)
mel_basis = torch.from_numpy(mel_basis).float() mel_basis = torch.from_numpy(mel_basis).float()
self.register_buffer("mel_basis", mel_basis) self.register_buffer("mel_basis", mel_basis)
self.n_fft = win_length if n_fft is None else n_fft self.n_fft = win_length if n_fft is None else n_fft
@ -218,9 +299,11 @@ class MelSpectrogram(torch.nn.Module):
n_fft_new = int(np.round(self.n_fft * factor)) n_fft_new = int(np.round(self.n_fft * factor))
win_length_new = int(np.round(self.win_length * factor)) win_length_new = int(np.round(self.win_length * factor))
hop_length_new = int(np.round(self.hop_length * speed)) hop_length_new = int(np.round(self.hop_length * speed))
keyshift_key = str(keyshift) + '_' + str(audio.device) keyshift_key = str(keyshift) + "_" + str(audio.device)
if keyshift_key not in self.hann_window: if keyshift_key not in self.hann_window:
self.hann_window[keyshift_key] = torch.hann_window(win_length_new).to(audio.device) self.hann_window[keyshift_key] = torch.hann_window(win_length_new).to(
audio.device
)
fft = torch.stft( fft = torch.stft(
audio, audio,
n_fft=n_fft_new, n_fft=n_fft_new,
@ -228,7 +311,8 @@ class MelSpectrogram(torch.nn.Module):
win_length=win_length_new, win_length=win_length_new,
window=self.hann_window[keyshift_key], window=self.hann_window[keyshift_key],
center=center, center=center,
return_complex=True) return_complex=True,
)
magnitude = torch.sqrt(fft.real.pow(2) + fft.imag.pow(2)) magnitude = torch.sqrt(fft.real.pow(2) + fft.imag.pow(2))
if keyshift != 0: if keyshift != 0:
size = self.n_fft // 2 + 1 size = self.n_fft // 2 + 1
@ -237,12 +321,12 @@ class MelSpectrogram(torch.nn.Module):
magnitude = F.pad(magnitude, (0, 0, 0, size - resize)) magnitude = F.pad(magnitude, (0, 0, 0, size - resize))
magnitude = magnitude[:, :size, :] * self.win_length / win_length_new magnitude = magnitude[:, :size, :] * self.win_length / win_length_new
mel_output = torch.matmul(self.mel_basis, magnitude) mel_output = torch.matmul(self.mel_basis, magnitude)
if(self.is_half==True):mel_output=mel_output.half() if self.is_half == True:
mel_output = mel_output.half()
log_mel_spec = torch.log(torch.clamp(mel_output, min=self.clamp)) log_mel_spec = torch.log(torch.clamp(mel_output, min=self.clamp))
return log_mel_spec return log_mel_spec
class RMVPE: class RMVPE:
def __init__(self, model_path, is_half, device=None): def __init__(self, model_path, is_half, device=None):
self.resample_kernel = {} self.resample_kernel = {}
@ -250,22 +334,27 @@ class RMVPE:
ckpt = torch.load(model_path, map_location="cpu") ckpt = torch.load(model_path, map_location="cpu")
model.load_state_dict(ckpt) model.load_state_dict(ckpt)
model.eval() model.eval()
if(is_half==True):model=model.half() if is_half == True:
model = model.half()
self.model = model self.model = model
self.resample_kernel = {} self.resample_kernel = {}
self.is_half = is_half self.is_half = is_half
if device is None: if device is None:
device = 'cuda' if torch.cuda.is_available() else 'cpu' device = "cuda" if torch.cuda.is_available() else "cpu"
self.device = device self.device = device
self.mel_extractor = MelSpectrogram(is_half,128, 16000, 1024, 160, None, 30, 8000).to(device) self.mel_extractor = MelSpectrogram(
is_half, 128, 16000, 1024, 160, None, 30, 8000
).to(device)
self.model = self.model.to(device) self.model = self.model.to(device)
cents_mapping = (20 * np.arange(360) + 1997.3794084376191) cents_mapping = 20 * np.arange(360) + 1997.3794084376191
self.cents_mapping = np.pad(cents_mapping, (4, 4)) # 368 self.cents_mapping = np.pad(cents_mapping, (4, 4)) # 368
def mel2hidden(self, mel): def mel2hidden(self, mel):
with torch.no_grad(): with torch.no_grad():
n_frames = mel.shape[-1] n_frames = mel.shape[-1]
mel = F.pad(mel, (0, 32 * ((n_frames - 1) // 32 + 1) - n_frames), mode='reflect') mel = F.pad(
mel, (0, 32 * ((n_frames - 1) // 32 + 1) - n_frames), mode="reflect"
)
hidden = self.model(mel) hidden = self.model(mel)
return hidden[:, :n_frames] return hidden[:, :n_frames]
@ -287,7 +376,8 @@ class RMVPE:
# torch.cuda.synchronize() # torch.cuda.synchronize()
# t2=ttime() # t2=ttime()
hidden = hidden.squeeze(0).cpu().numpy() hidden = hidden.squeeze(0).cpu().numpy()
if(self.is_half==True):hidden=hidden.astype("float32") if self.is_half == True:
hidden = hidden.astype("float32")
f0 = self.decode(hidden, thred=thred) f0 = self.decode(hidden, thred=thred)
# torch.cuda.synchronize() # torch.cuda.synchronize()
# t3=ttime() # t3=ttime()
@ -321,8 +411,6 @@ class RMVPE:
return devided return devided
# if __name__ == '__main__': # if __name__ == '__main__':
# audio, sampling_rate = sf.read("卢本伟语录~1.wav") # audio, sampling_rate = sf.read("卢本伟语录~1.wav")
# if len(audio.shape) > 1: # if len(audio.shape) > 1:

74
rvc_for_realtime.py
View File

@ -10,13 +10,16 @@ import os,sys
from time import time as ttime from time import time as ttime
import torch.nn.functional as F import torch.nn.functional as F
import scipy.signal as signal import scipy.signal as signal
now_dir = os.getcwd() now_dir = os.getcwd()
sys.path.append(now_dir) sys.path.append(now_dir)
from config import Config from config import Config
from multiprocessing import Manager as M from multiprocessing import Manager as M
mm = M() mm = M()
config = Config() config = Config()
class RVC: class RVC:
def __init__( def __init__(
self, key, pth_path, index_path, index_rate, n_cpu, inp_q, opt_q, device self, key, pth_path, index_path, index_rate, n_cpu, inp_q, opt_q, device
@ -102,9 +105,11 @@ class RVC:
def get_f0(self, x, f0_up_key, n_cpu, method="harvest"): def get_f0(self, x, f0_up_key, n_cpu, method="harvest"):
n_cpu = int(n_cpu) n_cpu = int(n_cpu)
if (method == "crepe"): return self.get_f0_crepe(x, f0_up_key) if method == "crepe":
if (method == "rmvpe"): return self.get_f0_rmvpe(x, f0_up_key) return self.get_f0_crepe(x, f0_up_key)
if (method == "pm"): if method == "rmvpe":
return self.get_f0_rmvpe(x, f0_up_key)
if method == "pm":
p_len = x.shape[0] // 160 p_len = x.shape[0] // 160
f0 = ( f0 = (
parselmouth.Sound(x, 16000) parselmouth.Sound(x, 16000)
@ -120,11 +125,13 @@ class RVC:
pad_size = (p_len - len(f0) + 1) // 2 pad_size = (p_len - len(f0) + 1) // 2
if pad_size > 0 or p_len - len(f0) - pad_size > 0: if pad_size > 0 or p_len - len(f0) - pad_size > 0:
print(pad_size, p_len - len(f0) - pad_size) print(pad_size, p_len - len(f0) - pad_size)
f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") f0 = np.pad(
f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant"
)
f0 *= pow(2, f0_up_key / 12) f0 *= pow(2, f0_up_key / 12)
return self.get_f0_post(f0) return self.get_f0_post(f0)
if (n_cpu == 1): if n_cpu == 1:
f0, t = pyworld.harvest( f0, t = pyworld.harvest(
x.astype(np.double), x.astype(np.double),
fs=16000, fs=16000,
@ -142,23 +149,27 @@ class RVC:
res_f0 = mm.dict() res_f0 = mm.dict()
for idx in range(n_cpu): for idx in range(n_cpu):
tail = part_length * (idx + 1) + 320 tail = part_length * (idx + 1) + 320
if (idx == 0): if idx == 0:
self.inp_q.put((idx, x[:tail], res_f0, n_cpu, ts)) self.inp_q.put((idx, x[:tail], res_f0, n_cpu, ts))
else: else:
self.inp_q.put((idx, x[part_length * idx - 320:tail], res_f0, n_cpu, ts)) self.inp_q.put(
while (1): (idx, x[part_length * idx - 320 : tail], res_f0, n_cpu, ts)
)
while 1:
res_ts = self.opt_q.get() res_ts = self.opt_q.get()
if (res_ts == ts): if res_ts == ts:
break break
f0s = [i[1] for i in sorted(res_f0.items(), key=lambda x: x[0])] f0s = [i[1] for i in sorted(res_f0.items(), key=lambda x: x[0])]
for idx, f0 in enumerate(f0s): for idx, f0 in enumerate(f0s):
if (idx == 0): if idx == 0:
f0 = f0[:-3] f0 = f0[:-3]
elif (idx != n_cpu - 1): elif idx != n_cpu - 1:
f0 = f0[2:-3] f0 = f0[2:-3]
else: else:
f0 = f0[2:-1] f0 = f0[2:-1]
f0bak[part_length * idx // 160:part_length * idx // 160 + f0.shape[0]] = f0 f0bak[
part_length * idx // 160 : part_length * idx // 160 + f0.shape[0]
] = f0
f0bak = signal.medfilt(f0bak, 3) f0bak = signal.medfilt(f0bak, 3)
f0bak *= pow(2, f0_up_key / 12) f0bak *= pow(2, f0_up_key / 12)
return self.get_f0_post(f0bak) return self.get_f0_post(f0bak)
@ -184,16 +195,28 @@ class RVC:
return self.get_f0_post(f0) return self.get_f0_post(f0)
def get_f0_rmvpe(self, x, f0_up_key): def get_f0_rmvpe(self, x, f0_up_key):
if (hasattr(self, "model_rmvpe") == False): if hasattr(self, "model_rmvpe") == False:
from rmvpe import RMVPE from rmvpe import RMVPE
print("loading rmvpe model") print("loading rmvpe model")
self.model_rmvpe = RMVPE("rmvpe.pt", is_half=self.is_half, device=self.device) self.model_rmvpe = RMVPE(
"rmvpe.pt", is_half=self.is_half, device=self.device
)
# self.model_rmvpe = RMVPE("aug2_58000_half.pt", is_half=self.is_half, device=self.device) # self.model_rmvpe = RMVPE("aug2_58000_half.pt", is_half=self.is_half, device=self.device)
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03) f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
f0 *= pow(2, f0_up_key / 12) f0 *= pow(2, f0_up_key / 12)
return self.get_f0_post(f0) return self.get_f0_post(f0)
def infer(self, feats: torch.Tensor, indata: np.ndarray, rate1, rate2, cache_pitch, cache_pitchf, f0method) -> np.ndarray: def infer(
self,
feats: torch.Tensor,
indata: np.ndarray,
rate1,
rate2,
cache_pitch,
cache_pitchf,
f0method,
) -> np.ndarray:
feats = feats.view(1, -1) feats = feats.view(1, -1)
if config.is_half: if config.is_half:
feats = feats.half() feats = feats.half()
@ -209,13 +232,12 @@ class RVC:
"output_layer": 9 if self.version == "v1" else 12, "output_layer": 9 if self.version == "v1" else 12,
} }
logits = self.model.extract_features(**inputs) logits = self.model.extract_features(**inputs)
feats = self.model.final_proj(logits[0]) if self.version == "v1" else logits[0] feats = (
self.model.final_proj(logits[0]) if self.version == "v1" else logits[0]
)
t2 = ttime() t2 = ttime()
try: try:
if ( if hasattr(self, "index") and self.index_rate != 0:
hasattr(self, "index")
and self.index_rate != 0
):
leng_replace_head = int(rate1 * feats[0].shape[0]) leng_replace_head = int(rate1 * feats[0].shape[0])
npy = feats[0][-leng_replace_head:].cpu().numpy().astype("float32") npy = feats[0][-leng_replace_head:].cpu().numpy().astype("float32")
score, ix = self.index.search(npy, k=8) score, ix = self.index.search(npy, k=8)
@ -238,7 +260,9 @@ class RVC:
if self.if_f0 == 1: if self.if_f0 == 1:
pitch, pitchf = self.get_f0(indata, self.f0_up_key, self.n_cpu, f0method) pitch, pitchf = self.get_f0(indata, self.f0_up_key, self.n_cpu, f0method)
cache_pitch[:] = np.append(cache_pitch[pitch[:-1].shape[0] :], pitch[:-1]) cache_pitch[:] = np.append(cache_pitch[pitch[:-1].shape[0] :], pitch[:-1])
cache_pitchf[:] = np.append(cache_pitchf[pitchf[:-1].shape[0]:], pitchf[:-1]) cache_pitchf[:] = np.append(
cache_pitchf[pitchf[:-1].shape[0] :], pitchf[:-1]
)
p_len = min(feats.shape[1], 13000, cache_pitch.shape[0]) p_len = min(feats.shape[1], 13000, cache_pitch.shape[0])
else: else:
cache_pitch, cache_pitchf = None, None cache_pitch, cache_pitchf = None, None
@ -256,13 +280,17 @@ class RVC:
with torch.no_grad(): with torch.no_grad():
if self.if_f0 == 1: if self.if_f0 == 1:
infered_audio = ( infered_audio = (
self.net_g.infer(feats, p_len, cache_pitch, cache_pitchf, sid, rate2)[0][0, 0] self.net_g.infer(
feats, p_len, cache_pitch, cache_pitchf, sid, rate2
)[0][0, 0]
.data.cpu() .data.cpu()
.float() .float()
) )
else: else:
infered_audio = ( infered_audio = (
self.net_g.infer(feats, p_len, sid, rate2)[0][0, 0].data.cpu().float() self.net_g.infer(feats, p_len, sid, rate2)[0][0, 0]
.data.cpu()
.float()
) )
t5 = ttime() t5 = ttime()
print("time->fea-index-f0-model:", t2 - t1, t3 - t2, t4 - t3, t5 - t4) print("time->fea-index-f0-model:", t2 - t1, t3 - t2, t4 - t3, t5 - t4)

8
vc_infer_pipeline.py
View File

@ -5,6 +5,7 @@ import scipy.signal as signal
import pyworld, os, traceback, faiss, librosa, torchcrepe import pyworld, os, traceback, faiss, librosa, torchcrepe
from scipy import signal from scipy import signal
from functools import lru_cache from functools import lru_cache
now_dir = os.getcwd() now_dir = os.getcwd()
sys.path.append(now_dir) sys.path.append(now_dir)
@ -127,10 +128,13 @@ class VC(object):
f0[pd < 0.1] = 0 f0[pd < 0.1] = 0
f0 = f0[0].cpu().numpy() f0 = f0[0].cpu().numpy()
elif f0_method == "rmvpe": elif f0_method == "rmvpe":
if(hasattr(self,"model_rmvpe")==False): if hasattr(self, "model_rmvpe") == False:
from rmvpe import RMVPE from rmvpe import RMVPE
print("loading rmvpe model") print("loading rmvpe model")
self.model_rmvpe = RMVPE("rmvpe.pt",is_half=self.is_half, device=self.device) self.model_rmvpe = RMVPE(
"rmvpe.pt", is_half=self.is_half, device=self.device
)
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03) f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
f0 *= pow(2, f0_up_key / 12) f0 *= pow(2, f0_up_key / 12)
# with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()])) # with open("test.txt","w")as f:f.write("\n".join([str(i)for i in f0.tolist()]))