From b8bc2b6d603830226c3e832ed75182c9778e53fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BA=90=E6=96=87=E9=9B=A8?= <41315874+fumiama@users.noreply.github.com> Date: Sun, 14 Jan 2024 16:37:55 +0900 Subject: [PATCH] drop: preprocess.py overwrite & optimize log printing --- configs/config.py | 16 +++++----------- infer-web.py | 15 +++++++-------- infer/modules/train/extract/extract_f0_print.py | 2 +- infer/modules/train/extract/extract_f0_rmvpe.py | 2 +- .../train/extract/extract_f0_rmvpe_dml.py | 2 +- infer/modules/train/extract_feature_print.py | 4 ++-- infer/modules/train/preprocess.py | 12 +++--------- 7 files changed, 20 insertions(+), 33 deletions(-) diff --git a/configs/config.py b/configs/config.py index af2350f..681a7a7 100644 --- a/configs/config.py +++ b/configs/config.py @@ -58,6 +58,7 @@ class Config: self.dml, ) = self.arg_parse() self.instead = "" + self.preprocess_per = 3.7 self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config() @staticmethod @@ -127,11 +128,8 @@ class Config: strr = f.read().replace("true", "false") with open(f"configs/{config_file}", "w") as f: f.write(strr) - with open("infer/modules/train/preprocess.py", "r") as f: - strr = f.read().replace("3.7", "3.0") - with open("infer/modules/train/preprocess.py", "w") as f: - f.write(strr) - print("overwrite preprocess and configs.json") + self.preprocess_per = 3.0 + logger.info("overwrite configs.json") def device_config(self) -> tuple: if torch.cuda.is_available(): @@ -160,11 +158,7 @@ class Config: / 1024 + 0.4 ) - if self.gpu_mem <= 4: - with open("infer/modules/train/preprocess.py", "r") as f: - strr = f.read().replace("3.7", "3.0") - with open("infer/modules/train/preprocess.py", "w") as f: - f.write(strr) + if self.gpu_mem <= 4: self.preprocess_per = 3.0 elif self.has_mps(): logger.info("No supported Nvidia GPU found") self.device = self.instead = "mps" @@ -247,5 +241,5 @@ class Config: ) except: pass - print("is_half:%s, device:%s" % (self.is_half, self.device)) + logger.info("Half-precision floating-point: %s, device: %s" % (self.is_half, self.device)) return x_pad, x_query, x_center, x_max diff --git a/infer-web.py b/infer-web.py index 99f413d..daaf631 100644 --- a/infer-web.py +++ b/infer-web.py @@ -207,7 +207,6 @@ def preprocess_dataset(trainset_dir, exp_dir, sr, n_p): os.makedirs("%s/logs/%s" % (now_dir, exp_dir), exist_ok=True) f = open("%s/logs/%s/preprocess.log" % (now_dir, exp_dir), "w") f.close() - per = 3.0 if config.is_half else 3.7 cmd = '"%s" infer/modules/train/preprocess.py "%s" %s %s "%s/logs/%s" %s %.1f' % ( config.python_cmd, trainset_dir, @@ -216,9 +215,9 @@ def preprocess_dataset(trainset_dir, exp_dir, sr, n_p): now_dir, exp_dir, config.noparallel, - per, + config.preprocess_per, ) - logger.info(cmd) + logger.info("Execute: "+cmd) # , stdin=PIPE, stdout=PIPE,stderr=PIPE,cwd=now_dir p = Popen(cmd, shell=True) # 煞笔gr, popen read都非得全跑完了再一次性读取, 不用gr就正常读一句输出一句;只能额外弄出一个文本流定时读 @@ -260,7 +259,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp f0method, ) ) - logger.info(cmd) + logger.info("Execute: "+cmd) p = Popen( cmd, shell=True, cwd=now_dir ) # , stdin=PIPE, stdout=PIPE,stderr=PIPE @@ -291,7 +290,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp config.is_half, ) ) - logger.info(cmd) + logger.info("Execute: "+cmd) p = Popen( cmd, shell=True, cwd=now_dir ) # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir @@ -314,7 +313,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp exp_dir, ) ) - logger.info(cmd) + logger.info("Execute: "+cmd) p = Popen( cmd, shell=True, cwd=now_dir ) # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir @@ -357,7 +356,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp config.is_half, ) ) - logger.info(cmd) + logger.info("Execute: "+cmd) p = Popen( cmd, shell=True, cwd=now_dir ) # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir @@ -590,7 +589,7 @@ def click_train( version19, ) ) - logger.info(cmd) + logger.info("Execute: "+cmd) p = Popen(cmd, shell=True, cwd=now_dir) p.wait() return "训练结束, 您可查看控制台训练日志或实验文件夹下的train.log" diff --git a/infer/modules/train/extract/extract_f0_print.py b/infer/modules/train/extract/extract_f0_print.py index ec5e1ec..9d231e4 100644 --- a/infer/modules/train/extract/extract_f0_print.py +++ b/infer/modules/train/extract/extract_f0_print.py @@ -143,7 +143,7 @@ if __name__ == "__main__": # exp_dir=r"E:\codes\py39\dataset\mi-test" # n_p=16 # f = open("%s/log_extract_f0.log"%exp_dir, "w") - printt(sys.argv) + printt(" ".join(sys.argv)) featureInput = FeatureInput() paths = [] inp_root = "%s/1_16k_wavs" % (exp_dir) diff --git a/infer/modules/train/extract/extract_f0_rmvpe.py b/infer/modules/train/extract/extract_f0_rmvpe.py index 0898b8d..90b2073 100644 --- a/infer/modules/train/extract/extract_f0_rmvpe.py +++ b/infer/modules/train/extract/extract_f0_rmvpe.py @@ -106,7 +106,7 @@ if __name__ == "__main__": # exp_dir=r"E:\codes\py39\dataset\mi-test" # n_p=16 # f = open("%s/log_extract_f0.log"%exp_dir, "w") - printt(sys.argv) + printt(" ".join(sys.argv)) featureInput = FeatureInput() paths = [] inp_root = "%s/1_16k_wavs" % (exp_dir) diff --git a/infer/modules/train/extract/extract_f0_rmvpe_dml.py b/infer/modules/train/extract/extract_f0_rmvpe_dml.py index 993945a..243e825 100644 --- a/infer/modules/train/extract/extract_f0_rmvpe_dml.py +++ b/infer/modules/train/extract/extract_f0_rmvpe_dml.py @@ -104,7 +104,7 @@ if __name__ == "__main__": # exp_dir=r"E:\codes\py39\dataset\mi-test" # n_p=16 # f = open("%s/log_extract_f0.log"%exp_dir, "w") - printt(sys.argv) + printt(" ".join(sys.argv)) featureInput = FeatureInput() paths = [] inp_root = "%s/1_16k_wavs" % (exp_dir) diff --git a/infer/modules/train/extract_feature_print.py b/infer/modules/train/extract_feature_print.py index 7b2254b..c423655 100644 --- a/infer/modules/train/extract_feature_print.py +++ b/infer/modules/train/extract_feature_print.py @@ -51,10 +51,10 @@ def printt(strr): f.flush() -printt(sys.argv) +printt(" ".join(sys.argv)) model_path = "assets/hubert/hubert_base.pt" -printt(exp_dir) +printt("exp_dir: " + exp_dir) wavPath = "%s/1_16k_wavs" % exp_dir outPath = ( "%s/3_feature256" % exp_dir if version == "v1" else "%s/3_feature768" % exp_dir diff --git a/infer/modules/train/preprocess.py b/infer/modules/train/preprocess.py index fbe8130..7c41817 100644 --- a/infer/modules/train/preprocess.py +++ b/infer/modules/train/preprocess.py @@ -6,14 +6,13 @@ from scipy import signal now_dir = os.getcwd() sys.path.append(now_dir) -print(sys.argv) +print(*sys.argv[1:]) inp_root = sys.argv[1] sr = int(sys.argv[2]) n_p = int(sys.argv[3]) exp_dir = sys.argv[4] noparallel = sys.argv[5] == "True" per = float(sys.argv[6]) -import multiprocessing import os import traceback @@ -24,17 +23,13 @@ from scipy.io import wavfile from infer.lib.audio import load_audio from infer.lib.slicer2 import Slicer -mutex = multiprocessing.Lock() f = open("%s/preprocess.log" % exp_dir, "a+") def println(strr): - mutex.acquire() print(strr) f.write("%s\n" % strr) f.flush() - mutex.release() - class PreProcess: def __init__(self, sr, exp_dir, per=3.7): @@ -104,9 +99,9 @@ class PreProcess: idx1 += 1 break self.norm_write(tmp_audio, idx0, idx1) - println("%s->Suc." % path) + println("%s\t-> Success" % path) except: - println("%s->%s" % (path, traceback.format_exc())) + println("%s\t-> %s" % (path, traceback.format_exc())) def pipeline_mp(self, infos): for path, idx0 in infos: @@ -138,7 +133,6 @@ class PreProcess: def preprocess_trainset(inp_root, sr, n_p, exp_dir, per): pp = PreProcess(sr, exp_dir, per) println("start preprocess") - println(sys.argv) pp.pipeline_mp_inp_dir(inp_root, n_p) println("end preprocess")