drop: preprocess.py overwrite & optimize log printing

configs/config.py

@@ -58,6 +58,7 @@ class Config:
             self.dml,
         ) = self.arg_parse()
         self.instead = ""
+        self.preprocess_per = 3.7
         self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()
 
     @staticmethod
@@ -127,11 +128,8 @@ class Config:
                 strr = f.read().replace("true", "false")
             with open(f"configs/{config_file}", "w") as f:
                 f.write(strr)
-        with open("infer/modules/train/preprocess.py", "r") as f:
+        self.preprocess_per = 3.0
-            strr = f.read().replace("3.7", "3.0")
+        logger.info("overwrite configs.json")
-        with open("infer/modules/train/preprocess.py", "w") as f:
-            f.write(strr)
-        print("overwrite preprocess and configs.json")
 
     def device_config(self) -> tuple:
         if torch.cuda.is_available():
@@ -160,11 +158,7 @@ class Config:
                 / 1024
                 + 0.4
             )
-            if self.gpu_mem <= 4:
+            if self.gpu_mem <= 4: self.preprocess_per = 3.0
-                with open("infer/modules/train/preprocess.py", "r") as f:
-                    strr = f.read().replace("3.7", "3.0")
-                with open("infer/modules/train/preprocess.py", "w") as f:
-                    f.write(strr)
         elif self.has_mps():
             logger.info("No supported Nvidia GPU found")
             self.device = self.instead = "mps"
@@ -247,5 +241,5 @@ class Config:
                     )
                 except:
                     pass
-        print("is_half:%s, device:%s" % (self.is_half, self.device))
+        logger.info("Half-precision floating-point: %s, device: %s" % (self.is_half, self.device))
         return x_pad, x_query, x_center, x_max

infer-web.py

@@ -207,7 +207,6 @@ def preprocess_dataset(trainset_dir, exp_dir, sr, n_p):
     os.makedirs("%s/logs/%s" % (now_dir, exp_dir), exist_ok=True)
     f = open("%s/logs/%s/preprocess.log" % (now_dir, exp_dir), "w")
     f.close()
-    per = 3.0 if config.is_half else 3.7
     cmd = '"%s" infer/modules/train/preprocess.py "%s" %s %s "%s/logs/%s" %s %.1f' % (
         config.python_cmd,
         trainset_dir,
@@ -216,9 +215,9 @@ def preprocess_dataset(trainset_dir, exp_dir, sr, n_p):
         now_dir,
         exp_dir,
         config.noparallel,
-        per,
+        config.preprocess_per,
     )
-    logger.info(cmd)
+    logger.info("Execute: "+cmd)
     # , stdin=PIPE, stdout=PIPE,stderr=PIPE,cwd=now_dir
     p = Popen(cmd, shell=True)
     # 煞笔gr, popen read都非得全跑完了再一次性读取, 不用gr就正常读一句输出一句;只能额外弄出一个文本流定时读
@@ -260,7 +259,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp
                     f0method,
                 )
             )
-            logger.info(cmd)
+            logger.info("Execute: "+cmd)
             p = Popen(
                 cmd, shell=True, cwd=now_dir
             )  # , stdin=PIPE, stdout=PIPE,stderr=PIPE
@@ -291,7 +290,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp
                             config.is_half,
                         )
                     )
-                    logger.info(cmd)
+                    logger.info("Execute: "+cmd)
                     p = Popen(
                         cmd, shell=True, cwd=now_dir
                     )  # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir
@@ -314,7 +313,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp
                         exp_dir,
                     )
                 )
-                logger.info(cmd)
+                logger.info("Execute: "+cmd)
                 p = Popen(
                     cmd, shell=True, cwd=now_dir
                 )  # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir
@@ -357,7 +356,7 @@ def extract_f0_feature(gpus, n_p, f0method, if_f0, exp_dir, version19, gpus_rmvp
                 config.is_half,
             )
         )
-        logger.info(cmd)
+        logger.info("Execute: "+cmd)
         p = Popen(
             cmd, shell=True, cwd=now_dir
         )  # , shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=now_dir
@@ -590,7 +589,7 @@ def click_train(
                 version19,
             )
         )
-    logger.info(cmd)
+    logger.info("Execute: "+cmd)
     p = Popen(cmd, shell=True, cwd=now_dir)
     p.wait()
     return "训练结束, 您可查看控制台训练日志或实验文件夹下的train.log"

infer/modules/train/extract/extract_f0_print.py

@@ -143,7 +143,7 @@ if __name__ == "__main__":
     # exp_dir=r"E:\codes\py39\dataset\mi-test"
     # n_p=16
     # f = open("%s/log_extract_f0.log"%exp_dir, "w")
-    printt(sys.argv)
+    printt(" ".join(sys.argv))
     featureInput = FeatureInput()
     paths = []
     inp_root = "%s/1_16k_wavs" % (exp_dir)

infer/modules/train/extract/extract_f0_rmvpe.py

@@ -106,7 +106,7 @@ if __name__ == "__main__":
     # exp_dir=r"E:\codes\py39\dataset\mi-test"
     # n_p=16
     # f = open("%s/log_extract_f0.log"%exp_dir, "w")
-    printt(sys.argv)
+    printt(" ".join(sys.argv))
     featureInput = FeatureInput()
     paths = []
     inp_root = "%s/1_16k_wavs" % (exp_dir)

infer/modules/train/extract/extract_f0_rmvpe_dml.py

@@ -104,7 +104,7 @@ if __name__ == "__main__":
     # exp_dir=r"E:\codes\py39\dataset\mi-test"
     # n_p=16
     # f = open("%s/log_extract_f0.log"%exp_dir, "w")
-    printt(sys.argv)
+    printt(" ".join(sys.argv))
     featureInput = FeatureInput()
     paths = []
     inp_root = "%s/1_16k_wavs" % (exp_dir)

infer/modules/train/extract_feature_print.py

@@ -51,10 +51,10 @@ def printt(strr):
     f.flush()
 
 
-printt(sys.argv)
+printt(" ".join(sys.argv))
 model_path = "assets/hubert/hubert_base.pt"
 
-printt(exp_dir)
+printt("exp_dir: " + exp_dir)
 wavPath = "%s/1_16k_wavs" % exp_dir
 outPath = (
     "%s/3_feature256" % exp_dir if version == "v1" else "%s/3_feature768" % exp_dir

infer/modules/train/preprocess.py

@@ -6,14 +6,13 @@ from scipy import signal
 
 now_dir = os.getcwd()
 sys.path.append(now_dir)
-print(sys.argv)
+print(*sys.argv[1:])
 inp_root = sys.argv[1]
 sr = int(sys.argv[2])
 n_p = int(sys.argv[3])
 exp_dir = sys.argv[4]
 noparallel = sys.argv[5] == "True"
 per = float(sys.argv[6])
-import multiprocessing
 import os
 import traceback
 
@@ -24,17 +23,13 @@ from scipy.io import wavfile
 from infer.lib.audio import load_audio
 from infer.lib.slicer2 import Slicer
 
-mutex = multiprocessing.Lock()
 f = open("%s/preprocess.log" % exp_dir, "a+")
 
 
 def println(strr):
-    mutex.acquire()
     print(strr)
     f.write("%s\n" % strr)
     f.flush()
-    mutex.release()
-
 
 class PreProcess:
     def __init__(self, sr, exp_dir, per=3.7):
@@ -104,9 +99,9 @@ class PreProcess:
                         idx1 += 1
                         break
                 self.norm_write(tmp_audio, idx0, idx1)
-            println("%s->Suc." % path)
+            println("%s\t-> Success" % path)
         except:
-            println("%s->%s" % (path, traceback.format_exc()))
+            println("%s\t-> %s" % (path, traceback.format_exc()))
 
     def pipeline_mp(self, infos):
         for path, idx0 in infos:
@@ -138,7 +133,6 @@ class PreProcess:
 def preprocess_trainset(inp_root, sr, n_p, exp_dir, per):
     pp = PreProcess(sr, exp_dir, per)
     println("start preprocess")
-    println(sys.argv)
     pp.pipeline_mp_inp_dir(inp_root, n_p)
     println("end preprocess")