Standardize loading and dumping routines for formats that use one file per chart

.flake8

@@ -1,19 +1,19 @@
 [flake8]
 ignore =
-    # break after binary op.
+    # First category : "ffs flake8 let black do its own formatting"
-    # black does its thing and I leave it that way
-    W503
-    # f-string is missing placeholders
-    # really don't want to be bothered for so little
-    F541
     # whitespace before ':'
-    # let black handle that
     E203
     # blank line contains whitespace
-    # let black handle that as well
     W293
+    # expected 2 blank lines, found 1
+    E302
+    # break after binary op.
+    W503
+    
+    # Second category : "I know what I'm doing leave me alone"
+    # f-string is missing placeholders
+    F541
     # do not assign a lambda expression, use a def
-    # I know when I need that one, thank you very much
     E731
 exclude =
     .git

jubeatools/formats/adapters.py

@@ -1,78 +0,0 @@
-"""Things that make it easier to integrate formats with different opinions
-on song folder structure"""
-
-from itertools import count
-from typing import TypedDict, Iterator, Any, Dict, AbstractSet
-from pathlib import Path
-
-from jubeatools.song import Song
-from jubeatools.formats.typing import ChartFileDumper, Dumper
-from jubeatools.formats.filetypes import ChartFile
-from jubeatools.formats.dump_tools import DIFFICULTY_INDEX, DIFFICULTY_NUMBER
-
-
-def make_dumper_from_chart_file_dumper(
-    internal_dumper: ChartFileDumper,
-    file_name_template: Path,
-) -> Dumper:
-    """Adapt a ChartFileDumper to the Dumper protocol, The resulting function
-    uses the file name template if it recieves an existing directory as an
-    output path"""
-
-    def dumper(song: Song, path: Path, **kwargs: Any) -> Dict[Path, bytes]:
-        res: Dict[Path, bytes] = {}
-        if path.is_dir():
-            file_path = file_name_template
-        else:
-            file_path = path
-
-        name_format = f"{file_path.stem}{{dedup}}{file_path.suffix}"
-        files = internal_dumper(song, **kwargs)
-        for chartfile in files:
-            filepath = choose_file_path(chartfile, name_format, path.parent, res.keys())
-            res[filepath] = chartfile.contents
-
-        return res
-
-    return dumper
-
-
-def choose_file_path(
-    chart_file: ChartFile,
-    name_format: str,
-    parent: Path,
-    already_chosen: AbstractSet[Path],
-) -> Path:
-    all_paths = iter_possible_paths(chart_file, name_format, parent)
-    not_on_filesystem = filter(lambda p: not p.exists(), all_paths)
-    not_already_chosen = filter(lambda p: p not in already_chosen, not_on_filesystem)
-    return next(not_already_chosen)
-
-
-def iter_possible_paths(
-    chart_file: ChartFile, name_format: str, parent: Path
-) -> Iterator[Path]:
-    for dedup_index in count(start=0):
-        params = extract_format_params(chart_file, dedup_index)
-        filename = name_format.format(**params).strip()
-        yield parent / filename
-
-
-class FormatParameters(TypedDict, total=False):
-    title: str
-    difficulty: str
-    # 0-based
-    difficulty_index: int
-    # 1-based
-    difficulty_number: int
-    dedup: str
-
-
-def extract_format_params(chartfile: ChartFile, dedup_index: int) -> FormatParameters:
-    return FormatParameters(
-        title=chartfile.song.metadata.title or "",
-        difficulty=chartfile.difficulty,
-        difficulty_index=DIFFICULTY_INDEX.get(chartfile.difficulty, 3),
-        difficulty_number=DIFFICULTY_NUMBER.get(chartfile.difficulty, 4),
-        dedup="" if dedup_index else f"-{dedup_index}",
-    )

jubeatools/formats/dump_tools.py

@@ -1,6 +1,10 @@
-from jubeatools.song import Difficulty
+from itertools import count
-from typing import Dict
+from pathlib import Path
+from typing import AbstractSet, Any, Dict, Iterator, TypedDict
 
+from jubeatools.formats.filetypes import ChartFile
+from jubeatools.formats.typing import ChartFileDumper, Dumper
+from jubeatools.song import Difficulty, Song
 
 DIFFICULTY_NUMBER: Dict[str, int] = {
     Difficulty.BASIC: 1,
@@ -13,3 +17,70 @@ DIFFICULTY_INDEX: Dict[str, int] = {
     Difficulty.ADVANCED: 1,
     Difficulty.EXTREME: 2,
 }
+
+
+def make_dumper_from_chart_file_dumper(
+    internal_dumper: ChartFileDumper,
+    file_name_template: Path,
+) -> Dumper:
+    """Adapt a ChartFileDumper to the Dumper protocol, The resulting function
+    uses the file name template if it recieves an existing directory as an
+    output path"""
+
+    def dumper(song: Song, path: Path, **kwargs: Any) -> Dict[Path, bytes]:
+        res: Dict[Path, bytes] = {}
+        if path.is_dir():
+            file_path = file_name_template
+        else:
+            file_path = path
+
+        name_format = f"{file_path.stem}{{dedup}}{file_path.suffix}"
+        files = internal_dumper(song, **kwargs)
+        for chartfile in files:
+            filepath = choose_file_path(chartfile, name_format, path.parent, res.keys())
+            res[filepath] = chartfile.contents
+
+        return res
+
+    return dumper
+
+
+def choose_file_path(
+    chart_file: ChartFile,
+    name_format: str,
+    parent: Path,
+    already_chosen: AbstractSet[Path],
+) -> Path:
+    all_paths = iter_possible_paths(chart_file, name_format, parent)
+    not_on_filesystem = filter(lambda p: not p.exists(), all_paths)
+    not_already_chosen = filter(lambda p: p not in already_chosen, not_on_filesystem)
+    return next(not_already_chosen)
+
+
+def iter_possible_paths(
+    chart_file: ChartFile, name_format: str, parent: Path
+) -> Iterator[Path]:
+    for dedup_index in count(start=0):
+        params = extract_format_params(chart_file, dedup_index)
+        filename = name_format.format(**params).strip()
+        yield parent / filename
+
+
+class FormatParameters(TypedDict, total=False):
+    title: str
+    difficulty: str
+    # 0-based
+    difficulty_index: int
+    # 1-based
+    difficulty_number: int
+    dedup: str
+
+
+def extract_format_params(chartfile: ChartFile, dedup_index: int) -> FormatParameters:
+    return FormatParameters(
+        title=chartfile.song.metadata.title or "",
+        difficulty=chartfile.difficulty,
+        difficulty_index=DIFFICULTY_INDEX.get(chartfile.difficulty, 3),
+        difficulty_number=DIFFICULTY_NUMBER.get(chartfile.difficulty, 4),
+        dedup="" if dedup_index else f"-{dedup_index}",
+    )

jubeatools/formats/jubeat_analyser/files.py

@@ -1,20 +0,0 @@
-from pathlib import Path
-from typing import Dict, List, Optional
-
-from jubeatools.formats.files import make_folder_loader
-
-def read_jubeat_analyser_file(path: Path) -> Optional[List[str]]:
-    try:
-        # The vast majority of memo files you will encounter will be propely
-        # decoded using shift-jis-2004. Get ready for endless fun with the small
-        # portion of files that won't
-        lines = path.read_text(encoding="shift-jis-2004").split("\n")
-    except UnicodeDecodeError:
-        return None
-    else:
-        return lines
-
-load_folder = make_folder_loader(
-    glob_pattern="*.txt",
-    file_loader=read_jubeat_analyser_file,
-)

jubeatools/formats/jubeat_analyser/load_tools.py

@@ -7,13 +7,15 @@ from copy import deepcopy
 from dataclasses import astuple, dataclass
 from decimal import Decimal
 from itertools import product, zip_longest
+from pathlib import Path
 from typing import AbstractSet, Dict, List, Optional, Set, Tuple, Union
 
 import constraint
 from parsimonious import Grammar, NodeVisitor, ParseError
 from parsimonious.nodes import Node
 
-from jubeatools.song import BeatsTime, BPMEvent, LongNote, NotePosition, Difficulty
+from jubeatools.formats.load_tools import make_folder_loader
+from jubeatools.song import BeatsTime, BPMEvent, Difficulty, LongNote, NotePosition
 
 from .symbols import (
     CIRCLE_FREE_SYMBOLS,
@@ -461,3 +463,21 @@ class DoubleColumnFrame:
                 line += [f"|{''.join(time)}|"]
             res += [" ".join(line)]
         return "\n".join(res)
+
+
+def read_jubeat_analyser_file(path: Path) -> Optional[List[str]]:
+    try:
+        # The vast majority of memo files you will encounter will be propely
+        # decoded using shift-jis-2004. Get ready for endless fun with the small
+        # portion of files that won't
+        lines = path.read_text(encoding="shift-jis-2004").split("\n")
+    except UnicodeDecodeError:
+        return None
+    else:
+        return lines
+
+
+load_folder = make_folder_loader(
+    glob_pattern="*.txt",
+    file_loader=read_jubeat_analyser_file,
+)

jubeatools/formats/load_tools.py

@@ -1,40 +1,41 @@
 from pathlib import Path
-from typing import Dict, List, TypeVar, Protocol, Generic, Optional
+from typing import Dict, Iterable, Optional, Protocol, TypeVar
-
 
 T = TypeVar("T")
+T_co = TypeVar("T_co", covariant=True)
 
-class FileLoader(Protocol, Generic[T]):
+
+class FileLoader(Protocol[T_co]):
     """Function that excepts a path to a file as a parameter and returns its
     contents in whatever form suitable for the current format. Returns None in
     case of error"""
-    def __call__(self, path: Path) -> Optional[T]:
+
+    def __call__(self, path: Path) -> T_co:
         ...
 
-class FolderLoader(Protocol, Generic[T]):
+
+class FolderLoader(Protocol[T]):
     """Function that expects a folder or a file path as a parameter. Loads
     either all valid files in the folder or just the given file depending on
     the argument"""
+
     def __call__(self, path: Path) -> Dict[Path, T]:
         ...
 
-def make_folder_loader(
-    glob_pattern: str,
-    file_loader: FileLoader
-) -> FolderLoader:
 
+def make_folder_loader(glob_pattern: str, file_loader: FileLoader) -> FolderLoader:
     def folder_loader(path: Path) -> Dict[Path, T]:
         files: Dict[Path, T] = {}
         if path.is_dir():
-            paths = path.glob(glob_pattern)
+            paths: Iterable[Path] = path.glob(glob_pattern)
         else:
             paths = [path]
-        
+
         for p in paths:
             value = file_loader(p)
             if value is not None:
                 files[p] = value
 
         return files
-    
+
     return folder_loader