From 25b6529245260a8db97db119e5318cafbb1aa36f Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Sun, 9 May 2021 16:31:53 +0200 Subject: [PATCH] Standardize loading and dumping routines for formats that use one file per chart --- .flake8 | 18 ++--- jubeatools/formats/adapters.py | 78 ------------------- jubeatools/formats/dump_tools.py | 75 +++++++++++++++++- jubeatools/formats/jubeat_analyser/files.py | 20 ----- .../formats/jubeat_analyser/load_tools.py | 22 +++++- .../formats/{files.py => load_tools.py} | 25 +++--- 6 files changed, 116 insertions(+), 122 deletions(-) delete mode 100644 jubeatools/formats/adapters.py delete mode 100644 jubeatools/formats/jubeat_analyser/files.py rename jubeatools/formats/{files.py => load_tools.py} (68%) diff --git a/.flake8 b/.flake8 index 8e6fd7c..454b978 100644 --- a/.flake8 +++ b/.flake8 @@ -1,19 +1,19 @@ [flake8] ignore = - # break after binary op. - # black does its thing and I leave it that way - W503 - # f-string is missing placeholders - # really don't want to be bothered for so little - F541 + # First category : "ffs flake8 let black do its own formatting" # whitespace before ':' - # let black handle that E203 # blank line contains whitespace - # let black handle that as well W293 + # expected 2 blank lines, found 1 + E302 + # break after binary op. + W503 + + # Second category : "I know what I'm doing leave me alone" + # f-string is missing placeholders + F541 # do not assign a lambda expression, use a def - # I know when I need that one, thank you very much E731 exclude = .git diff --git a/jubeatools/formats/adapters.py b/jubeatools/formats/adapters.py deleted file mode 100644 index f6de369..0000000 --- a/jubeatools/formats/adapters.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Things that make it easier to integrate formats with different opinions -on song folder structure""" - -from itertools import count -from typing import TypedDict, Iterator, Any, Dict, AbstractSet -from pathlib import Path - -from jubeatools.song import Song -from jubeatools.formats.typing import ChartFileDumper, Dumper -from jubeatools.formats.filetypes import ChartFile -from jubeatools.formats.dump_tools import DIFFICULTY_INDEX, DIFFICULTY_NUMBER - - -def make_dumper_from_chart_file_dumper( - internal_dumper: ChartFileDumper, - file_name_template: Path, -) -> Dumper: - """Adapt a ChartFileDumper to the Dumper protocol, The resulting function - uses the file name template if it recieves an existing directory as an - output path""" - - def dumper(song: Song, path: Path, **kwargs: Any) -> Dict[Path, bytes]: - res: Dict[Path, bytes] = {} - if path.is_dir(): - file_path = file_name_template - else: - file_path = path - - name_format = f"{file_path.stem}{{dedup}}{file_path.suffix}" - files = internal_dumper(song, **kwargs) - for chartfile in files: - filepath = choose_file_path(chartfile, name_format, path.parent, res.keys()) - res[filepath] = chartfile.contents - - return res - - return dumper - - -def choose_file_path( - chart_file: ChartFile, - name_format: str, - parent: Path, - already_chosen: AbstractSet[Path], -) -> Path: - all_paths = iter_possible_paths(chart_file, name_format, parent) - not_on_filesystem = filter(lambda p: not p.exists(), all_paths) - not_already_chosen = filter(lambda p: p not in already_chosen, not_on_filesystem) - return next(not_already_chosen) - - -def iter_possible_paths( - chart_file: ChartFile, name_format: str, parent: Path -) -> Iterator[Path]: - for dedup_index in count(start=0): - params = extract_format_params(chart_file, dedup_index) - filename = name_format.format(**params).strip() - yield parent / filename - - -class FormatParameters(TypedDict, total=False): - title: str - difficulty: str - # 0-based - difficulty_index: int - # 1-based - difficulty_number: int - dedup: str - - -def extract_format_params(chartfile: ChartFile, dedup_index: int) -> FormatParameters: - return FormatParameters( - title=chartfile.song.metadata.title or "", - difficulty=chartfile.difficulty, - difficulty_index=DIFFICULTY_INDEX.get(chartfile.difficulty, 3), - difficulty_number=DIFFICULTY_NUMBER.get(chartfile.difficulty, 4), - dedup="" if dedup_index else f"-{dedup_index}", - ) diff --git a/jubeatools/formats/dump_tools.py b/jubeatools/formats/dump_tools.py index c19d7ad..601f0f5 100644 --- a/jubeatools/formats/dump_tools.py +++ b/jubeatools/formats/dump_tools.py @@ -1,6 +1,10 @@ -from jubeatools.song import Difficulty -from typing import Dict +from itertools import count +from pathlib import Path +from typing import AbstractSet, Any, Dict, Iterator, TypedDict +from jubeatools.formats.filetypes import ChartFile +from jubeatools.formats.typing import ChartFileDumper, Dumper +from jubeatools.song import Difficulty, Song DIFFICULTY_NUMBER: Dict[str, int] = { Difficulty.BASIC: 1, @@ -13,3 +17,70 @@ DIFFICULTY_INDEX: Dict[str, int] = { Difficulty.ADVANCED: 1, Difficulty.EXTREME: 2, } + + +def make_dumper_from_chart_file_dumper( + internal_dumper: ChartFileDumper, + file_name_template: Path, +) -> Dumper: + """Adapt a ChartFileDumper to the Dumper protocol, The resulting function + uses the file name template if it recieves an existing directory as an + output path""" + + def dumper(song: Song, path: Path, **kwargs: Any) -> Dict[Path, bytes]: + res: Dict[Path, bytes] = {} + if path.is_dir(): + file_path = file_name_template + else: + file_path = path + + name_format = f"{file_path.stem}{{dedup}}{file_path.suffix}" + files = internal_dumper(song, **kwargs) + for chartfile in files: + filepath = choose_file_path(chartfile, name_format, path.parent, res.keys()) + res[filepath] = chartfile.contents + + return res + + return dumper + + +def choose_file_path( + chart_file: ChartFile, + name_format: str, + parent: Path, + already_chosen: AbstractSet[Path], +) -> Path: + all_paths = iter_possible_paths(chart_file, name_format, parent) + not_on_filesystem = filter(lambda p: not p.exists(), all_paths) + not_already_chosen = filter(lambda p: p not in already_chosen, not_on_filesystem) + return next(not_already_chosen) + + +def iter_possible_paths( + chart_file: ChartFile, name_format: str, parent: Path +) -> Iterator[Path]: + for dedup_index in count(start=0): + params = extract_format_params(chart_file, dedup_index) + filename = name_format.format(**params).strip() + yield parent / filename + + +class FormatParameters(TypedDict, total=False): + title: str + difficulty: str + # 0-based + difficulty_index: int + # 1-based + difficulty_number: int + dedup: str + + +def extract_format_params(chartfile: ChartFile, dedup_index: int) -> FormatParameters: + return FormatParameters( + title=chartfile.song.metadata.title or "", + difficulty=chartfile.difficulty, + difficulty_index=DIFFICULTY_INDEX.get(chartfile.difficulty, 3), + difficulty_number=DIFFICULTY_NUMBER.get(chartfile.difficulty, 4), + dedup="" if dedup_index else f"-{dedup_index}", + ) diff --git a/jubeatools/formats/jubeat_analyser/files.py b/jubeatools/formats/jubeat_analyser/files.py deleted file mode 100644 index ba3649b..0000000 --- a/jubeatools/formats/jubeat_analyser/files.py +++ /dev/null @@ -1,20 +0,0 @@ -from pathlib import Path -from typing import Dict, List, Optional - -from jubeatools.formats.files import make_folder_loader - -def read_jubeat_analyser_file(path: Path) -> Optional[List[str]]: - try: - # The vast majority of memo files you will encounter will be propely - # decoded using shift-jis-2004. Get ready for endless fun with the small - # portion of files that won't - lines = path.read_text(encoding="shift-jis-2004").split("\n") - except UnicodeDecodeError: - return None - else: - return lines - -load_folder = make_folder_loader( - glob_pattern="*.txt", - file_loader=read_jubeat_analyser_file, -) diff --git a/jubeatools/formats/jubeat_analyser/load_tools.py b/jubeatools/formats/jubeat_analyser/load_tools.py index 2ffa6d6..b3c8dde 100644 --- a/jubeatools/formats/jubeat_analyser/load_tools.py +++ b/jubeatools/formats/jubeat_analyser/load_tools.py @@ -7,13 +7,15 @@ from copy import deepcopy from dataclasses import astuple, dataclass from decimal import Decimal from itertools import product, zip_longest +from pathlib import Path from typing import AbstractSet, Dict, List, Optional, Set, Tuple, Union import constraint from parsimonious import Grammar, NodeVisitor, ParseError from parsimonious.nodes import Node -from jubeatools.song import BeatsTime, BPMEvent, LongNote, NotePosition, Difficulty +from jubeatools.formats.load_tools import make_folder_loader +from jubeatools.song import BeatsTime, BPMEvent, Difficulty, LongNote, NotePosition from .symbols import ( CIRCLE_FREE_SYMBOLS, @@ -461,3 +463,21 @@ class DoubleColumnFrame: line += [f"|{''.join(time)}|"] res += [" ".join(line)] return "\n".join(res) + + +def read_jubeat_analyser_file(path: Path) -> Optional[List[str]]: + try: + # The vast majority of memo files you will encounter will be propely + # decoded using shift-jis-2004. Get ready for endless fun with the small + # portion of files that won't + lines = path.read_text(encoding="shift-jis-2004").split("\n") + except UnicodeDecodeError: + return None + else: + return lines + + +load_folder = make_folder_loader( + glob_pattern="*.txt", + file_loader=read_jubeat_analyser_file, +) diff --git a/jubeatools/formats/files.py b/jubeatools/formats/load_tools.py similarity index 68% rename from jubeatools/formats/files.py rename to jubeatools/formats/load_tools.py index 0894942..a662923 100644 --- a/jubeatools/formats/files.py +++ b/jubeatools/formats/load_tools.py @@ -1,40 +1,41 @@ from pathlib import Path -from typing import Dict, List, TypeVar, Protocol, Generic, Optional - +from typing import Dict, Iterable, Optional, Protocol, TypeVar T = TypeVar("T") +T_co = TypeVar("T_co", covariant=True) -class FileLoader(Protocol, Generic[T]): + +class FileLoader(Protocol[T_co]): """Function that excepts a path to a file as a parameter and returns its contents in whatever form suitable for the current format. Returns None in case of error""" - def __call__(self, path: Path) -> Optional[T]: + + def __call__(self, path: Path) -> T_co: ... -class FolderLoader(Protocol, Generic[T]): + +class FolderLoader(Protocol[T]): """Function that expects a folder or a file path as a parameter. Loads either all valid files in the folder or just the given file depending on the argument""" + def __call__(self, path: Path) -> Dict[Path, T]: ... -def make_folder_loader( - glob_pattern: str, - file_loader: FileLoader -) -> FolderLoader: +def make_folder_loader(glob_pattern: str, file_loader: FileLoader) -> FolderLoader: def folder_loader(path: Path) -> Dict[Path, T]: files: Dict[Path, T] = {} if path.is_dir(): - paths = path.glob(glob_pattern) + paths: Iterable[Path] = path.glob(glob_pattern) else: paths = [path] - + for p in paths: value = file_loader(p) if value is not None: files[p] = value return files - + return folder_loader