From d67df121dad3254c550fd84b12a92f18012b4900 Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Wed, 26 May 2021 22:49:33 +0200 Subject: [PATCH 1/3] [malody] beta load support ! --- jubeatools/formats/konami/commons.py | 3 +- jubeatools/formats/konami/dump_tools.py | 2 +- .../formats/konami/eve/tests/test_events.py | 2 +- .../formats/konami/eve/tests/test_timemap.py | 2 +- jubeatools/formats/konami/load_tools.py | 2 +- jubeatools/formats/load_tools.py | 4 +- jubeatools/formats/malody/__init__.py | 6 + jubeatools/formats/malody/load.py | 120 ++++++++++++++++++ jubeatools/formats/malody/schema.py | 100 +++++++++++++++ jubeatools/formats/{konami => }/timemap.py | 102 +++++++++------ mypy.ini | 1 + poetry.lock | 80 +++++++++++- pyproject.toml | 1 + 13 files changed, 374 insertions(+), 51 deletions(-) create mode 100644 jubeatools/formats/malody/__init__.py create mode 100644 jubeatools/formats/malody/load.py create mode 100644 jubeatools/formats/malody/schema.py rename jubeatools/formats/{konami => }/timemap.py (62%) diff --git a/jubeatools/formats/konami/commons.py b/jubeatools/formats/konami/commons.py index 6818d6a..b93c275 100644 --- a/jubeatools/formats/konami/commons.py +++ b/jubeatools/formats/konami/commons.py @@ -8,8 +8,7 @@ from itertools import count from typing import Iterator, Union from jubeatools import song - -from .timemap import TimeMap +from jubeatools.formats.timemap import TimeMap AnyNote = Union[song.TapNote, song.LongNote] diff --git a/jubeatools/formats/konami/dump_tools.py b/jubeatools/formats/konami/dump_tools.py index c94cad9..c44097e 100644 --- a/jubeatools/formats/konami/dump_tools.py +++ b/jubeatools/formats/konami/dump_tools.py @@ -6,9 +6,9 @@ from typing import List from more_itertools import numeric_range from jubeatools import song +from jubeatools.formats.timemap import TimeMap from .commons import AnyNote, Command, Event, bpm_to_value, ticks_at_beat -from .timemap import TimeMap def make_events_from_chart(notes: List[AnyNote], timing: song.Timing) -> List[Event]: diff --git a/jubeatools/formats/konami/eve/tests/test_events.py b/jubeatools/formats/konami/eve/tests/test_events.py index 4e518a1..cac1614 100644 --- a/jubeatools/formats/konami/eve/tests/test_events.py +++ b/jubeatools/formats/konami/eve/tests/test_events.py @@ -3,7 +3,7 @@ from hypothesis import strategies as st from jubeatools import song from jubeatools.formats.konami.commons import EveLong -from jubeatools.formats.konami.timemap import TimeMap +from jubeatools.formats.timemap import TimeMap from jubeatools.testutils import strategies as jbst diff --git a/jubeatools/formats/konami/eve/tests/test_timemap.py b/jubeatools/formats/konami/eve/tests/test_timemap.py index ddf6f84..41c5481 100644 --- a/jubeatools/formats/konami/eve/tests/test_timemap.py +++ b/jubeatools/formats/konami/eve/tests/test_timemap.py @@ -3,7 +3,7 @@ from fractions import Fraction from hypothesis import given from jubeatools import song -from jubeatools.formats.konami.timemap import TimeMap +from jubeatools.formats.timemap import TimeMap from jubeatools.testutils import strategies as jbst from jubeatools.utils import group_by diff --git a/jubeatools/formats/konami/load_tools.py b/jubeatools/formats/konami/load_tools.py index 4c8784b..415acb8 100644 --- a/jubeatools/formats/konami/load_tools.py +++ b/jubeatools/formats/konami/load_tools.py @@ -3,6 +3,7 @@ from typing import Iterable, List from jubeatools import song from jubeatools.formats.load_tools import round_beats +from jubeatools.formats.timemap import BPMAtSecond, TimeMap from jubeatools.utils import group_by from .commons import ( @@ -14,7 +15,6 @@ from .commons import ( ticks_to_seconds, value_to_truncated_bpm, ) -from .timemap import BPMAtSecond, TimeMap def make_chart_from_events(events: Iterable[Event], beat_snap: int = 240) -> song.Chart: diff --git a/jubeatools/formats/load_tools.py b/jubeatools/formats/load_tools.py index af723a6..991fdb8 100644 --- a/jubeatools/formats/load_tools.py +++ b/jubeatools/formats/load_tools.py @@ -1,7 +1,7 @@ from decimal import Decimal from fractions import Fraction from pathlib import Path -from typing import Dict, Iterable, Protocol, TypeVar, Union +from typing import Dict, Iterable, Optional, Protocol, TypeVar, Union from jubeatools import song @@ -14,7 +14,7 @@ class FileLoader(Protocol[T_co]): contents in whatever form suitable for the current format. Returns None in case of error""" - def __call__(self, path: Path) -> T_co: + def __call__(self, path: Path) -> Optional[T_co]: ... diff --git a/jubeatools/formats/malody/__init__.py b/jubeatools/formats/malody/__init__.py new file mode 100644 index 0000000..b3b1426 --- /dev/null +++ b/jubeatools/formats/malody/__init__.py @@ -0,0 +1,6 @@ +"""Malody is a multiplatform rhythm game that mainly lives off content created +by its community, as is common in the rhythm game simulator scene. It support +many different games or "Modes", including jubeat (known as "Pad" Mode) + +The file format it uses is not that well documented but is simple enough to +make sense of without docs. It's a json file with some defined schema""" diff --git a/jubeatools/formats/malody/load.py b/jubeatools/formats/malody/load.py new file mode 100644 index 0000000..a244ae6 --- /dev/null +++ b/jubeatools/formats/malody/load.py @@ -0,0 +1,120 @@ +import json +import warnings +from decimal import Decimal +from fractions import Fraction +from functools import reduce, singledispatch +from pathlib import Path +from typing import Any, List, Optional, Tuple, Union + +from jubeatools import song +from jubeatools.formats import timemap +from jubeatools.formats.load_tools import make_folder_loader +from jubeatools.utils import none_or + +from . import schema as malody + + +def load_malody(path: Path, **kwargs: Any) -> song.Song: + files = load_folder(path) + charts = [load_malody_file(d) for d in files.values()] + return reduce(song.Song.merge, charts) + + +def load_file(path: Path) -> Any: + with path.open() as f: + return json.load(f) + + +load_folder = make_folder_loader("*.mc", load_file) + + +def load_malody_file(raw_dict: dict) -> song.Song: + file: malody.Chart = malody.Chart.Schema().load(raw_dict) + if file.meta.mode != malody.Mode.PAD: + raise ValueError("This file is not a Malody Pad Chart (Malody's jubeat mode)") + + bgm = find_bgm(file.note) + metadata = load_metadata(file.meta, bgm) + time_map = load_timing_info(file.time, bgm) + timing = time_map.convert_to_timing_info() + chart = song.Chart(level=Decimal(0), timing=timing, notes=load_notes(file.note)) + dif = file.meta.version or song.Difficulty.EXTREME + return song.Song(metadata=metadata, charts={dif: chart}) + + +def find_bgm(events: List[malody.Event]) -> Optional[malody.Sound]: + sounds = [e for e in events if isinstance(e, malody.Sound)] + bgms = [s for s in sounds if s.type == malody.SoundType.BACKGROUND_MUSIC] + if not bgms: + return None + + if len(bgms) > 1: + warnings.warn( + "This file defines more than one background music, the first one " + "will be used" + ) + + return min(bgms, key=lambda b: tuple_to_beats(b.beat)) + + +def load_metadata(meta: malody.Metadata, bgm: Optional[malody.Sound]) -> song.Metadata: + return song.Metadata( + title=meta.song.title, + artist=meta.song.artist, + audio=none_or(lambda b: Path(b.sound), bgm), + cover=none_or(Path, meta.background), + ) + + +def load_timing_info( + bpm_changes: List[malody.BPMEvent], bgm: Optional[malody.Sound] +) -> timemap.TimeMap: + if bgm is None: + offset = timemap.SecondsAtBeat(seconds=Fraction(0), beats=Fraction(0)) + else: + offset = timemap.SecondsAtBeat( + seconds=-Fraction(bgm.offset) / 1000, beats=tuple_to_beats(bgm.beat) + ) + return timemap.TimeMap.from_beats( + events=[ + timemap.BPMAtBeat(beats=tuple_to_beats(b.beat), BPM=Fraction(b.bpm)) + for b in bpm_changes + ], + offset=offset, + ) + + +def load_notes(events: List[malody.Event]) -> List[Union[song.TapNote, song.LongNote]]: + # filter out sound events + notes = filter(lambda e: isinstance(e, (malody.TapNote, malody.TapNote)), events) + return [load_note(n) for n in notes] + + +@singledispatch +def load_note( + n: Union[malody.TapNote, malody.LongNote] +) -> Union[song.TapNote, song.LongNote]: + raise NotImplementedError(f"Unknown note type : {type(n)}") + + +@load_note.register +def load_tap_note(n: malody.TapNote) -> song.TapNote: + return song.TapNote( + time=tuple_to_beats(n.beat), position=song.NotePosition.from_index(n.index) + ) + + +@load_note.register +def load_long_note(n: malody.LongNote) -> song.LongNote: + start = tuple_to_beats(n.beat) + end = tuple_to_beats(n.endbeat) + return song.LongNote( + time=start, + position=song.NotePosition.from_index(n.index), + duration=end - start, + tail_tip=song.NotePosition.from_index(n.endindex), + ) + + +def tuple_to_beats(b: Tuple[int, int, int]) -> song.BeatsTime: + return b[0] + song.BeatsTime(b[1], b[2]) diff --git a/jubeatools/formats/malody/schema.py b/jubeatools/formats/malody/schema.py new file mode 100644 index 0000000..875a527 --- /dev/null +++ b/jubeatools/formats/malody/schema.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import field +from decimal import Decimal +from enum import Enum +from typing import ClassVar, List, Optional, Tuple, Type, Union + +from marshmallow import Schema as ms_Schema +from marshmallow.validate import Range +from marshmallow_dataclass import NewType, dataclass + + +@dataclass +class Chart: + meta: Metadata + time: List[BPMEvent] = field(default_factory=list) + note: List[Event] = field(default_factory=list) + + Schema: ClassVar[Type[ms_Schema]] = ms_Schema + + +@dataclass +class Metadata: + cover: Optional[str] # path to album art ? + creator: Optional[str] # Chart author + background: Optional[str] # path to background image + version: Optional[str] # freeform difficulty name + id: Optional[int] + mode: int + time: int # creation timestamp ? + song: SongInfo + + +class Mode(int, Enum): + KEY = 0 # Vertical Scrolling Rhythm Game + # 1 : Unused + # 2 : Unused + CATCH = 3 # EZ2CATCH / Catch the Beat + PAD = 4 # Jubeat + TAIKO = 5 # Taiko no Tatsujin + RING = 6 # Reminds me of Beatstream ? + + +@dataclass +class SongInfo: + title: Optional[str] + artist: Optional[str] + id: Optional[int] + + +PositiveInt = NewType("PositiveInt", int, validate=Range(min=0)) +BeatTime = Tuple[PositiveInt, PositiveInt, PositiveInt] + +StrictlyPositiveDecimal = NewType( + "StrictlyPositiveDecimal", Decimal, validate=Range(min=0, min_inclusive=False) +) + + +@dataclass +class BPMEvent: + beat: BeatTime + bpm: StrictlyPositiveDecimal + + +ButtonIndex = NewType("ButtonIndex", int, validate=Range(min=0, max=15)) + + +@dataclass +class TapNote: + beat: BeatTime + index: ButtonIndex + + +@dataclass +class LongNote: + beat: BeatTime + index: ButtonIndex + endbeat: BeatTime + endindex: ButtonIndex + + +@dataclass +class Sound: + """Used both for the background music and keysounds""" + + beat: BeatTime + sound: str # audio file path + type: int + offset: int + isBgm: Optional[bool] + vol: Optional[int] # Volume, out of 100 + x: Optional[int] + + +# TODO: find a keysounded chart to discovery the other values +class SoundType(int, Enum): + BACKGROUND_MUSIC = 1 + + +Event = Union[Sound, LongNote, TapNote] diff --git a/jubeatools/formats/konami/timemap.py b/jubeatools/formats/timemap.py similarity index 62% rename from jubeatools/formats/konami/timemap.py rename to jubeatools/formats/timemap.py index cd6edfb..8a380a6 100644 --- a/jubeatools/formats/konami/timemap.py +++ b/jubeatools/formats/timemap.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace from fractions import Fraction from typing import List, Union @@ -18,6 +18,18 @@ class BPMAtSecond: BPM: Fraction +@dataclass +class BPMAtBeat: + beats: Fraction + BPM: Fraction + + +@dataclass +class SecondsAtBeat: + seconds: Fraction + beats: Fraction + + @dataclass class BPMChange: beats: song.BeatsTime @@ -30,48 +42,69 @@ class TimeMap: """Wraps a song.Timing to allow converting symbolic time (in beats) to clock time (in seconds) and back""" - beat_zero_offset: song.SecondsTime events_by_beats: SortedKeyList[BPMChange, song.BeatsTime] events_by_seconds: SortedKeyList[BPMChange, Fraction] @classmethod - def from_timing(cls, beats: song.Timing) -> TimeMap: + def from_timing(cls, timing: song.Timing) -> TimeMap: """Create a time map from a song.Timing object""" - if not beats.events: + return cls.from_beats( + events=[ + BPMAtBeat(beats=e.time, BPM=Fraction(e.BPM)) for e in timing.events + ], + offset=SecondsAtBeat( + seconds=Fraction(timing.beat_zero_offset), beats=Fraction(0) + ), + ) + + @classmethod + def from_beats(cls, events: List[BPMAtBeat], offset: SecondsAtBeat) -> TimeMap: + """Create a time map from a list of BPM changes with times given in + beats, the offset parameter is more flexible than a "regular" beat zero + offset as it accepts non-zero beats""" + if not events: raise ValueError("No BPM defined") - grouped_by_time = group_by(beats.events, key=lambda e: e.time) - for time, events in grouped_by_time.items(): - if len(events) > 1: - raise ValueError(f"Multiple BPMs defined on beat {time} : {events}") + grouped_by_time = group_by(events, key=lambda e: e.beats) + for time, events_at_time in grouped_by_time.items(): + if len(events_at_time) > 1: + raise ValueError(f"Multiple BPMs defined at beat {time} : {events}") - sorted_events = sorted(beats.events, key=lambda e: e.time) + # First compute everything as if the first BPM change happened at + # zero seconds, then shift according to the offset + sorted_events = sorted(events, key=lambda e: e.beats) first_event = sorted_events[0] - if first_event.time != song.BeatsTime(0): - raise ValueError("First BPM event is not on beat zero") - - # set first BPM change then compute from there - current_second = Fraction(beats.beat_zero_offset) + current_second = Fraction(0) bpm_changes = [ - BPMChange(first_event.time, current_second, Fraction(first_event.BPM)) + BPMChange(first_event.beats, current_second, Fraction(first_event.BPM)) ] for previous, current in windowed(sorted_events, 2): if previous is None or current is None: continue - beats_since_last_event = current.time - previous.time + beats_since_last_event = current.beats - previous.beats seconds_since_last_event = (60 * beats_since_last_event) / Fraction( previous.BPM ) current_second += seconds_since_last_event - bpm_change = BPMChange(current.time, current_second, Fraction(current.BPM)) + bpm_change = BPMChange(current.beats, current_second, Fraction(current.BPM)) bpm_changes.append(bpm_change) - return cls( - beat_zero_offset=beats.beat_zero_offset, + not_shifted = cls( events_by_beats=SortedKeyList(bpm_changes, key=lambda b: b.beats), events_by_seconds=SortedKeyList(bpm_changes, key=lambda b: b.seconds), ) + unshifted_seconds_at_offset = not_shifted.fractional_seconds_at(offset.beats) + shift = offset.seconds - unshifted_seconds_at_offset + shifted_bpm_changes = [ + replace(b, seconds=b.seconds + shift) for b in bpm_changes + ] + return cls( + events_by_beats=SortedKeyList(shifted_bpm_changes, key=lambda b: b.beats), + events_by_seconds=SortedKeyList( + shifted_bpm_changes, key=lambda b: b.seconds + ), + ) @classmethod def from_seconds(cls, events: List[BPMAtSecond]) -> TimeMap: @@ -103,7 +136,6 @@ class TimeMap: bpm_changes.append(bpm_change) return cls( - beat_zero_offset=fraction_to_decimal(first_event.seconds), events_by_beats=SortedKeyList(bpm_changes, key=lambda b: b.beats), events_by_seconds=SortedKeyList(bpm_changes, key=lambda b: b.seconds), ) @@ -113,31 +145,21 @@ class TimeMap: return fraction_to_decimal(frac_seconds) def fractional_seconds_at(self, beat: song.BeatsTime) -> Fraction: - if beat < 0: - raise ValueError("Can't compute seconds at negative beat") - - # find previous bpm change - index = self.events_by_beats.bisect_key_right(beat) - 1 - bpm_change: BPMChange = self.events_by_beats[index] - - # compute seconds since last bpm change + """Before the first bpm change, compute backwards from the first bpm, + after the first bpm change, compute forwards from the previous bpm + change""" + index = self.events_by_beats.bisect_key_right(beat) + first_or_previous_index = max(0, index - 1) + bpm_change: BPMChange = self.events_by_beats[first_or_previous_index] beats_since_last_event = beat - bpm_change.beats seconds_since_last_event = (60 * beats_since_last_event) / bpm_change.BPM return bpm_change.seconds + seconds_since_last_event def beats_at(self, seconds: Union[song.SecondsTime, Fraction]) -> song.BeatsTime: - if seconds < self.beat_zero_offset: - raise ValueError( - f"Can't compute beat time at {seconds} seconds, since it predates " - f"beat zero, which happens at {self.beat_zero_offset} seconds" - ) - - # find previous bpm change frac_seconds = Fraction(seconds) - index = self.events_by_seconds.bisect_key_right(frac_seconds) - 1 - bpm_change: BPMChange = self.events_by_seconds[index] - - # compute beats since last bpm change + index = self.events_by_seconds.bisect_key_right(frac_seconds) + first_or_previous_index = max(0, index - 1) + bpm_change: BPMChange = self.events_by_seconds[first_or_previous_index] seconds_since_last_event = frac_seconds - bpm_change.seconds beats_since_last_event = (bpm_change.BPM * seconds_since_last_event) / Fraction( 60 @@ -153,5 +175,5 @@ class TimeMap: ) for e in self.events_by_beats ], - beat_zero_offset=self.beat_zero_offset, + beat_zero_offset=self.seconds_at(song.BeatsTime(0)), ) diff --git a/mypy.ini b/mypy.ini index 2699243..76fedad 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,6 +9,7 @@ warn_redundant_casts = True warn_unused_ignores = True warn_return_any = True warn_unreachable = True +plugins = marshmallow_dataclass.mypy [mypy-constraint] ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 47c7775..bc07ecb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -189,6 +189,39 @@ docs = ["sphinx (==3.4.3)", "sphinx-issues (==1.2.0)", "alabaster (==0.7.12)", " lint = ["mypy (==0.812)", "flake8 (==3.9.0)", "flake8-bugbear (==21.3.2)", "pre-commit (>=2.4,<3.0)"] tests = ["pytest", "pytz", "simplejson"] +[[package]] +name = "marshmallow-dataclass" +version = "8.4.1" +description = "Python library to convert dataclasses into marshmallow schemas." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +marshmallow = ">=3.0.0,<4.0" +marshmallow-enum = {version = "*", optional = true, markers = "extra == \"enum\""} +typeguard = {version = "*", optional = true, markers = "extra == \"union\""} +typing-inspect = "*" + +[package.extras] +dev = ["marshmallow-enum", "typeguard", "pre-commit (>=1.18,<2.0)", "sphinx", "pytest (>=5.4)", "pytest-mypy-plugins (>=1.2.0)", "typing-extensions (>=3.7.2,<3.8.0)"] +docs = ["sphinx"] +enum = ["marshmallow-enum"] +lint = ["pre-commit (>=1.18,<2.0)"] +tests = ["pytest (>=5.4)", "pytest-mypy-plugins (>=1.2.0)", "typing-extensions (>=3.7.2,<3.8.0)"] +union = ["typeguard"] + +[[package]] +name = "marshmallow-enum" +version = "1.5.1" +description = "Enum field for Marshmallow" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +marshmallow = ">=2.0.0" + [[package]] name = "mccabe" version = "0.6.1" @@ -233,7 +266,7 @@ dmypy = ["psutil (>=4.0)"] name = "mypy-extensions" version = "0.4.3" description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -430,18 +463,42 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "typeguard" +version = "2.12.0" +description = "Run-time type checker for Python" +category = "main" +optional = false +python-versions = ">=3.5.3" + +[package.extras] +doc = ["sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"] +test = ["pytest", "typing-extensions", "mypy"] + [[package]] name = "typing-extensions" version = "3.7.4.3" description = "Backported and Experimental Type Hints for Python 3.5+" -category = "dev" +category = "main" optional = false python-versions = "*" +[[package]] +name = "typing-inspect" +version = "0.6.0" +description = "Runtime inspection utilities for typing module." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +mypy-extensions = ">=0.3.0" +typing-extensions = ">=3.7.4" + [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "dd2989bf15b7c389b4d6d8f28335f4fea3cb064b41723e5fe500f38dd92c39bb" +content-hash = "0e7f4c7f3e3554861ffd76a0e4143dd5c5218904234ee5641ad106a26f17b2bb" [metadata.files] appdirs = [ @@ -502,6 +559,14 @@ marshmallow = [ {file = "marshmallow-3.11.1-py2.py3-none-any.whl", hash = "sha256:0dd42891a5ef288217ed6410917f3c6048f585f8692075a0052c24f9bfff9dfd"}, {file = "marshmallow-3.11.1.tar.gz", hash = "sha256:16e99cb7f630c0ef4d7d364ed0109ac194268dde123966076ab3dafb9ae3906b"}, ] +marshmallow-dataclass = [ + {file = "marshmallow_dataclass-8.4.1-py3-none-any.whl", hash = "sha256:035f4aa9f516ca3c14c9ae3905fe8370b14cea6462ec1a9d4451209a6117976e"}, + {file = "marshmallow_dataclass-8.4.1.tar.gz", hash = "sha256:26b6ef76646c6cd71df3163c7106ddeaab27d9fac355cad41046627d5c15cda0"}, +] +marshmallow-enum = [ + {file = "marshmallow-enum-1.5.1.tar.gz", hash = "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58"}, + {file = "marshmallow_enum-1.5.1-py2.py3-none-any.whl", hash = "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -764,8 +829,17 @@ typed-ast = [ {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, ] +typeguard = [ + {file = "typeguard-2.12.0-py3-none-any.whl", hash = "sha256:7d1cf82b35e9ff3cd083133ebda54ad1d7a40296471397e6c6b229cf07fe5307"}, + {file = "typeguard-2.12.0.tar.gz", hash = "sha256:fca77fd4ccba63465b421cdbbab5a1a8e3994e6d6f18b45da2bb475c09f147ef"}, +] typing-extensions = [ {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, ] +typing-inspect = [ + {file = "typing_inspect-0.6.0-py2-none-any.whl", hash = "sha256:de08f50a22955ddec353876df7b2545994d6df08a2f45d54ac8c05e530372ca0"}, + {file = "typing_inspect-0.6.0-py3-none-any.whl", hash = "sha256:3b98390df4d999a28cf5b35d8b333425af5da2ece8a4ea9e98f71e7591347b4f"}, + {file = "typing_inspect-0.6.0.tar.gz", hash = "sha256:8f1b1dd25908dbfd81d3bebc218011531e7ab614ba6e5bf7826d887c834afab7"}, +] diff --git a/pyproject.toml b/pyproject.toml index 5dcae40..752270e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ sortedcontainers = "^2.3.0" python-constraint = "^1.4.0" construct = "~=2.10" construct-typing = "^0.4.2" +marshmallow-dataclass = {extras = ["union", "enum"], version = "^8.4.1"} [tool.poetry.dev-dependencies] pytest = "^6.2.3" From 7b110e246287f0efcd49cac012abe38afb3425bf Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Thu, 27 May 2021 01:31:41 +0200 Subject: [PATCH 2/3] [malody] beta dump support ! --- jubeatools/formats/malody/dump.py | 108 ++++++++++++++++++++++++++++ jubeatools/formats/malody/schema.py | 2 +- 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 jubeatools/formats/malody/dump.py diff --git a/jubeatools/formats/malody/dump.py b/jubeatools/formats/malody/dump.py new file mode 100644 index 0000000..6e770b4 --- /dev/null +++ b/jubeatools/formats/malody/dump.py @@ -0,0 +1,108 @@ +from pathlib import Path +from typing import List +import json +import time + +from jubeatools import song +from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper +from jubeatools.formats.filetypes import ChartFile + +from . import schema as malody + + +def dump_malody_song(song: song.Song, **kwargs: dict) -> List[ChartFile]: + res = [] + for dif, chart, timing in song.iter_charts_with_timing(): + malody_chart = dump_malody_chart(song.metadata, dif, chart, timing) + json_chart = malody.Chart.Schema().dump(malody_chart) + chart_bytes = json.dumps(json_chart).encode("utf-8") + res.append(ChartFile(chart_bytes, song, dif, chart)) + + return res + + +dump_malody = make_dumper_from_chart_file_dumper( + internal_dumper=dump_malody_song, file_name_template=Path("{difficulty:l}.mc") +) + + +def dump_malody_chart(metadata: song.Metadata, dif: str, chart: song.Chart, timing: song.Timing) -> malody.Chart: + meta = dump_metadata(metadata, dif) + time = dump_timing(timing) + notes = dump_notes(chart.notes) + if metadata.audio is not None: + notes += dump_bgm(metadata.audio, timing) + return malody.Chart(meta=meta, time=time, note=notes) + +def dump_metadata(metadata: song.Metadata, dif: str) -> malody.Metadata: + return malody.Metadata( + cover="", + creator="", + background=none_or(str, metadata.cover), + version=dif, + id=0, + mode=malody.Mode.PAD, + time=int(time.time()), + song=malody.SongInfo( + title=metadata.title, + artist=metadata.artist, + id=0, + ) + ) + +def dump_timing(timing: song.Timing) -> List[malody.BPMEvent]: + sorted_events = sorted(timing.events, key=lambda e: e.time) + return [dump_bpm_change(e) for e in sorted_events] + +def dump_bpm_change(b: song.BPMEvent) -> malody.BPMEvent: + return malody.BPMEvent( + beat=beats_to_tuple(b.time), + bpm=b.BPM, + ) + +def dump_notes(notes: List[Union[song.TapNote, song.LongNote]]) -> List[Union[malody.TapNote, malody.LongNote]]: + return [dump_note(n) for n in notes] + + +@singledispatch +def dump_note( + n: Union[song.TapNote, song.LongNote] +) -> Union[malody.TapNote, malody.LongNote]: + raise NotImplementedError(f"Unknown note type : {type(n)}") + + +@dump_note.register +def dump_tap_note(n: song.TapNote) -> malody.TapNote: + return malody.TapNote( + beat=beats_to_tuple(n.time), + index=n.position.index, + ) + +@dump_note.register +def dump_long_note(n: song.LongNote) -> malody.LongNote: + return malody.LongNote( + beat=beats_to_tuple(n.time) + index=n.position.index, + endbeat=beats_to_tuple(n.time + n.duration), + endindex=n.tail_tip.index, + ) + +def dump_bgm(audio: Path, timing: song.Timing) -> malody.Sound: + return malody.Sound( + beat=beats_to_tuple(song.BeatsTime(0)), + sound=str(audio), + vol=100, + offset=-int(timing.beat_zero_offset*1000), + type=malody.SoundType.BACKGROUND_MUSIC, + isBgm=None, + x=None, + ) + +def beats_to_tuple(b: song.BeatsTime) -> Tuple[int, int, int]: + integer_part = int(b) + remainder = b % 1 + return ( + integer_part, + remainder.numerator, + remainder.denominator, + ) \ No newline at end of file diff --git a/jubeatools/formats/malody/schema.py b/jubeatools/formats/malody/schema.py index 875a527..a48dd1d 100644 --- a/jubeatools/formats/malody/schema.py +++ b/jubeatools/formats/malody/schema.py @@ -27,7 +27,7 @@ class Metadata: version: Optional[str] # freeform difficulty name id: Optional[int] mode: int - time: int # creation timestamp ? + time: Optional[int] # creation timestamp ? song: SongInfo From b613a2d96062e05f62bd6c7ea145a003e505d933 Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Fri, 28 May 2021 01:11:27 +0200 Subject: [PATCH 3/3] [malody] test pass ! --- CHANGELOG.md | 4 ++ jubeatools/formats/enum.py | 1 + jubeatools/formats/formats.py | 3 + jubeatools/formats/guess.py | 21 ++++-- .../konami/jbsq/{tests => }/test_jbsq.py | 2 +- .../formats/konami/jbsq/tests/__init__.py | 0 jubeatools/formats/malody/__init__.py | 3 + jubeatools/formats/malody/dump.py | 36 +++++++---- jubeatools/formats/malody/load.py | 9 +-- jubeatools/formats/malody/schema.py | 64 ++++++++++--------- jubeatools/formats/malody/test_malody.py | 32 ++++++++++ 11 files changed, 120 insertions(+), 55 deletions(-) rename jubeatools/formats/konami/jbsq/{tests => }/test_jbsq.py (94%) delete mode 100644 jubeatools/formats/konami/jbsq/tests/__init__.py create mode 100644 jubeatools/formats/malody/test_malody.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b32de65..614d157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.2.0 +## Added +- [malody] 🎉 initial malody support ! + # v1.1.3 ## Fixed - [jubeat-analyser] All files are read and written in `surrogateescape` error diff --git a/jubeatools/formats/enum.py b/jubeatools/formats/enum.py index c4d30ae..2681ad1 100644 --- a/jubeatools/formats/enum.py +++ b/jubeatools/formats/enum.py @@ -4,6 +4,7 @@ from enum import Enum class Format(str, Enum): EVE = "eve" JBSQ = "jbsq" + MALODY = "malody" MEMON_LEGACY = "memon:legacy" MEMON_0_1_0 = "memon:v0.1.0" MEMON_0_2_0 = "memon:v0.2.0" diff --git a/jubeatools/formats/formats.py b/jubeatools/formats/formats.py index 970bf25..3efcd53 100644 --- a/jubeatools/formats/formats.py +++ b/jubeatools/formats/formats.py @@ -12,6 +12,7 @@ from .jubeat_analyser import ( load_mono_column, ) from .konami import dump_eve, dump_jbsq, load_eve, load_jbsq +from .malody import dump_malody, load_malody from .memon import ( dump_memon_0_1_0, dump_memon_0_2_0, @@ -25,6 +26,7 @@ from .typing import Dumper, Loader LOADERS: Dict[Format, Loader] = { Format.EVE: load_eve, Format.JBSQ: load_jbsq, + Format.MALODY: load_malody, Format.MEMON_LEGACY: load_memon_legacy, Format.MEMON_0_1_0: load_memon_0_1_0, Format.MEMON_0_2_0: load_memon_0_2_0, @@ -37,6 +39,7 @@ LOADERS: Dict[Format, Loader] = { DUMPERS: Dict[Format, Dumper] = { Format.EVE: dump_eve, Format.JBSQ: dump_jbsq, + Format.MALODY: dump_malody, Format.MEMON_LEGACY: dump_memon_legacy, Format.MEMON_0_1_0: dump_memon_0_1_0, Format.MEMON_0_2_0: dump_memon_0_2_0, diff --git a/jubeatools/formats/guess.py b/jubeatools/formats/guess.py index c4577f7..6b1542a 100644 --- a/jubeatools/formats/guess.py +++ b/jubeatools/formats/guess.py @@ -10,7 +10,7 @@ def guess_format(path: Path) -> Format: raise ValueError("Can't guess chart format for a folder") try: - return recognize_memon_version(path) + return recognize_json_formats(path) except (json.JSONDecodeError, UnicodeDecodeError, ValueError): pass @@ -28,19 +28,26 @@ def guess_format(path: Path) -> Format: raise ValueError("Unrecognized file format") -def recognize_memon_version(path: Path) -> Format: +def recognize_json_formats(path: Path) -> Format: with path.open() as f: obj = json.load(f) + if not isinstance(obj, dict): + raise ValueError("Top level value is not an object") + + if obj.keys() >= {"metadata", "data"}: + return recognize_memon_version(obj) + elif obj.keys() >= {"meta", "time", "note"}: + return Format.MALODY + else: + raise ValueError("Unrecognized file format") + + +def recognize_memon_version(obj: dict) -> Format: try: version = obj["version"] except KeyError: return Format.MEMON_LEGACY - except TypeError: - raise ValueError( - "This JSON file is not a correct memon file : the top-level " - "value is not an object" - ) if version == "0.1.0": return Format.MEMON_0_1_0 diff --git a/jubeatools/formats/konami/jbsq/tests/test_jbsq.py b/jubeatools/formats/konami/jbsq/test_jbsq.py similarity index 94% rename from jubeatools/formats/konami/jbsq/tests/test_jbsq.py rename to jubeatools/formats/konami/jbsq/test_jbsq.py index c44b00e..d8b0749 100644 --- a/jubeatools/formats/konami/jbsq/tests/test_jbsq.py +++ b/jubeatools/formats/konami/jbsq/test_jbsq.py @@ -5,7 +5,7 @@ from jubeatools.formats import Format from jubeatools.formats.konami.testutils import eve_compatible_song, open_temp_dir from jubeatools.testutils.test_patterns import dump_and_load_then_compare -from ..construct import jbsq +from .construct import jbsq @given(eve_compatible_song()) diff --git a/jubeatools/formats/konami/jbsq/tests/__init__.py b/jubeatools/formats/konami/jbsq/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/jubeatools/formats/malody/__init__.py b/jubeatools/formats/malody/__init__.py index b3b1426..d9f4090 100644 --- a/jubeatools/formats/malody/__init__.py +++ b/jubeatools/formats/malody/__init__.py @@ -4,3 +4,6 @@ many different games or "Modes", including jubeat (known as "Pad" Mode) The file format it uses is not that well documented but is simple enough to make sense of without docs. It's a json file with some defined schema""" + +from .dump import dump_malody +from .load import load_malody diff --git a/jubeatools/formats/malody/dump.py b/jubeatools/formats/malody/dump.py index 6e770b4..2d6dac6 100644 --- a/jubeatools/formats/malody/dump.py +++ b/jubeatools/formats/malody/dump.py @@ -1,11 +1,14 @@ -from pathlib import Path -from typing import List -import json import time +from functools import singledispatch +from pathlib import Path +from typing import List, Tuple, Union + +import simplejson as json from jubeatools import song from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper from jubeatools.formats.filetypes import ChartFile +from jubeatools.utils import none_or from . import schema as malody @@ -14,8 +17,8 @@ def dump_malody_song(song: song.Song, **kwargs: dict) -> List[ChartFile]: res = [] for dif, chart, timing in song.iter_charts_with_timing(): malody_chart = dump_malody_chart(song.metadata, dif, chart, timing) - json_chart = malody.Chart.Schema().dump(malody_chart) - chart_bytes = json.dumps(json_chart).encode("utf-8") + json_chart = malody.CHART_SCHEMA.dump(malody_chart) + chart_bytes = json.dumps(json_chart, indent=4, use_decimal=True).encode("utf-8") res.append(ChartFile(chart_bytes, song, dif, chart)) return res @@ -26,14 +29,17 @@ dump_malody = make_dumper_from_chart_file_dumper( ) -def dump_malody_chart(metadata: song.Metadata, dif: str, chart: song.Chart, timing: song.Timing) -> malody.Chart: +def dump_malody_chart( + metadata: song.Metadata, dif: str, chart: song.Chart, timing: song.Timing +) -> malody.Chart: meta = dump_metadata(metadata, dif) time = dump_timing(timing) notes = dump_notes(chart.notes) if metadata.audio is not None: - notes += dump_bgm(metadata.audio, timing) + notes += [dump_bgm(metadata.audio, timing)] return malody.Chart(meta=meta, time=time, note=notes) + def dump_metadata(metadata: song.Metadata, dif: str) -> malody.Metadata: return malody.Metadata( cover="", @@ -47,20 +53,23 @@ def dump_metadata(metadata: song.Metadata, dif: str) -> malody.Metadata: title=metadata.title, artist=metadata.artist, id=0, - ) + ), ) + def dump_timing(timing: song.Timing) -> List[malody.BPMEvent]: sorted_events = sorted(timing.events, key=lambda e: e.time) return [dump_bpm_change(e) for e in sorted_events] + def dump_bpm_change(b: song.BPMEvent) -> malody.BPMEvent: return malody.BPMEvent( beat=beats_to_tuple(b.time), bpm=b.BPM, ) -def dump_notes(notes: List[Union[song.TapNote, song.LongNote]]) -> List[Union[malody.TapNote, malody.LongNote]]: + +def dump_notes(notes: List[Union[song.TapNote, song.LongNote]]) -> List[malody.Event]: return [dump_note(n) for n in notes] @@ -78,26 +87,29 @@ def dump_tap_note(n: song.TapNote) -> malody.TapNote: index=n.position.index, ) + @dump_note.register def dump_long_note(n: song.LongNote) -> malody.LongNote: return malody.LongNote( - beat=beats_to_tuple(n.time) + beat=beats_to_tuple(n.time), index=n.position.index, endbeat=beats_to_tuple(n.time + n.duration), endindex=n.tail_tip.index, ) + def dump_bgm(audio: Path, timing: song.Timing) -> malody.Sound: return malody.Sound( beat=beats_to_tuple(song.BeatsTime(0)), sound=str(audio), vol=100, - offset=-int(timing.beat_zero_offset*1000), + offset=-int(timing.beat_zero_offset * 1000), type=malody.SoundType.BACKGROUND_MUSIC, isBgm=None, x=None, ) + def beats_to_tuple(b: song.BeatsTime) -> Tuple[int, int, int]: integer_part = int(b) remainder = b % 1 @@ -105,4 +117,4 @@ def beats_to_tuple(b: song.BeatsTime) -> Tuple[int, int, int]: integer_part, remainder.numerator, remainder.denominator, - ) \ No newline at end of file + ) diff --git a/jubeatools/formats/malody/load.py b/jubeatools/formats/malody/load.py index a244ae6..ca7fe15 100644 --- a/jubeatools/formats/malody/load.py +++ b/jubeatools/formats/malody/load.py @@ -1,4 +1,3 @@ -import json import warnings from decimal import Decimal from fractions import Fraction @@ -6,6 +5,8 @@ from functools import reduce, singledispatch from pathlib import Path from typing import Any, List, Optional, Tuple, Union +import simplejson as json + from jubeatools import song from jubeatools.formats import timemap from jubeatools.formats.load_tools import make_folder_loader @@ -22,14 +23,14 @@ def load_malody(path: Path, **kwargs: Any) -> song.Song: def load_file(path: Path) -> Any: with path.open() as f: - return json.load(f) + return json.load(f, use_decimal=True) load_folder = make_folder_loader("*.mc", load_file) def load_malody_file(raw_dict: dict) -> song.Song: - file: malody.Chart = malody.Chart.Schema().load(raw_dict) + file: malody.Chart = malody.CHART_SCHEMA.load(raw_dict) if file.meta.mode != malody.Mode.PAD: raise ValueError("This file is not a Malody Pad Chart (Malody's jubeat mode)") @@ -86,7 +87,7 @@ def load_timing_info( def load_notes(events: List[malody.Event]) -> List[Union[song.TapNote, song.LongNote]]: # filter out sound events - notes = filter(lambda e: isinstance(e, (malody.TapNote, malody.TapNote)), events) + notes = filter(lambda e: isinstance(e, (malody.TapNote, malody.LongNote)), events) return [load_note(n) for n in notes] diff --git a/jubeatools/formats/malody/schema.py b/jubeatools/formats/malody/schema.py index a48dd1d..8263b6a 100644 --- a/jubeatools/formats/malody/schema.py +++ b/jubeatools/formats/malody/schema.py @@ -1,34 +1,22 @@ -from __future__ import annotations - -from dataclasses import field +from dataclasses import dataclass, field from decimal import Decimal from enum import Enum -from typing import ClassVar, List, Optional, Tuple, Type, Union +from typing import List, Optional, Tuple, Union -from marshmallow import Schema as ms_Schema from marshmallow.validate import Range -from marshmallow_dataclass import NewType, dataclass +from marshmallow_dataclass import NewType, class_schema + + +class Ordered: + class Meta: + ordered = True @dataclass -class Chart: - meta: Metadata - time: List[BPMEvent] = field(default_factory=list) - note: List[Event] = field(default_factory=list) - - Schema: ClassVar[Type[ms_Schema]] = ms_Schema - - -@dataclass -class Metadata: - cover: Optional[str] # path to album art ? - creator: Optional[str] # Chart author - background: Optional[str] # path to background image - version: Optional[str] # freeform difficulty name +class SongInfo(Ordered): + title: Optional[str] + artist: Optional[str] id: Optional[int] - mode: int - time: Optional[int] # creation timestamp ? - song: SongInfo class Mode(int, Enum): @@ -42,22 +30,26 @@ class Mode(int, Enum): @dataclass -class SongInfo: - title: Optional[str] - artist: Optional[str] +class Metadata(Ordered): + cover: Optional[str] # path to album art ? + creator: Optional[str] # Chart author + background: Optional[str] # path to background image + version: Optional[str] # freeform difficulty name id: Optional[int] + mode: int + time: Optional[int] # creation timestamp ? + song: SongInfo PositiveInt = NewType("PositiveInt", int, validate=Range(min=0)) BeatTime = Tuple[PositiveInt, PositiveInt, PositiveInt] - StrictlyPositiveDecimal = NewType( "StrictlyPositiveDecimal", Decimal, validate=Range(min=0, min_inclusive=False) ) @dataclass -class BPMEvent: +class BPMEvent(Ordered): beat: BeatTime bpm: StrictlyPositiveDecimal @@ -66,13 +58,13 @@ ButtonIndex = NewType("ButtonIndex", int, validate=Range(min=0, max=15)) @dataclass -class TapNote: +class TapNote(Ordered): beat: BeatTime index: ButtonIndex @dataclass -class LongNote: +class LongNote(Ordered): beat: BeatTime index: ButtonIndex endbeat: BeatTime @@ -80,7 +72,7 @@ class LongNote: @dataclass -class Sound: +class Sound(Ordered): """Used both for the background music and keysounds""" beat: BeatTime @@ -98,3 +90,13 @@ class SoundType(int, Enum): Event = Union[Sound, LongNote, TapNote] + + +@dataclass +class Chart(Ordered): + meta: Metadata + time: List[BPMEvent] = field(default_factory=list) + note: List[Event] = field(default_factory=list) + + +CHART_SCHEMA = class_schema(Chart)() diff --git a/jubeatools/formats/malody/test_malody.py b/jubeatools/formats/malody/test_malody.py new file mode 100644 index 0000000..b41395d --- /dev/null +++ b/jubeatools/formats/malody/test_malody.py @@ -0,0 +1,32 @@ +from decimal import Decimal + +from hypothesis import given +from hypothesis import strategies as st + +from jubeatools import song +from jubeatools.formats import Format +from jubeatools.formats.konami.testutils import open_temp_dir +from jubeatools.testutils import strategies as jbst +from jubeatools.testutils.test_patterns import dump_and_load_then_compare +from jubeatools.testutils.typing import DrawFunc + + +@st.composite +def malody_compatible_song(draw: DrawFunc) -> song.Song: + """Malody files only hold one chart and have limited metadata""" + diff = draw(st.sampled_from(list(song.Difficulty))).value + chart = draw(jbst.chart(level_strat=st.just(Decimal(0)))) + metadata = draw(jbst.metadata()) + metadata.preview = None + metadata.preview_file = None + return song.Song(metadata=metadata, charts={diff: chart}) + + +@given(malody_compatible_song()) +def test_that_full_chart_roundtrips(song: song.Song) -> None: + dump_and_load_then_compare( + Format.MALODY, + song, + temp_path=open_temp_dir(), + bytes_decoder=lambda b: b.decode("utf-8"), + )