From 42d2a3c118a63c1840eda69ab48c76e15555d712 Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Mon, 17 May 2021 13:58:06 +0200 Subject: [PATCH] [jbsq] initial support ! Closes #6 --- CHANGELOG.md | 4 + README.md | 10 +- jubeatools/formats/enum.py | 1 + jubeatools/formats/eve/load.py | 139 ------------------ jubeatools/formats/formats.py | 4 +- jubeatools/formats/guess.py | 14 +- jubeatools/formats/konami/__init__.py | 2 + jubeatools/formats/{eve => konami}/commons.py | 3 +- .../{eve/dump.py => konami/dump_tools.py} | 25 +--- .../formats/{ => konami}/eve/__init__.py | 0 jubeatools/formats/konami/eve/dump.py | 24 +++ jubeatools/formats/konami/eve/load.py | 80 ++++++++++ .../{ => konami}/eve/tests/__init__.py | 0 .../{ => konami}/eve/tests/test_bpm_value.py | 2 +- .../formats/konami/eve/tests/test_eve.py | 17 +++ .../{ => konami}/eve/tests/test_events.py | 5 +- .../{ => konami}/eve/tests/test_timemap.py | 2 +- jubeatools/formats/konami/jbsq/__init__.py | 8 + jubeatools/formats/konami/jbsq/construct.py | 43 ++++++ jubeatools/formats/konami/jbsq/dump.py | 102 +++++++++++++ jubeatools/formats/konami/jbsq/jbsq.ksy | 105 +++++++++++++ jubeatools/formats/konami/jbsq/load.py | 51 +++++++ .../formats/konami/jbsq/tests/__init__.py | 0 .../formats/konami/jbsq/tests/test_jbsq.py | 19 +++ jubeatools/formats/konami/load_tools.py | 69 +++++++++ .../tests/test_eve.py => konami/testutils.py} | 14 -- jubeatools/formats/{eve => konami}/timemap.py | 0 jubeatools/utils.py | 4 +- poetry.lock | 31 +++- pyproject.toml | 2 + 30 files changed, 588 insertions(+), 192 deletions(-) delete mode 100644 jubeatools/formats/eve/load.py create mode 100644 jubeatools/formats/konami/__init__.py rename jubeatools/formats/{eve => konami}/commons.py (98%) rename jubeatools/formats/{eve/dump.py => konami/dump_tools.py} (82%) rename jubeatools/formats/{ => konami}/eve/__init__.py (100%) create mode 100644 jubeatools/formats/konami/eve/dump.py create mode 100644 jubeatools/formats/konami/eve/load.py rename jubeatools/formats/{ => konami}/eve/tests/__init__.py (100%) rename jubeatools/formats/{ => konami}/eve/tests/test_bpm_value.py (83%) create mode 100644 jubeatools/formats/konami/eve/tests/test_eve.py rename jubeatools/formats/{ => konami}/eve/tests/test_events.py (86%) rename jubeatools/formats/{ => konami}/eve/tests/test_timemap.py (97%) create mode 100644 jubeatools/formats/konami/jbsq/__init__.py create mode 100644 jubeatools/formats/konami/jbsq/construct.py create mode 100644 jubeatools/formats/konami/jbsq/dump.py create mode 100644 jubeatools/formats/konami/jbsq/jbsq.ksy create mode 100644 jubeatools/formats/konami/jbsq/load.py create mode 100644 jubeatools/formats/konami/jbsq/tests/__init__.py create mode 100644 jubeatools/formats/konami/jbsq/tests/test_jbsq.py create mode 100644 jubeatools/formats/konami/load_tools.py rename jubeatools/formats/{eve/tests/test_eve.py => konami/testutils.py} (82%) rename jubeatools/formats/{eve => konami}/timemap.py (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c37f6a..03a86e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Unreleased +## Added +- [jbsq] 🎉 initial .jbsq support ! + # 1.0.1 ## Fixed - Remove debug `print(locals())` mistakenly left in diff --git a/README.md b/README.md index f185be3..a1c0201 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ jubeatools ${source} ${destination} -f ${output format} (... format specific opt | | | input | output | |-----------------|----------------------|:-----:|:------:| | memon | v0.2.0 | ✔️ | ✔️ | -| . | v0.1.0 | ✔️ | ✔️ | -| . | legacy | ✔️ | ✔️ | +| | v0.1.0 | ✔️ | ✔️ | +| | legacy | ✔️ | ✔️ | | jubeat analyser | #memo2 | ✔️ | ✔️ | -| . | #memo1 | ✔️ | ✔️ | -| . | #memo | ✔️ | ✔️ | -| . | mono-column (1列形式) | ✔️ | ✔️ | +| | #memo1 | ✔️ | ✔️ | +| | #memo | ✔️ | ✔️ | +| | mono-column (1列形式) | ✔️ | ✔️ | | jubeat (arcade) | .eve | ✔️ | ✔️ | diff --git a/jubeatools/formats/enum.py b/jubeatools/formats/enum.py index a0052a9..c4d30ae 100644 --- a/jubeatools/formats/enum.py +++ b/jubeatools/formats/enum.py @@ -3,6 +3,7 @@ from enum import Enum class Format(str, Enum): EVE = "eve" + JBSQ = "jbsq" MEMON_LEGACY = "memon:legacy" MEMON_0_1_0 = "memon:v0.1.0" MEMON_0_2_0 = "memon:v0.2.0" diff --git a/jubeatools/formats/eve/load.py b/jubeatools/formats/eve/load.py deleted file mode 100644 index 13ba2cf..0000000 --- a/jubeatools/formats/eve/load.py +++ /dev/null @@ -1,139 +0,0 @@ -from decimal import Decimal -from functools import reduce -from pathlib import Path -from typing import Any, Iterator, List, Optional - -from jubeatools import song -from jubeatools.formats.load_tools import make_folder_loader, round_beats -from jubeatools.utils import group_by - -from .commons import ( - VALUE_TO_DIRECTION, - AnyNote, - Command, - EveLong, - Event, - ticks_to_seconds, - value_to_truncated_bpm, -) -from .timemap import BPMAtSecond, TimeMap - - -def load_eve(path: Path, *, beat_snap: int = 240, **kwargs: Any) -> song.Song: - files = load_folder(path) - charts = [_load_eve(l, p, beat_snap=beat_snap) for p, l in files.items()] - return reduce(song.Song.merge, charts) - - -def load_file(path: Path) -> List[str]: - return path.read_text(encoding="ascii").split("\n") - - -load_folder = make_folder_loader("*.eve", load_file) - - -def _load_eve(lines: List[str], file_path: Path, *, beat_snap: int = 240) -> song.Song: - events = list(iter_events(lines)) - events_by_command = group_by(events, lambda e: e.command) - bpms = [ - BPMAtSecond( - seconds=ticks_to_seconds(e.time), BPM=value_to_truncated_bpm(e.value) - ) - for e in sorted(events_by_command[Command.TEMPO]) - ] - time_map = TimeMap.from_seconds(bpms) - tap_notes: List[AnyNote] = [ - make_tap_note(e.time, e.value, time_map, beat_snap) - for e in events_by_command[Command.PLAY] - ] - long_notes: List[AnyNote] = [ - make_long_note(e.time, e.value, time_map, beat_snap) - for e in events_by_command[Command.LONG] - ] - all_notes = sorted(tap_notes + long_notes, key=lambda n: (n.time, n.position)) - timing = time_map.convert_to_timing_info(beat_snap=beat_snap) - chart = song.Chart(level=Decimal(0), timing=timing, notes=all_notes) - dif = guess_difficulty(file_path.stem) or song.Difficulty.EXTREME - return song.Song(metadata=song.Metadata(), charts={dif: chart}) - - -def iter_events(lines: List[str]) -> Iterator[Event]: - for i, raw_line in enumerate(lines, start=1): - line = raw_line.strip() - if not line: - continue - - try: - yield parse_event(line) - except ValueError as e: - raise ValueError(f"Error on line {i} : {e}") - - -def parse_event(line: str) -> Event: - columns = line.split(",") - if len(columns) != 3: - raise ValueError(f"Expected 3 comma-separated values but found {len(columns)}") - - raw_tick, raw_command, raw_value = map(str.strip, columns) - try: - tick = int(raw_tick) - except ValueError: - raise ValueError( - f"The first column should contain an integer but {raw_tick!r} was " - f"found, which python could not understand as an integer" - ) - - try: - command = Command[raw_command] - except KeyError: - raise ValueError( - f"The second column should contain one of " - f"{list(Command.__members__)}, but {raw_command!r} was found" - ) - - try: - value = int(raw_value) - except ValueError: - raise ValueError( - f"The third column should contain an integer but {raw_tick!r} was " - f"found, which python could not understand as an integer" - ) - - return Event(tick, command, value) - - -def make_tap_note( - ticks: int, value: int, time_map: TimeMap, beat_snap: int -) -> song.TapNote: - seconds = ticks_to_seconds(ticks) - raw_beats = time_map.beats_at(seconds) - beats = round_beats(raw_beats, beat_snap) - position = song.NotePosition.from_index(value) - return song.TapNote(time=beats, position=position) - - -def make_long_note( - ticks: int, value: int, time_map: TimeMap, beat_snap: int -) -> song.LongNote: - seconds = ticks_to_seconds(ticks) - raw_beats = time_map.beats_at(seconds) - beats = round_beats(raw_beats, beat_snap) - eve_long = EveLong.from_value(value) - seconds_duration = ticks_to_seconds(eve_long.duration) - raw_beats_duration = time_map.beats_at(seconds + seconds_duration) - raw_beats - beats_duration = round_beats(raw_beats_duration, beat_snap) - position = song.NotePosition.from_index(eve_long.position) - direction = VALUE_TO_DIRECTION[eve_long.direction] - step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction] - raw_tail_pos = position + (eve_long.length * step_vector) - tail_pos = song.NotePosition.from_raw_position(raw_tail_pos) - return song.LongNote( - time=beats, position=position, duration=beats_duration, tail_tip=tail_pos - ) - - -def guess_difficulty(filename: str) -> Optional[song.Difficulty]: - try: - return song.Difficulty(filename.upper()) - except ValueError: - return None diff --git a/jubeatools/formats/formats.py b/jubeatools/formats/formats.py index d6197fb..970bf25 100644 --- a/jubeatools/formats/formats.py +++ b/jubeatools/formats/formats.py @@ -1,7 +1,6 @@ from typing import Dict from .enum import Format -from .eve import dump_eve, load_eve from .jubeat_analyser import ( dump_memo, dump_memo1, @@ -12,6 +11,7 @@ from .jubeat_analyser import ( load_memo2, load_mono_column, ) +from .konami import dump_eve, dump_jbsq, load_eve, load_jbsq from .memon import ( dump_memon_0_1_0, dump_memon_0_2_0, @@ -24,6 +24,7 @@ from .typing import Dumper, Loader LOADERS: Dict[Format, Loader] = { Format.EVE: load_eve, + Format.JBSQ: load_jbsq, Format.MEMON_LEGACY: load_memon_legacy, Format.MEMON_0_1_0: load_memon_0_1_0, Format.MEMON_0_2_0: load_memon_0_2_0, @@ -35,6 +36,7 @@ LOADERS: Dict[Format, Loader] = { DUMPERS: Dict[Format, Dumper] = { Format.EVE: dump_eve, + Format.JBSQ: dump_jbsq, Format.MEMON_LEGACY: dump_memon_legacy, Format.MEMON_0_1_0: dump_memon_0_1_0, Format.MEMON_0_2_0: dump_memon_0_2_0, diff --git a/jubeatools/formats/guess.py b/jubeatools/formats/guess.py index fb06661..7c04dc1 100644 --- a/jubeatools/formats/guess.py +++ b/jubeatools/formats/guess.py @@ -22,6 +22,9 @@ def guess_format(path: Path) -> Format: if looks_like_eve(path): return Format.EVE + if looks_like_jbsq(path): + return Format.JBSQ + raise ValueError("Unrecognized file format") @@ -94,7 +97,11 @@ def recognize_jubeat_analyser_format(path: Path) -> Format: def looks_like_eve(path: Path) -> bool: with path.open(encoding="ascii") as f: - line = f.readline() + try: + line = f.readline() + except UnicodeDecodeError: + return False + if line.strip(): return looks_like_eve_line(next(f)) @@ -131,3 +138,8 @@ def looks_like_eve_line(line: str) -> bool: return False return True + + +def looks_like_jbsq(path: Path) -> bool: + magic = path.open(mode="rb").read(4) + return magic in (b"IJBQ", b"IJSQ", b"JBSQ") diff --git a/jubeatools/formats/konami/__init__.py b/jubeatools/formats/konami/__init__.py new file mode 100644 index 0000000..d502719 --- /dev/null +++ b/jubeatools/formats/konami/__init__.py @@ -0,0 +1,2 @@ +from .eve import dump_eve, load_eve +from .jbsq import dump_jbsq, load_jbsq diff --git a/jubeatools/formats/eve/commons.py b/jubeatools/formats/konami/commons.py similarity index 98% rename from jubeatools/formats/eve/commons.py rename to jubeatools/formats/konami/commons.py index de4268b..6818d6a 100644 --- a/jubeatools/formats/eve/commons.py +++ b/jubeatools/formats/konami/commons.py @@ -13,6 +13,7 @@ from .timemap import TimeMap AnyNote = Union[song.TapNote, song.LongNote] + DIRECTION_TO_VALUE = { song.Direction.DOWN: 0, song.Direction.UP: 1, @@ -34,7 +35,7 @@ class Command(int, Enum): @dataclass(order=True) class Event: - """Represents a line in an .eve file""" + """Represents a line in an .eve file or an event struct in a .jbsq file""" time: int command: Command diff --git a/jubeatools/formats/eve/dump.py b/jubeatools/formats/konami/dump_tools.py similarity index 82% rename from jubeatools/formats/eve/dump.py rename to jubeatools/formats/konami/dump_tools.py index fff48a3..c94cad9 100644 --- a/jubeatools/formats/eve/dump.py +++ b/jubeatools/formats/konami/dump_tools.py @@ -1,42 +1,21 @@ -from __future__ import annotations - import math from fractions import Fraction from functools import singledispatch -from pathlib import Path from typing import List from more_itertools import numeric_range from jubeatools import song -from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper -from jubeatools.formats.filetypes import ChartFile from .commons import AnyNote, Command, Event, bpm_to_value, ticks_at_beat from .timemap import TimeMap -def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]: - res = [] - for dif, chart, timing in song.iter_charts_with_timing(): - chart_text = dump_chart(chart.notes, timing) - chart_bytes = chart_text.encode("ascii") - res.append(ChartFile(chart_bytes, song, dif, chart)) - - return res - - -dump_eve = make_dumper_from_chart_file_dumper( - internal_dumper=_dump_eve, file_name_template=Path("{difficulty:l}.eve") -) - - -def dump_chart(notes: List[AnyNote], timing: song.Timing) -> str: +def make_events_from_chart(notes: List[AnyNote], timing: song.Timing) -> List[Event]: time_map = TimeMap.from_timing(timing) note_events = make_note_events(notes, time_map) timing_events = make_timing_events(notes, timing, time_map) - sorted_events = sorted(note_events + timing_events) - return "\n".join(e.dump() for e in sorted_events) + return sorted(note_events + timing_events) def make_note_events(notes: List[AnyNote], time_map: TimeMap) -> List[Event]: diff --git a/jubeatools/formats/eve/__init__.py b/jubeatools/formats/konami/eve/__init__.py similarity index 100% rename from jubeatools/formats/eve/__init__.py rename to jubeatools/formats/konami/eve/__init__.py diff --git a/jubeatools/formats/konami/eve/dump.py b/jubeatools/formats/konami/eve/dump.py new file mode 100644 index 0000000..c2a99c7 --- /dev/null +++ b/jubeatools/formats/konami/eve/dump.py @@ -0,0 +1,24 @@ +from pathlib import Path +from typing import List + +from jubeatools import song +from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper +from jubeatools.formats.filetypes import ChartFile + +from ..dump_tools import make_events_from_chart + + +def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]: + res = [] + for dif, chart, timing in song.iter_charts_with_timing(): + events = make_events_from_chart(chart.notes, timing) + chart_text = "\n".join(e.dump() for e in events) + chart_bytes = chart_text.encode("ascii") + res.append(ChartFile(chart_bytes, song, dif, chart)) + + return res + + +dump_eve = make_dumper_from_chart_file_dumper( + internal_dumper=_dump_eve, file_name_template=Path("{difficulty:l}.eve") +) diff --git a/jubeatools/formats/konami/eve/load.py b/jubeatools/formats/konami/eve/load.py new file mode 100644 index 0000000..536588a --- /dev/null +++ b/jubeatools/formats/konami/eve/load.py @@ -0,0 +1,80 @@ +from functools import reduce +from pathlib import Path +from typing import Any, Iterator, List, Optional + +from jubeatools import song +from jubeatools.formats.load_tools import make_folder_loader + +from ..commons import Command, Event +from ..load_tools import make_chart_from_events + + +def load_eve(path: Path, *, beat_snap: int = 240, **kwargs: Any) -> song.Song: + files = load_folder(path) + charts = [_load_eve(l, p, beat_snap=beat_snap) for p, l in files.items()] + return reduce(song.Song.merge, charts) + + +def load_file(path: Path) -> List[str]: + return path.read_text(encoding="ascii").split("\n") + + +load_folder = make_folder_loader("*.eve", load_file) + + +def _load_eve(lines: List[str], file_path: Path, *, beat_snap: int = 240) -> song.Song: + chart = make_chart_from_events(iter_events(lines), beat_snap=beat_snap) + dif = guess_difficulty(file_path.stem) or song.Difficulty.EXTREME + return song.Song(metadata=song.Metadata(), charts={dif: chart}) + + +def iter_events(lines: List[str]) -> Iterator[Event]: + for i, raw_line in enumerate(lines, start=1): + line = raw_line.strip() + if not line: + continue + + try: + yield parse_event(line) + except ValueError as e: + raise ValueError(f"Error on line {i} : {e}") + + +def parse_event(line: str) -> Event: + columns = line.split(",") + if len(columns) != 3: + raise ValueError(f"Expected 3 comma-separated values but found {len(columns)}") + + raw_tick, raw_command, raw_value = map(str.strip, columns) + try: + tick = int(raw_tick) + except ValueError: + raise ValueError( + f"The first column should contain an integer but {raw_tick!r} was " + f"found, which python could not understand as an integer" + ) + + try: + command = Command[raw_command] + except KeyError: + raise ValueError( + f"The second column should contain one of " + f"{list(Command.__members__)}, but {raw_command!r} was found" + ) + + try: + value = int(raw_value) + except ValueError: + raise ValueError( + f"The third column should contain an integer but {raw_tick!r} was " + f"found, which python could not understand as an integer" + ) + + return Event(tick, command, value) + + +def guess_difficulty(filename: str) -> Optional[song.Difficulty]: + try: + return song.Difficulty(filename.upper()) + except ValueError: + return None diff --git a/jubeatools/formats/eve/tests/__init__.py b/jubeatools/formats/konami/eve/tests/__init__.py similarity index 100% rename from jubeatools/formats/eve/tests/__init__.py rename to jubeatools/formats/konami/eve/tests/__init__.py diff --git a/jubeatools/formats/eve/tests/test_bpm_value.py b/jubeatools/formats/konami/eve/tests/test_bpm_value.py similarity index 83% rename from jubeatools/formats/eve/tests/test_bpm_value.py rename to jubeatools/formats/konami/eve/tests/test_bpm_value.py index 977f803..9cae00d 100644 --- a/jubeatools/formats/eve/tests/test_bpm_value.py +++ b/jubeatools/formats/konami/eve/tests/test_bpm_value.py @@ -3,7 +3,7 @@ import math from hypothesis import given from hypothesis import strategies as st -from ..commons import bpm_to_value, value_to_truncated_bpm +from jubeatools.formats.konami.commons import bpm_to_value, value_to_truncated_bpm @given(st.integers(min_value=1, max_value=6 * 10 ** 7)) diff --git a/jubeatools/formats/konami/eve/tests/test_eve.py b/jubeatools/formats/konami/eve/tests/test_eve.py new file mode 100644 index 0000000..d9e82c2 --- /dev/null +++ b/jubeatools/formats/konami/eve/tests/test_eve.py @@ -0,0 +1,17 @@ +from hypothesis import given + +from jubeatools import song +from jubeatools.formats import Format +from jubeatools.formats.konami.testutils import eve_compatible_song, open_temp_dir +from jubeatools.testutils.test_patterns import dump_and_load_then_compare + + +@given(eve_compatible_song()) +def test_that_full_chart_roundtrips(song: song.Song) -> None: + dump_and_load_then_compare( + Format.EVE, + song, + temp_path=open_temp_dir(), + bytes_decoder=lambda b: b.decode("ascii"), + load_options={"beat_snap": 24}, + ) diff --git a/jubeatools/formats/eve/tests/test_events.py b/jubeatools/formats/konami/eve/tests/test_events.py similarity index 86% rename from jubeatools/formats/eve/tests/test_events.py rename to jubeatools/formats/konami/eve/tests/test_events.py index 39d47f1..4e518a1 100644 --- a/jubeatools/formats/eve/tests/test_events.py +++ b/jubeatools/formats/konami/eve/tests/test_events.py @@ -2,11 +2,10 @@ from hypothesis import given from hypothesis import strategies as st from jubeatools import song +from jubeatools.formats.konami.commons import EveLong +from jubeatools.formats.konami.timemap import TimeMap from jubeatools.testutils import strategies as jbst -from ..commons import EveLong -from ..timemap import TimeMap - @given( jbst.long_note(), diff --git a/jubeatools/formats/eve/tests/test_timemap.py b/jubeatools/formats/konami/eve/tests/test_timemap.py similarity index 97% rename from jubeatools/formats/eve/tests/test_timemap.py rename to jubeatools/formats/konami/eve/tests/test_timemap.py index 195901f..ddf6f84 100644 --- a/jubeatools/formats/eve/tests/test_timemap.py +++ b/jubeatools/formats/konami/eve/tests/test_timemap.py @@ -3,7 +3,7 @@ from fractions import Fraction from hypothesis import given from jubeatools import song -from jubeatools.formats.eve.timemap import TimeMap +from jubeatools.formats.konami.timemap import TimeMap from jubeatools.testutils import strategies as jbst from jubeatools.utils import group_by diff --git a/jubeatools/formats/konami/jbsq/__init__.py b/jubeatools/formats/konami/jbsq/__init__.py new file mode 100644 index 0000000..7b80af5 --- /dev/null +++ b/jubeatools/formats/konami/jbsq/__init__.py @@ -0,0 +1,8 @@ +""".jbsq is the file format used in the iOS and Android app Jubeat plus + +It's a binary version of the .eve format, with the same limitations. + +I wanted to try kaitai for this but it doesn't support serializing right now""" + +from .dump import dump_jbsq +from .load import load_jbsq diff --git a/jubeatools/formats/konami/jbsq/construct.py b/jubeatools/formats/konami/jbsq/construct.py new file mode 100644 index 0000000..8899f19 --- /dev/null +++ b/jubeatools/formats/konami/jbsq/construct.py @@ -0,0 +1,43 @@ +"""The JBSQ format described using construct. +see https://construct.readthedocs.io/en/latest/index.html""" + +from dataclasses import dataclass +from typing import List, Optional + +import construct as c +import construct_typed as ct + + +class EventType(ct.EnumBase): + PLAY = 1 + END = 2 + MEASURE = 3 + HAKU = 4 + TEMPO = 5 + LONG = 6 + + +@dataclass +class Event(ct.TContainerMixin): + type_: EventType = ct.sfield(ct.TEnum(c.Byte, EventType)) + time_in_ticks: int = ct.sfield(c.Int24ul) + value: int = ct.sfield(c.Int32ul) + + +@dataclass +class JBSQ(ct.TContainerMixin): + magic: Optional[bytes] = ct.sfield( + c.Select(c.Const(b"IJBQ"), c.Const(b"IJSQ"), c.Const(b"JBSQ")) + ) + num_events: int = ct.sfield(c.Int32ul) + combo: int = ct.sfield(c.Int32ul) + end_time: int = ct.sfield(c.Int32ul) + _1: None = ct.sfield(c.Padding(2)) + starting_buttons: int = ct.sfield(c.Int16ul) + start_time: int = ct.sfield(c.Int32ul) + _2: None = ct.sfield(c.Padding(12)) + density_graph: List[int] = ct.sfield(c.Byte[60]) + events: List[Event] = ct.sfield(c.Array(c.this.num_events, ct.TStruct(Event))) + + +jbsq = ct.TStruct(JBSQ) diff --git a/jubeatools/formats/konami/jbsq/dump.py b/jubeatools/formats/konami/jbsq/dump.py new file mode 100644 index 0000000..4638490 --- /dev/null +++ b/jubeatools/formats/konami/jbsq/dump.py @@ -0,0 +1,102 @@ +from collections import defaultdict +from pathlib import Path +from typing import DefaultDict, List + +from jubeatools import song +from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper +from jubeatools.formats.filetypes import ChartFile +from jubeatools.utils import group_by + +from .. import commons as konami +from ..commons import AnyNote +from ..dump_tools import make_events_from_chart +from . import construct + + +def _dump_jbsq(song: song.Song, **kwargs: dict) -> List[ChartFile]: + res = [] + for dif, chart, timing in song.iter_charts_with_timing(): + events = make_events_from_chart(chart.notes, timing) + jbsq_chart = make_jbsq_chart(events, chart.notes) + chart_bytes = construct.jbsq.build(jbsq_chart) + res.append(ChartFile(chart_bytes, song, dif, chart)) + + return res + + +dump_jbsq = make_dumper_from_chart_file_dumper( + internal_dumper=_dump_jbsq, file_name_template=Path("seq_{difficulty:l}.jbsq") +) + + +def make_jbsq_chart(events: List[konami.Event], notes: List[AnyNote]) -> construct.JBSQ: + jbsq_events = [convert_event_to_jbsq(e) for e in events] + num_events = len(events) + combo = compute_max_combo(notes) + end_time = next(e for e in events if e.command == konami.Command.END).time + first_note_time_in_beats = min((n.time for n in notes), default=0) + starting_notes = [n for n in notes if n.time == first_note_time_in_beats] + starting_buttons = sum(1 << n.position.index for n in starting_notes) + first_note_time = min( + ( + e.time + for e in events + if e.command in (konami.Command.PLAY, konami.Command.LONG) + ), + default=0, + ) + densities = compute_density_graph(events, end_time) + jbsq_chart = construct.JBSQ( + num_events=num_events, + combo=combo, + end_time=end_time, + starting_buttons=starting_buttons, + start_time=first_note_time, + density_graph=densities, + events=jbsq_events, + ) + jbsq_chart.magic = b"JBSQ" + return jbsq_chart + + +def convert_event_to_jbsq(event: konami.Event) -> construct.Event: + return construct.Event( + type_=construct.EventType[event.command.name], + time_in_ticks=event.time, + value=event.value, + ) + + +def compute_max_combo(notes: List[AnyNote]) -> int: + notes_by_type = group_by(notes, type) + tap_notes = len(notes_by_type[song.TapNote]) + long_notes = len(notes_by_type[song.LongNote]) + return tap_notes + 2 * long_notes + + +def compute_density_graph(events: List[konami.Event], end_time: int) -> List[int]: + events_by_type = group_by(events, lambda e: e.command) + buckets: DefaultDict[int, int] = defaultdict(int) + for tap in events_by_type[konami.Command.PLAY]: + bucket = int((tap.time / end_time) * 120) + buckets[bucket] += 1 + + for long in events_by_type[konami.Command.LONG]: + press_bucket = int((long.time / end_time) * 120) + buckets[press_bucket] += 1 + duration = konami.EveLong.from_value(long.value).duration + release_time = long.time + duration + release_bucket = int((release_time / end_time) * 120) + buckets[release_bucket] += 1 + + res = [] + for i in range(0, 120, 2): + # The jbsq density graph in a array of nibbles, the twist is that for + # some obscure reason each pair of nibbles is swapped in the byte ... + # little-endianness is a hell of a drug, don't do drugs kids ... + first_nibble = min(buckets[i], 15) + second_nibble = min(buckets[i + 1], 15) + density_byte = (second_nibble << 4) + first_nibble + res.append(density_byte) + + return res diff --git a/jubeatools/formats/konami/jbsq/jbsq.ksy b/jubeatools/formats/konami/jbsq/jbsq.ksy new file mode 100644 index 0000000..c4cf1f1 --- /dev/null +++ b/jubeatools/formats/konami/jbsq/jbsq.ksy @@ -0,0 +1,105 @@ +meta: + id: jbsq + endian: le + bit-endian: le +seq: + - id: magic + type: u4be + enum: magic_bytes + - id: num_events + type: u4 + - id: combo + type: u4 + - id: end_time + type: u4 + - id: reserved1 + size: 2 + - id: starting_buttons + type: u2 + - id: first_sector + type: u4 + - id: reserved2 + size: 12 + - id: density_graph + type: b4 + repeat: expr + repeat-expr: 120 + - id: events + type: event + repeat: expr + repeat-expr: num_events +enums: + magic_bytes: + 0x494a4251: ijbq + 0x494a5351: ijsq + 0x4a425351: jbsq + command: + 1: play + 2: end + 3: measure + 4: haku + 5: tempo + 6: long + direction: + 0: down + 1: up + 2: right + 3: left +types: + event: + seq: + - id: type + type: b8 + enum: command + - id: time_in_ticks + type: b24 + - id: value + type: + switch-on: type + cases: + 'command::play': play_value + 'command::end': zero_value + 'command::measure': zero_value + 'command::haku': zero_value + 'command::tempo': tempo_value + 'command::long': long_value + instances: + time_in_seconds: + value: time_in_ticks / 300.0 + zero_value: + seq: + - id: value + contents: [0, 0, 0, 0] + play_value: + seq: + - id: note_position + type: position + - id: reserved + type: b28 + tempo_value: + seq: + - id: tempo + type: u4 + instances: + bpm: + value: 60000000.0 / tempo + long_value: + seq: + - id: note_position + type: position + - id: tail_direction + type: b2 + enum: direction + - id: tail_length + type: b2 + - id: duration_in_ticks + type: b24 + instances: + duration_in_seconds: + value: duration_in_ticks / 300.0 + position: + seq: + - id: x + type: b2 + - id: y + type: b2 \ No newline at end of file diff --git a/jubeatools/formats/konami/jbsq/load.py b/jubeatools/formats/konami/jbsq/load.py new file mode 100644 index 0000000..8617055 --- /dev/null +++ b/jubeatools/formats/konami/jbsq/load.py @@ -0,0 +1,51 @@ +from functools import reduce +from pathlib import Path +from typing import Any, Optional + +from jubeatools import song +from jubeatools.formats.load_tools import make_folder_loader + +from .. import commons as konami +from ..load_tools import make_chart_from_events +from . import construct + + +def load_jbsq(path: Path, *, beat_snap: int = 240, **kwargs: Any) -> song.Song: + files = load_folder(path) + charts = [ + load_jbsq_file(bytes_, path, beat_snap=beat_snap) + for path, bytes_ in files.items() + ] + return reduce(song.Song.merge, charts) + + +def load_file(path: Path) -> bytes: + return path.read_bytes() + + +load_folder = make_folder_loader("*.jbsq", load_file) + + +def load_jbsq_file( + bytes_: bytes, file_path: Path, *, beat_snap: int = 240 +) -> song.Song: + raw_data = construct.jbsq.parse(bytes_) + events = [make_event_from_construct(e) for e in raw_data.events] + chart = make_chart_from_events(events, beat_snap=beat_snap) + dif = guess_difficulty(file_path.stem) or song.Difficulty.EXTREME + return song.Song(metadata=song.Metadata(), charts={dif: chart}) + + +def make_event_from_construct(e: construct.Event) -> konami.Event: + return konami.Event( + time=e.time_in_ticks, + command=konami.Command[e.type_.name], + value=e.value, + ) + + +def guess_difficulty(filename: str) -> Optional[song.Difficulty]: + try: + return song.Difficulty(filename[-3:].upper()) + except ValueError: + return None diff --git a/jubeatools/formats/konami/jbsq/tests/__init__.py b/jubeatools/formats/konami/jbsq/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jubeatools/formats/konami/jbsq/tests/test_jbsq.py b/jubeatools/formats/konami/jbsq/tests/test_jbsq.py new file mode 100644 index 0000000..c44b00e --- /dev/null +++ b/jubeatools/formats/konami/jbsq/tests/test_jbsq.py @@ -0,0 +1,19 @@ +from hypothesis import given + +from jubeatools import song +from jubeatools.formats import Format +from jubeatools.formats.konami.testutils import eve_compatible_song, open_temp_dir +from jubeatools.testutils.test_patterns import dump_and_load_then_compare + +from ..construct import jbsq + + +@given(eve_compatible_song()) +def test_that_full_chart_roundtrips(song: song.Song) -> None: + dump_and_load_then_compare( + Format.JBSQ, + song, + temp_path=open_temp_dir(), + bytes_decoder=lambda b: str(jbsq.parse(b)), + load_options={"beat_snap": 24}, + ) diff --git a/jubeatools/formats/konami/load_tools.py b/jubeatools/formats/konami/load_tools.py new file mode 100644 index 0000000..4c8784b --- /dev/null +++ b/jubeatools/formats/konami/load_tools.py @@ -0,0 +1,69 @@ +from decimal import Decimal +from typing import Iterable, List + +from jubeatools import song +from jubeatools.formats.load_tools import round_beats +from jubeatools.utils import group_by + +from .commons import ( + VALUE_TO_DIRECTION, + AnyNote, + Command, + EveLong, + Event, + ticks_to_seconds, + value_to_truncated_bpm, +) +from .timemap import BPMAtSecond, TimeMap + + +def make_chart_from_events(events: Iterable[Event], beat_snap: int = 240) -> song.Chart: + events_by_command = group_by(events, lambda e: e.command) + bpms = [ + BPMAtSecond( + seconds=ticks_to_seconds(e.time), BPM=value_to_truncated_bpm(e.value) + ) + for e in sorted(events_by_command[Command.TEMPO]) + ] + time_map = TimeMap.from_seconds(bpms) + tap_notes: List[AnyNote] = [ + make_tap_note(e.time, e.value, time_map, beat_snap) + for e in events_by_command[Command.PLAY] + ] + long_notes: List[AnyNote] = [ + make_long_note(e.time, e.value, time_map, beat_snap) + for e in events_by_command[Command.LONG] + ] + all_notes = sorted(tap_notes + long_notes, key=lambda n: (n.time, n.position)) + timing = time_map.convert_to_timing_info(beat_snap=beat_snap) + return song.Chart(level=Decimal(0), timing=timing, notes=all_notes) + + +def make_tap_note( + ticks: int, value: int, time_map: TimeMap, beat_snap: int +) -> song.TapNote: + seconds = ticks_to_seconds(ticks) + raw_beats = time_map.beats_at(seconds) + beats = round_beats(raw_beats, beat_snap) + position = song.NotePosition.from_index(value) + return song.TapNote(time=beats, position=position) + + +def make_long_note( + ticks: int, value: int, time_map: TimeMap, beat_snap: int +) -> song.LongNote: + seconds = ticks_to_seconds(ticks) + raw_beats = time_map.beats_at(seconds) + beats = round_beats(raw_beats, beat_snap) + eve_long = EveLong.from_value(value) + seconds_duration = ticks_to_seconds(eve_long.duration) + raw_beats_duration = time_map.beats_at(seconds + seconds_duration) - raw_beats + beats_duration = round_beats(raw_beats_duration, beat_snap) + position = song.NotePosition.from_index(eve_long.position) + direction = VALUE_TO_DIRECTION[eve_long.direction] + step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction] + raw_tail_pos = position + (eve_long.length * step_vector) + tail_pos = song.NotePosition.from_raw_position(raw_tail_pos) + return song.LongNote( + time=beats, position=position, duration=beats_duration, tail_tip=tail_pos + ) diff --git a/jubeatools/formats/eve/tests/test_eve.py b/jubeatools/formats/konami/testutils.py similarity index 82% rename from jubeatools/formats/eve/tests/test_eve.py rename to jubeatools/formats/konami/testutils.py index 61f63d1..d26f447 100644 --- a/jubeatools/formats/eve/tests/test_eve.py +++ b/jubeatools/formats/konami/testutils.py @@ -4,13 +4,10 @@ from decimal import Decimal from pathlib import Path from typing import Iterator -from hypothesis import given from hypothesis import strategies as st from jubeatools import song -from jubeatools.formats import Format from jubeatools.testutils import strategies as jbst -from jubeatools.testutils.test_patterns import dump_and_load_then_compare from jubeatools.testutils.typing import DrawFunc simple_beat_strat = jbst.beat_time( @@ -62,14 +59,3 @@ def eve_compatible_song(draw: DrawFunc) -> song.Song: def open_temp_dir() -> Iterator[Path]: with tempfile.TemporaryDirectory() as temp_dir: yield Path(temp_dir) - - -@given(eve_compatible_song()) -def test_that_full_chart_roundtrips(song: song.Song) -> None: - dump_and_load_then_compare( - Format.EVE, - song, - temp_path=open_temp_dir(), - bytes_decoder=lambda b: b.decode("ascii"), - load_options={"beat_snap": 24}, - ) diff --git a/jubeatools/formats/eve/timemap.py b/jubeatools/formats/konami/timemap.py similarity index 100% rename from jubeatools/formats/eve/timemap.py rename to jubeatools/formats/konami/timemap.py diff --git a/jubeatools/utils.py b/jubeatools/utils.py index c0cb690..cd32877 100644 --- a/jubeatools/utils.py +++ b/jubeatools/utils.py @@ -6,7 +6,7 @@ from decimal import Decimal from fractions import Fraction from functools import reduce from math import gcd -from typing import Callable, Dict, Hashable, List, Optional, TypeVar +from typing import Callable, Dict, Hashable, Iterable, List, Optional, TypeVar def single_lcm(a: int, b: int) -> int: @@ -44,7 +44,7 @@ K = TypeVar("K", bound=Hashable) V = TypeVar("V") -def group_by(elements: List[V], key: Callable[[V], K]) -> Dict[K, List[V]]: +def group_by(elements: Iterable[V], key: Callable[[V], K]) -> Dict[K, List[V]]: res = defaultdict(list) for e in elements: res[key(e)].append(e) diff --git a/poetry.lock b/poetry.lock index a10c0c0..3741f4e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -90,6 +90,28 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "construct" +version = "2.10.66" +description = "A powerful declarative symmetric parser/builder for binary data" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +extras = ["arrow", "cloudpickle", "enum34", "lz4", "numpy", "ruamel.yaml"] + +[[package]] +name = "construct-typing" +version = "0.4.2" +description = "Extension for the python package 'construct' that adds typing features" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +construct = "2.10.66" + [[package]] name = "flake8" version = "3.9.1" @@ -419,7 +441,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "4065f5a647b6cfb73d1aeceaae1fee4f94ecccdaa807aa134d3c307c6ecfbbd9" +content-hash = "8831aec0d2e864364d6b1681b19f918072cb4acf2fc35fdac8934d107335ce0a" [metadata.files] appdirs = [ @@ -453,6 +475,13 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +construct = [ + {file = "construct-2.10.66.tar.gz", hash = "sha256:268ab261e8d6a34247038d25e6c7bad831c2b0fc9ef782308c232ecadc89e5d0"}, +] +construct-typing = [ + {file = "construct-typing-0.4.2.tar.gz", hash = "sha256:e19be4f4ad437e28a11d1675c52d3485fe37b4f49edbfe94e219059954004094"}, + {file = "construct_typing-0.4.2-py3-none-any.whl", hash = "sha256:5b6b7322005f4859cb6b6bcb5dec965de29d21168c599a9100429b19fa85aaee"}, +] flake8 = [ {file = "flake8-3.9.1-py2.py3-none-any.whl", hash = "sha256:3b9f848952dddccf635be78098ca75010f073bfe14d2c6bda867154bea728d2a"}, {file = "flake8-3.9.1.tar.gz", hash = "sha256:1aa8990be1e689d96c745c5682b687ea49f2e05a443aff1f8251092b0014e378"}, diff --git a/pyproject.toml b/pyproject.toml index a3ba8f9..8c7c999 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ parsimonious = "^0.8.1" more-itertools = "^8.4.0" sortedcontainers = "^2.3.0" python-constraint = "^1.4.0" +construct = "~=2.10" [tool.poetry.dev-dependencies] pytest = "^6.2.3" @@ -28,6 +29,7 @@ isort = "^4.3.21" toml = "^0.10.2" flake8 = "^3.9.1" autoimport = "^0.7.0" +construct-typing = "^0.4.2" [tool.poetry.scripts] jubeatools = 'jubeatools.cli.cli:convert'