From 8322747a15faa5825827ab944ff61ae6dffb2207 Mon Sep 17 00:00:00 2001 From: Stepland <16676308+Stepland@users.noreply.github.com> Date: Tue, 11 May 2021 01:43:21 +0200 Subject: [PATCH] [eve] full support ! --- jubeatools/formats/eve/__init__.py | 13 +- jubeatools/formats/eve/commons.py | 76 +++++++---- jubeatools/formats/eve/dump.py | 21 +-- jubeatools/formats/eve/load.py | 125 +++++++++++++++--- jubeatools/formats/eve/tests/__init__.py | 0 .../formats/eve/tests/test_bpm_value.py | 15 +++ jubeatools/formats/eve/tests/test_eve.py | 50 +++++++ jubeatools/formats/eve/tests/test_timemap.py | 2 +- jubeatools/formats/eve/timemap.py | 14 +- jubeatools/song.py | 11 +- 10 files changed, 269 insertions(+), 58 deletions(-) create mode 100644 jubeatools/formats/eve/tests/__init__.py create mode 100644 jubeatools/formats/eve/tests/test_bpm_value.py create mode 100644 jubeatools/formats/eve/tests/test_eve.py diff --git a/jubeatools/formats/eve/__init__.py b/jubeatools/formats/eve/__init__.py index 570dcb1..5d5f6fc 100644 --- a/jubeatools/formats/eve/__init__.py +++ b/jubeatools/formats/eve/__init__.py @@ -1,5 +1,14 @@ -""" -.eve is the file format used in arcade releases of jubeat +""".eve is the file format used in arcade releases of jubeat + +.eve files are CSVs with three columns : time, command, value + +A small but annoying amount of precision is lost when using this format : +- time is stored already "rendered" as a whole number of ticks on a 300Hz clock + instead of using symbolic time +- while some symbolic time information remains in the form of TEMPO, MEASURE + and HAKU commands (respectively a BPM change, a measure marker and a beat + marker), BPMs are stored as `int((6*10^7)/BPM)` which makes it hard to + recover many significant digits """ from .dump import dump_eve diff --git a/jubeatools/formats/eve/commons.py b/jubeatools/formats/eve/commons.py index 407069a..a0bb09b 100644 --- a/jubeatools/formats/eve/commons.py +++ b/jubeatools/formats/eve/commons.py @@ -1,10 +1,11 @@ from __future__ import annotations + +from dataclasses import astuple, dataclass from enum import Enum -from dataclasses import dataclass, astuple -from jubeatools import song -from typing import Union -from functools import singledispatchmethod from fractions import Fraction +from typing import Union + +from jubeatools import song from .timemap import TimeMap @@ -19,7 +20,8 @@ DIRECTION_TO_VALUE = { VALUE_TO_DIRECTION = {v: k for k, v in DIRECTION_TO_VALUE.items()} -class Command(Enum): +# int is here to allow sorting +class Command(int, Enum): END = 1 MEASURE = 2 HAKU = 3 @@ -38,17 +40,22 @@ class Event: def __post_init__(self) -> None: try: - check_func = VALUES_CHECKERS[self.command](self.value) + check_func = VALUES_CHECKERS[self.command] except KeyError: # most likely no check function associated : forget about it pass + + try: + check_func(self.value) except ValueError as e: - raise ValueError(f"Invalid value for the {self.command!r} command. {e}") + raise ValueError( + f"{self.value} is not a valid value for the {self.command!r} " + f"command. {e}" + ) def dump(self) -> str: return f"{self.time:>8},{self.command.name:<8},{self.value:>8}" - @classmethod def from_tap_note(cls, note: song.TapNote, time_map: TimeMap) -> Event: ticks = ticks_at_beat(note.time, time_map) @@ -60,36 +67,36 @@ class Event: if not note.has_straight_tail(): raise ValueError("Diagonal tails cannot be represented in eve format") - duration = duration_in_ticks(note, time_map) - direction = DIRECTION_TO_VALUE[note.tail_direction()] - length = len(list(note.positions_covered())) - 1 - if not (1 <= length <= 3): - raise ValueError( - f"Given note has a length of {length}, which is not representable " - "in the eve format" - ) - position_index = note.position.index - long_note_value = duration << 8 + length << 6 + direction << 4 + position_index + eve_long = EveLong( + duration=duration_in_ticks(note, time_map), + length=len(list(note.positions_covered())) - 1, + direction=DIRECTION_TO_VALUE[note.tail_direction()], + position=note.position.index, + ) ticks = ticks_at_beat(note.time, time_map) - return Event(time=ticks, command=Command.LONG, value=long_note_value) + return Event(time=ticks, command=Command.LONG, value=eve_long.value) def is_zero(value: int) -> None: if value != 0: - raise ValueError(f"Value should be zero but {value} found") + raise ValueError(f"Value should be zero") + def is_valid_button_index(value: int) -> None: # Should raise ValueError if invalid _ = song.NotePosition.from_index(value) + def is_valid_tail_position(value: int) -> None: # Should raise ValueError if invalid _ = EveLong.from_value(value) + def is_not_zero(value: int) -> None: if value == 0: raise ValueError(f"Value cannot be zero") + VALUES_CHECKERS = { Command.END: is_zero, Command.MEASURE: is_zero, @@ -104,7 +111,7 @@ VALUES_CHECKERS = { class EveLong: duration: int length: int - direction: song.Direction + direction: int position: int def __post_init__(self) -> None: @@ -112,18 +119,37 @@ class EveLong: raise ValueError("Duration can't be negative") if not 1 <= self.length < 4: raise ValueError("Tail length must be between 1 and 3 inclusive") + if not 0 <= self.position < 16: + raise ValueError("Note Position must be between 0 and 15 inclusive") + if not 0 <= self.direction < 4: + raise ValueError("direction value must be between 0 and 3 inclusive") + pos = song.NotePosition.from_index(self.position) - step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[self.direction] + direction = VALUE_TO_DIRECTION[self.direction] + step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction] tail_pos = pos + (self.length * step_vector) if not ((0 <= tail_pos.x < 4) and (0 <= tail_pos.y < 4)): raise ValueError( f"Long note tail starts on {astuple(tail_pos)} which is " "outside the screen" ) - + @classmethod def from_value(cls, value: int) -> EveLong: - ... + if value < 0: + raise ValueError("Value cannot be negative") + + position = value & 0b1111 # first 4 bits + direction = (value >> 4) & 0b11 # next 2 bits + length = (value >> 6) & 0b11 # next 2 bits + duration = value >> 8 # remaining bits + return cls(duration, length, direction, position) + + @property + def value(self) -> int: + return ( + self.duration << 8 + self.length << 6 + self.direction << 4 + self.position + ) def ticks_at_beat(time: song.BeatsTime, time_map: TimeMap) -> int: @@ -145,4 +171,4 @@ def ticks_to_seconds(tick: int) -> Fraction: def seconds_to_ticks(time: Fraction) -> int: """Convert fractional seconds to eve ticks (300 Hz)""" - return round(time * 300) \ No newline at end of file + return round(time * 300) diff --git a/jubeatools/formats/eve/dump.py b/jubeatools/formats/eve/dump.py index 9c81227..e07397b 100644 --- a/jubeatools/formats/eve/dump.py +++ b/jubeatools/formats/eve/dump.py @@ -1,11 +1,10 @@ from __future__ import annotations -from dataclasses import dataclass -from enum import Enum +import math from fractions import Fraction from functools import singledispatch from pathlib import Path -from typing import List, Union +from typing import List from more_itertools import numeric_range @@ -13,8 +12,8 @@ from jubeatools import song from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper from jubeatools.formats.filetypes import ChartFile +from .commons import AnyNote, Command, Event, ticks_at_beat from .timemap import TimeMap -from .commons import AnyNote, Command, Event, DIRECTION_TO_VALUE def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]: @@ -28,7 +27,7 @@ def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]: dump_eve = make_dumper_from_chart_file_dumper( - internal_dumper=_dump_eve, file_name_template=Path("{difficulty_index}.eve") + internal_dumper=_dump_eve, file_name_template=Path("{difficulty:l}.eve") ) @@ -41,17 +40,19 @@ def dump_chart(notes: List[AnyNote], timing: song.Timing) -> str: def make_note_events(notes: List[AnyNote], time_map: TimeMap) -> List[Event]: - return [Event.from_note(note, time_map) for note in notes] + return [make_note_event(note, time_map) for note in notes] @singledispatch def make_note_event(note: AnyNote, time_map: TimeMap) -> Event: raise NotImplementedError(f"Unknown note type : {type(note)}") + @make_note_event.register def make_tap_note_event(note: song.TapNote, time_map: TimeMap) -> Event: return Event.from_tap_note(note, time_map) + @make_note_event.register def make_long_note_event(note: song.LongNote, time_map: TimeMap) -> Event: return Event.from_long_note(note, time_map) @@ -70,10 +71,14 @@ def make_timing_events( def make_bpm_event(bpm_change: song.BPMEvent, time_map: TimeMap) -> Event: ticks = ticks_at_beat(bpm_change.time, time_map) - bpm_value = round(60 * 10 ** 6 / Fraction(bpm_change.BPM)) + bpm_value = bpm_to_value(Fraction(bpm_change.BPM)) return Event(time=ticks, command=Command.TEMPO, value=bpm_value) +def bpm_to_value(bpm: Fraction) -> int: + return math.floor(60 * 10 ** 6 / bpm) + + def choose_end_beat(notes: List[AnyNote]) -> song.BeatsTime: """Leave 2 empty measures (4 beats) after the last event""" last_note_beat = compute_last_note_beat(notes) @@ -93,7 +98,7 @@ def compute_last_note_beat(notes: List[AnyNote]) -> song.BeatsTime: n.time + n.duration for n in notes if isinstance(n, song.LongNote) ) all_note_times = note_times | long_note_ends - return max(all_note_times) + return max(all_note_times, default=song.BeatsTime(0)) def make_end_event(end_beat: song.BeatsTime, time_map: TimeMap) -> Event: diff --git a/jubeatools/formats/eve/load.py b/jubeatools/formats/eve/load.py index ab42706..11b245a 100644 --- a/jubeatools/formats/eve/load.py +++ b/jubeatools/formats/eve/load.py @@ -1,45 +1,80 @@ -from typing import List, Iterator -from pathlib import Path +import math +from decimal import Decimal +from fractions import Fraction from functools import reduce -from dataclasses import dataclass +from pathlib import Path +from typing import Iterator, List, Optional from jubeatools import song -from jubeatools.formats.load_tools import make_folder_loader +from jubeatools.formats.load_tools import make_folder_loader, round_beats +from jubeatools.utils import group_by + +from .commons import ( + VALUE_TO_DIRECTION, + AnyNote, + Command, + EveLong, + Event, + ticks_to_seconds, +) +from .timemap import BPMAtSecond, TimeMap -from .commons import Event, Command def load_eve(path: Path) -> song.Song: files = load_folder(path) charts = [_load_eve(l, p) for p, l in files.items()] return reduce(song.Song.merge, charts) + def load_file(path: Path) -> List[str]: return path.read_text(encoding="ascii").split("\n") + load_folder = make_folder_loader("*.eve", load_file) + def _load_eve(lines: List[str], file_path: Path) -> song.Song: events = list(iter_events(lines)) - ... + events_by_command = group_by(events, lambda e: e.command) + bpms = [ + BPMAtSecond( + seconds=ticks_to_seconds(e.time), BPM=value_to_truncated_bpm(e.value) + ) + for e in sorted(events_by_command[Command.TEMPO]) + ] + time_map = TimeMap.from_seconds(bpms) + tap_notes: List[AnyNote] = [ + make_tap_note(e.time, e.value, time_map) + for e in events_by_command[Command.PLAY] + ] + long_notes: List[AnyNote] = [ + make_long_notes(e.time, e.value, time_map) + for e in events_by_command[Command.LONG] + ] + all_notes = sorted(tap_notes + long_notes, key=lambda n: (n.time, n.position)) + timing = time_map.convert_to_timing_info() + chart = song.Chart(level=Decimal(0), timing=timing, notes=all_notes) + dif = guess_difficulty(file_path.stem) or song.Difficulty.EXTREME + return song.Song(metadata=song.Metadata(), charts={dif: chart}) + def iter_events(lines: List[str]) -> Iterator[Event]: for i, raw_line in enumerate(lines, start=1): line = raw_line.strip() if not line: continue - + try: - event = parser_event(line) + yield parse_event(line) except ValueError as e: - raise ValueError( - f"Error on line {i} : {e}" - ) + raise ValueError(f"Error on line {i} : {e}") + def parse_event(line: str) -> Event: columns = line.split(",") if len(columns) != 3: raise ValueError(f"Expected 3 comma-separated values but found {len(columns)}") - + raw_tick, raw_command, raw_value = map(str.strip, columns) try: tick = int(raw_tick) @@ -56,7 +91,7 @@ def parse_event(line: str) -> Event: f"The second column should contain one of " f"{list(Command.__members__)}, but {raw_command!r} was found" ) - + try: value = int(raw_value) except ValueError: @@ -64,6 +99,66 @@ def parse_event(line: str) -> Event: f"The third column should contain an integer but {raw_tick!r} was " f"found, which python could not understand as an integer" ) - - return Event(time, command, value) + return Event(tick, command, value) + + +def value_to_truncated_bpm(value: int) -> Fraction: + """Only keeps enough significant digits to allow recovering the original + TEMPO line value from the bpm""" + precision = tempo_precision(value) + places = significant_decimal_places(precision) + 1 + raw_bpm = value_to_bpm(value) + return truncate_fraction(raw_bpm, places) + + +def value_to_bpm(value: int) -> Fraction: + return 6 * 10 ** 7 / Fraction(value) + + +def significant_decimal_places(max_error: float) -> int: + return int(-(math.log(max_error / 5) / math.log(10))) + + +def tempo_precision(value: int) -> float: + """Maximum error on the bpm this tempo value corresponds to""" + return (6 * 10 ** 7) / (value * (value + 1)) + + +def truncate_fraction(f: Fraction, places: int) -> Fraction: + """Truncates a fraction to the given number of decimal places""" + exponent = Fraction(10) ** places + return Fraction(math.floor(f * exponent), exponent) + + +def make_tap_note(ticks: int, value: int, time_map: TimeMap) -> song.TapNote: + seconds = ticks_to_seconds(ticks) + raw_beats = time_map.beats_at(seconds) + beats = round_beats(raw_beats) + position = song.NotePosition.from_index(value) + return song.TapNote(time=beats, position=position) + + +def make_long_notes(ticks: int, value: int, time_map: TimeMap) -> song.LongNote: + seconds = ticks_to_seconds(ticks) + raw_beats = time_map.beats_at(seconds) + beats = round_beats(raw_beats) + eve_long = EveLong.from_value(value) + seconds_duration = ticks_to_seconds(eve_long.duration) + raw_beats_duration = time_map.beats_at(seconds + seconds_duration) - raw_beats + beats_duration = round_beats(raw_beats_duration) + position = song.NotePosition.from_index(eve_long.position) + direction = VALUE_TO_DIRECTION[eve_long.direction] + step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction] + raw_tail_pos = position + (eve_long.length * step_vector) + tail_pos = song.NotePosition.from_raw_position(raw_tail_pos) + return song.LongNote( + time=beats, position=position, duration=beats_duration, tail_tip=tail_pos + ) + + +def guess_difficulty(filename: str) -> Optional[song.Difficulty]: + try: + return song.Difficulty(filename.upper()) + except ValueError: + return None diff --git a/jubeatools/formats/eve/tests/__init__.py b/jubeatools/formats/eve/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/jubeatools/formats/eve/tests/test_bpm_value.py b/jubeatools/formats/eve/tests/test_bpm_value.py new file mode 100644 index 0000000..7bb8a04 --- /dev/null +++ b/jubeatools/formats/eve/tests/test_bpm_value.py @@ -0,0 +1,15 @@ +import math + +from hypothesis import given +from hypothesis import strategies as st + +from ..dump import bpm_to_value +from ..load import value_to_truncated_bpm + + +@given(st.integers(min_value=1, max_value=6 * 10 ** 7)) +def test_that_truncating_preserves_tempo_value(original_value: int) -> None: + truncated_bpm = value_to_truncated_bpm(original_value) + raw_recovered_value = bpm_to_value(truncated_bpm) + recovered_value = math.floor(raw_recovered_value) + assert recovered_value == original_value diff --git a/jubeatools/formats/eve/tests/test_eve.py b/jubeatools/formats/eve/tests/test_eve.py new file mode 100644 index 0000000..5b20b27 --- /dev/null +++ b/jubeatools/formats/eve/tests/test_eve.py @@ -0,0 +1,50 @@ +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +from hypothesis import Verbosity, given, settings +from hypothesis import strategies as st + +from jubeatools import song +from jubeatools.formats import Format +from jubeatools.testutils import strategies as jbst +from jubeatools.testutils.test_patterns import dump_and_load_then_compare +from jubeatools.testutils.typing import DrawFunc + + +@st.composite +def eve_compatible_song(draw: DrawFunc) -> song.Song: + """eve only keeps notes, timing info and difficulty""" + diff = draw(st.sampled_from(list(song.Difficulty))) + chart = draw( + jbst.chart( + timing_strat=jbst.timing_info( + with_bpm_changes=True, + bpm_strat=st.decimals(min_value=1, max_value=1000, places=2), + beat_zero_offset_strat=st.decimals(min_value=0, max_value=20, places=2), + ), + notes_strat=jbst.notes(jbst.NoteOption.LONGS), + ) + ) + return song.Song( + metadata=song.Metadata(), + charts={diff: chart}, + ) + + +@contextmanager +def open_temp_dir() -> Iterator[Path]: + with tempfile.TemporaryDirectory() as temp_dir: + yield Path(temp_dir) + + +@given(eve_compatible_song()) +@settings(verbosity=Verbosity.debug) +def test_that_full_chart_roundtrips(song: song.Song) -> None: + dump_and_load_then_compare( + Format.EVE, + song, + temp_path=open_temp_dir(), + bytes_decoder=lambda b: b.decode("ascii"), + ) diff --git a/jubeatools/formats/eve/tests/test_timemap.py b/jubeatools/formats/eve/tests/test_timemap.py index e89a48f..195901f 100644 --- a/jubeatools/formats/eve/tests/test_timemap.py +++ b/jubeatools/formats/eve/tests/test_timemap.py @@ -8,7 +8,7 @@ from jubeatools.testutils import strategies as jbst from jubeatools.utils import group_by -@given(jbst.timing_info(bpm_changes=True), jbst.beat_time()) +@given(jbst.timing_info(with_bpm_changes=True), jbst.beat_time()) def test_that_seconds_at_beat_works_like_the_naive_approach( timing: song.Timing, beat: song.BeatsTime ) -> None: diff --git a/jubeatools/formats/eve/timemap.py b/jubeatools/formats/eve/timemap.py index 681a407..dd54b94 100644 --- a/jubeatools/formats/eve/timemap.py +++ b/jubeatools/formats/eve/timemap.py @@ -2,12 +2,13 @@ from __future__ import annotations from dataclasses import dataclass from fractions import Fraction -from typing import List +from typing import List, Union from more_itertools import windowed from sortedcontainers import SortedKeyList from jubeatools import song +from jubeatools.formats.load_tools import round_beats from jubeatools.utils import fraction_to_decimal, group_by @@ -124,7 +125,7 @@ class TimeMap: seconds_since_last_event = (60 * beats_since_last_event) / bpm_change.BPM return bpm_change.seconds + seconds_since_last_event - def beats_at(self, seconds: song.SecondsTime) -> song.BeatsTime: + def beats_at(self, seconds: Union[song.SecondsTime, Fraction]) -> song.BeatsTime: if seconds < self.beat_zero_offset: raise ValueError( f"Can't compute beat time at {seconds} seconds, since it predates " @@ -142,3 +143,12 @@ class TimeMap: 60 ) return bpm_change.beats + beats_since_last_event + + def convert_to_timing_info(self) -> song.Timing: + return song.Timing( + events=[ + song.BPMEvent(time=round_beats(e.beats), BPM=fraction_to_decimal(e.BPM)) + for e in self.events_by_beats + ], + beat_zero_offset=self.beat_zero_offset, + ) diff --git a/jubeatools/song.py b/jubeatools/song.py index cabab13..e442635 100644 --- a/jubeatools/song.py +++ b/jubeatools/song.py @@ -44,6 +44,7 @@ def convert_other( return wrapped + @dataclass(frozen=True, order=True) class Position: """2D integer vector""" @@ -53,7 +54,7 @@ class Position: def __iter__(self) -> Iterator[int]: yield from astuple(self) - + @convert_other def __add__(self, other: Position) -> Position: return Position(self.x + other.x, self.y + other.y) @@ -61,10 +62,10 @@ class Position: @convert_other def __sub__(self, other: Position) -> Position: return Position(self.x - other.x, self.y - other.y) - + def __mul__(self, other: int) -> Position: return Position(self.x * other, self.y * other) - + __rmul__ = __mul__ @@ -79,7 +80,7 @@ class NotePosition(Position): ↓ 1 □ □ □ □ 2 □ □ □ □ 3 □ □ □ □ - + The main difference with Position is that x and y MUST be between 0 and 3 """ @@ -99,7 +100,7 @@ class NotePosition(Position): raise ValueError(f"Note position index out of range : {index}") return cls(x=index % 4, y=index // 4) - + @classmethod def from_raw_position(cls, pos: Position) -> NotePosition: return cls(x=pos.x, y=pos.y)