[eve] full support !

jubeatools/formats/eve/__init__.py

@@ -1,5 +1,14 @@
-"""
-.eve is the file format used in arcade releases of jubeat
+""".eve is the file format used in arcade releases of jubeat
+
+.eve files are CSVs with three columns : time, command, value
+
+A small but annoying amount of precision is lost when using this format :
+- time is stored already "rendered" as a whole number of ticks on a 300Hz clock
+  instead of using symbolic time
+- while some symbolic time information remains in the form of TEMPO, MEASURE
+  and HAKU commands (respectively a BPM change, a measure marker and a beat
+  marker), BPMs are stored as `int((6*10^7)/BPM)` which makes it hard to
+  recover many significant digits
 """
 
 from .dump import dump_eve

jubeatools/formats/eve/commons.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
+
+from dataclasses import astuple, dataclass
 from enum import Enum
-from dataclasses import dataclass, astuple
-from jubeatools import song
-from typing import Union
-from functools import singledispatchmethod
 from fractions import Fraction
+from typing import Union
+
+from jubeatools import song
 
 from .timemap import TimeMap
 
@@ -19,7 +20,8 @@ DIRECTION_TO_VALUE = {
 
 VALUE_TO_DIRECTION = {v: k for k, v in DIRECTION_TO_VALUE.items()}
 
-class Command(Enum):
+# int is here to allow sorting
+class Command(int, Enum):
     END = 1
     MEASURE = 2
     HAKU = 3
@@ -38,17 +40,22 @@ class Event:
 
     def __post_init__(self) -> None:
         try:
-            check_func = VALUES_CHECKERS[self.command](self.value)
+            check_func = VALUES_CHECKERS[self.command]
         except KeyError:
             # most likely no check function associated : forget about it
             pass
+
+        try:
+            check_func(self.value)
         except ValueError as e:
-            raise ValueError(f"Invalid value for the {self.command!r} command. {e}")
+            raise ValueError(
+                f"{self.value} is not a valid value for the {self.command!r} "
+                f"command. {e}"
+            )
 
     def dump(self) -> str:
         return f"{self.time:>8},{self.command.name:<8},{self.value:>8}"
 
-
     @classmethod
     def from_tap_note(cls, note: song.TapNote, time_map: TimeMap) -> Event:
         ticks = ticks_at_beat(note.time, time_map)
@@ -60,36 +67,36 @@ class Event:
         if not note.has_straight_tail():
             raise ValueError("Diagonal tails cannot be represented in eve format")
 
-        duration = duration_in_ticks(note, time_map)
-        direction = DIRECTION_TO_VALUE[note.tail_direction()]
-        length = len(list(note.positions_covered())) - 1
-        if not (1 <= length <= 3):
-            raise ValueError(
-                f"Given note has a length of {length}, which is not representable "
-                "in the eve format"
-            )
-        position_index = note.position.index
-        long_note_value = duration << 8 + length << 6 + direction << 4 + position_index
+        eve_long = EveLong(
+            duration=duration_in_ticks(note, time_map),
+            length=len(list(note.positions_covered())) - 1,
+            direction=DIRECTION_TO_VALUE[note.tail_direction()],
+            position=note.position.index,
+        )
         ticks = ticks_at_beat(note.time, time_map)
-        return Event(time=ticks, command=Command.LONG, value=long_note_value)
+        return Event(time=ticks, command=Command.LONG, value=eve_long.value)
 
 
 def is_zero(value: int) -> None:
     if value != 0:
-        raise ValueError(f"Value should be zero but {value} found")
+        raise ValueError(f"Value should be zero")
+
 
 def is_valid_button_index(value: int) -> None:
     # Should raise ValueError if invalid
     _ = song.NotePosition.from_index(value)
 
+
 def is_valid_tail_position(value: int) -> None:
     # Should raise ValueError if invalid
     _ = EveLong.from_value(value)
 
+
 def is_not_zero(value: int) -> None:
     if value == 0:
         raise ValueError(f"Value cannot be zero")
 
+
 VALUES_CHECKERS = {
     Command.END: is_zero,
     Command.MEASURE: is_zero,
@@ -104,7 +111,7 @@ VALUES_CHECKERS = {
 class EveLong:
     duration: int
     length: int
-    direction: song.Direction
+    direction: int
     position: int
 
     def __post_init__(self) -> None:
@@ -112,18 +119,37 @@ class EveLong:
             raise ValueError("Duration can't be negative")
         if not 1 <= self.length < 4:
             raise ValueError("Tail length must be between 1 and 3 inclusive")
+        if not 0 <= self.position < 16:
+            raise ValueError("Note Position must be between 0 and 15 inclusive")
+        if not 0 <= self.direction < 4:
+            raise ValueError("direction value must be between 0 and 3 inclusive")
+
         pos = song.NotePosition.from_index(self.position)
-        step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[self.direction]
+        direction = VALUE_TO_DIRECTION[self.direction]
+        step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction]
         tail_pos = pos + (self.length * step_vector)
         if not ((0 <= tail_pos.x < 4) and (0 <= tail_pos.y < 4)):
             raise ValueError(
                 f"Long note tail starts on {astuple(tail_pos)} which is "
                 "outside the screen"
             )
-    
+
     @classmethod
     def from_value(cls, value: int) -> EveLong:
-        ...
+        if value < 0:
+            raise ValueError("Value cannot be negative")
+
+        position = value & 0b1111  # first 4 bits
+        direction = (value >> 4) & 0b11  # next 2 bits
+        length = (value >> 6) & 0b11  # next 2 bits
+        duration = value >> 8  # remaining bits
+        return cls(duration, length, direction, position)
+
+    @property
+    def value(self) -> int:
+        return (
+            self.duration << 8 + self.length << 6 + self.direction << 4 + self.position
+        )
 
 
 def ticks_at_beat(time: song.BeatsTime, time_map: TimeMap) -> int:
@@ -145,4 +171,4 @@ def ticks_to_seconds(tick: int) -> Fraction:
 
 def seconds_to_ticks(time: Fraction) -> int:
     """Convert fractional seconds to eve ticks (300 Hz)"""
-    return round(time * 300)
+    return round(time * 300)

jubeatools/formats/eve/dump.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
-from enum import Enum
+import math
 from fractions import Fraction
 from functools import singledispatch
 from pathlib import Path
-from typing import List, Union
+from typing import List
 
 from more_itertools import numeric_range
 
@@ -13,8 +12,8 @@ from jubeatools import song
 from jubeatools.formats.dump_tools import make_dumper_from_chart_file_dumper
 from jubeatools.formats.filetypes import ChartFile
 
+from .commons import AnyNote, Command, Event, ticks_at_beat
 from .timemap import TimeMap
-from .commons import AnyNote, Command, Event, DIRECTION_TO_VALUE
 
 
 def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]:
@@ -28,7 +27,7 @@ def _dump_eve(song: song.Song, **kwargs: dict) -> List[ChartFile]:
 
 
 dump_eve = make_dumper_from_chart_file_dumper(
-    internal_dumper=_dump_eve, file_name_template=Path("{difficulty_index}.eve")
+    internal_dumper=_dump_eve, file_name_template=Path("{difficulty:l}.eve")
 )
 
 
@@ -41,17 +40,19 @@ def dump_chart(notes: List[AnyNote], timing: song.Timing) -> str:
 
 
 def make_note_events(notes: List[AnyNote], time_map: TimeMap) -> List[Event]:
-    return [Event.from_note(note, time_map) for note in notes]
+    return [make_note_event(note, time_map) for note in notes]
 
 
 @singledispatch
 def make_note_event(note: AnyNote, time_map: TimeMap) -> Event:
     raise NotImplementedError(f"Unknown note type : {type(note)}")
 
+
 @make_note_event.register
 def make_tap_note_event(note: song.TapNote, time_map: TimeMap) -> Event:
     return Event.from_tap_note(note, time_map)
 
+
 @make_note_event.register
 def make_long_note_event(note: song.LongNote, time_map: TimeMap) -> Event:
     return Event.from_long_note(note, time_map)
@@ -70,10 +71,14 @@ def make_timing_events(
 
 def make_bpm_event(bpm_change: song.BPMEvent, time_map: TimeMap) -> Event:
     ticks = ticks_at_beat(bpm_change.time, time_map)
-    bpm_value = round(60 * 10 ** 6 / Fraction(bpm_change.BPM))
+    bpm_value = bpm_to_value(Fraction(bpm_change.BPM))
     return Event(time=ticks, command=Command.TEMPO, value=bpm_value)
 
 
+def bpm_to_value(bpm: Fraction) -> int:
+    return math.floor(60 * 10 ** 6 / bpm)
+
+
 def choose_end_beat(notes: List[AnyNote]) -> song.BeatsTime:
     """Leave 2 empty measures (4 beats) after the last event"""
     last_note_beat = compute_last_note_beat(notes)
@@ -93,7 +98,7 @@ def compute_last_note_beat(notes: List[AnyNote]) -> song.BeatsTime:
         n.time + n.duration for n in notes if isinstance(n, song.LongNote)
     )
     all_note_times = note_times | long_note_ends
-    return max(all_note_times)
+    return max(all_note_times, default=song.BeatsTime(0))
 
 
 def make_end_event(end_beat: song.BeatsTime, time_map: TimeMap) -> Event:

jubeatools/formats/eve/load.py

@@ -1,45 +1,80 @@
-from typing import List, Iterator
-from pathlib import Path
+import math
+from decimal import Decimal
+from fractions import Fraction
 from functools import reduce
-from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, List, Optional
 
 from jubeatools import song
-from jubeatools.formats.load_tools import make_folder_loader
+from jubeatools.formats.load_tools import make_folder_loader, round_beats
+from jubeatools.utils import group_by
+
+from .commons import (
+    VALUE_TO_DIRECTION,
+    AnyNote,
+    Command,
+    EveLong,
+    Event,
+    ticks_to_seconds,
+)
+from .timemap import BPMAtSecond, TimeMap
 
-from .commons import Event, Command
 
 def load_eve(path: Path) -> song.Song:
     files = load_folder(path)
     charts = [_load_eve(l, p) for p, l in files.items()]
     return reduce(song.Song.merge, charts)
 
+
 def load_file(path: Path) -> List[str]:
     return path.read_text(encoding="ascii").split("\n")
 
+
 load_folder = make_folder_loader("*.eve", load_file)
 
+
 def _load_eve(lines: List[str], file_path: Path) -> song.Song:
     events = list(iter_events(lines))
-    ...
+    events_by_command = group_by(events, lambda e: e.command)
+    bpms = [
+        BPMAtSecond(
+            seconds=ticks_to_seconds(e.time), BPM=value_to_truncated_bpm(e.value)
+        )
+        for e in sorted(events_by_command[Command.TEMPO])
+    ]
+    time_map = TimeMap.from_seconds(bpms)
+    tap_notes: List[AnyNote] = [
+        make_tap_note(e.time, e.value, time_map)
+        for e in events_by_command[Command.PLAY]
+    ]
+    long_notes: List[AnyNote] = [
+        make_long_notes(e.time, e.value, time_map)
+        for e in events_by_command[Command.LONG]
+    ]
+    all_notes = sorted(tap_notes + long_notes, key=lambda n: (n.time, n.position))
+    timing = time_map.convert_to_timing_info()
+    chart = song.Chart(level=Decimal(0), timing=timing, notes=all_notes)
+    dif = guess_difficulty(file_path.stem) or song.Difficulty.EXTREME
+    return song.Song(metadata=song.Metadata(), charts={dif: chart})
+
 
 def iter_events(lines: List[str]) -> Iterator[Event]:
     for i, raw_line in enumerate(lines, start=1):
         line = raw_line.strip()
         if not line:
             continue
-        
+
         try:
-            event = parser_event(line)
+            yield parse_event(line)
         except ValueError as e:
-            raise ValueError(
-                f"Error on line {i} : {e}"
-            )
+            raise ValueError(f"Error on line {i} : {e}")
+
 
 def parse_event(line: str) -> Event:
     columns = line.split(",")
     if len(columns) != 3:
         raise ValueError(f"Expected 3 comma-separated values but found {len(columns)}")
-    
+
     raw_tick, raw_command, raw_value = map(str.strip, columns)
     try:
         tick = int(raw_tick)
@@ -56,7 +91,7 @@ def parse_event(line: str) -> Event:
             f"The second column should contain one of "
             f"{list(Command.__members__)}, but {raw_command!r} was found"
         )
-    
+
     try:
         value = int(raw_value)
     except ValueError:
@@ -64,6 +99,66 @@ def parse_event(line: str) -> Event:
             f"The third column should contain an integer but {raw_tick!r} was "
             f"found, which python could not understand as an integer"
         )
-    
-    return Event(time, command, value)
 
+    return Event(tick, command, value)
+
+
+def value_to_truncated_bpm(value: int) -> Fraction:
+    """Only keeps enough significant digits to allow recovering the original
+    TEMPO line value from the bpm"""
+    precision = tempo_precision(value)
+    places = significant_decimal_places(precision) + 1
+    raw_bpm = value_to_bpm(value)
+    return truncate_fraction(raw_bpm, places)
+
+
+def value_to_bpm(value: int) -> Fraction:
+    return 6 * 10 ** 7 / Fraction(value)
+
+
+def significant_decimal_places(max_error: float) -> int:
+    return int(-(math.log(max_error / 5) / math.log(10)))
+
+
+def tempo_precision(value: int) -> float:
+    """Maximum error on the bpm this tempo value corresponds to"""
+    return (6 * 10 ** 7) / (value * (value + 1))
+
+
+def truncate_fraction(f: Fraction, places: int) -> Fraction:
+    """Truncates a fraction to the given number of decimal places"""
+    exponent = Fraction(10) ** places
+    return Fraction(math.floor(f * exponent), exponent)
+
+
+def make_tap_note(ticks: int, value: int, time_map: TimeMap) -> song.TapNote:
+    seconds = ticks_to_seconds(ticks)
+    raw_beats = time_map.beats_at(seconds)
+    beats = round_beats(raw_beats)
+    position = song.NotePosition.from_index(value)
+    return song.TapNote(time=beats, position=position)
+
+
+def make_long_notes(ticks: int, value: int, time_map: TimeMap) -> song.LongNote:
+    seconds = ticks_to_seconds(ticks)
+    raw_beats = time_map.beats_at(seconds)
+    beats = round_beats(raw_beats)
+    eve_long = EveLong.from_value(value)
+    seconds_duration = ticks_to_seconds(eve_long.duration)
+    raw_beats_duration = time_map.beats_at(seconds + seconds_duration) - raw_beats
+    beats_duration = round_beats(raw_beats_duration)
+    position = song.NotePosition.from_index(eve_long.position)
+    direction = VALUE_TO_DIRECTION[eve_long.direction]
+    step_vector = song.TAIL_DIRECTION_TO_OUTWARDS_VECTOR[direction]
+    raw_tail_pos = position + (eve_long.length * step_vector)
+    tail_pos = song.NotePosition.from_raw_position(raw_tail_pos)
+    return song.LongNote(
+        time=beats, position=position, duration=beats_duration, tail_tip=tail_pos
+    )
+
+
+def guess_difficulty(filename: str) -> Optional[song.Difficulty]:
+    try:
+        return song.Difficulty(filename.upper())
+    except ValueError:
+        return None

jubeatools/formats/eve/tests/test_bpm_value.py

@@ -0,0 +1,15 @@
+import math
+
+from hypothesis import given
+from hypothesis import strategies as st
+
+from ..dump import bpm_to_value
+from ..load import value_to_truncated_bpm
+
+
+@given(st.integers(min_value=1, max_value=6 * 10 ** 7))
+def test_that_truncating_preserves_tempo_value(original_value: int) -> None:
+    truncated_bpm = value_to_truncated_bpm(original_value)
+    raw_recovered_value = bpm_to_value(truncated_bpm)
+    recovered_value = math.floor(raw_recovered_value)
+    assert recovered_value == original_value

jubeatools/formats/eve/tests/test_eve.py

@@ -0,0 +1,50 @@
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+from hypothesis import Verbosity, given, settings
+from hypothesis import strategies as st
+
+from jubeatools import song
+from jubeatools.formats import Format
+from jubeatools.testutils import strategies as jbst
+from jubeatools.testutils.test_patterns import dump_and_load_then_compare
+from jubeatools.testutils.typing import DrawFunc
+
+
+@st.composite
+def eve_compatible_song(draw: DrawFunc) -> song.Song:
+    """eve only keeps notes, timing info and difficulty"""
+    diff = draw(st.sampled_from(list(song.Difficulty)))
+    chart = draw(
+        jbst.chart(
+            timing_strat=jbst.timing_info(
+                with_bpm_changes=True,
+                bpm_strat=st.decimals(min_value=1, max_value=1000, places=2),
+                beat_zero_offset_strat=st.decimals(min_value=0, max_value=20, places=2),
+            ),
+            notes_strat=jbst.notes(jbst.NoteOption.LONGS),
+        )
+    )
+    return song.Song(
+        metadata=song.Metadata(),
+        charts={diff: chart},
+    )
+
+
+@contextmanager
+def open_temp_dir() -> Iterator[Path]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        yield Path(temp_dir)
+
+
+@given(eve_compatible_song())
+@settings(verbosity=Verbosity.debug)
+def test_that_full_chart_roundtrips(song: song.Song) -> None:
+    dump_and_load_then_compare(
+        Format.EVE,
+        song,
+        temp_path=open_temp_dir(),
+        bytes_decoder=lambda b: b.decode("ascii"),
+    )

jubeatools/formats/eve/tests/test_timemap.py

@@ -8,7 +8,7 @@ from jubeatools.testutils import strategies as jbst
 from jubeatools.utils import group_by
 
 
-@given(jbst.timing_info(bpm_changes=True), jbst.beat_time())
+@given(jbst.timing_info(with_bpm_changes=True), jbst.beat_time())
 def test_that_seconds_at_beat_works_like_the_naive_approach(
     timing: song.Timing, beat: song.BeatsTime
 ) -> None:

jubeatools/formats/eve/timemap.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from fractions import Fraction
-from typing import List
+from typing import List, Union
 
 from more_itertools import windowed
 from sortedcontainers import SortedKeyList
 
 from jubeatools import song
+from jubeatools.formats.load_tools import round_beats
 from jubeatools.utils import fraction_to_decimal, group_by
 
 
@@ -124,7 +125,7 @@ class TimeMap:
         seconds_since_last_event = (60 * beats_since_last_event) / bpm_change.BPM
         return bpm_change.seconds + seconds_since_last_event
 
-    def beats_at(self, seconds: song.SecondsTime) -> song.BeatsTime:
+    def beats_at(self, seconds: Union[song.SecondsTime, Fraction]) -> song.BeatsTime:
         if seconds < self.beat_zero_offset:
             raise ValueError(
                 f"Can't compute beat time at {seconds} seconds, since it predates "
@@ -142,3 +143,12 @@ class TimeMap:
             60
         )
         return bpm_change.beats + beats_since_last_event
+
+    def convert_to_timing_info(self) -> song.Timing:
+        return song.Timing(
+            events=[
+                song.BPMEvent(time=round_beats(e.beats), BPM=fraction_to_decimal(e.BPM))
+                for e in self.events_by_beats
+            ],
+            beat_zero_offset=self.beat_zero_offset,
+        )

jubeatools/song.py

@@ -44,6 +44,7 @@ def convert_other(
 
     return wrapped
 
+
 @dataclass(frozen=True, order=True)
 class Position:
     """2D integer vector"""
@@ -53,7 +54,7 @@ class Position:
 
     def __iter__(self) -> Iterator[int]:
         yield from astuple(self)
-    
+
     @convert_other
     def __add__(self, other: Position) -> Position:
         return Position(self.x + other.x, self.y + other.y)
@@ -61,10 +62,10 @@ class Position:
     @convert_other
     def __sub__(self, other: Position) -> Position:
         return Position(self.x - other.x, self.y - other.y)
-    
+
     def __mul__(self, other: int) -> Position:
         return Position(self.x * other, self.y * other)
-    
+
     __rmul__ = __mul__
 
 
@@ -79,7 +80,7 @@ class NotePosition(Position):
     ↓ 1 □ □ □ □
       2 □ □ □ □
       3 □ □ □ □
-    
+
     The main difference with Position is that x and y MUST be between 0 and 3
     """
 
@@ -99,7 +100,7 @@ class NotePosition(Position):
             raise ValueError(f"Note position index out of range : {index}")
 
         return cls(x=index % 4, y=index // 4)
-    
+
     @classmethod
     def from_raw_position(cls, pos: Position) -> NotePosition:
         return cls(x=pos.x, y=pos.y)