#memo wip

jubeatools/formats/__init__.py

@@ -35,7 +35,6 @@ LOADERS: Dict[Format, Callable[[Path], Song]] = {
     Format.MONO_COLUMN: load_mono_column,
 }
 
-# Dumpers serialize a Song object into a (filename -> file) mapping
 DUMPERS: Dict[str, Callable[[Song], Dict[str, IO]]] = {
     Format.MEMON_LEGACY: dump_memon_legacy,
     Format.MEMON_0_1_0: dump_memon_0_1_0,

jubeatools/formats/filetypes.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+from typing import IO
+
+from jubeatools.song import Chart, Song
+
+
+@dataclass
+class JubeatFile:
+    contents: IO
+
+
+@dataclass
+class SongFile(JubeatFile):
+    """File representing a collection of charts with metadata,
+    like a .memon file for example"""
+
+    song: Song
+
+
+@dataclass
+class ChartFile(SongFile):
+    """File representing a single chart (with possibly some metadata),
+    Used in jubeat analyser formats for instance"""
+
+    difficulty: str
+    chart: Chart

jubeatools/formats/jubeat_analyser/__init__.py

@@ -11,7 +11,7 @@ on these pages :
 - http://yosh52.web.fc2.com/jubeat/holdmarker.html
 """
 
-from .mono_column.dump import dump_mono_column
-from .mono_column.load import load_mono_column
 from .memo.dump import dump_memo
 from .memo.load import load_memo
+from .mono_column.dump import dump_mono_column
+from .mono_column.load import load_mono_column

jubeatools/formats/jubeat_analyser/files.py

@@ -1,6 +1,8 @@
-from path import Path
 from typing import Dict, List
 
+from path import Path
+
+
 def load_files(path: Path) -> Dict[Path, List[str]]:
     # The vast majority of memo files you will encounter will be propely
     # decoded using shift_jis_2004. Get ready for endless fun with the small
@@ -13,8 +15,9 @@ def load_files(path: Path) -> Dict[Path, List[str]]:
         _load_file(path, files)
     return files
 
+
 def _load_file(path: Path, files: Dict[Path, List[str]]):
     try:
-        files[path] = path.lines('shift_jis_2004')
+        files[path] = path.lines("shift_jis_2004")
     except UnicodeDecodeError:
-        pass
+        pass

jubeatools/formats/jubeat_analyser/memo/__init__.py

@@ -1,10 +1,18 @@
 """
 memo
 
-#memo is the first (and probably oldest) youbeat-like format jubeat analyser
+memo is the first (and probably oldest) youbeat-like format jubeat analyser
 supports, the japanese docs give a good overview of what it looks like :
 
 http://yosh52.web.fc2.com/jubeat/fumenformat.html
+
+A chart in this format needs to have a `#memo` line somewhere to indicate its format
+
+Like youbeat chart files, the position and timing information is split between
+the left and right columns. However the timing column works differently.
+In this precise format, the symbols in the timing part must only be interpeted
+as quarter notes, even in the rare cases where a timing line is shorter than 4
+symbols. It can never be longer than 4 either.
 """
 
 from .dump import dump_memo

jubeatools/formats/jubeat_analyser/memo/dump.py

@@ -1,2 +1,2 @@
 def dump_memo(song):
-    ...
+    ...

jubeatools/formats/jubeat_analyser/memo/load.py

@@ -1,43 +1,413 @@
-from jubeatools.song import Song
-from path import Path
-from ..parser import JubeatAnalyserParser
+import warnings
+from collections import ChainMap
+from copy import deepcopy
 from dataclasses import dataclass
-from typing import List
+from decimal import Decimal
+from functools import reduce
+from itertools import chain, product
+from typing import Mapping, Dict, Iterator, List, Optional, Set, Tuple, Union
 
+import constraint
+from parsimonious import Grammar, NodeVisitor, ParseError
+from path import Path
+
+from jubeatools.song import (
+    Chart,
+    LongNote,
+    Metadata,
+    NotePosition,
+    SecondsTime,
+    Song,
+    TapNote,
+    Timing,
+)
+
+from ..command import is_command, parse_command
+from ..files import load_files
+from ..parser import (
+    CIRCLE_FREE_TO_DECIMAL_TIME,
+    LONG_ARROWS,
+    LONG_DIRECTION,
+    JubeatAnalyserParser,
+    UnfinishedLongNote,
+    decimal_to_beats,
+    is_empty_line,
+    is_simple_solution,
+    long_note_solution_heuristic,
+    split_double_byte_line,
+)
+from ..symbol_definition import is_symbol_definition, parse_symbol_definition
+from ..symbols import CIRCLE_FREE_SYMBOLS
+
+memo_chart_line_grammar = Grammar(
+    r"""
+    line            = ws position_part ws (timing_part ws)? comment?
+    position_part   = ~r"[^*#:|/\s]{4,8}"
+    timing_part     = "|" ~r"[^*#:|/\s]+" "|"
+    ws              = ~r"[\t ]*"
+    comment         = ~r"//.*"
+"""
+)
+
+
+@dataclass
+class MemoChartLine:
+    position: str
+    timing: Optional[str]
+
+    def __str__(self):
+        return f"{self.position} |{self.timing}|"
+
+
+class MemoChartLineVisitor(NodeVisitor):
+    def __init__(self):
+        super().__init__()
+        self.pos_part = None
+        self.time_part = None
+
+    def visit_line(self, node, visited_children):
+        return MemoChartLine(self.pos_part, self.time_part)
+
+    def visit_position_part(self, node, visited_children):
+        self.pos_part = node.text
+
+    def visit_timing_part(self, node, visited_children):
+        _, time_part, _ = node.children
+        self.time_part = time_part.text
+
+    def generic_visit(self, node, visited_children):
+        ...
+
+
+def is_memo_chart_line(line: str) -> bool:
+    try:
+        memo_chart_line_grammar.parse(line)
+    except ParseError:
+        return False
+    else:
+        return True
+
+
+def parse_memo_chart_line(line: str) -> MemoChartLine:
+    return MemoChartLineVisitor().visit(memo_chart_line_grammar.parse(line))
+
+
+@dataclass
+class MemoFrame:
+    position_part: List[List[str]]
+    timing_part: List[List[str]]
+
+    @property
+    def duration(self) -> Decimal:
+        res = 0
+        for t in self.timing_part:
+            res += len(t)
+        return Decimal("0.25") * res
+
+
+@dataclass
+class MemoLoadedSection:
+    frames: List[MemoFrame]
+    symbols: Dict[str, Decimal]
+    length: Decimal
+    tempo: Decimal
+
+
+# Any unicode character that's both :
+#  - confusable with a dash/hyphen
+#  - encodable in shift_jis_2004
+# Gets added to the list of characters to be ignored in the timing section
+EMPTY_BEAT_SYMBOLS = {
+    "一",  # U+04E00 - CJK UNIFIED IDEOGRAPH-4E00
+    "－",  # U+0FF0D - FULLWIDTH HYPHEN-MINUS
+    "ー",  # U+030FC - KATAKANA-HIRAGANA PROLONGED SOUND MARK
+    "─",  # U+02500 - BOX DRAWINGS LIGHT HORIZONTAL
+    "―",  # U+02015 - HORIZONTAL BAR
+    "━",  # U+02501 - BOX DRAWINGS HEAVY HORIZONTAL
+    "–",  # U+02013 - EN DASH
+    "‐",  # U+02010 - HYPHEN
+    "-",  # U+0002D - HYPHEN-MINUS
+    "−",  # U+02212 - MINUS SIGN
+}
 
 
 class MemoParser(JubeatAnalyserParser):
     def __init__(self):
         super().__init__()
+        self.symbols: Dict[str, Decimal] = {}
+        self.frames: List[MemoFrame] = []
         self.sections: List[MemoLoadedSection] = []
+        self.only_timingless_frames = False
     
     def do_memo(self):
         ...
-        
+
     def do_memo1(self):
         raise ValueError("This is not a memo file")
 
     do_boogie = do_memo2 = do_memo1
 
     def do_bpp(self, value):
-        if self.sections:
+        if self.sections or self.frames:
             raise ValueError(
-                "jubeatools does not handle changing the bytes per panel value halfway"
+                "jubeatools does not handle changes of #bpp halfway"
             )
         else:
-            super().do_bpp(value)
-    
+            self._do_bpp(value)
 
-    def append_chart_line(self, position: str):
-        if self.bytes_per_panel == 1 and len(line) != 4:
-            raise SyntaxError(f"Invalid chart line for #bpp=1 : {line}")
-        elif self.bytes_per_panel == 2 and len(line.encode("shift_jis_2004")) != 8:
-            raise SyntaxError(f"Invalid chart line for #bpp=2 : {line}")
+    def append_chart_line(self, line: MemoChartLine):
+        if len(line.position.encode("shift_jis_2004")) != 4 * self.bytes_per_panel:
+            raise SyntaxError(f"Invalid chart line for #bpp={self.bytes_per_panel} : {line}")
+        if line.timing is not None and self.bytes_per_panel == 2:
+            if len(line.timing.encode("shift_jis_2004")) % 2 != 0:
+                raise SyntaxError(f"Invalid chart line for #bpp=2 : {line}")
         self.current_chart_lines.append(line)
+        if len(self.current_chart_lines) == 4:
+            self._push_frame()
+
+    def _split_chart_line(self, line: str):
+        if self.bytes_per_panel == 2:
+            return split_double_byte_line(line)
+        else:
+            return list(line)
+    
+    def _frames_duration(self) -> Decimal:
+        return sum(frame.duration for frame in self.frames)
+
+    def _push_frame(self):
+        position_part = [
+            self._split_chart_line(memo_line.position)
+            for memo_line in self.current_chart_lines
+        ]
+        timing_part = [
+            self._split_chart_line(memo_line.timing)
+            for memo_line in self.current_chart_lines
+            if memo_line.timing is not None
+        ]
+        frame = MemoFrame(position_part, timing_part)
+        # if the current frame has some timing info
+        if frame.duration > 0:
+            # and the previous frames already cover enough beats
+            if self._frames_duration() >= self.beats_per_section:
+                # then the current frame starts a new section
+                self._push_section()
+        
+        self.frames.append(frame)
+
+    def _push_section(self):
+        self.sections.append(
+            MemoLoadedSection(
+                frames=deepcopy(self.frames),
+                symbols=deepcopy(self.symbols),
+                length=self.beats_per_section,
+                tempo=self.current_tempo,
+            )
+        )
+        self.frames = []
+        self.current_chart_lines = []
+        self.section_starting_beat += self.beats_per_section
+
+    def finish_last_few_notes(self):
+        """Call this once when the end of the file is reached,
+        flushes the chart line and chart frame buffers to create the last chart
+        section"""
+        if self.current_chart_lines:
+            if len(self.current_chart_lines) != 4:
+                raise SyntaxError(
+                    f"Unfinished chart frame when flushing : {self.current_chart_lines}"
+                )
+            self._push_frame()
+        self._push_section()
+
+    def load_line(self, raw_line: str):
+        line = raw_line.strip()
+        if is_command(line):
+            command, value = parse_command(line)
+            self.handle_command(command, value)
+        elif is_symbol_definition(line):
+            symbol, timing = parse_symbol_definition(line)
+            self.define_symbol(symbol, timing)
+        elif is_memo_chart_line(line):
+            memo_chart_line = parse_memo_chart_line(line)
+            self.append_chart_line(memo_chart_line)
+        elif not (is_empty_line(line) or self.is_short_line(line)):
+            raise SyntaxError(f"not a valid mono-column file line : {line}")
+
+    def notes(self) -> Iterator[Union[TapNote, LongNote]]:
+        if self.hold_by_arrow:
+            yield from self._iter_notes()
+        else:
+            yield from self._iter_notes_without_longs()
+    
+    def _iter_frames(self) -> Iterator[Tuple[Decimal, Mapping[str, Decimal], MemoLoadedSection]]:
+        """iterate over tuples of frame_starting_beat, frame, section_starting_beat, section"""
+        section_starting_beat = Decimal(0)
+        for section in self.sections:
+            frame_starting_beat = Decimal(0)
+            for frame in section.frames:
+                yield frame_starting_beat, frame, section_starting_beat, section
+                frame_starting_beat += frame.duration
+            section_starting_beat += section.length
+
+    def _iter_notes(self) -> Iterator[Union[TapNote, LongNote]]:
+        unfinished_longs: Dict[NotePosition, UnfinishedLongNote] = {}
+        for section_starting_beat, section, bloc in self._iter_blocs():
+            should_skip: Set[NotePosition] = set()
+
+            # 1/3 : look for ends to unfinished long notes
+            for pos, unfinished_long in unfinished_longs.items():
+                x, y = pos.as_tuple()
+                symbol = bloc[y][x]
+                if self.circle_free:
+                    if symbol in CIRCLE_FREE_SYMBOLS:
+                        should_skip.add(pos)
+                        symbol_time = CIRCLE_FREE_TO_DECIMAL_TIME[symbol]
+                        note_time = decimal_to_beats(section_starting_beat + symbol_time)
+                        yield unfinished_long.ends_at(note_time)
+                    elif symbol in section.symbols:
+                        raise SyntaxError(
+                            "Can't have a note symbol on the holding square of"
+                            " an unfinished long note when #circlefree is on"
+                        )
+                else:
+                    if symbol in section.symbols:
+                        should_skip.add(pos)
+                        symbol_time = section.symbols[symbol]
+                        note_time = decimal_to_beats(section_starting_beat + symbol_time)
+                        yield unfinished_long.ends_at(note_time)
+
+            unfinished_longs = {
+                k: unfinished_longs[k] for k in unfinished_longs.keys() - should_skip
+            }
+
+            # 2/3 : look for new long notes starting on this bloc
+            arrow_to_note_candidates: Dict[NotePosition, Set[NotePosition]] = {}
+            for y, x in product(range(4), range(4)):
+                pos = NotePosition(x, y)
+                if pos in should_skip:
+                    continue
+                symbol = bloc[y][x]
+                if symbol not in LONG_ARROWS:
+                    continue
+                # at this point we are sure we have a long arrow
+                # we need to check in its direction for note candidates
+                note_candidates: Set[Tuple[int, int]] = set()
+                𝛿pos = LONG_DIRECTION[symbol]
+                candidate = NotePosition(x, y) + 𝛿pos
+                while 0 <= candidate.x < 4 and 0 <= candidate.y < 4:
+                    if candidate in should_skip:
+                        continue
+                    new_symbol = bloc[candidate.y][candidate.x]
+                    if new_symbol in section.symbols:
+                        note_candidates.add(candidate)
+                    candidate += 𝛿pos
+                # if no notes have been crossed, we just ignore the arrow
+                if note_candidates:
+                    arrow_to_note_candidates[pos] = note_candidates
+
+            # Believe it or not, assigning each arrow to a valid note candidate
+            # involves whipping out a CSP solver
+            if arrow_to_note_candidates:
+                problem = constraint.Problem()
+                for arrow_pos, note_candidates in arrow_to_note_candidates.items():
+                    problem.addVariable(arrow_pos, list(note_candidates))
+                problem.addConstraint(constraint.AllDifferentConstraint())
+                solutions = problem.getSolutions()
+                if not solutions:
+                    raise SyntaxError(
+                        "Invalid long note arrow pattern in bloc :\n"
+                        + "\n".join("".join(line) for line in bloc)
+                    )
+                solution = min(solutions, key=long_note_solution_heuristic)
+                if len(solutions) > 1 and not is_simple_solution(
+                    solution, arrow_to_note_candidates
+                ):
+                    warnings.warn(
+                        "Ambiguous arrow pattern in bloc :\n"
+                        + "\n".join("".join(line) for line in bloc)
+                        + "\n"
+                        "The resulting long notes might not be what you expect"
+                    )
+                for arrow_pos, note_pos in solution.items():
+                    should_skip.add(arrow_pos)
+                    should_skip.add(note_pos)
+                    symbol = bloc[note_pos.y][note_pos.x]
+                    symbol_time = section.symbols[symbol]
+                    note_time = decimal_to_beats(section_starting_beat + symbol_time)
+                    unfinished_longs[note_pos] = UnfinishedLongNote(
+                        time=note_time, position=note_pos, tail_tip=arrow_pos,
+                    )
+
+            # 3/3 : find regular notes
+            for y, x in product(range(4), range(4)):
+                position = NotePosition(x, y)
+                if position in should_skip:
+                    continue
+                symbol = bloc[y][x]
+                if symbol in section.symbols:
+                    symbol_time = section.symbols[symbol]
+                    note_time = decimal_to_beats(section_starting_beat + symbol_time)
+                    yield TapNote(note_time, position)
+
+    def _iter_notes_without_longs(self) -> Iterator[TapNote]:
+        local_symbols: Dict[str, Decimal] = {}
+        for frame_starting_beat, frame, section_starting_beat, section in self._iter_frames():
+            # define local note symbols according to what's found in the timing part
+            if frame.timing_part:
+                local_symbols = {
+                    symbol: Decimal("0.25") * i
+                    for i, symbol in enumerate(chain(frame.timing_part))
+                    if symbol not in EMPTY_BEAT_SYMBOLS
+                }
+            currently_defined_symbols = ChainMap(local_symbols, section.symbols)
+            # cross compare with the position information
+            for y, x in product(range(4), range(4)):
+                symbol = frame.position_part[y][x]
+                try:
+                    symbol_time = currently_defined_symbols[symbol]
+                except KeyError:
+                    continue
+                note_time = decimal_to_beats(section_starting_beat + frame_starting_beat + symbol_time)
+                position = NotePosition(x, y)
+                yield TapNote(note_time, position)
+
+
+def _load_memo_file(lines: List[str]) -> Song:
+    parser = MemoParser()
+    for i, raw_line in enumerate(lines):
+        try:
+            parser.load_line(raw_line)
+        except Exception as e:
+            raise SyntaxError(
+                f"Error while parsing memo line {i} :\n" f"{type(e).__name__}: {e}"
+            ) from None
+
+    # finish the current section
+    parser.finish_last_few_notes()
+    metadata = Metadata(
+        title=parser.title,
+        artist=parser.artist,
+        audio=parser.music,
+        cover=parser.jacket,
+    )
+    if parser.preview_start is not None:
+        metadata.preview_start = SecondsTime(parser.preview_start) / 1000
+        metadata.preview_length = SecondsTime(10)
+
+    timing = Timing(
+        events=parser.timing_events, beat_zero_offset=SecondsTime(parser.offset) / 1000
+    )
+    charts = {
+        parser.difficulty: Chart(
+            level=parser.level,
+            timing=timing,
+            notes=sorted(parser.notes(), key=lambda n: (n.time, n.position)),
+        )
+    }
+    return Song(metadata=metadata, charts=charts)
+
 
 def load_memo(path: Path) -> Song:
-    # The vast majority of memo files you will encounter will be propely
-    # decoded using shift_jis_2004. Get ready for endless fun with the small
-    # portion of files that won't
-    with open(path, encoding="shift_jis_2004") as f:
-        lines = f.readlines()
+    files = load_files(path)
+    charts = [_load_memo_file(lines) for _, lines in files.items()]
+    return reduce(lambda a, b: a.merge(b), charts)

jubeatools/formats/jubeat_analyser/mono_column/dump.py

@@ -5,12 +5,14 @@ from decimal import Decimal
 from fractions import Fraction
 from io import StringIO
 from itertools import chain
-from typing import IO, Dict, Iterator, List, Optional, Tuple
+from typing import Dict, Iterator, List, Optional, Tuple
 
 from more_itertools import collapse, intersperse, mark_ends, windowed
+from path import Path
 from sortedcontainers import SortedDict, SortedKeyList, SortedSet
 
 from jubeatools import __version__
+from jubeatools.formats.filetypes import ChartFile, JubeatFile
 from jubeatools.song import (
     BeatsTime,
     Chart,
@@ -86,7 +88,7 @@ class MonoColumnDumpedSection:
     symbols: Dict[BeatsTime, str] = field(default_factory=dict)
     notes: List[TapNote] = field(default_factory=list)
 
-    def render(self, circle_free: bool = False,) -> str:
+    def render(self, circle_free: bool = False) -> str:
         blocs = []
         commands = list(self._dump_commands())
         if commands:
@@ -135,7 +137,6 @@ class MonoColumnDumpedSection:
                         frame[pos] = arrow
                     else:
                         frame[pos] = line
-
             elif isinstance(note, TapNote):
                 if note.position in frame:
                     frames.append(frame)
@@ -143,7 +144,6 @@ class MonoColumnDumpedSection:
                 time_in_section = note.time - self.current_beat
                 symbol = self.symbols[time_in_section]
                 frame[note.position] = symbol
-
             elif isinstance(note, LongNoteEnd):
                 if note.position in frame:
                     frames.append(frame)
@@ -219,16 +219,22 @@ def _raise_if_unfit_for_mono_column(
 
 
 def _dump_mono_column_chart(
-    difficulty: str, chart: Chart, metadata: Metadata, timing: Timing
+    difficulty: str,
+    chart: Chart,
+    metadata: Metadata,
+    timing: Timing,
+    circle_free: bool = False,
 ) -> StringIO:
 
-    _raise_if_unfit_for_mono_column(chart, timing)
+    _raise_if_unfit_for_mono_column(chart, timing, circle_free)
 
     timing_events = sorted(timing.events, key=lambda e: e.time)
     notes = SortedKeyList(set(chart.notes), key=lambda n: n.time)
+
     for note in chart.notes:
         if isinstance(note, LongNote):
             notes.add(LongNoteEnd(note.time + note.duration, note.position))
+
     all_events = SortedKeyList(timing_events + notes, key=lambda n: n.time)
     last_event = all_events[-1]
     last_measure = last_event.time // 4
@@ -236,6 +242,7 @@ def _dump_mono_column_chart(
     for i in range(last_measure + 1):
         beat = BeatsTime(4) * i
         sections.add_section(beat)
+
     header = sections[0].commands
     header["o"] = int(timing.beat_zero_offset * 1000)
     header["lev"] = int(chart.level)
@@ -264,6 +271,7 @@ def _dump_mono_column_chart(
             sections[key].commands["b"] = 4
         else:
             sections[key].commands["b"] = fraction_to_decimal(next_key - key)
+
     # Then, trim all the redundant b=…
     last_b = 4
     for section in sections.values():
@@ -272,11 +280,13 @@ def _dump_mono_column_chart(
             del section.commands["b"]
         else:
             last_b = current_b
+
     # Fill sections with notes
     for key, next_key in windowed(chain(sections.keys(), [None]), 2):
         sections[key].notes = list(
             notes.irange_key(min_key=key, max_key=next_key, inclusive=(True, False))
         )
+
     # Define extra symbols
     existing_symbols = deepcopy(BEATS_TIME_TO_SYMBOL)
     extra_symbols = iter(DEFAULT_EXTRA_SYMBOLS)
@@ -296,16 +306,30 @@ def _dump_mono_column_chart(
     file.write(f"// Converted using jubeatools {__version__}\n")
     file.write(f"// https://github.com/Stepland/jubeatools\n\n")
     for section_start, section in sections.items():
-        file.write(section.render() + "\n")
+        file.write(section.render(circle_free) + "\n")
 
     return file
 
 
-def dump_mono_column(song: Song) -> Dict[str, IO]:
-    files = {}
-    for difname, chart in song.charts.items():
-        filename = f"{song.metadata.title} [{difname}].txt"
-        files[filename] = _dump_mono_column_chart(
-            difname, chart, song.metadata, chart.timing or song.global_timing,
+def _dump_mono_column_internal(
+    song: Song, circle_free: bool = False
+) -> List[JubeatFile]:
+    files = []
+    for difficulty, chart in song.charts.items():
+        contents = _dump_mono_column_chart(
+            difficulty,
+            chart,
+            song.metadata,
+            chart.timing or song.global_timing,
+            circle_free,
         )
+        files.append(ChartFile(contents, song, difficulty, chart))
+
     return files
+
+
+def dump_mono_column(
+    song: Song, circle_free: bool, folder: Path, name_pattern: str = None
+):
+    if not folder.isdir():
+        raise ValueError(f"{folder} is not a directory")

jubeatools/formats/jubeat_analyser/mono_column/load.py

@@ -5,9 +5,9 @@ from copy import deepcopy
 from dataclasses import dataclass
 from decimal import Decimal
 from enum import Enum
+from functools import reduce
 from itertools import product
 from typing import Dict, Iterator, List, Set, Tuple
-from functools import reduce
 
 import constraint
 from parsimonious import Grammar, NodeVisitor, ParseError
@@ -29,7 +29,18 @@ from jubeatools.song import (
 
 from ..command import is_command, parse_command
 from ..files import load_files
-from ..parser import JubeatAnalyserParser
+from ..parser import (
+    CIRCLE_FREE_TO_DECIMAL_TIME,
+    LONG_ARROWS,
+    LONG_DIRECTION,
+    JubeatAnalyserParser,
+    UnfinishedLongNote,
+    decimal_to_beats,
+    is_empty_line,
+    is_simple_solution,
+    long_note_solution_heuristic,
+    split_double_byte_line,
+)
 from ..symbol_definition import is_symbol_definition, parse_symbol_definition
 from ..symbols import CIRCLE_FREE_SYMBOLS, NOTE_SYMBOLS
 
@@ -74,13 +85,6 @@ def is_separator(line: str) -> bool:
     return bool(SEPARATOR.match(line))
 
 
-EMPTY_LINE = re.compile(r"\s*(//.*)?")
-
-
-def is_empty_line(line: str) -> bool:
-    return bool(EMPTY_LINE.match(line))
-
-
 DIFFICULTIES = {1: "BSC", 2: "ADV", 3: "EXT"}
 
 SYMBOL_TO_DECIMAL_TIME = {
@@ -88,23 +92,6 @@ SYMBOL_TO_DECIMAL_TIME = {
 }
 
 
-def split_chart_line(line: str) -> List[str]:
-    """Split a #bpp=2 chart line into symbols :
-    Given the symbol definition : *25:6
-    >>> split_chart_line("25口口25")
-    ... ["25","口","口","25"]
-    >>> split_chart_line("口⑪①25")
-    ... ["口","⑪","①","25"]
-    """
-    encoded_line = line.encode("shift_jis_2004")
-    if len(encoded_line) % 2 != 0:
-        raise ValueError(f"Invalid chart line : {line}")
-    symbols = []
-    for i in range(0, len(encoded_line), 2):
-        symbols.append(encoded_line[i : i + 2].decode("shift_jis_2004"))
-    return symbols
-
-
 @dataclass
 class MonoColumnLoadedSection:
     """
@@ -125,7 +112,7 @@ class MonoColumnLoadedSection:
         if bpp not in (1, 2):
             raise ValueError(f"Invalid bpp : {bpp}")
         elif bpp == 2:
-            split_line = split_chart_line
+            split_line = split_double_byte_line
         else:
             split_line = lambda l: list(l)
 
@@ -133,87 +120,6 @@ class MonoColumnLoadedSection:
             yield [split_line(self.chart_lines[i + j]) for j in range(4)]
 
 
-@dataclass(frozen=True)
-class UnfinishedLongNote:
-    time: BeatsTime
-    position: NotePosition
-    tail_tip: NotePosition
-
-    def ends_at(self, end: BeatsTime) -> LongNote:
-        if end < self.time:
-            raise ValueError(
-                f"Invalid end time ({end}) for long note starting at {self.time}"
-            )
-        return LongNote(
-            time=self.time,
-            position=self.position,
-            duration=end - self.time,
-            tail_tip=self.tail_tip,
-        )
-
-
-LONG_ARROW_RIGHT = {
-    ">",  # U+003E : GREATER-THAN SIGN
-    "＞",  # U+FF1E : FULLWIDTH GREATER-THAN SIGN
-}
-
-LONG_ARROW_LEFT = {
-    "<",  # U+003C : LESS-THAN SIGN
-    "＜",  # U+FF1C : FULLWIDTH LESS-THAN SIGN
-}
-
-LONG_ARROW_DOWN = {
-    "V",  # U+0056 : LATIN CAPITAL LETTER V
-    "v",  # U+0076 : LATIN SMALL LETTER V
-    "Ⅴ",  # U+2164 : ROMAN NUMERAL FIVE
-    "ⅴ",  # U+2174 : SMALL ROMAN NUMERAL FIVE
-    "∨",  # U+2228 : LOGICAL OR
-    "Ｖ",  # U+FF36 : FULLWIDTH LATIN CAPITAL LETTER V
-    "ｖ",  # U+FF56 : FULLWIDTH LATIN SMALL LETTER V
-}
-
-LONG_ARROW_UP = {
-    "^",  # U+005E : CIRCUMFLEX ACCENT
-    "∧",  # U+2227 : LOGICAL AND
-}
-
-LONG_ARROWS = LONG_ARROW_LEFT | LONG_ARROW_DOWN | LONG_ARROW_UP | LONG_ARROW_RIGHT
-
-LONG_DIRECTION = {
-    **{c: (1, 0) for c in LONG_ARROW_RIGHT},
-    **{c: (-1, 0) for c in LONG_ARROW_LEFT},
-    **{c: (0, 1) for c in LONG_ARROW_DOWN},
-    **{c: (0, -1) for c in LONG_ARROW_UP},
-}
-
-CIRCLE_FREE_TO_DECIMAL_TIME = {
-    c: Decimal("0.25") * i for i, c in enumerate(CIRCLE_FREE_SYMBOLS)
-}
-
-
-def _distance(a: NotePosition, b: NotePosition) -> float:
-    return abs(complex(*a.as_tuple()) - complex(*b.as_tuple()))
-
-
-def _long_note_solution_heuristic(
-    solution: Dict[NotePosition, NotePosition]
-) -> Tuple[int, int, int]:
-    c = Counter(int(_distance(k, v)) for k, v in solution.items())
-    return (c[3], c[2], c[1])
-
-
-def _is_simple_solution(solution, domains) -> bool:
-    return all(
-        solution[v] == min(domains[v], key=lambda e: _distance(e, v))
-        for v in solution.keys()
-    )
-
-
-def decimal_to_beats(current_beat: Decimal, symbol_timing: Decimal) -> BeatsTime:
-    decimal_time = current_beat + symbol_timing
-    return BeatsTime(decimal_time).limit_denominator(240)
-
-
 class MonoColumnParser(JubeatAnalyserParser):
     def __init__(self):
         super().__init__()
@@ -230,7 +136,7 @@ class MonoColumnParser(JubeatAnalyserParser):
                 "jubeatools does not handle changing the bytes per panel value halfway"
             )
         else:
-            super().do_bpp(value)
+            self._do_bpp(value)
 
     def move_to_next_section(self):
         if len(self.current_chart_lines) % 4 != 0:
@@ -245,7 +151,7 @@ class MonoColumnParser(JubeatAnalyserParser):
                 )
             )
             self.current_chart_lines = []
-            self.current_beat += self.beats_per_section
+            self.section_starting_beat += self.beats_per_section
 
     def append_chart_line(self, line: str):
         if self.bytes_per_panel == 1 and len(line) != 4:
@@ -267,7 +173,7 @@ class MonoColumnParser(JubeatAnalyserParser):
             self.append_chart_line(chart_line)
         elif is_separator(line):
             self.move_to_next_section()
-        elif not is_empty_line(line):
+        elif not (is_empty_line(line) or self.is_short_line(line)):
             raise SyntaxError(f"not a valid mono-column file line : {line}")
 
     def notes(self) -> Iterator[Union[TapNote, LongNote]]:
@@ -279,15 +185,15 @@ class MonoColumnParser(JubeatAnalyserParser):
     def _iter_blocs(
         self,
     ) -> Iterator[Tuple[Decimal, MonoColumnLoadedSection, List[List[str]]]]:
-        current_beat = Decimal(0)
+        section_starting_beat = Decimal(0)
         for section in self.sections:
             for bloc in section.blocs():
-                yield current_beat, section, bloc
-            current_beat += section.length
+                yield section_starting_beat, section, bloc
+            section_starting_beat += section.length
 
     def _iter_notes(self) -> Iterator[Union[TapNote, LongNote]]:
         unfinished_longs: Dict[NotePosition, UnfinishedLongNote] = {}
-        for current_beat, section, bloc in self._iter_blocs():
+        for section_starting_beat, section, bloc in self._iter_blocs():
             should_skip: Set[NotePosition] = set()
 
             # 1/3 : look for ends to unfinished long notes
@@ -298,7 +204,7 @@ class MonoColumnParser(JubeatAnalyserParser):
                     if symbol in CIRCLE_FREE_SYMBOLS:
                         should_skip.add(pos)
                         symbol_time = CIRCLE_FREE_TO_DECIMAL_TIME[symbol]
-                        note_time = decimal_to_beats(current_beat, symbol_time)
+                        note_time = decimal_to_beats(section_starting_beat + symbol_time)
                         yield unfinished_long.ends_at(note_time)
                     elif symbol in section.symbols:
                         raise SyntaxError(
@@ -309,7 +215,7 @@ class MonoColumnParser(JubeatAnalyserParser):
                     if symbol in section.symbols:
                         should_skip.add(pos)
                         symbol_time = section.symbols[symbol]
-                        note_time = decimal_to_beats(current_beat, symbol_time)
+                        note_time = decimal_to_beats(section_starting_beat + symbol_time)
                         yield unfinished_long.ends_at(note_time)
 
             unfinished_longs = {
@@ -354,8 +260,8 @@ class MonoColumnParser(JubeatAnalyserParser):
                         "Invalid long note arrow pattern in bloc :\n"
                         + "\n".join("".join(line) for line in bloc)
                     )
-                solution = min(solutions, key=_long_note_solution_heuristic)
-                if len(solutions) > 1 and not _is_simple_solution(
+                solution = min(solutions, key=long_note_solution_heuristic)
+                if len(solutions) > 1 and not is_simple_solution(
                     solution, arrow_to_note_candidates
                 ):
                     warnings.warn(
@@ -369,7 +275,7 @@ class MonoColumnParser(JubeatAnalyserParser):
                     should_skip.add(note_pos)
                     symbol = bloc[note_pos.y][note_pos.x]
                     symbol_time = section.symbols[symbol]
-                    note_time = decimal_to_beats(current_beat, symbol_time)
+                    note_time = decimal_to_beats(section_starting_beat + symbol_time)
                     unfinished_longs[note_pos] = UnfinishedLongNote(
                         time=note_time, position=note_pos, tail_tip=arrow_pos,
                     )
@@ -382,36 +288,33 @@ class MonoColumnParser(JubeatAnalyserParser):
                 symbol = bloc[y][x]
                 if symbol in section.symbols:
                     symbol_time = section.symbols[symbol]
-                    note_time = decimal_to_beats(current_beat, symbol_time)
+                    note_time = decimal_to_beats(section_starting_beat + symbol_time)
                     yield TapNote(note_time, position)
 
     def _iter_notes_without_longs(self) -> Iterator[TapNote]:
-        current_beat = Decimal(0)
+        section_starting_beat = Decimal(0)
         for section in self.sections:
             for bloc, y, x in product(section.blocs(), range(4), range(4)):
                 symbol = bloc[y][x]
                 if symbol in section.symbols:
                     symbol_time = section.symbols[symbol]
-                    note_time = decimal_to_beats(current_beat, symbol_time)
+                    note_time = decimal_to_beats(section_starting_beat + symbol_time)
                     position = NotePosition(x, y)
                     yield TapNote(note_time, position)
-            current_beat += section.length
+            section_starting_beat += section.length
 
 
 def load_mono_column(path: Path) -> Song:
     files = load_files(path)
-    charts = [
-        _load_mono_column_file(lines)
-        for _, lines in files.items()
-    ]
+    charts = [_load_mono_column_file(lines) for _, lines in files.items()]
     return reduce(lambda a, b: a.merge(b), charts)
-    
+
 
 def _load_mono_column_file(lines: List[str]) -> Song:
-    state = MonoColumnParser()
+    parser = MonoColumnParser()
     for i, raw_line in enumerate(lines):
         try:
-            state.load_line(raw_line)
+            parser.load_line(raw_line)
         except Exception as e:
             raise SyntaxError(
                 f"Error while parsing mono column line {i} :\n"
@@ -419,20 +322,23 @@ def _load_mono_column_file(lines: List[str]) -> Song:
             ) from None
 
     metadata = Metadata(
-        title=state.title, artist=state.artist, audio=state.music, cover=state.jacket
+        title=parser.title,
+        artist=parser.artist,
+        audio=parser.music,
+        cover=parser.jacket,
     )
-    if state.preview_start is not None:
-        metadata.preview_start = SecondsTime(state.preview_start) / 1000
+    if parser.preview_start is not None:
+        metadata.preview_start = SecondsTime(parser.preview_start) / 1000
         metadata.preview_length = SecondsTime(10)
 
     timing = Timing(
-        events=state.timing_events, beat_zero_offset=SecondsTime(state.offset) / 1000
+        events=parser.timing_events, beat_zero_offset=SecondsTime(parser.offset) / 1000
     )
     charts = {
-        state.difficulty: Chart(
-            level=state.level,
+        parser.difficulty: Chart(
+            level=parser.level,
             timing=timing,
-            notes=sorted(state.notes(), key=lambda n: (n.time, n.position)),
+            notes=sorted(parser.notes(), key=lambda n: (n.time, n.position)),
         )
     }
     return Song(metadata=metadata, charts=charts)

jubeatools/formats/jubeat_analyser/parser.py

@@ -1,21 +1,110 @@
-"""Base class and tools for the different parsers"""
+"""Collection of tools that are common to all the jubeat analyser formats"""
+import re
+from collections import Counter
 from copy import deepcopy
-from .symbols import NOTE_SYMBOLS
+from dataclasses import dataclass
 from decimal import Decimal
-from jubeatools.song import BPMEvent
+from typing import Dict, List, Tuple
+
+from jubeatools.song import BeatsTime, BPMEvent, LongNote, NotePosition
+
+from .symbols import (
+    CIRCLE_FREE_SYMBOLS,
+    LONG_ARROW_DOWN,
+    LONG_ARROW_LEFT,
+    LONG_ARROW_RIGHT,
+    LONG_ARROW_UP,
+    NOTE_SYMBOLS,
+)
 
 DIFFICULTIES = {1: "BSC", 2: "ADV", 3: "EXT"}
 
-SYMBOL_TO_DECIMAL_TIME = {
-    symbol: Decimal("0.25") * index for index, symbol in enumerate(NOTE_SYMBOLS)
+SYMBOL_TO_DECIMAL_TIME = {c: Decimal("0.25") * i for i, c in enumerate(NOTE_SYMBOLS)}
+
+CIRCLE_FREE_TO_DECIMAL_TIME = {
+    c: Decimal("0.25") * i for i, c in enumerate(CIRCLE_FREE_SYMBOLS)
 }
 
+LONG_ARROWS = LONG_ARROW_LEFT | LONG_ARROW_DOWN | LONG_ARROW_UP | LONG_ARROW_RIGHT
+
+LONG_DIRECTION = {
+    **{c: (1, 0) for c in LONG_ARROW_RIGHT},
+    **{c: (-1, 0) for c in LONG_ARROW_LEFT},
+    **{c: (0, 1) for c in LONG_ARROW_DOWN},
+    **{c: (0, -1) for c in LONG_ARROW_UP},
+}
+
+
+EMPTY_LINE = re.compile(r"\s*(//.*)?")
+
+
+def is_empty_line(line: str) -> bool:
+    return bool(EMPTY_LINE.match(line))
+
+
+def split_double_byte_line(line: str) -> List[str]:
+    """Split a #bpp=2 chart line into symbols.
+    For example, Assuming "25" was defined as a symbol earlier :
+    >>> split_chart_line("25口口25")
+    ... ["25","口","口","25"]
+    >>> split_chart_line("口⑪①25")
+    ... ["口","⑪","①","25"]
+    """
+    encoded_line = line.encode("shift_jis_2004")
+    if len(encoded_line) % 2 != 0:
+        raise ValueError(f"Invalid chart line : {line}")
+    symbols = []
+    for i in range(0, len(encoded_line), 2):
+        symbols.append(encoded_line[i : i + 2].decode("shift_jis_2004"))
+    return symbols
+
+
+def decimal_to_beats(decimal_time: Decimal) -> BeatsTime:
+    return BeatsTime(decimal_time).limit_denominator(240)
+
+
+def note_distance(a: NotePosition, b: NotePosition) -> float:
+    return abs(complex(*a.as_tuple()) - complex(*b.as_tuple()))
+
+
+def long_note_solution_heuristic(
+    solution: Dict[NotePosition, NotePosition]
+) -> Tuple[int, int, int]:
+    c = Counter(int(note_distance(k, v)) for k, v in solution.items())
+    return (c[3], c[2], c[1])
+
+
+def is_simple_solution(solution, domains) -> bool:
+    return all(
+        solution[v] == min(domains[v], key=lambda e: note_distance(e, v))
+        for v in solution.keys()
+    )
+
+
+@dataclass(frozen=True)
+class UnfinishedLongNote:
+    time: BeatsTime
+    position: NotePosition
+    tail_tip: NotePosition
+
+    def ends_at(self, end: BeatsTime) -> LongNote:
+        if end < self.time:
+            raise ValueError(
+                f"Invalid end time ({end}) for long note starting at {self.time}"
+            )
+        return LongNote(
+            time=self.time,
+            position=self.position,
+            duration=end - self.time,
+            tail_tip=self.tail_tip,
+        )
+
 
 class JubeatAnalyserParser:
     def __init__(self):
         self.music = None
         self.symbols = deepcopy(SYMBOL_TO_DECIMAL_TIME)
-        self.current_beat = Decimal("0")
+        self.section_starting_beat = Decimal("0")
         self.current_tempo = None
         self.current_chart_lines = []
         self.timing_events = []
@@ -47,7 +136,7 @@ class JubeatAnalyserParser:
 
     def do_t(self, value):
         self.current_tempo = Decimal(value)
-        self.timing_events.append(BPMEvent(self.current_beat, BPM=self.current_tempo))
+        self.timing_events.append(BPMEvent(self.section_starting_beat, BPM=self.current_tempo))
 
     def do_o(self, value):
         self.offset = int(value)
@@ -85,7 +174,7 @@ class JubeatAnalyserParser:
     def do_prevpos(self, value):
         self.preview_start = int(value)
 
-    def do_bpp(self, value):
+    def _do_bpp(self, value):
         bpp = int(value)
         if bpp not in (1, 2):
             raise ValueError(f"Unexcpected bpp value : {value}")
@@ -121,4 +210,7 @@ class JubeatAnalyserParser:
                 f"*{symbol}:{timing}"
             )
             raise ValueError(message)
-        self.symbols[symbol] = timing
+        self.symbols[symbol] = timing
+
+    def is_short_line(self, line: str) -> bool:
+        return len(line.encode("shift_jis_2004")) < self.bytes_per_panel * 4

jubeatools/formats/jubeat_analyser/symbols.py

@@ -1,4 +1,4 @@
-"""Usual symbols for memo files"""
+"""Common symbols for jubeat analyser files"""
 
 NOTE_SYMBOLS = [
     "①",
@@ -75,3 +75,28 @@ CIRCLE_FREE_SYMBOLS = [
     "19",  #  ⎪
     "20",  #  ⎭
 ]
+
+LONG_ARROW_RIGHT = {
+    ">",  # U+003E : GREATER-THAN SIGN
+    "＞",  # U+FF1E : FULLWIDTH GREATER-THAN SIGN
+}
+
+LONG_ARROW_LEFT = {
+    "<",  # U+003C : LESS-THAN SIGN
+    "＜",  # U+FF1C : FULLWIDTH LESS-THAN SIGN
+}
+
+LONG_ARROW_DOWN = {
+    "V",  # U+0056 : LATIN CAPITAL LETTER V
+    "v",  # U+0076 : LATIN SMALL LETTER V
+    "Ⅴ",  # U+2164 : ROMAN NUMERAL FIVE
+    "ⅴ",  # U+2174 : SMALL ROMAN NUMERAL FIVE
+    "∨",  # U+2228 : LOGICAL OR
+    "Ｖ",  # U+FF36 : FULLWIDTH LATIN CAPITAL LETTER V
+    "ｖ",  # U+FF56 : FULLWIDTH LATIN SMALL LETTER V
+}
+
+LONG_ARROW_UP = {
+    "^",  # U+005E : CIRCUMFLEX ACCENT
+    "∧",  # U+2227 : LOGICAL AND
+}

jubeatools/formats/jubeat_analyser/tests/test_memo.py

@@ -12,4 +12,4 @@ from ..mono_column.load import MonoColumnParser
 
 @given(notes_strat(NoteOption.LONGS))
 def test_many_notes(notes):
-    ...
+    ...

jubeatools/song.py

@@ -185,4 +185,3 @@ class Song:
             raise ValueError("Can't merge songs with differing global timings")
         global_timing = self.global_timing or other.global_timing
         return Song(self.metadata, charts, global_timing)
-