From 69af60e4e305573ecd41189c5013ba486da74dd3 Mon Sep 17 00:00:00 2001 From: Viv Date: Sun, 25 Jun 2023 13:04:22 -0400 Subject: [PATCH] Improve readability of header processing code - header -> headerPadding - headerUnknown -> headerMetadata - Add more thorough comments about the meaning of various header bytes --- src/tja2fumen/constants.py | 58 ++++++++++++++++++++----------------- src/tja2fumen/converters.py | 7 +++-- src/tja2fumen/parsers.py | 8 +++-- src/tja2fumen/writers.py | 6 ++-- testing/test_songs.py | 4 +-- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/src/tja2fumen/constants.py b/src/tja2fumen/constants.py index bb99974..eacda67 100644 --- a/src/tja2fumen/constants.py +++ b/src/tja2fumen/constants.py @@ -55,9 +55,12 @@ typeNotes = {v: k for k, v in noteTypes.items()} branchNames = ("normal", "advanced", "master") -# Fumen headers are made up of smaller substrings of bytes +# The first 432 bytes of fumen headers are made up of combinations of byte substrings +# Commonly repeated byte substrings are listed below: byte_strings = { + # 6-byte substring 'x00': b'\x00\x00\x00\x00\x00\x00', + # 12-byte substrings '431': b'43\xc8Ag&\x96B"\xe2\xd8B', '432': b'43\xc8Ag&\x96BD\x84\xb7B', '433': b'43\xc8A"\xe2\xd8B\x00@\xfaB', @@ -67,34 +70,35 @@ byte_strings = { 'V2': b'V\xd5&B"\xe2\xd8B\x00@\xfaB', 'V3': b'V\xd5&B\x00@\xfaB\xf0\xce\rC', } - +# The simplest 432-byte headers are just 36 repeated copies of the specific 12-byte substrings below simpleHeaders = [b * 36 for b in [byte_strings['431'], byte_strings['V1'], byte_strings['V2']]] -# Create a sample header pre-filled with known bytes -unknownHeaderSample = [0] * 80 -# The following bytes are hardcoded by tja2fumen.exe (implying they have little/no effect on how the song is parsed) -unknownHeaderSample[4] = 16 -unknownHeaderSample[5] = 39 -unknownHeaderSample[12] = 10 -unknownHeaderSample[16] = 8 -unknownHeaderSample[21] = 255 -unknownHeaderSample[22] = 255 -unknownHeaderSample[23] = 255 -unknownHeaderSample[26] = 1 -unknownHeaderSample[30] = 1 -unknownHeaderSample[34] = 1 -unknownHeaderSample[36] = 20 -unknownHeaderSample[40] = 10 -unknownHeaderSample[48] = 1 -unknownHeaderSample[52] = 20 -unknownHeaderSample[56] = 10 -unknownHeaderSample[60] = 1 -unknownHeaderSample[64] = 30 -unknownHeaderSample[68] = 30 -unknownHeaderSample[72] = 20 -unknownHeaderSample[76] = 78 -unknownHeaderSample[77] = 97 -unknownHeaderSample[78] = 188 +# The next 80 bytes of fumen headers contain mostly 0s, with a few non-zero bytes thrown in +sampleHeaderMetadata = [0] * 80 +# The following bytes are hardcoded by tja2bin.exe (implying they have little/no effect on how the song is parsed) +sampleHeaderMetadata[4] = 16 +sampleHeaderMetadata[5] = 39 +sampleHeaderMetadata[12] = 10 +sampleHeaderMetadata[16] = 8 +sampleHeaderMetadata[21] = 255 +sampleHeaderMetadata[22] = 255 +sampleHeaderMetadata[23] = 255 +sampleHeaderMetadata[26] = 1 +sampleHeaderMetadata[30] = 1 +sampleHeaderMetadata[34] = 1 +sampleHeaderMetadata[36] = 20 +sampleHeaderMetadata[40] = 10 +sampleHeaderMetadata[48] = 1 +sampleHeaderMetadata[52] = 20 +sampleHeaderMetadata[56] = 10 +sampleHeaderMetadata[60] = 1 +sampleHeaderMetadata[64] = 30 +sampleHeaderMetadata[68] = 30 +sampleHeaderMetadata[72] = 20 +sampleHeaderMetadata[76] = 78 +sampleHeaderMetadata[77] = 97 +sampleHeaderMetadata[78] = 188 +# Certain other bytes (8+9, 20) will need to be filled in on a song-by-song basis NORMALIZE_COURSE = { '0': 'Easy', diff --git a/src/tja2fumen/converters.py b/src/tja2fumen/converters.py index ef3adfe..07be507 100644 --- a/src/tja2fumen/converters.py +++ b/src/tja2fumen/converters.py @@ -1,7 +1,7 @@ from copy import deepcopy from tja2fumen.utils import computeSoulGaugeByte -from tja2fumen.constants import TJA_NOTE_TYPES, DIFFICULTY_BYTES, unknownHeaderSample +from tja2fumen.constants import TJA_NOTE_TYPES, DIFFICULTY_BYTES, sampleHeaderMetadata, simpleHeaders # Filler metadata that the `writeFumen` function expects # TODO: Determine how to properly set the item byte (https://github.com/vivaria/tja2fumen/issues/17) @@ -210,11 +210,12 @@ def convertTJAToFumen(tja): total_notes += note_counter # Take a stock header metadata sample and add song-specific metadata - headerMetadata = unknownHeaderSample + headerMetadata = sampleHeaderMetadata headerMetadata[8] = DIFFICULTY_BYTES[tja['metadata']['course']][0] headerMetadata[9] = DIFFICULTY_BYTES[tja['metadata']['course']][1] headerMetadata[20] = computeSoulGaugeByte(total_notes) - tjaConverted['headerUnknown'] = b"".join(i.to_bytes(1, 'little') for i in headerMetadata) + tjaConverted['headerMetadata'] = b"".join(i.to_bytes(1, 'little') for i in headerMetadata) + tjaConverted['headerPadding'] = simpleHeaders[0] # Use a basic, known set of header bytes tjaConverted['order'] = '<' tjaConverted['length'] = len(tjaConverted['measures']) tjaConverted['unknownMetadata'] = 0 diff --git a/src/tja2fumen/parsers.py b/src/tja2fumen/parsers.py index 7937f0c..fb2d51e 100644 --- a/src/tja2fumen/parsers.py +++ b/src/tja2fumen/parsers.py @@ -316,14 +316,16 @@ def readFumen(fumenFile, byteOrder=None, debug=False): else: order = "<" totalMeasures = measuresLittle - unknownMetadata = readStruct(file, order, format_string="I", seek=0x204)[0] # Initialize the dict that will contain the chart information song = {'measures': []} - song['header'] = fumenHeader - song['headerUnknown'] = fumenHeaderUnknown + song['headerPadding'] = fumenHeader + song['headerMetadata'] = fumenHeaderUnknown song['order'] = order song["length"] = totalMeasures + + # I am unsure what byte this represents + unknownMetadata = readStruct(file, order, format_string="I", seek=0x204)[0] song["unknownMetadata"] = unknownMetadata # Determine whether the song has branches from byte 0x1b0 (decimal 432) diff --git a/src/tja2fumen/writers.py b/src/tja2fumen/writers.py index 712ebcb..7c244cb 100644 --- a/src/tja2fumen/writers.py +++ b/src/tja2fumen/writers.py @@ -1,5 +1,5 @@ from tja2fumen.utils import writeStruct, putBool -from tja2fumen.constants import simpleHeaders, branchNames, typeNotes +from tja2fumen.constants import branchNames, typeNotes def writeFumen(file, song): @@ -7,8 +7,8 @@ def writeFumen(file, song): order = song['order'] # Write the header - file.write(simpleHeaders[0]) # Write known, valid header - file.write(song['headerUnknown']) # Write unknown header + file.write(song['headerPadding']) # Write header padding bytes + file.write(song['headerMetadata']) # Write header metadata bytes # Preallocate space in the file len_metadata = 8 diff --git a/testing/test_songs.py b/testing/test_songs.py index 07f6aec..8d62c43 100644 --- a/testing/test_songs.py +++ b/testing/test_songs.py @@ -61,8 +61,8 @@ def test_converted_tja_vs_cached_fumen(id_song, tmp_path): co_song = readFumen(path_out) ca_song = readFumen(os.path.join(path_bin, os.path.basename(path_out))) # 1. Check song headers - assert_song_property(co_song, ca_song, 'header', func=len) - assert_song_property(co_song, ca_song, 'headerUnknown', func=len) + assert_song_property(co_song, ca_song, 'headerPadding', func=len) + assert_song_property(co_song, ca_song, 'headerMetadata', func=len) # 2. Check song metadata assert_song_property(co_song, ca_song, 'order') assert_song_property(co_song, ca_song, 'branches')