Improve readability of header processing code

- header -> headerPadding
- headerUnknown -> headerMetadata
- Add more thorough comments about the meaning of various header bytes

src/tja2fumen/constants.py

@@ -55,9 +55,12 @@ typeNotes = {v: k for k, v in noteTypes.items()}
 
 branchNames = ("normal", "advanced", "master")
 
-# Fumen headers are made up of smaller substrings of bytes
+# The first 432 bytes of fumen headers are made up of combinations of byte substrings
+# Commonly repeated byte substrings are listed below:
 byte_strings = {
+    # 6-byte substring
     'x00': b'\x00\x00\x00\x00\x00\x00',
+    # 12-byte substrings
     '431': b'43\xc8Ag&\x96B"\xe2\xd8B',
     '432': b'43\xc8Ag&\x96BD\x84\xb7B',
     '433': b'43\xc8A"\xe2\xd8B\x00@\xfaB',
@@ -67,34 +70,35 @@ byte_strings = {
     'V2': b'V\xd5&B"\xe2\xd8B\x00@\xfaB',
     'V3': b'V\xd5&B\x00@\xfaB\xf0\xce\rC',
 }
-
+# The simplest 432-byte headers are just 36 repeated copies of the specific 12-byte substrings below
 simpleHeaders = [b * 36 for b in [byte_strings['431'], byte_strings['V1'], byte_strings['V2']]]
 
-# Create a sample header pre-filled with known bytes
+# The next 80 bytes of fumen headers contain mostly 0s, with a few non-zero bytes thrown in
-unknownHeaderSample = [0] * 80
+sampleHeaderMetadata = [0] * 80
-# The following bytes are hardcoded by tja2fumen.exe (implying they have little/no effect on how the song is parsed)
+# The following bytes are hardcoded by tja2bin.exe (implying they have little/no effect on how the song is parsed)
-unknownHeaderSample[4] = 16
+sampleHeaderMetadata[4] = 16
-unknownHeaderSample[5] = 39
+sampleHeaderMetadata[5] = 39
-unknownHeaderSample[12] = 10
+sampleHeaderMetadata[12] = 10
-unknownHeaderSample[16] = 8
+sampleHeaderMetadata[16] = 8
-unknownHeaderSample[21] = 255
+sampleHeaderMetadata[21] = 255
-unknownHeaderSample[22] = 255
+sampleHeaderMetadata[22] = 255
-unknownHeaderSample[23] = 255
+sampleHeaderMetadata[23] = 255
-unknownHeaderSample[26] = 1
+sampleHeaderMetadata[26] = 1
-unknownHeaderSample[30] = 1
+sampleHeaderMetadata[30] = 1
-unknownHeaderSample[34] = 1
+sampleHeaderMetadata[34] = 1
-unknownHeaderSample[36] = 20
+sampleHeaderMetadata[36] = 20
-unknownHeaderSample[40] = 10
+sampleHeaderMetadata[40] = 10
-unknownHeaderSample[48] = 1
+sampleHeaderMetadata[48] = 1
-unknownHeaderSample[52] = 20
+sampleHeaderMetadata[52] = 20
-unknownHeaderSample[56] = 10
+sampleHeaderMetadata[56] = 10
-unknownHeaderSample[60] = 1
+sampleHeaderMetadata[60] = 1
-unknownHeaderSample[64] = 30
+sampleHeaderMetadata[64] = 30
-unknownHeaderSample[68] = 30
+sampleHeaderMetadata[68] = 30
-unknownHeaderSample[72] = 20
+sampleHeaderMetadata[72] = 20
-unknownHeaderSample[76] = 78
+sampleHeaderMetadata[76] = 78
-unknownHeaderSample[77] = 97
+sampleHeaderMetadata[77] = 97
-unknownHeaderSample[78] = 188
+sampleHeaderMetadata[78] = 188
+# Certain other bytes (8+9, 20) will need to be filled in on a song-by-song basis
 
 NORMALIZE_COURSE = {
     '0': 'Easy',

src/tja2fumen/converters.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 
 from tja2fumen.utils import computeSoulGaugeByte
-from tja2fumen.constants import TJA_NOTE_TYPES, DIFFICULTY_BYTES, unknownHeaderSample
+from tja2fumen.constants import TJA_NOTE_TYPES, DIFFICULTY_BYTES, sampleHeaderMetadata, simpleHeaders
 
 # Filler metadata that the `writeFumen` function expects
 # TODO: Determine how to properly set the item byte (https://github.com/vivaria/tja2fumen/issues/17)
@@ -210,11 +210,12 @@ def convertTJAToFumen(tja):
         total_notes += note_counter
 
     # Take a stock header metadata sample and add song-specific metadata
-    headerMetadata = unknownHeaderSample
+    headerMetadata = sampleHeaderMetadata
     headerMetadata[8] = DIFFICULTY_BYTES[tja['metadata']['course']][0]
     headerMetadata[9] = DIFFICULTY_BYTES[tja['metadata']['course']][1]
     headerMetadata[20] = computeSoulGaugeByte(total_notes)
-    tjaConverted['headerUnknown'] = b"".join(i.to_bytes(1, 'little') for i in headerMetadata)
+    tjaConverted['headerMetadata'] = b"".join(i.to_bytes(1, 'little') for i in headerMetadata)
+    tjaConverted['headerPadding'] = simpleHeaders[0]  # Use a basic, known set of header bytes
     tjaConverted['order'] = '<'
     tjaConverted['length'] = len(tjaConverted['measures'])
     tjaConverted['unknownMetadata'] = 0

src/tja2fumen/parsers.py

@@ -316,14 +316,16 @@ def readFumen(fumenFile, byteOrder=None, debug=False):
         else:
             order = "<"
             totalMeasures = measuresLittle
-    unknownMetadata = readStruct(file, order, format_string="I", seek=0x204)[0]
 
     # Initialize the dict that will contain the chart information
     song = {'measures': []}
-    song['header'] = fumenHeader
+    song['headerPadding'] = fumenHeader
-    song['headerUnknown'] = fumenHeaderUnknown
+    song['headerMetadata'] = fumenHeaderUnknown
     song['order'] = order
     song["length"] = totalMeasures
+
+    # I am unsure what byte this represents
+    unknownMetadata = readStruct(file, order, format_string="I", seek=0x204)[0]
     song["unknownMetadata"] = unknownMetadata
 
     # Determine whether the song has branches from byte 0x1b0 (decimal 432)

src/tja2fumen/writers.py

@@ -1,5 +1,5 @@
 from tja2fumen.utils import writeStruct, putBool
-from tja2fumen.constants import simpleHeaders, branchNames, typeNotes
+from tja2fumen.constants import branchNames, typeNotes
 
 
 def writeFumen(file, song):
@@ -7,8 +7,8 @@ def writeFumen(file, song):
     order = song['order']
 
     # Write the header
-    file.write(simpleHeaders[0])       # Write known, valid header
+    file.write(song['headerPadding'])   # Write header padding bytes
-    file.write(song['headerUnknown'])  # Write unknown header
+    file.write(song['headerMetadata'])  # Write header metadata bytes
 
     # Preallocate space in the file
     len_metadata = 8

testing/test_songs.py

@@ -61,8 +61,8 @@ def test_converted_tja_vs_cached_fumen(id_song, tmp_path):
         co_song = readFumen(path_out)
         ca_song = readFumen(os.path.join(path_bin, os.path.basename(path_out)))
         # 1. Check song headers
-        assert_song_property(co_song, ca_song, 'header', func=len)
+        assert_song_property(co_song, ca_song, 'headerPadding', func=len)
-        assert_song_property(co_song, ca_song, 'headerUnknown', func=len)
+        assert_song_property(co_song, ca_song, 'headerMetadata', func=len)
         # 2. Check song metadata
         assert_song_property(co_song, ca_song, 'order')
         assert_song_property(co_song, ca_song, 'branches')