diff --git a/bemani/format/afp.py b/bemani/format/afp.py index 15873ef..ceabc5a 100644 --- a/bemani/format/afp.py +++ b/bemani/format/afp.py @@ -1,5 +1,6 @@ import io from hashlib import md5 +import os import struct import sys from PIL import Image # type: ignore @@ -98,6 +99,17 @@ class TextureRegion: 'bottom': self.bottom, } + def __repr__(self) -> str: + return ( + f"texture: {self.textureno}, " + + f"left: {self.left / 2}, " + + f"top: {self.top / 2}, " + + f"right: {self.right / 2}, " + + f"bottom: {self.bottom / 2}, " + + f"width: {(self.right - self.left) / 2}, " + + f"height: {(self.bottom - self.top) / 2}" + ) + class Matrix: def __init__(self, a: float, b: float, c: float, d: float, tx: float, ty: float) -> None: @@ -112,6 +124,9 @@ class Matrix: def identity() -> "Matrix": return Matrix(1.0, 0.0, 0.0, 1.0, 0.0, 0.0) + def __repr__(self) -> str: + return f"a: {round(self.a, 5)}, b: {round(self.b, 5)}, c: {round(self.c, 5)}, d: {round(self.d, 5)}, tx: {round(self.tx, 5)}, ty: {round(self.ty, 5)}" + class Color: def __init__(self, r: float, g: float, b: float, a: float) -> None: @@ -120,8 +135,32 @@ class Color: self.b = b self.a = a + def as_dict(self) -> Dict[str, Any]: + return { + 'r': self.r, + 'g': self.g, + 'b': self.b, + 'a': self.a, + } + def __repr__(self) -> str: - return f"{round(self.r, 5)}, {round(self.g, 5)}, {round(self.b, 5)}, {round(self.a, 5)}" + return f"r: {round(self.r, 5)}, g: {round(self.g, 5)}, b: {round(self.b, 5)}, a: {round(self.a, 5)}" + + +class Point: + def __init__(self, x: float, y: float) -> None: + self.x = x + self.y = y + + def as_dict(self) -> Dict[str, Any]: + return { + 'x': self.x, + 'y': self.y, + } + + def __repr__(self) -> str: + return f"x: {round(self.x, 5)}, y: {round(self.y, 5)}" + class Tag: END = 0x0 @@ -874,7 +913,7 @@ class SWF: add_coverage(28, 4) if flags & 0x2: - # I think this is FPS given the output of this bit of code. + # FPS can be either an integer or a float. fps = struct.unpack(" None: + self.flags = flags + self.region = region + self.vertexes = vertexes + self.blend = blend + + def as_dict(self) -> Dict[str, Any]: + return { + 'flags': self.flags, + 'region': self.region, + 'vertexes': self.vertexes, + 'blend': self.blend.as_dict() if self.blend else None, + } + + def __repr__(self) -> str: + flagbits: List[str] = [] + if self.flags & 0x1: + flagbits.append("(Instantiable)") + if self.flags & 0x2: + flagbits.append("(Includes Texture)") + if self.flags & 0x8: + flagbits.append("(Includes Blend Color)") + if self.flags & 0x40: + flagbits.append("(Needs Tex Point Normalization)") + + flagspart = f"flags: {hex(self.flags)} {' '.join(flagbits)}" + if self.flags & 0x2: + texpart = f", region: {self.region}, vertexes: {', '.join(str(x) for x in self.vertexes)}" + else: + texpart = "" + + if self.flags & 0x8: + blendpart = f", blend: {self.blend}" + else: + blendpart = "" + + return f"{flagspart}{texpart}{blendpart}" + + class Shape: def __init__( self, @@ -1002,12 +1087,144 @@ class Shape: self.name = name self.data = data + # Rectangle points outlining this shape. + self.rect_points: List[Point] = [] + + # Texture points, as used alongside vertex chunks when the shape contains a texture. + self.tex_points: List[Point] = [] + + # Actual shape drawing parameters. + self.draw_params: List[DrawParams] = [] + def as_dict(self) -> Dict[str, Any]: return { 'name': self.name, - 'data': "".join(_hex(x) for x in self.data), + 'rect_points': [p.as_dict() for p in self.rect_points], + 'tex_points': [p.as_dict() for p in self.tex_points], + 'draw_params': [d.as_dict() for d in self.draw_params], } + def __repr__(self) -> str: + return os.linesep.join([ + *[f"rect point: {rect}" for rect in self.rect_points], + *[f"tex point: {tex}" for tex in self.tex_points], + *[f"draw params: {params}" for params in self.draw_params], + ]) + + def get_until_null(self, offset: int) -> bytes: + out = b"" + while self.data[offset] != 0: + out += self.data[offset:(offset + 1)] + offset += 1 + return out + + def parse(self, text_obfuscated: bool = True) -> None: + # First, grab the header bytes. + magic = self.data[0:4] + + if magic == b"D2EG": + endian = "<" + elif magic == b"GE2D": + endian = ">" + else: + raise Exception("Invalid magic value in GE2D structure!") + + filesize = struct.unpack(f"{endian}I", self.data[12:16])[0] + if filesize != len(self.data): + raise Exception("Unexpected file size for GE2D structure!") + + rect_count, tex_count, unk1_count, label_count, render_params_count, _ = struct.unpack( + f"{endian}HHHHHH", + self.data[20:32], + ) + + rect_offset, tex_offset, unk1_offset, label_offset, render_params_offset = struct.unpack( + f"{endian}IIIII", + self.data[32:52], + ) + + rect_points: List[Point] = [] + if rect_offset != 0: + for rectno in range(rect_count): + rectno_offset = rect_offset + (8 * rectno) + x, y = struct.unpack(f"{endian}ff", self.data[rectno_offset:rectno_offset + 8]) + rect_points.append(Point(x, y)) + self.rect_points = rect_points + + tex_points: List[Point] = [] + if tex_offset != 0: + for texno in range(tex_count): + texno_offset = tex_offset + (8 * texno) + x, y = struct.unpack(f"{endian}ff", self.data[texno_offset:texno_offset + 8]) + tex_points.append(Point(x, y)) + self.tex_points = tex_points + + if unk1_offset != 0: + raise Exception("Unknown offset pointer data present!") + + labels: List[str] = [] + if label_offset != 0: + for labelno in range(label_count): + labelno_offset = label_offset + (4 * labelno) + labelptr = struct.unpack(f"{endian}I", self.data[labelno_offset:labelno_offset + 4])[0] + + bytedata = self.get_until_null(labelptr) + labels.append(AFPFile.descramble_text(bytedata, text_obfuscated)) + + draw_params: List[DrawParams] = [] + if render_params_offset != 0: + # The actual render parameters for the shape. This dictates how the texture values + # are used when drawing shapes, whether to use a blend value or draw a primitive, etc. + for render_paramsno in range(render_params_count): + render_paramsno_offset = render_params_offset + (16 * render_paramsno) + points, flags, label, _, trianglecount, _, rgba, triangleoffset = struct.unpack( + f"{endian}BBBBHHII", + self.data[(render_paramsno_offset):(render_paramsno_offset + 16)] + ) + + if points != 4: + raise Exception("Unexpected number of points in GE2D structure!") + if (flags & 0x2) and len(labels) == 0: + raise Exception("GE2D structure has a texture, but no region labels present!") + + color = Color( + r=(rgba & 0xFF) / 255.0, + g=((rgba >> 8) & 0xFF) / 255.0, + b=((rgba >> 16) & 0xFF) / 255.0, + a=((rgba >> 24) & 0xFF) / 255.0, + ) + + verticies: List[int] = [] + for render_paramstriangleno in range(trianglecount): + render_paramstriangleno_offset = triangleoffset + (2 * render_paramstriangleno) + tex_offset = struct.unpack(f"{endian}H", self.data[render_paramstriangleno_offset:(render_paramstriangleno_offset + 2)])[0] + verticies.append(tex_offset) + + # Seen bits are 0x1, 0x2, 0x8 so far. + # 0x1 Is a "this shape is instantiable/drawable" bit. + # 0x2 Is the shape having a texture. + # 0x8 Is "draw background color/blend" flag. + # 0x40 Is a "normalize texture coordinates" flag. It performs the below algorithm. + + if (flags & (0x2 | 0x40)) == (0x2 | 0x40): + # The tex offsets point at the tex vals parsed above, and are used in conjunction with + # texture/region metrics to calcuate some offsets. First, the region left/right/top/bottom + # is divided by 2 (looks like a scaling of 2 for regions to textures is hardcoded) and then + # divided by the texture width/height (as relevant). The returned metrics are in texture space + # where 0.0 is the origin and 1.0 is the furthest right/down. The metrics are then multiplied + # by the texture point pairs that appear above, meaning they should be treated as percentages. + pass + + draw_params.append( + DrawParams( + flags=flags, + region=labels[label] if (flags & 0x2) else None, + vertexes=verticies if (flags & 0x2) else [], + blend=color if (flags & 0x8) else None, + ) + ) + self.draw_params = draw_params + class Unknown1: def __init__( @@ -1320,7 +1537,7 @@ class AFPFile: if length != len(self.data): raise Exception(f"Invalid graphic file length, expecting {length} bytes!") - # I think that offset 16-20 are the file data offset, but I'm not sure? + # This is always the header length, or the offset of the data payload. header_length = struct.unpack(f"{self.endian}I", self.data[16:20])[0] add_coverage(16, 4) @@ -1435,8 +1652,6 @@ class AFPFile: # flags1 = (fmtflags >> 24) & 0xFF # flags2 = (fmtflags >> 16) & 0xFF - # These flags may have some significance, such as - # the unk3/unk4 possibly indicating texture doubling? # unk1 = 3 if (flags1 & 0xF == 1) else 1 # unk2 = 3 if ((flags1 >> 4) & 0xF == 1) else 1 # unk3 = 1 if (flags2 & 0xF == 1) else 2 @@ -1612,8 +1827,7 @@ class AFPFile: # Mapping between texture index and the name of the texture. if feature_mask & 0x02: - # Seems to be a structure that duplicates texture names? I am pretty - # sure this is used to map texture names to file indexes used elsewhere. + # Mapping of texture name to texture index. This is used by regions to look up textures. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 @@ -1653,18 +1867,19 @@ class AFPFile: if texture_no < 0 or texture_no >= len(self.texturemap.entries): raise Exception(f"Out of bounds texture {texture_no}") - vprint(f" length: 10, offset: {hex(offset + (10 * i))}") - # TODO: The offsets here seem to be off by a power of 2, there - # might be more flags in the above texture format that specify - # device scaling and such? - self.texture_to_region.append(TextureRegion(texture_no, left, top, right, bottom)) + # Texture regions are multiplied by a power of 2. Not sure why, but the games I + # looked at hardcode a divide by 2 when loading regions. + region = TextureRegion(texture_no, left, top, right, bottom) + self.texture_to_region.append(region) + + vprint(f" {region}, offset: {hex(descriptor_offset)}") else: vprint("Bit 0x000008 - regions; NOT PRESENT") if feature_mask & 0x10: # Names of the graphics regions, so we can look into the texture_to_region - # mapping above. + # mapping above. Used by shapes to find the right region offset given a name. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 @@ -1779,7 +1994,8 @@ class AFPFile: vprint("Bit 0x000400 - unknown; NOT PRESENT") if feature_mask & 0x800: - # This is the names of the SWF data as far as I can tell. + # SWF raw data that is loaded and passed to AFP core. It is equivalent to the + # afp files in an IFS container. length, offset = struct.unpack(f"{self.endian}II", self.data[header_offset:(header_offset + 8)]) add_coverage(header_offset, 8) header_offset += 8 @@ -1813,7 +2029,7 @@ class AFPFile: vprint("Bit 0x000800 - swfdata; NOT PRESENT") if feature_mask & 0x1000: - # Seems to be a secondary structure mirroring the above. + # A mapping structure that allows looking up SWF data by name. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 @@ -1826,16 +2042,15 @@ class AFPFile: vprint("Bit 0x001000 - swfmapping; NOT PRESENT") if feature_mask & 0x2000: - # I am making a very preliminary guess that these are shapes used along - # with SWF data specified below. The names in these sections tend to - # have the word "shape" in them. + # These are shapes as used with the SWF data above. They contain mappings between a + # loaded texture shape and the region that contains data. They are equivalent to the + # geo files found in an IFS container. length, offset = struct.unpack(f"{self.endian}II", self.data[header_offset:(header_offset + 8)]) add_coverage(header_offset, 8) header_offset += 8 vprint(f"Bit 0x002000 - shapes; count: {length}, offset: {hex(offset)}") - # TODO: We do a LOT of extra stuff with this one, if count > 0... for x in range(length): shape_base_offset = offset + (x * 12) if shape_base_offset != 0: @@ -1845,85 +2060,32 @@ class AFPFile: ) add_coverage(shape_base_offset, 12) - # TODO: At the shape offset is a "D2EG" structure of some sort. - # I have no idea what these do. I would have to look into it - # more if its important. - if name_offset != 0: # Let's decode this until the first null. bytedata = self.get_until_null(name_offset) add_coverage(name_offset, len(bytedata) + 1, unique=False) name = AFPFile.descramble_text(bytedata, self.text_obfuscated) - vprint(f" {name}, length: {shape_length}, offset: {hex(shape_offset)}") + else: + name = "" if shape_offset != 0: - shape_data = self.data[shape_offset:(shape_offset + shape_length)] + shape = Shape( + name, + self.data[shape_offset:(shape_offset + shape_length)], + ) + shape.parse(text_obfuscated=self.text_obfuscated) + self.shapes.append(shape) add_coverage(shape_offset, shape_length) - magic, header1, header2, filesize, header3 = struct.unpack( - f"{self.endian}4sIIII", - shape_data[0:20], - ) + vprint(f" {name}, length: {shape_length}, offset: {hex(shape_offset)}") + for line in str(shape).split(os.linesep): + vprint(f" {line}") - if self.endian == "<" and magic != b"D2EG": - raise Exception("Invalid magic value in D2EG structure!") - if self.endian == ">" and magic != b"GE2D": - raise Exception("Invalid magic value in D2EG structure!") - if filesize != len(shape_data): - raise Exception("Unexpected file size for D2EG structure!") - - # Get width/height - endian = "<" if self.endian == ">" else ">" - width, height = struct.unpack(f"{endian}HH", shape_data[20:24]) - - header4, header5 = struct.unpack( - f"{self.endian}II", - shape_data[24:32], - ) - - rect_offset, tex_offset, unk1_offset, label_offset, unk2_offset = struct.unpack( - f"{self.endian}IIIII", - shape_data[32:52], - ) - - label = None - if label_offset != 0: - labelptr = struct.unpack(f"{self.endian}I", shape_data[label_offset:label_offset + 4])[0] - if labelptr is not None: - bytedata = self.get_until_null(shape_offset + labelptr) - label = AFPFile.descramble_text(bytedata, self.text_obfuscated) # NOQA: F841 - - if rect_offset != 0: - floats = struct.unpack( - f"{self.endian}ffffffff", - shape_data[(rect_offset):(rect_offset + 32)] - ) - _rect_offsets = [x for x in floats] # NOQA: F841 - if tex_offset != 0: - floats = struct.unpack( - f"{self.endian}ffffffff", - shape_data[(tex_offset):(tex_offset + 32)] - ) - tex_offsets = [] - for i, flt in enumerate(floats): - tex_offsets.append(flt * (width if ((i & 1) == 0) else height)) - if unk2_offset != 0: - test = struct.unpack( # NOQA: F841 - f"{endian}iii", - shape_data[(unk2_offset):(unk2_offset + 12)] - ) - - self.shapes.append( - Shape( - name, - self.data[shape_offset:(shape_offset + shape_length)], - ) - ) else: vprint("Bit 0x002000 - shapes; NOT PRESENT") if feature_mask & 0x4000: - # Seems to be a secondary section mirroring the names from above. + # Mapping so that shapes can be looked up by name to get their offset. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 @@ -1951,7 +2113,8 @@ class AFPFile: vprint("Bit 0x008000 - unknown; NOT PRESENT") if feature_mask & 0x10000: - # Included font package, BINXRPC encoded. + # Included font package, BINXRPC encoded. This is basically a texture sheet with an XML + # pointing at the region in the texture sheet for every renderable character. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 @@ -1977,8 +2140,8 @@ class AFPFile: vprint("Bit 0x010000 - fontinfo; NOT PRESENT") if feature_mask & 0x20000: - # I am beginning to suspect that this is SWF data/level data. I have - # no idea what "afp" is. Games refer to these as "afp streams". + # This is the byteswapping headers that allow us to byteswap the SWF data before passing it + # to AFP core. It is equivalent to the bsi files in an IFS container. offset = struct.unpack(f"{self.endian}I", self.data[header_offset:(header_offset + 4)])[0] add_coverage(header_offset, 4) header_offset += 4 diff --git a/bemani/utils/afputils.py b/bemani/utils/afputils.py index 550a34d..420710e 100644 --- a/bemani/utils/afputils.py +++ b/bemani/utils/afputils.py @@ -8,7 +8,7 @@ import textwrap from PIL import Image, ImageDraw # type: ignore from typing import Any, Dict -from bemani.format.afp import AFPFile, SWF +from bemani.format.afp import AFPFile, Shape, SWF def main() -> int: @@ -100,18 +100,31 @@ def main() -> int: help="Display verbuse debugging output", ) - parse_parser = subparsers.add_parser('parse', help='Parse a raw AFP/BSI file pair from an IFS container') - parse_parser.add_argument( + parseafp_parser = subparsers.add_parser('parseafp', help='Parse a raw AFP/BSI file pair extracted from an IFS container') + parseafp_parser.add_argument( "afp", metavar="AFPFILE", help="The AFP file to parse", ) - parse_parser.add_argument( + parseafp_parser.add_argument( "bsi", metavar="BSIFILE", help="The BSI file to parse", ) - parse_parser.add_argument( + parseafp_parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Display verbuse debugging output", + ) + + parsegeo_parser = subparsers.add_parser('parsegeo', help='Parse a raw GEO file extracted from an IFS container') + parsegeo_parser.add_argument( + "geo", + metavar="GEOFILE", + help="The GEO file to parse", + ) + parsegeo_parser.add_argument( "-v", "--verbose", action="store_true", @@ -331,7 +344,7 @@ def main() -> int: # Now, print it print(json.dumps(afpfile.as_dict(), sort_keys=True, indent=4)) - if args.action == "parse": + if args.action == "parseafp": # First, load the AFP and BSI files with open(args.afp, "rb") as bafp: with open(args.bsi, "rb") as bbsi: @@ -341,6 +354,17 @@ def main() -> int: swf.parse(verbose=args.verbose) print(json.dumps(swf.as_dict(), sort_keys=True, indent=4)) + if args.action == "parsegeo": + # First, load the AFP and BSI files + with open(args.geo, "rb") as bfp: + geo = Shape("", bfp.read()) + + # Now, print it + geo.parse() + if args.verbose: + print(geo, file=sys.stderr) + print(json.dumps(geo.as_dict(), sort_keys=True, indent=4)) + return 0