From 56498f615426909775725e9e50bbe5953c82e112 Mon Sep 17 00:00:00 2001 From: Jennifer Taylor Date: Sun, 23 May 2021 20:37:18 +0000 Subject: [PATCH] Implement masking support. --- bemani/format/afp/blend.py | 57 ++++++++++++- bemani/format/afp/blendalt.pyi | 3 +- bemani/format/afp/blendalt.pyx | 17 +++- bemani/format/afp/blendaltimpl.cxx | 41 +++++++++ bemani/format/afp/render.py | 130 ++++++++++++++++++++++------- 5 files changed, 215 insertions(+), 33 deletions(-) diff --git a/bemani/format/afp/blend.py b/bemani/format/afp/blend.py index b54f3ad..2e823e5 100644 --- a/bemani/format/afp/blend.py +++ b/bemani/format/afp/blend.py @@ -1,7 +1,7 @@ import multiprocessing import signal from PIL import Image # type: ignore -from typing import Any, List, Sequence +from typing import Any, List, Optional, Sequence from .types.generic import Color, Matrix, Point @@ -122,6 +122,7 @@ except ImportError: add_color: Color, mult_color: Color, transform: Matrix, + mask: Optional[Image.Image], blendfunc: int, texture: Image.Image, single_threaded: bool = False, @@ -136,7 +137,7 @@ except ImportError: return img # Warn if we have an unsupported blend. - if blendfunc not in {0, 1, 2, 3, 8, 9, 70}: + if blendfunc not in {0, 1, 2, 3, 8, 9, 70, 256, 257}: print(f"WARNING: Unsupported blend {blendfunc}") return img @@ -168,6 +169,11 @@ except ImportError: # Get the data in an easier to manipulate and faster to update fashion. imgmap = list(img.getdata()) texmap = list(texture.getdata()) + if mask: + alpha = mask.split()[-1] + maskmap = alpha.tobytes('raw', 'L') + else: + maskmap = None # We don't have enough CPU cores to bother multiprocessing. for imgy in range(miny, maxy): @@ -185,12 +191,20 @@ except ImportError: # Blend it. texoff = texx + (texy * texwidth) + if maskmap is not None and maskmap[imgoff] == 0: + # This pixel is masked off! + continue imgmap[imgoff] = blend_point(add_color, mult_color, texmap[texoff], imgmap[imgoff], blendfunc) img.putdata(imgmap) else: imgbytes = img.tobytes('raw', 'RGBA') texbytes = texture.tobytes('raw', 'RGBA') + if mask: + alpha = mask.split()[-1] + maskbytes = alpha.tobytes('raw', 'L') + else: + maskbytes = None # Let's spread the load across multiple processors. procs: List[multiprocessing.Process] = [] @@ -223,6 +237,7 @@ except ImportError: blendfunc, imgbytes, texbytes, + maskbytes, ), ) procs.append(proc) @@ -256,6 +271,33 @@ except ImportError: img = Image.frombytes('RGBA', (imgwidth, imgheight), b''.join(lines)) return img + def blend_mask_create( + # RGBA color tuple representing what's already at the dest. + dest: Sequence[int], + # RGBA color tuple representing the source we want to blend to the dest. + src: Sequence[int], + ) -> Sequence[int]: + # Mask creating just allows a pixel to be drawn if the source image has a nonzero + # alpha, according to the SWF spec. + if src[3] != 0: + return (255, 0, 0, 255) + else: + return (0, 0, 0, 0) + + def blend_mask_combine( + # RGBA color tuple representing what's already at the dest. + dest: Sequence[int], + # RGBA color tuple representing the source we want to blend to the dest. + src: Sequence[int], + ) -> Sequence[int]: + # Mask blending just takes the source and destination and ands them together, making + # a final mask that is the intersection of the original mask and the new mask. The + # reason we even have a color component to this is for debugging visibility. + if dest[3] != 0 and src[3] != 0: + return (255, 0, 0, 255) + else: + return (0, 0, 0, 0) + def pixel_renderer( work: multiprocessing.Queue, results: multiprocessing.Queue, @@ -270,6 +312,7 @@ except ImportError: blendfunc: int, imgbytes: bytes, texbytes: bytes, + maskbytes: Optional[bytes], ) -> None: while True: imgy = work.get() @@ -295,6 +338,10 @@ except ImportError: # Blend it. texoff = texx + (texy * texwidth) + if maskbytes is not None and maskbytes[imgoff] == 0: + # This pixel is masked off! + result.append(imgbytes[(imgoff * 4):((imgoff + 1) * 4)]) + continue result.append(blend_point(add_color, mult_color, texbytes[(texoff * 4):((texoff + 1) * 4)], imgbytes[(imgoff * 4):((imgoff + 1) * 4)], blendfunc)) linebytes = bytes([channel for pixel in result for channel in pixel]) @@ -335,5 +382,11 @@ except ImportError: return blend_subtraction(dest_color, src_color) # TODO: blend mode 75, which is not in the SWF spec and appears to have the equation # Src * (1 - Dst) + Dst * (1 - Src). + elif blendfunc == 256: + # Dummy blend function for calculating masks. + return blend_mask_combine(dest_color, src_color) + elif blendfunc == 257: + # Dummy blend function for calculating masks. + return blend_mask_create(dest_color, src_color) else: return blend_normal(dest_color, src_color) diff --git a/bemani/format/afp/blendalt.pyi b/bemani/format/afp/blendalt.pyi index c552ac4..1df4492 100644 --- a/bemani/format/afp/blendalt.pyi +++ b/bemani/format/afp/blendalt.pyi @@ -1,5 +1,5 @@ from PIL import Image # type: ignore -from typing import Tuple +from typing import Optional, Tuple from .types.generic import Color, Matrix, Point @@ -8,6 +8,7 @@ def affine_composite( add_color: Color, mult_color: Color, transform: Matrix, + mask: Optional[Image.Image], blendfunc: int, texture: Image.Image, single_threaded: bool = False, diff --git a/bemani/format/afp/blendalt.pyx b/bemani/format/afp/blendalt.pyx index f8df54e..1dcafdb 100644 --- a/bemani/format/afp/blendalt.pyx +++ b/bemani/format/afp/blendalt.pyx @@ -1,6 +1,6 @@ import multiprocessing from PIL import Image # type: ignore -from typing import Tuple +from typing import Optional, Tuple from .types.generic import Color, Matrix, Point @@ -24,6 +24,7 @@ cdef extern struct point_t: cdef extern int affine_composite_fast( unsigned char *imgdata, + unsigned char *maskdata, unsigned int imgwidth, unsigned int imgheight, unsigned int minx, @@ -45,6 +46,7 @@ def affine_composite( add_color: Color, mult_color: Color, transform: Matrix, + mask: Optional[Image.Image], blendfunc: int, texture: Image.Image, single_threaded: bool = False, @@ -58,7 +60,7 @@ def affine_composite( # be drawn. return img - if blendfunc not in {0, 1, 2, 3, 8, 9, 70}: + if blendfunc not in {0, 1, 2, 3, 8, 9, 70, 256, 257}: print(f"WARNING: Unsupported blend {blendfunc}") return img @@ -89,6 +91,16 @@ def affine_composite( imgbytes = img.tobytes('raw', 'RGBA') texbytes = texture.tobytes('raw', 'RGBA') + # Grab the mask data. + if mask is not None: + alpha = mask.split()[-1] + maskdata = alpha.tobytes('raw', 'L') + else: + maskdata = None + cdef unsigned char *maskbytes = NULL + if maskdata is not None: + maskbytes = maskdata + # Convert classes to C structs. cdef floatcolor_t c_addcolor = floatcolor_t(r=add_color.r, g=add_color.g, b=add_color.b, a=add_color.a) cdef floatcolor_t c_multcolor = floatcolor_t(r=mult_color.r, g=mult_color.g, b=mult_color.b, a=mult_color.a) @@ -98,6 +110,7 @@ def affine_composite( # Call the C++ function. errors = affine_composite_fast( imgbytes, + maskbytes, imgwidth, imgheight, minx, diff --git a/bemani/format/afp/blendaltimpl.cxx b/bemani/format/afp/blendaltimpl.cxx index 2b8e1fb..d130e6d 100644 --- a/bemani/format/afp/blendaltimpl.cxx +++ b/bemani/format/afp/blendaltimpl.cxx @@ -51,6 +51,7 @@ extern "C" typedef struct work { intcolor_t *imgdata; + unsigned char *maskdata; unsigned int imgwidth; unsigned int minx; unsigned int maxx; @@ -174,6 +175,33 @@ extern "C" }; } + intcolor_t blend_mask_create( + intcolor_t dest, + intcolor_t src + ) { + // Mask creating just allows a pixel to be drawn if the source image has a nonzero + // alpha, according to the SWF spec. + if (src.a != 0) { + return (intcolor_t){255, 0, 0, 255}; + } else { + return (intcolor_t){0, 0, 0, 0}; + } + } + + intcolor_t blend_mask_combine( + intcolor_t dest, + intcolor_t src + ) { + // Mask blending just takes the source and destination and ands them together, making + // a final mask that is the intersection of the original mask and the new mask. The + // reason we even have a color component to this is for debugging visibility. + if (dest.a != 0 && src.a != 0) { + return (intcolor_t){255, 0, 0, 255}; + } else { + return (intcolor_t){0, 0, 0, 0}; + } + } + intcolor_t blend_point( floatcolor_t add_color, floatcolor_t mult_color, @@ -208,6 +236,12 @@ extern "C" if (blendfunc == 9 || blendfunc == 70) { return blend_subtraction(dest_color, src_color); } + if (blendfunc == 256) { + return blend_mask_combine(dest_color, src_color); + } + if (blendfunc == 257) { + return blend_mask_create(dest_color, src_color); + } // TODO: blend mode 75, which is not in the SWF spec and appears to have the equation // Src * (1 - Dst) + Dst * (1 - Src). return blend_normal(dest_color, src_color); @@ -231,6 +265,10 @@ extern "C" // Blend it. unsigned int texoff = texx + (texy * work->texwidth); + if (work->maskdata != NULL && work->maskdata[imgoff] == 0) { + // This pixel is masked off! + continue; + } work->imgdata[imgoff] = blend_point(work->add_color, work->mult_color, work->texdata[texoff], work->imgdata[imgoff], work->blendfunc); } } @@ -244,6 +282,7 @@ extern "C" int affine_composite_fast( unsigned char *imgbytes, + unsigned char *maskbytes, unsigned int imgwidth, unsigned int imgheight, unsigned int minx, @@ -267,6 +306,7 @@ extern "C" // Just create a local work structure so we can call the common function. work_t work; work.imgdata = imgdata; + work.maskdata = maskbytes; work.imgwidth = imgwidth; work.minx = minx; work.maxx = maxx; @@ -308,6 +348,7 @@ extern "C" // Pass to it all of the params it needs. work->imgdata = imgdata; + work->maskdata = maskbytes; work->imgwidth = imgwidth; work->minx = minx; work->maxx = maxx; diff --git a/bemani/format/afp/render.py b/bemani/format/afp/render.py index 7e01bd6..8195c12 100644 --- a/bemani/format/afp/render.py +++ b/bemani/format/afp/render.py @@ -4,7 +4,7 @@ from PIL import Image # type: ignore from .blend import affine_composite from .swf import SWF, Frame, Tag, AP2ShapeTag, AP2DefineSpriteTag, AP2PlaceObjectTag, AP2RemoveObjectTag, AP2DoActionTag, AP2DefineFontTag, AP2DefineEditTextTag, AP2PlaceCameraTag from .decompile import ByteCode -from .types import Color, Matrix, Point, Rectangle, AP2Trigger, AP2Action, PushAction, StoreRegisterAction, StringConstant, Register, THIS, UNDEFINED, GLOBAL +from .types import Color, Matrix, Point, Rectangle, AP2Trigger, AP2Action, PushAction, StoreRegisterAction, StringConstant, Register, NULL, UNDEFINED, GLOBAL, ROOT, PARENT, THIS, CLIP from .geo import Shape, DrawParams from .util import VerboseOutput @@ -46,9 +46,15 @@ class RegisteredDummy: return f"RegisteredDummy(tag_id={self.tag_id})" +class Mask: + def __init__(self, bounds: Rectangle) -> None: + self.bounds = bounds + self.rectangle: Optional[Image.Image] = None + + class PlacedObject: # An object that occupies the screen at some depth. - def __init__(self, object_id: int, depth: int, rotation_offset: Point, transform: Matrix, mult_color: Color, add_color: Color, blend: int, mask: Optional[Rectangle]) -> None: + def __init__(self, object_id: int, depth: int, rotation_offset: Point, transform: Matrix, mult_color: Color, add_color: Color, blend: int, mask: Optional[Mask]) -> None: self.__object_id = object_id self.__depth = depth self.rotation_offset = rotation_offset @@ -86,7 +92,7 @@ class PlacedShape(PlacedObject): mult_color: Color, add_color: Color, blend: int, - mask: Optional[Rectangle], + mask: Optional[Mask], source: RegisteredShape, ) -> None: super().__init__(object_id, depth, rotation_offset, transform, mult_color, add_color, blend, mask) @@ -112,7 +118,7 @@ class PlacedClip(PlacedObject): mult_color: Color, add_color: Color, blend: int, - mask: Optional[Rectangle], + mask: Optional[Mask], source: RegisteredClip, ) -> None: super().__init__(object_id, depth, rotation_offset, transform, mult_color, add_color, blend, mask) @@ -193,7 +199,7 @@ class PlacedDummy(PlacedObject): mult_color: Color, add_color: Color, blend: int, - mask: Optional[Rectangle], + mask: Optional[Mask], source: RegisteredDummy, ) -> None: super().__init__(object_id, depth, rotation_offset, transform, mult_color, add_color, blend, mask) @@ -206,7 +212,7 @@ class PlacedDummy(PlacedObject): class Movie: def __init__(self, root: PlacedClip) -> None: - self.__root = root + self.root = root def getInstanceAtDepth(self, depth: Any) -> Any: if not isinstance(depth, int): @@ -216,7 +222,7 @@ class Movie: # stored added to -0x4000, so let's reverse that. depth = depth + 0x4000 - for obj in self.__root.placed_objects: + for obj in self.root.placed_objects: if obj.depth == depth: return obj @@ -225,29 +231,30 @@ class Movie: class AEPLib: - def __init__(self, this: PlacedObject, movie: Movie) -> None: - self.__this = this - self.__movie = movie - def aep_set_rect_mask(self, thisptr: Any, left: Any, right: Any, top: Any, bottom: Any) -> None: if not isinstance(left, (int, float)) or not isinstance(right, (int, float)) or not isinstance(top, (int, float)) or not isinstance(bottom, (int, float)): - print("WARNING: Ignoring aeplib.aep_set_rect_mask call with invalid parameters!") + print(f"WARNING: Ignoring aeplib.aep_set_rect_mask call with invalid parameters {left}, {right}, {top}, {bottom}!") return - if thisptr is THIS: - self.__this.mask = Rectangle( - left=float(left), - right=float(right), - top=float(top), - bottom=float(bottom), + if isinstance(thisptr, PlacedObject): + thisptr.mask = Mask( + Rectangle( + left=float(left), + right=float(right), + top=float(top), + bottom=float(bottom), + ), ) else: - print("WARNING: Ignoring aeplib.aep_set_rect_mask call with unrecognized target!") + print(f"WARNING: Ignoring aeplib.aep_set_rect_mask call with unrecognized target {thisptr}!") def aep_set_set_frame(self, thisptr: Any, frame: Any) -> None: # I have no idea what this should do, so let's ignore it. pass +MissingThis = object() + + class AFPRenderer(VerboseOutput): def __init__(self, shapes: Dict[str, Shape] = {}, textures: Dict[str, Image.Image] = {}, swfs: Dict[str, SWF] = {}, single_threaded: bool = False) -> None: super().__init__() @@ -319,15 +326,15 @@ class AFPRenderer(VerboseOutput): return paths - def __execute_bytecode(self, bytecode: ByteCode, clip: PlacedClip) -> None: + def __execute_bytecode(self, bytecode: ByteCode, clip: PlacedClip, thisptr: Optional[Any] = MissingThis) -> None: if self.__movie is None: raise Exception("Logic error, executing bytecode outside of a rendering movie clip!") + this = clip if (thisptr is MissingThis) else thisptr location: int = 0 stack: List[Any] = [] variables: Dict[str, Any] = { - 'aeplib': AEPLib(clip, self.__movie), - 'GLOBAL': self.__movie, + 'aeplib': AEPLib(), } registers: List[Any] = [UNDEFINED] * 256 @@ -386,7 +393,7 @@ class AFPRenderer(VerboseOutput): funcname = stack.pop() # Grab the object to perform the call on. - obj = variables['GLOBAL'] + obj = self.__movie # Grab the parameters to pass to the function. num_params = stack.pop() @@ -415,10 +422,34 @@ class AFPRenderer(VerboseOutput): stack.append(obj.alias) else: stack.append(StringConstant.property_to_name(obj.const)) + elif obj is NULL: + stack.append(None) elif obj is THIS: - stack.append(clip) + stack.append(this) elif obj is GLOBAL: stack.append(self.__movie) + elif obj is ROOT: + stack.append(self.__movie.root) + elif obj is CLIP: + # I am not sure this is correct? Maybe it works out + # in circumstances where "THIS" is pointed at something + # else, such as defined function calls maybe? + stack.append(clip) + elif obj is PARENT: + # Find the parent of this clip. + def find_parent(parent: PlacedClip, child: PlacedClip) -> Any: + for obj in parent.placed_objects: + if obj is child: + # This is us, so the parent is our parent. + return parent + if isinstance(obj, PlacedClip): + maybe_parent = find_parent(obj, child) + if maybe_parent is not None: + return maybe_parent + + return None + + stack.append(find_parent(self.__movie.root, clip) or UNDEFINED) else: stack.append(obj) elif isinstance(action, StoreRegisterAction): @@ -683,11 +714,49 @@ class AFPRenderer(VerboseOutput): else: raise Exception(f"Failed to process tag: {tag}") + def __apply_mask( + self, + parent_mask: Image.Image, + transform: Matrix, + mask: Mask, + ) -> Image.Image: + if mask.rectangle is None: + # Calculate the new mask rectangle. + mask.rectangle = Image.new('RGBA', (int(mask.bounds.width), int(mask.bounds.height)), (255, 0, 0, 255)) + + # Offset it by its top/left. + transform = transform.translate(Point(mask.bounds.left, mask.bounds.top)) + + # Draw the mask onto a new image. + calculated_mask = affine_composite( + Image.new('RGBA', (parent_mask.width, parent_mask.height), (0, 0, 0, 0)), + Color(0.0, 0.0, 0.0, 0.0), + Color(1.0, 1.0, 1.0, 1.0), + transform, + None, + 257, + mask.rectangle, + single_threaded=self.__single_threaded, + ) + + # Composite it onto the current mask. + return affine_composite( + parent_mask.copy(), + Color(0.0, 0.0, 0.0, 0.0), + Color(1.0, 1.0, 1.0, 1.0), + Matrix.identity(), + None, + 256, + calculated_mask, + single_threaded=self.__single_threaded, + ) + def __render_object( self, img: Image.Image, renderable: PlacedObject, parent_transform: Matrix, + parent_mask: Image.Image, parent_mult_color: Color, parent_add_color: Color, parent_blend: int, @@ -707,7 +776,9 @@ class AFPRenderer(VerboseOutput): blend = parent_blend if renderable.mask: - print(f"WARNING: Unsupported mask Rectangle({renderable.mask})!") + mask = self.__apply_mask(parent_mask, transform, renderable.mask) + else: + mask = parent_mask # Render individual shapes if this is a sprite. if isinstance(renderable, PlacedClip): @@ -729,7 +800,7 @@ class AFPRenderer(VerboseOutput): for obj in renderable.placed_objects: if obj.depth != depth: continue - img = self.__render_object(img, obj, transform, mult_color, add_color, blend, only_depths=new_only_depths, prefix=prefix + " ") + img = self.__render_object(img, obj, transform, mask, mult_color, add_color, blend, only_depths=new_only_depths, prefix=prefix + " ") elif isinstance(renderable, PlacedShape): if only_depths is not None and renderable.depth not in only_depths: # Not on the correct depth plane. @@ -790,7 +861,7 @@ class AFPRenderer(VerboseOutput): texture = shape.rectangle if texture is not None: - img = affine_composite(img, add_color, mult_color, transform, blend, texture, single_threaded=self.__single_threaded) + img = affine_composite(img, add_color, mult_color, transform, mask, blend, texture, single_threaded=self.__single_threaded) elif isinstance(renderable, PlacedDummy): # Nothing to do! pass @@ -968,6 +1039,9 @@ class AFPRenderer(VerboseOutput): ) self.__movie = Movie(root_clip) + # Create the root mask for where to draw the root clip. + movie_mask = Image.new("RGBA", actual_size, color=(255, 0, 0, 255)) + # These could possibly be overwritten from an external source of we wanted. actual_mult_color = Color(1.0, 1.0, 1.0, 1.0) actual_add_color = Color(0.0, 0.0, 0.0, 0.0) @@ -987,7 +1061,7 @@ class AFPRenderer(VerboseOutput): if changed or frameno == 0: # Now, render out the placed objects. curimage = Image.new("RGBA", actual_size, color=color.as_tuple()) - curimage = self.__render_object(curimage, root_clip, movie_transform, actual_mult_color, actual_add_color, actual_blend, only_depths=only_depths) + curimage = self.__render_object(curimage, root_clip, movie_transform, movie_mask, actual_mult_color, actual_add_color, actual_blend, only_depths=only_depths) else: # Nothing changed, make a copy of the previous render. self.vprint(" Using previous frame render")